diff --git a/apps/desktop/scripts/fixtures/preview-keyboard.cjs b/apps/desktop/scripts/fixtures/preview-keyboard.cjs
new file mode 100644
index 000000000000..340f2113e8b0
--- /dev/null
+++ b/apps/desktop/scripts/fixtures/preview-keyboard.cjs
@@ -0,0 +1,213 @@
+const { app, BrowserWindow, ClipboardItem, clipboard } = require("electron");
+const assert = require("node:assert/strict");
+const { once } = require("node:events");
+const http = require("node:http");
+const Effect = require("effect/Effect");
+const Stream = require("effect/Stream");
+const { chromium } = require("playwright-core");
+const { wsServer } = require("playwright-core/lib/utilsBundle");
+const ServerBrowserPage = require("../../../server/src/preview/ServerBrowserPage.ts");
+const DesktopBrowserHost = require("../../src/preview/DesktopBrowserHost.ts");
+
+app.setPath("userData", process.argv[2]);
+const server = http.createServer((request, response) => {
+ response.setHeader("Content-Type", "text/html");
+ response.end(
+ request.url === "/child"
+ ? ''
+ : `
+
+
+
+
+ `,
+ );
+});
+
+const run = async () => {
+ server.listen(0, "0.0.0.0");
+ await once(server, "listening");
+ await app.whenReady();
+ const originalClipboard = await Promise.all(
+ (await clipboard.read()).map(
+ async (item) =>
+ new ClipboardItem(
+ Object.fromEntries(
+ await Promise.all(item.types.map(async (type) => [type, await item.getType(type)])),
+ ),
+ ),
+ ),
+ );
+ try {
+ const win = new BrowserWindow({
+ width: 800,
+ height: 500,
+ webPreferences: { webviewTag: true, backgroundThrottling: false },
+ });
+ const attached = once(win.webContents, "did-attach-webview");
+ await win.loadURL(
+ `data:text/html,${encodeURIComponent(``)}`,
+ );
+ const [, guest] = await attached;
+ if (guest.isLoading()) await once(guest, "did-stop-loading");
+ guest.debugger.attach("1.3");
+ await guest.debugger.sendCommand("Emulation.setFocusEmulationEnabled", { enabled: true });
+ const host = await Effect.runPromise(DesktopBrowserHost.make);
+ const key = { threadId: "regression", tabId: "guest" };
+ let socket;
+ Effect.runFork(
+ host.events.pipe(
+ Stream.runForEach((line) =>
+ Effect.sync(() => {
+ const event = JSON.parse(new TextDecoder().decode(line));
+ if (event.type === "cdp") socket?.send(event.message);
+ }),
+ ),
+ ),
+ );
+ host.attach(key, { webContents: guest, debugger: guest.debugger });
+ const sockets = new wsServer({ server });
+ sockets.on("connection", (connection) => {
+ socket = connection;
+ connection.on("message", (message) =>
+ Effect.runPromise(
+ host.handleCommandLine(
+ JSON.stringify({ type: "cdp", ...key, message: message.toString() }),
+ ),
+ ),
+ );
+ });
+ const browser = await chromium.connectOverCDP(`ws://127.0.0.1:${server.address().port}`);
+ const page = browser.contexts()[0].pages()[0];
+ const cdp = await page.context().newCDPSession(page);
+ const send = (method, params = {}) => cdp.send(method, params);
+ const press = (key, modifiers = []) => ServerBrowserPage.press(page, { key, modifiers });
+ const focus = async (expression, frame = guest.mainFrame) => {
+ await frame.executeJavaScript(`${expression}.focus()`);
+ win.focus();
+ win.webContents.focus();
+ await win.webContents.executeJavaScript('document.getElementById("draft").focus()');
+ };
+ const value = (selector, frame = guest.mainFrame) =>
+ frame.executeJavaScript(`document.querySelector(${JSON.stringify(selector)}).value`);
+ const preserved = async () => {
+ assert.equal(
+ await win.webContents.executeJavaScript('document.getElementById("draft").value'),
+ "unfinished draft",
+ );
+ assert.equal(await win.webContents.executeJavaScript("document.activeElement.id"), "draft");
+ assert.deepEqual(await win.webContents.executeJavaScript("hostKeys"), []);
+ };
+ const results = [];
+ for (const hidden of [true, false]) {
+ await win.webContents.executeJavaScript(
+ `document.querySelector("webview").style.visibility = ${JSON.stringify(hidden ? "hidden" : "visible")}`,
+ );
+ await focus('document.getElementById("name")');
+ await ServerBrowserPage.type(page, { locator: 'role=textbox[name="Name"]', text: "Jason" });
+ await press("J");
+ await press("o");
+ assert.equal(await value("#name"), "JasonJo");
+ await press("Meta");
+ await press("a", ["Meta"]);
+ await press("Delete");
+ assert.equal(await value("#name"), "");
+ await ServerBrowserPage.type(page, {
+ locator: 'role=textbox[name="Name"]',
+ text: "replacement",
+ clear: true,
+ });
+ assert.equal(await value("#name"), "replacement");
+ await press("Enter");
+ await press("a", ["Meta"]);
+ await press("Delete");
+ await preserved();
+ results.push(hidden ? "hidden input and keys" : "visible input and keys");
+ }
+ await focus('document.getElementById("area")');
+ await send("Input.insertText", { text: "textarea" });
+ await press("!");
+ assert.equal(await value("#area"), "textarea!");
+ await focus('document.getElementById("edit")');
+ await send("Input.insertText", { text: "editor" });
+ await press("x");
+ assert.equal(
+ await guest.executeJavaScript('document.getElementById("edit").textContent'),
+ "editorx",
+ );
+ await focus('document.getElementById("shadow").shadowRoot.querySelector("input")');
+ await send("Input.insertText", { text: "shadow" });
+ await press("x");
+ assert.equal(
+ await guest.executeJavaScript(
+ 'document.getElementById("shadow").shadowRoot.querySelector("input").value',
+ ),
+ "shadowx",
+ );
+ results.push("textarea, contenteditable, and shadow input");
+ for (const frame of guest.mainFrame.frames) {
+ await focus('document.querySelector("input")', frame);
+ await send("Input.insertText", { text: "frame" });
+ await press("x");
+ assert.equal(await value("input", frame), "framex");
+ await preserved();
+ results.push(frame.url.includes("localhost") ? "cross-site frame" : "same-site frame");
+ }
+ await focus('document.getElementById("name")');
+ await send("Input.insertText", { text: "history" });
+ await press("z", ["Meta"]);
+ assert.equal(await value("#name"), "");
+ await press("z", ["Meta", "Shift"]);
+ assert.equal(await value("#name"), "history");
+ await press("a", ["Meta"]);
+ await press("c", ["Meta"]);
+ assert.equal(await clipboard.readText(), "history");
+ await clipboard.writeText("paste");
+ await press("v", ["Meta"]);
+ assert.equal(await value("#name"), "paste");
+ await guest.executeJavaScript(
+ 'document.getElementById("name").addEventListener("keydown", e => { if(e.key === "q") e.preventDefault(); })',
+ );
+ await press("q");
+ assert.equal(await value("#name"), "paste");
+ await guest.executeJavaScript(
+ 'addEventListener("keyup", e => { if(e.key === "y") e.stopImmediatePropagation(); }, true)',
+ );
+ await press("y");
+ assert.equal(await value("#name"), "pastey");
+ await preserved();
+ assert.equal(await guest.executeJavaScript("keys.every(key => key.trusted)"), true);
+ results.push("editing shortcuts, canceled keys, and trusted events");
+ await guest.executeJavaScript("document.body.tabIndex = -1");
+ await focus("document.body");
+ await assert.rejects(send("Input.insertText", { text: "lost" }), /did not accept/);
+ await focus('document.getElementById("name")');
+ await send("Input.insertText", { text: " recovered" });
+ assert.equal(await value("#name"), "pastey recovered");
+ results.push("undeliverable text fails and the queue recovers");
+ await focus('document.getElementById("submit")');
+ await ServerBrowserPage.type(page, { text: "submitted" });
+ const navigated = page.waitForURL(/\/submitted/);
+ await press("Enter");
+ await navigated;
+ await preserved();
+ results.push("Enter navigation preserves the host draft");
+ console.log(JSON.stringify({ results }));
+ await browser.close();
+ sockets.close();
+ win.destroy();
+ server.close();
+ } finally {
+ await clipboard.write(originalClipboard);
+ }
+ app.quit();
+};
+run().catch((error) => {
+ console.error(error);
+ server.close();
+ app.exit(1);
+});
diff --git a/apps/desktop/scripts/preview-keyboard.test.mjs b/apps/desktop/scripts/preview-keyboard.test.mjs
new file mode 100644
index 000000000000..9847590684ad
--- /dev/null
+++ b/apps/desktop/scripts/preview-keyboard.test.mjs
@@ -0,0 +1,46 @@
+import * as NodeChildProcess from "node:child_process";
+import * as NodeFS from "node:fs";
+import * as NodeOS from "node:os";
+import * as NodePath from "node:path";
+import * as NodeModule from "node:module";
+import * as NodeURL from "node:url";
+import { describe, expect, it } from "vite-plus/test";
+
+const require = NodeModule.createRequire(import.meta.url);
+// Opt in because this regression needs a real Electron window and a desktop session.
+describe.skipIf(process.env.T3CODE_TEST_ELECTRON !== "1")("desktop preview keyboard", () => {
+ it("delivers input to visible and hidden guests while preserving the host draft", async () => {
+ const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-preview-keyboard-"));
+ const environment = { ...process.env };
+ delete environment.ELECTRON_RUN_AS_NODE;
+ try {
+ const output = await new Promise((resolve, reject) => {
+ NodeChildProcess.execFile(
+ require("electron"),
+ [
+ NodeURL.fileURLToPath(new URL("./fixtures/preview-keyboard.cjs", import.meta.url)),
+ directory,
+ ],
+ { env: environment, timeout: 30_000 },
+ (error, stdout, stderr) => {
+ if (error) reject(new Error(`${error.message}\n${stdout}\n${stderr}`));
+ else resolve(stdout);
+ },
+ );
+ });
+ const result = JSON.parse(output.trim().split("\n").at(-1));
+ expect(result.results).toEqual([
+ "hidden input and keys",
+ "visible input and keys",
+ "textarea, contenteditable, and shadow input",
+ "same-site frame",
+ "cross-site frame",
+ "editing shortcuts, canceled keys, and trusted events",
+ "undeliverable text fails and the queue recovers",
+ "Enter navigation preserves the host draft",
+ ]);
+ } finally {
+ NodeFS.rmSync(directory, { recursive: true, force: true });
+ }
+ });
+});
diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts
index 81ea040701f0..48c2a98913b2 100644
--- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts
+++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts
@@ -3,6 +3,7 @@ import { describe, expect, it } from "@effect/vitest";
import { DesktopBrowserEvent } from "@t3tools/contracts";
import * as Effect from "effect/Effect";
import * as Fiber from "effect/Fiber";
+import * as Queue from "effect/Queue";
import * as Schema from "effect/Schema";
import * as Stream from "effect/Stream";
import * as NodeEvents from "node:events";
@@ -92,6 +93,68 @@ describe("DesktopBrowserHost", () => {
}).pipe(Effect.scoped),
);
+ it.effect("does not type deferred commands from a released connection", () =>
+ Effect.gen(function* () {
+ const host = yield* DesktopBrowserHost.make;
+ const debuggee = makeDebuggee();
+ const started = Promise.withResolvers();
+ const resume = Promise.withResolvers();
+ const edits: Array = [];
+ let first = true;
+ Object.assign(debuggee.tab.webContents, {
+ mainFrame: {
+ frames: [],
+ executeJavaScript: async (source: string) => {
+ if (source.includes("indexOf(element.contentWindow)")) {
+ if (first) {
+ first = false;
+ started.resolve();
+ await resume.promise;
+ }
+ return -1;
+ }
+ edits.push(source);
+ return true;
+ },
+ },
+ });
+ const replies = yield* Queue.unbounded();
+ yield* host.events.pipe(
+ Stream.runForEach((line) =>
+ Queue.offer(replies, decodeEvent(new TextDecoder().decode(line))),
+ ),
+ Effect.forkScoped,
+ );
+ host.attach(key, debuggee.tab);
+ expect(yield* Queue.take(replies)).toEqual({ type: "attached", ...key });
+ const type = (id: number, text: string) =>
+ host.handleCommandLine(
+ encodeJson({
+ type: "cdp",
+ ...key,
+ message: encodeJson({
+ id,
+ method: "Input.insertText",
+ params: { text },
+ sessionId: "t3-preview-page",
+ }),
+ }),
+ );
+ yield* type(1, "obsolete active input");
+ yield* Effect.promise(() => started.promise);
+ yield* type(2, "obsolete queued input");
+ yield* host.handleCommandLine(encodeJson({ type: "release", ...key }));
+ yield* type(3, "fresh input");
+ resume.resolve();
+ const reply = yield* Queue.take(replies);
+ expect(reply).toMatchObject({ type: "cdp" });
+ expect(decodeCdpReply((reply as { message: string }).message).id).toBe(3);
+ expect(edits).toHaveLength(1);
+ expect(edits[0]).toContain("fresh input");
+ expect(host.isDispatchingKeyboard(debuggee.tab.webContents)).toBe(false);
+ }).pipe(Effect.scoped),
+ );
+
it.effect("saves a server tab's download under its CDP guid where the server asked", () =>
Effect.gen(function* () {
const host = yield* DesktopBrowserHost.make;
diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts
index cfb1ff753c47..1c39b41a4224 100644
--- a/apps/desktop/src/preview/DesktopBrowserHost.ts
+++ b/apps/desktop/src/preview/DesktopBrowserHost.ts
@@ -22,8 +22,10 @@ import * as Option from "effect/Option";
import * as PubSub from "effect/PubSub";
import * as Schema from "effect/Schema";
import * as Stream from "effect/Stream";
+import { clipboard } from "electron";
import { createCdpRelayConnection, type CdpRelayConnection } from "./CdpRelay.ts";
+import { createDesktopBrowserKeyboard } from "./DesktopBrowserKeyboard.ts";
const encodeEvent = Schema.encodeSync(Schema.fromJsonString(DesktopBrowserEvent));
const decodeCommand = Schema.decodeUnknownOption(Schema.fromJsonString(DesktopBrowserCommand));
@@ -50,6 +52,7 @@ const keyOf = ({ threadId, tabId }: DesktopBrowserTabKey) => `${threadId}\u0000$
interface AttachedTab {
readonly key: DesktopBrowserTabKey;
readonly debuggee: DesktopBrowserTabDebugger;
+ readonly keyboard: ReturnType;
relay: CdpRelayConnection | null;
/** Where the server wants this tab's downloads; null keeps Electron's own handling. */
downloadDirectory: string | null;
@@ -84,6 +87,8 @@ export class DesktopBrowserHost extends Context.Service<
* the server sent the page no input just before it started.
*/
readonly humanStartedDownload: (source: Electron.WebContents) => boolean;
+ /** Suppresses desktop shortcuts while a native agent key is being delivered. */
+ readonly isDispatchingKeyboard: (source: Electron.WebContents) => boolean;
/** Points a server tab's download at the server; false for any other download. */
readonly placeDownload: (source: Electron.WebContents, item: Electron.DownloadItem) => boolean;
/** The agent's cursor positions for attached tabs, keyed by their server tab. */
@@ -114,9 +119,12 @@ export const make = Effect.gen(function* () {
const relay: CdpRelayConnection = createCdpRelayConnection(
{
send: (method, params, sessionId) =>
- sessionId === undefined
- ? debuggee.sendCommand(method, params)
- : debuggee.sendCommand(method, params, sessionId),
+ sessionId === undefined &&
+ (method === "Input.insertText" || method === "Input.dispatchKeyEvent")
+ ? tab.keyboard.send(method, params)
+ : sessionId === undefined
+ ? debuggee.sendCommand(method, params)
+ : debuggee.sendCommand(method, params, sessionId),
targetId: () =>
debuggee
.sendCommand("Target.getTargetInfo")
@@ -158,6 +166,7 @@ export const make = Effect.gen(function* () {
const id = keyOf(key);
const tab = tabs.get(id);
if (!tab) return;
+ tab.keyboard.cancel();
tabs.delete(id);
tab.debuggee.debugger.off("message", tab.onMessage);
emit({ type: "detached", ...key });
@@ -170,6 +179,9 @@ export const make = Effect.gen(function* () {
const tab: AttachedTab = {
key,
debuggee,
+ keyboard: createDesktopBrowserKeyboard(debuggee.webContents, debuggee.debugger, () =>
+ clipboard.read(),
+ ),
relay: null,
downloadDirectory: null,
pendingDownloadGuid: null,
@@ -200,6 +212,7 @@ export const make = Effect.gen(function* () {
}
if (command.value.type === "release") {
// A new server connection starts with a fresh relay and fresh sessions.
+ tab.keyboard.cancel();
tab.relay = null;
return;
}
@@ -217,6 +230,7 @@ export const make = Effect.gen(function* () {
Effect.forEach(
[...tabs.values()],
(tab) => {
+ tab.keyboard.cancel();
tab.relay = null;
return PubSub.publish(outbox, { type: "attached", ...tab.key });
},
@@ -239,6 +253,10 @@ export const make = Effect.gen(function* () {
detach,
placeDownload,
humanStartedDownload,
+ isDispatchingKeyboard: (source) =>
+ [...tabs.values()].some(
+ (tab) => tab.debuggee.webContents === source && tab.keyboard.isDispatching(),
+ ),
});
});
diff --git a/apps/desktop/src/preview/DesktopBrowserKeyboard.ts b/apps/desktop/src/preview/DesktopBrowserKeyboard.ts
new file mode 100644
index 000000000000..2f86b3c82ab7
--- /dev/null
+++ b/apps/desktop/src/preview/DesktopBrowserKeyboard.ts
@@ -0,0 +1,339 @@
+// @effect-diagnostics globalTimers:off -- The Electron CDP callback adapter bounds native renderer receipts outside an Effect runtime.
+/** Routes Playwright's keyboard packets to the guest, without changing desktop focus. */
+export function createDesktopBrowserKeyboard(
+ contents: Electron.WebContents,
+ debuggee: Electron.Debugger,
+ readClipboard: () => Promise>,
+) {
+ let pending = 0;
+ let sequence = 0;
+ let generation = 0;
+ let tail: Promise = Promise.resolve();
+
+ const focusedFrame = async () => {
+ // Electron's focusedFrame follows native focus and can be null while the
+ // desktop owns it. Follow DOM focus instead, including cross-site frames.
+ let frame = contents.mainFrame;
+ while (true) {
+ const index: unknown = await frame.executeJavaScript(`(() => {
+ let element = document.activeElement;
+ while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement;
+ if (element?.tagName !== "IFRAME" && element?.tagName !== "FRAME") return -1;
+ return Array.from({ length: window.length }, (_, i) => window[i]).indexOf(element.contentWindow);
+ })()`);
+ if (index === -1) return frame;
+ if (typeof index !== "number" || !frame.frames[index]) {
+ throw new Error("The focused preview frame is unavailable.");
+ }
+ frame = frame.frames[index]!;
+ }
+ };
+
+ const dispatch = async (event: Electron.KeyboardInputEvent, checkCurrent: () => void) => {
+ checkCurrent();
+ const packet = { ...event, skipIfUnhandled: true };
+ contents.setIgnoreMenuShortcuts(true);
+ // Chromium suppresses the char packet when keydown is canceled. The
+ // following keyup receipt confirms the whole native sequence was handled.
+ if (event.type === "char") {
+ contents.sendInputEvent(packet);
+ return false;
+ }
+ const frame = await focusedFrame();
+ const receipt = `__t3KeyboardReceipt${++sequence}`;
+ const type = event.type === "keyUp" ? "keyup" : "keydown";
+ await frame.executeJavaScript(`(() => {
+ const name = ${JSON.stringify(receipt)};
+ const type = ${JSON.stringify(type)};
+ let finish;
+ const promise = new Promise(resolve => { finish = resolve; });
+ let observed = false;
+ let received;
+ const count = performance.eventCounts.get(type) ?? 0;
+ const channel = new MessageChannel();
+ channel.port1.onmessage = () => done({ prevented: received.defaultPrevented });
+ const listener = event => {
+ if (!event.isTrusted) return;
+ observed = true;
+ received = event;
+ // A task runs after every page listener and the default edit. A
+ // microtask can run between listeners, before preventDefault.
+ channel.port2.postMessage(null);
+ };
+ const pagehide = () => done(observed ? { prevented: false } : null);
+ const timer = setTimeout(() => done(null), 5000);
+ const check = setInterval(() => {
+ // Trusted event counts also confirm delivery when page code hides a
+ // key with stopImmediatePropagation before our listener sees it.
+ if (!observed && (performance.eventCounts.get(type) ?? 0) > count) done({ prevented: false });
+ }, 16);
+ const done = result => {
+ clearTimeout(timer);
+ clearInterval(check);
+ channel.port1.close();
+ channel.port2.close();
+ removeEventListener(type, listener, true);
+ removeEventListener("pagehide", pagehide, true);
+ finish(result);
+ };
+ addEventListener(type, listener, true);
+ addEventListener("pagehide", pagehide, true);
+ globalThis[name] = { promise, dispose: () => done(null) };
+ })()`);
+ let timeout: ReturnType | undefined;
+ try {
+ // CDP's root Input commands can target the embedder's focused renderer.
+ // Native packets address this guest widget; skipIfUnhandled also keeps
+ // keys such as Enter from falling back to the desktop composer.
+ checkCurrent();
+ contents.sendInputEvent(packet);
+ const result: unknown = await Promise.race([
+ frame.executeJavaScript(`globalThis[${JSON.stringify(receipt)}]?.promise`),
+ new Promise((_, reject) => {
+ timeout = setTimeout(
+ () => reject(new Error("The preview page did not receive the keyboard event.")),
+ 5000,
+ );
+ }),
+ ]);
+ if (typeof result !== "object" || result === null || !("prevented" in result)) {
+ throw new Error("The preview page did not receive the keyboard event.");
+ }
+ return result.prevented === true;
+ } finally {
+ clearTimeout(timeout);
+ if (!frame.detached) {
+ void frame
+ .executeJavaScript(
+ `globalThis[${JSON.stringify(receipt)}]?.dispose(); delete globalThis[${JSON.stringify(receipt)}]`,
+ )
+ .catch(() => {});
+ }
+ }
+ };
+
+ const press = async (params: Record, checkCurrent: () => void) => {
+ const type = params["type"];
+ const key = params["key"];
+ if (
+ (type !== "keyDown" && type !== "rawKeyDown" && type !== "keyUp" && type !== "char") ||
+ typeof key !== "string"
+ ) {
+ throw new Error("Invalid preview keyboard event.");
+ }
+ const mask = typeof params["modifiers"] === "number" ? params["modifiers"] : 0;
+ const target = await focusedFrame();
+ if (target.processId !== contents.mainFrame.processId) {
+ // A cross-site frame has its own renderer session. Its Input commands
+ // bypass the root target's embedder-focus lookup.
+ const sessions: Array = [];
+ let sessionId: string | undefined;
+ let contextId: number | undefined;
+ try {
+ while (true) {
+ const evaluated = (await debuggee.sendCommand(
+ "Runtime.evaluate",
+ {
+ expression: `(() => {
+ let element = document.activeElement;
+ while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement;
+ return element?.tagName === "IFRAME" || element?.tagName === "FRAME" ? element : null;
+ })()`,
+ ...(contextId === undefined ? {} : { contextId }),
+ },
+ sessionId,
+ )) as { result: { objectId?: string; subtype?: string } };
+ if (evaluated.result.subtype === "null") break;
+ const objectId = evaluated.result.objectId;
+ if (!objectId) throw new Error("The focused preview frame is unavailable.");
+ let frameId: string | undefined;
+ try {
+ const described = (await debuggee.sendCommand(
+ "DOM.describeNode",
+ { objectId },
+ sessionId,
+ )) as { node: { frameId?: string } };
+ frameId = described.node.frameId;
+ } finally {
+ await debuggee
+ .sendCommand("Runtime.releaseObject", { objectId }, sessionId)
+ .catch(() => {});
+ }
+ if (!frameId) throw new Error("The focused preview frame is unavailable.");
+ const targets = (await debuggee.sendCommand("Target.getTargets")) as {
+ targetInfos: ReadonlyArray<{ targetId: string; type: string }>;
+ };
+ if (
+ targets.targetInfos.some(
+ (target) => target.type === "iframe" && target.targetId === frameId,
+ )
+ ) {
+ const attached = (await debuggee.sendCommand("Target.attachToTarget", {
+ targetId: frameId,
+ flatten: true,
+ })) as { sessionId: string };
+ sessionId = attached.sessionId;
+ sessions.push(sessionId);
+ contextId = undefined;
+ } else {
+ const world = (await debuggee.sendCommand(
+ "Page.createIsolatedWorld",
+ { frameId, worldName: "t3-preview-keyboard" },
+ sessionId,
+ )) as { executionContextId: number };
+ contextId = world.executionContextId;
+ }
+ }
+ if (!sessionId) throw new Error("The focused preview frame is unavailable.");
+ await debuggee.sendCommand(
+ "Emulation.setFocusEmulationEnabled",
+ { enabled: true },
+ sessionId,
+ );
+ checkCurrent();
+ await debuggee.sendCommand("Input.dispatchKeyEvent", params, sessionId);
+ } finally {
+ if (sessionId)
+ await debuggee
+ .sendCommand("Emulation.setFocusEmulationEnabled", { enabled: false }, sessionId)
+ .catch(() => {});
+ for (const id of sessions)
+ await debuggee.sendCommand("Target.detachFromTarget", { sessionId: id }).catch(() => {});
+ }
+ return;
+ }
+ const modifiers: NonNullable = [];
+ for (const [bit, modifier] of [
+ [1, "alt"],
+ [2, "control"],
+ [4, "meta"],
+ [8, "shift"],
+ ] as const) {
+ if (mask & bit) modifiers.push(modifier);
+ }
+ if (params["isKeypad"] === true) modifiers.push("iskeypad");
+ if (params["autoRepeat"] === true) modifiers.push("isautorepeat");
+ if (params["location"] === 1) modifiers.push("left");
+ if (params["location"] === 2) modifiers.push("right");
+ const keyCode = key.startsWith("Arrow") ? key.slice(5) : key === " " ? "Space" : key;
+ const prevented = await dispatch(
+ {
+ type: type === "rawKeyDown" ? "keyDown" : type,
+ keyCode,
+ modifiers,
+ },
+ checkCurrent,
+ );
+ if (type === "keyUp" || type === "char" || prevented) return;
+ const commands = params["commands"];
+ if (Array.isArray(commands) && commands.length > 0) {
+ const frame = await focusedFrame();
+ const clipboardData: Array<{ type: string; data: string }> = [];
+ if (commands.includes("paste")) {
+ for (const item of await readClipboard()) {
+ for (const type of item.types) {
+ if (type.startsWith("electron ")) continue;
+ const blob = await item.getType(type);
+ if (!("arrayBuffer" in blob)) continue;
+ clipboardData.push({
+ type,
+ data: type.startsWith("text/")
+ ? await blob.text()
+ : Buffer.from(await blob.arrayBuffer()).toString("base64"),
+ });
+ }
+ }
+ }
+ checkCurrent();
+ await frame.executeJavaScript(
+ `(() => {
+ for (const command of ${JSON.stringify(commands)}) {
+ let element = document.activeElement;
+ while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement;
+ if (command === "paste") {
+ const transfer = new DataTransfer();
+ for (const { type, data } of ${JSON.stringify(clipboardData)}) {
+ if (type === "text/html") {
+ const container = document.createElement("div");
+ container.setHTML(data);
+ transfer.setData(type, container.innerHTML);
+ } else if (type.startsWith("text/")) transfer.setData(type, data);
+ else transfer.items.add(new File([Uint8Array.from(atob(data), c => c.charCodeAt(0))], "clipboard", { type }));
+ }
+ if (!element?.dispatchEvent(new ClipboardEvent("paste", { clipboardData: transfer, bubbles: true, cancelable: true, composed: true }))) continue;
+ const text = transfer.getData("text/plain");
+ if (!element.dispatchEvent(new InputEvent("beforeinput", { inputType: "insertFromPaste", data: text, dataTransfer: transfer, bubbles: true, cancelable: true, composed: true }))) continue;
+ const html = element.isContentEditable ? transfer.getData("text/html") : "";
+ document.execCommand(html ? "insertHTML" : "insertText", false, html || text);
+ } else if (command.startsWith("moveTo")) {
+ document.getSelection()?.modify(
+ command.endsWith("AndModifySelection") ? "extend" : "move",
+ command.includes("Beginning") ? "backward" : command.includes("Left") ? "left" : command.includes("Right") ? "right" : "forward",
+ command.includes("Document") ? "documentboundary" : "lineboundary",
+ );
+ } else if (command === "deleteToBeginningOfLine") {
+ const selection = document.getSelection();
+ if (!element?.dispatchEvent(new InputEvent("beforeinput", { inputType: "deleteSoftLineBackward", bubbles: true, cancelable: true, composed: true }))) continue;
+ const collapsed = typeof element?.selectionStart === "number" ? element.selectionStart === element.selectionEnd : selection?.isCollapsed;
+ if (collapsed) selection?.modify("extend", "backward", "lineboundary");
+ document.execCommand("delete");
+ } else {
+ const inputType = command === "undo" ? "historyUndo" : command === "redo" ? "historyRedo" : null;
+ if (inputType && !element?.dispatchEvent(new InputEvent("beforeinput", { inputType, bubbles: true, cancelable: true, composed: true }))) continue;
+ document.execCommand(command);
+ }
+ }
+ })()`,
+ true,
+ );
+ }
+ const text = params["text"];
+ if (type === "keyDown" && typeof text === "string" && text.length > 0) {
+ await dispatch({ type: "char", keyCode: text, modifiers }, checkCurrent);
+ }
+ };
+
+ return {
+ isDispatching: () => pending > 0,
+ cancel: () => {
+ generation++;
+ },
+ send: (method: string, params: Record) => {
+ pending++;
+ const current = generation;
+ const checkCurrent = () => {
+ if (current !== generation)
+ throw new Error("The preview keyboard connection was released.");
+ };
+ const result = tail.then(async () => {
+ checkCurrent();
+ if (method === "Input.insertText") {
+ const text = params["text"];
+ if (typeof text !== "string") throw new Error("Invalid preview text.");
+ if (text.length > 0) {
+ const frame = await focusedFrame();
+ checkCurrent();
+ const inserted: unknown = await frame.executeJavaScript(`(() => {
+ let element = document.activeElement;
+ while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement;
+ const text = ${JSON.stringify(text)};
+ const nonText = ["button", "checkbox", "color", "file", "hidden", "image", "radio", "range", "reset", "submit"];
+ const textControl = element?.tagName === "TEXTAREA" || (element?.tagName === "INPUT" && !nonText.includes(element.type));
+ if (!(textControl || element?.isContentEditable) || element.disabled || element.readOnly) return false;
+ if (!element?.dispatchEvent(new InputEvent("beforeinput", { inputType: "insertText", data: text, bubbles: true, cancelable: true, composed: true }))) return true;
+ return document.execCommand("insertText", false, text);
+ })()`);
+ if (inserted !== true) throw new Error("The preview page did not accept the text.");
+ }
+ } else {
+ await press(params, checkCurrent);
+ }
+ return {};
+ });
+ tail = result.catch(() => {});
+ return result.finally(() => {
+ pending--;
+ });
+ },
+ };
+}
diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts
index 84c6273aad62..8d2f3d3b069e 100644
--- a/apps/desktop/src/preview/Manager.test.ts
+++ b/apps/desktop/src/preview/Manager.test.ts
@@ -685,6 +685,83 @@ describe("PreviewManager", () => {
),
);
+ effectIt.effect(
+ "agent keyboard input does not run desktop shortcuts, while human input still does",
+ () =>
+ Effect.gen(function* () {
+ const manager = yield* PreviewManager.PreviewManager;
+ const preview = makeFaviconWebContents();
+ const frame = {
+ processId: 1,
+ frames: [],
+ detached: false,
+ executeJavaScript: async (source: string) =>
+ source.includes("indexOf(element.contentWindow)")
+ ? -1
+ : source.endsWith("?.promise")
+ ? { prevented: false }
+ : undefined,
+ };
+ const input = {
+ type: "keyDown",
+ key: "r",
+ code: "KeyR",
+ meta: true,
+ control: false,
+ shift: false,
+ alt: false,
+ };
+ Object.assign(preview.webContents, {
+ mainFrame: frame,
+ sendInputEvent: () =>
+ preview.listeners.get("before-input-event")!(
+ { preventDefault: vi.fn() } as never,
+ input as never,
+ ),
+ });
+ fromId.mockReturnValue(preview.webContents);
+ getFocusedWebContents.mockReturnValue(preview.webContents as never);
+ yield* manager.createTab("tab_agent_keys");
+ yield* manager.registerWebview("tab_agent_keys", 42);
+ const host = yield* DesktopBrowserHost.DesktopBrowserHost;
+ const key = { threadId: "thread-keys", tabId: "server-keys" };
+ const replies = yield* Queue.unbounded();
+ yield* host.events.pipe(
+ Stream.runForEach((line) => Queue.offer(replies, new TextDecoder().decode(line))),
+ Effect.forkScoped,
+ );
+ host.attach(key, {
+ webContents: preview.webContents as Electron.WebContents,
+ debugger: (preview.webContents as Electron.WebContents).debugger,
+ });
+ expect(JSON.parse(yield* Queue.take(replies)).type).toBe("attached");
+ yield* host.handleCommandLine(
+ JSON.stringify({
+ type: "cdp",
+ ...key,
+ message: JSON.stringify({
+ id: 1,
+ method: "Input.dispatchKeyEvent",
+ params: { type: "rawKeyDown", key: "r", modifiers: 4 },
+ sessionId: "t3-preview-page",
+ }),
+ }),
+ );
+ let reply: string;
+ do {
+ reply = yield* Queue.take(replies);
+ } while (JSON.parse(reply).type !== "cdp");
+ expect(JSON.parse(JSON.parse(reply).message)).toMatchObject({ id: 1, result: {} });
+ expect(preview.reload).not.toHaveBeenCalled();
+ expect(host.isDispatchingKeyboard(preview.webContents as Electron.WebContents)).toBe(false);
+ const preventDefault = vi.fn();
+ preview.listeners.get("before-input-event")!({ preventDefault } as never, input as never);
+ expect(preventDefault).toHaveBeenCalledOnce();
+ yield* Effect.yieldNow;
+ expect(preview.reload).toHaveBeenCalledOnce();
+ }).pipe(Effect.provide(managerLayer()), Effect.scoped),
+ );
+
effectIt.effect.each([
["mod+shift+t", "view.reopenClosed"],
["ctrl+alt+u", "sidebar.toggle"],
diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts
index 123943f70f9a..5c47cfbd9923 100644
--- a/apps/desktop/src/preview/Manager.ts
+++ b/apps/desktop/src/preview/Manager.ts
@@ -1527,6 +1527,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
});
};
const beforeInput = (event: Electron.Event, input: Electron.Input): void => {
+ if (browserHost.isDispatchingKeyboard(wc)) {
+ wc.setIgnoreMenuShortcuts(true);
+ return;
+ }
syncMenuShortcuts(wc, input);
const host = wc.hostWebContents;
const forwarded =