diff --git a/apps/desktop/scripts/fixtures/preview-keyboard.cjs b/apps/desktop/scripts/fixtures/preview-keyboard.cjs new file mode 100644 index 000000000000..340f2113e8b0 --- /dev/null +++ b/apps/desktop/scripts/fixtures/preview-keyboard.cjs @@ -0,0 +1,213 @@ +const { app, BrowserWindow, ClipboardItem, clipboard } = require("electron"); +const assert = require("node:assert/strict"); +const { once } = require("node:events"); +const http = require("node:http"); +const Effect = require("effect/Effect"); +const Stream = require("effect/Stream"); +const { chromium } = require("playwright-core"); +const { wsServer } = require("playwright-core/lib/utilsBundle"); +const ServerBrowserPage = require("../../../server/src/preview/ServerBrowserPage.ts"); +const DesktopBrowserHost = require("../../src/preview/DesktopBrowserHost.ts"); + +app.setPath("userData", process.argv[2]); +const server = http.createServer((request, response) => { + response.setHeader("Content-Type", "text/html"); + response.end( + request.url === "/child" + ? '' + : ` +
+
+ + + `, + ); +}); + +const run = async () => { + server.listen(0, "0.0.0.0"); + await once(server, "listening"); + await app.whenReady(); + const originalClipboard = await Promise.all( + (await clipboard.read()).map( + async (item) => + new ClipboardItem( + Object.fromEntries( + await Promise.all(item.types.map(async (type) => [type, await item.getType(type)])), + ), + ), + ), + ); + try { + const win = new BrowserWindow({ + width: 800, + height: 500, + webPreferences: { webviewTag: true, backgroundThrottling: false }, + }); + const attached = once(win.webContents, "did-attach-webview"); + await win.loadURL( + `data:text/html,${encodeURIComponent(``)}`, + ); + const [, guest] = await attached; + if (guest.isLoading()) await once(guest, "did-stop-loading"); + guest.debugger.attach("1.3"); + await guest.debugger.sendCommand("Emulation.setFocusEmulationEnabled", { enabled: true }); + const host = await Effect.runPromise(DesktopBrowserHost.make); + const key = { threadId: "regression", tabId: "guest" }; + let socket; + Effect.runFork( + host.events.pipe( + Stream.runForEach((line) => + Effect.sync(() => { + const event = JSON.parse(new TextDecoder().decode(line)); + if (event.type === "cdp") socket?.send(event.message); + }), + ), + ), + ); + host.attach(key, { webContents: guest, debugger: guest.debugger }); + const sockets = new wsServer({ server }); + sockets.on("connection", (connection) => { + socket = connection; + connection.on("message", (message) => + Effect.runPromise( + host.handleCommandLine( + JSON.stringify({ type: "cdp", ...key, message: message.toString() }), + ), + ), + ); + }); + const browser = await chromium.connectOverCDP(`ws://127.0.0.1:${server.address().port}`); + const page = browser.contexts()[0].pages()[0]; + const cdp = await page.context().newCDPSession(page); + const send = (method, params = {}) => cdp.send(method, params); + const press = (key, modifiers = []) => ServerBrowserPage.press(page, { key, modifiers }); + const focus = async (expression, frame = guest.mainFrame) => { + await frame.executeJavaScript(`${expression}.focus()`); + win.focus(); + win.webContents.focus(); + await win.webContents.executeJavaScript('document.getElementById("draft").focus()'); + }; + const value = (selector, frame = guest.mainFrame) => + frame.executeJavaScript(`document.querySelector(${JSON.stringify(selector)}).value`); + const preserved = async () => { + assert.equal( + await win.webContents.executeJavaScript('document.getElementById("draft").value'), + "unfinished draft", + ); + assert.equal(await win.webContents.executeJavaScript("document.activeElement.id"), "draft"); + assert.deepEqual(await win.webContents.executeJavaScript("hostKeys"), []); + }; + const results = []; + for (const hidden of [true, false]) { + await win.webContents.executeJavaScript( + `document.querySelector("webview").style.visibility = ${JSON.stringify(hidden ? "hidden" : "visible")}`, + ); + await focus('document.getElementById("name")'); + await ServerBrowserPage.type(page, { locator: 'role=textbox[name="Name"]', text: "Jason" }); + await press("J"); + await press("o"); + assert.equal(await value("#name"), "JasonJo"); + await press("Meta"); + await press("a", ["Meta"]); + await press("Delete"); + assert.equal(await value("#name"), ""); + await ServerBrowserPage.type(page, { + locator: 'role=textbox[name="Name"]', + text: "replacement", + clear: true, + }); + assert.equal(await value("#name"), "replacement"); + await press("Enter"); + await press("a", ["Meta"]); + await press("Delete"); + await preserved(); + results.push(hidden ? "hidden input and keys" : "visible input and keys"); + } + await focus('document.getElementById("area")'); + await send("Input.insertText", { text: "textarea" }); + await press("!"); + assert.equal(await value("#area"), "textarea!"); + await focus('document.getElementById("edit")'); + await send("Input.insertText", { text: "editor" }); + await press("x"); + assert.equal( + await guest.executeJavaScript('document.getElementById("edit").textContent'), + "editorx", + ); + await focus('document.getElementById("shadow").shadowRoot.querySelector("input")'); + await send("Input.insertText", { text: "shadow" }); + await press("x"); + assert.equal( + await guest.executeJavaScript( + 'document.getElementById("shadow").shadowRoot.querySelector("input").value', + ), + "shadowx", + ); + results.push("textarea, contenteditable, and shadow input"); + for (const frame of guest.mainFrame.frames) { + await focus('document.querySelector("input")', frame); + await send("Input.insertText", { text: "frame" }); + await press("x"); + assert.equal(await value("input", frame), "framex"); + await preserved(); + results.push(frame.url.includes("localhost") ? "cross-site frame" : "same-site frame"); + } + await focus('document.getElementById("name")'); + await send("Input.insertText", { text: "history" }); + await press("z", ["Meta"]); + assert.equal(await value("#name"), ""); + await press("z", ["Meta", "Shift"]); + assert.equal(await value("#name"), "history"); + await press("a", ["Meta"]); + await press("c", ["Meta"]); + assert.equal(await clipboard.readText(), "history"); + await clipboard.writeText("paste"); + await press("v", ["Meta"]); + assert.equal(await value("#name"), "paste"); + await guest.executeJavaScript( + 'document.getElementById("name").addEventListener("keydown", e => { if(e.key === "q") e.preventDefault(); })', + ); + await press("q"); + assert.equal(await value("#name"), "paste"); + await guest.executeJavaScript( + 'addEventListener("keyup", e => { if(e.key === "y") e.stopImmediatePropagation(); }, true)', + ); + await press("y"); + assert.equal(await value("#name"), "pastey"); + await preserved(); + assert.equal(await guest.executeJavaScript("keys.every(key => key.trusted)"), true); + results.push("editing shortcuts, canceled keys, and trusted events"); + await guest.executeJavaScript("document.body.tabIndex = -1"); + await focus("document.body"); + await assert.rejects(send("Input.insertText", { text: "lost" }), /did not accept/); + await focus('document.getElementById("name")'); + await send("Input.insertText", { text: " recovered" }); + assert.equal(await value("#name"), "pastey recovered"); + results.push("undeliverable text fails and the queue recovers"); + await focus('document.getElementById("submit")'); + await ServerBrowserPage.type(page, { text: "submitted" }); + const navigated = page.waitForURL(/\/submitted/); + await press("Enter"); + await navigated; + await preserved(); + results.push("Enter navigation preserves the host draft"); + console.log(JSON.stringify({ results })); + await browser.close(); + sockets.close(); + win.destroy(); + server.close(); + } finally { + await clipboard.write(originalClipboard); + } + app.quit(); +}; +run().catch((error) => { + console.error(error); + server.close(); + app.exit(1); +}); diff --git a/apps/desktop/scripts/preview-keyboard.test.mjs b/apps/desktop/scripts/preview-keyboard.test.mjs new file mode 100644 index 000000000000..9847590684ad --- /dev/null +++ b/apps/desktop/scripts/preview-keyboard.test.mjs @@ -0,0 +1,46 @@ +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeModule from "node:module"; +import * as NodeURL from "node:url"; +import { describe, expect, it } from "vite-plus/test"; + +const require = NodeModule.createRequire(import.meta.url); +// Opt in because this regression needs a real Electron window and a desktop session. +describe.skipIf(process.env.T3CODE_TEST_ELECTRON !== "1")("desktop preview keyboard", () => { + it("delivers input to visible and hidden guests while preserving the host draft", async () => { + const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-preview-keyboard-")); + const environment = { ...process.env }; + delete environment.ELECTRON_RUN_AS_NODE; + try { + const output = await new Promise((resolve, reject) => { + NodeChildProcess.execFile( + require("electron"), + [ + NodeURL.fileURLToPath(new URL("./fixtures/preview-keyboard.cjs", import.meta.url)), + directory, + ], + { env: environment, timeout: 30_000 }, + (error, stdout, stderr) => { + if (error) reject(new Error(`${error.message}\n${stdout}\n${stderr}`)); + else resolve(stdout); + }, + ); + }); + const result = JSON.parse(output.trim().split("\n").at(-1)); + expect(result.results).toEqual([ + "hidden input and keys", + "visible input and keys", + "textarea, contenteditable, and shadow input", + "same-site frame", + "cross-site frame", + "editing shortcuts, canceled keys, and trusted events", + "undeliverable text fails and the queue recovers", + "Enter navigation preserves the host draft", + ]); + } finally { + NodeFS.rmSync(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 81ea040701f0..48c2a98913b2 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "@effect/vitest"; import { DesktopBrowserEvent } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; +import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import * as NodeEvents from "node:events"; @@ -92,6 +93,68 @@ describe("DesktopBrowserHost", () => { }).pipe(Effect.scoped), ); + it.effect("does not type deferred commands from a released connection", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + const started = Promise.withResolvers(); + const resume = Promise.withResolvers(); + const edits: Array = []; + let first = true; + Object.assign(debuggee.tab.webContents, { + mainFrame: { + frames: [], + executeJavaScript: async (source: string) => { + if (source.includes("indexOf(element.contentWindow)")) { + if (first) { + first = false; + started.resolve(); + await resume.promise; + } + return -1; + } + edits.push(source); + return true; + }, + }, + }); + const replies = yield* Queue.unbounded(); + yield* host.events.pipe( + Stream.runForEach((line) => + Queue.offer(replies, decodeEvent(new TextDecoder().decode(line))), + ), + Effect.forkScoped, + ); + host.attach(key, debuggee.tab); + expect(yield* Queue.take(replies)).toEqual({ type: "attached", ...key }); + const type = (id: number, text: string) => + host.handleCommandLine( + encodeJson({ + type: "cdp", + ...key, + message: encodeJson({ + id, + method: "Input.insertText", + params: { text }, + sessionId: "t3-preview-page", + }), + }), + ); + yield* type(1, "obsolete active input"); + yield* Effect.promise(() => started.promise); + yield* type(2, "obsolete queued input"); + yield* host.handleCommandLine(encodeJson({ type: "release", ...key })); + yield* type(3, "fresh input"); + resume.resolve(); + const reply = yield* Queue.take(replies); + expect(reply).toMatchObject({ type: "cdp" }); + expect(decodeCdpReply((reply as { message: string }).message).id).toBe(3); + expect(edits).toHaveLength(1); + expect(edits[0]).toContain("fresh input"); + expect(host.isDispatchingKeyboard(debuggee.tab.webContents)).toBe(false); + }).pipe(Effect.scoped), + ); + it.effect("saves a server tab's download under its CDP guid where the server asked", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index cfb1ff753c47..1c39b41a4224 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -22,8 +22,10 @@ import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; +import { clipboard } from "electron"; import { createCdpRelayConnection, type CdpRelayConnection } from "./CdpRelay.ts"; +import { createDesktopBrowserKeyboard } from "./DesktopBrowserKeyboard.ts"; const encodeEvent = Schema.encodeSync(Schema.fromJsonString(DesktopBrowserEvent)); const decodeCommand = Schema.decodeUnknownOption(Schema.fromJsonString(DesktopBrowserCommand)); @@ -50,6 +52,7 @@ const keyOf = ({ threadId, tabId }: DesktopBrowserTabKey) => `${threadId}\u0000$ interface AttachedTab { readonly key: DesktopBrowserTabKey; readonly debuggee: DesktopBrowserTabDebugger; + readonly keyboard: ReturnType; relay: CdpRelayConnection | null; /** Where the server wants this tab's downloads; null keeps Electron's own handling. */ downloadDirectory: string | null; @@ -84,6 +87,8 @@ export class DesktopBrowserHost extends Context.Service< * the server sent the page no input just before it started. */ readonly humanStartedDownload: (source: Electron.WebContents) => boolean; + /** Suppresses desktop shortcuts while a native agent key is being delivered. */ + readonly isDispatchingKeyboard: (source: Electron.WebContents) => boolean; /** Points a server tab's download at the server; false for any other download. */ readonly placeDownload: (source: Electron.WebContents, item: Electron.DownloadItem) => boolean; /** The agent's cursor positions for attached tabs, keyed by their server tab. */ @@ -114,9 +119,12 @@ export const make = Effect.gen(function* () { const relay: CdpRelayConnection = createCdpRelayConnection( { send: (method, params, sessionId) => - sessionId === undefined - ? debuggee.sendCommand(method, params) - : debuggee.sendCommand(method, params, sessionId), + sessionId === undefined && + (method === "Input.insertText" || method === "Input.dispatchKeyEvent") + ? tab.keyboard.send(method, params) + : sessionId === undefined + ? debuggee.sendCommand(method, params) + : debuggee.sendCommand(method, params, sessionId), targetId: () => debuggee .sendCommand("Target.getTargetInfo") @@ -158,6 +166,7 @@ export const make = Effect.gen(function* () { const id = keyOf(key); const tab = tabs.get(id); if (!tab) return; + tab.keyboard.cancel(); tabs.delete(id); tab.debuggee.debugger.off("message", tab.onMessage); emit({ type: "detached", ...key }); @@ -170,6 +179,9 @@ export const make = Effect.gen(function* () { const tab: AttachedTab = { key, debuggee, + keyboard: createDesktopBrowserKeyboard(debuggee.webContents, debuggee.debugger, () => + clipboard.read(), + ), relay: null, downloadDirectory: null, pendingDownloadGuid: null, @@ -200,6 +212,7 @@ export const make = Effect.gen(function* () { } if (command.value.type === "release") { // A new server connection starts with a fresh relay and fresh sessions. + tab.keyboard.cancel(); tab.relay = null; return; } @@ -217,6 +230,7 @@ export const make = Effect.gen(function* () { Effect.forEach( [...tabs.values()], (tab) => { + tab.keyboard.cancel(); tab.relay = null; return PubSub.publish(outbox, { type: "attached", ...tab.key }); }, @@ -239,6 +253,10 @@ export const make = Effect.gen(function* () { detach, placeDownload, humanStartedDownload, + isDispatchingKeyboard: (source) => + [...tabs.values()].some( + (tab) => tab.debuggee.webContents === source && tab.keyboard.isDispatching(), + ), }); }); diff --git a/apps/desktop/src/preview/DesktopBrowserKeyboard.ts b/apps/desktop/src/preview/DesktopBrowserKeyboard.ts new file mode 100644 index 000000000000..2f86b3c82ab7 --- /dev/null +++ b/apps/desktop/src/preview/DesktopBrowserKeyboard.ts @@ -0,0 +1,339 @@ +// @effect-diagnostics globalTimers:off -- The Electron CDP callback adapter bounds native renderer receipts outside an Effect runtime. +/** Routes Playwright's keyboard packets to the guest, without changing desktop focus. */ +export function createDesktopBrowserKeyboard( + contents: Electron.WebContents, + debuggee: Electron.Debugger, + readClipboard: () => Promise>, +) { + let pending = 0; + let sequence = 0; + let generation = 0; + let tail: Promise = Promise.resolve(); + + const focusedFrame = async () => { + // Electron's focusedFrame follows native focus and can be null while the + // desktop owns it. Follow DOM focus instead, including cross-site frames. + let frame = contents.mainFrame; + while (true) { + const index: unknown = await frame.executeJavaScript(`(() => { + let element = document.activeElement; + while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement; + if (element?.tagName !== "IFRAME" && element?.tagName !== "FRAME") return -1; + return Array.from({ length: window.length }, (_, i) => window[i]).indexOf(element.contentWindow); + })()`); + if (index === -1) return frame; + if (typeof index !== "number" || !frame.frames[index]) { + throw new Error("The focused preview frame is unavailable."); + } + frame = frame.frames[index]!; + } + }; + + const dispatch = async (event: Electron.KeyboardInputEvent, checkCurrent: () => void) => { + checkCurrent(); + const packet = { ...event, skipIfUnhandled: true }; + contents.setIgnoreMenuShortcuts(true); + // Chromium suppresses the char packet when keydown is canceled. The + // following keyup receipt confirms the whole native sequence was handled. + if (event.type === "char") { + contents.sendInputEvent(packet); + return false; + } + const frame = await focusedFrame(); + const receipt = `__t3KeyboardReceipt${++sequence}`; + const type = event.type === "keyUp" ? "keyup" : "keydown"; + await frame.executeJavaScript(`(() => { + const name = ${JSON.stringify(receipt)}; + const type = ${JSON.stringify(type)}; + let finish; + const promise = new Promise(resolve => { finish = resolve; }); + let observed = false; + let received; + const count = performance.eventCounts.get(type) ?? 0; + const channel = new MessageChannel(); + channel.port1.onmessage = () => done({ prevented: received.defaultPrevented }); + const listener = event => { + if (!event.isTrusted) return; + observed = true; + received = event; + // A task runs after every page listener and the default edit. A + // microtask can run between listeners, before preventDefault. + channel.port2.postMessage(null); + }; + const pagehide = () => done(observed ? { prevented: false } : null); + const timer = setTimeout(() => done(null), 5000); + const check = setInterval(() => { + // Trusted event counts also confirm delivery when page code hides a + // key with stopImmediatePropagation before our listener sees it. + if (!observed && (performance.eventCounts.get(type) ?? 0) > count) done({ prevented: false }); + }, 16); + const done = result => { + clearTimeout(timer); + clearInterval(check); + channel.port1.close(); + channel.port2.close(); + removeEventListener(type, listener, true); + removeEventListener("pagehide", pagehide, true); + finish(result); + }; + addEventListener(type, listener, true); + addEventListener("pagehide", pagehide, true); + globalThis[name] = { promise, dispose: () => done(null) }; + })()`); + let timeout: ReturnType | undefined; + try { + // CDP's root Input commands can target the embedder's focused renderer. + // Native packets address this guest widget; skipIfUnhandled also keeps + // keys such as Enter from falling back to the desktop composer. + checkCurrent(); + contents.sendInputEvent(packet); + const result: unknown = await Promise.race([ + frame.executeJavaScript(`globalThis[${JSON.stringify(receipt)}]?.promise`), + new Promise((_, reject) => { + timeout = setTimeout( + () => reject(new Error("The preview page did not receive the keyboard event.")), + 5000, + ); + }), + ]); + if (typeof result !== "object" || result === null || !("prevented" in result)) { + throw new Error("The preview page did not receive the keyboard event."); + } + return result.prevented === true; + } finally { + clearTimeout(timeout); + if (!frame.detached) { + void frame + .executeJavaScript( + `globalThis[${JSON.stringify(receipt)}]?.dispose(); delete globalThis[${JSON.stringify(receipt)}]`, + ) + .catch(() => {}); + } + } + }; + + const press = async (params: Record, checkCurrent: () => void) => { + const type = params["type"]; + const key = params["key"]; + if ( + (type !== "keyDown" && type !== "rawKeyDown" && type !== "keyUp" && type !== "char") || + typeof key !== "string" + ) { + throw new Error("Invalid preview keyboard event."); + } + const mask = typeof params["modifiers"] === "number" ? params["modifiers"] : 0; + const target = await focusedFrame(); + if (target.processId !== contents.mainFrame.processId) { + // A cross-site frame has its own renderer session. Its Input commands + // bypass the root target's embedder-focus lookup. + const sessions: Array = []; + let sessionId: string | undefined; + let contextId: number | undefined; + try { + while (true) { + const evaluated = (await debuggee.sendCommand( + "Runtime.evaluate", + { + expression: `(() => { + let element = document.activeElement; + while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement; + return element?.tagName === "IFRAME" || element?.tagName === "FRAME" ? element : null; + })()`, + ...(contextId === undefined ? {} : { contextId }), + }, + sessionId, + )) as { result: { objectId?: string; subtype?: string } }; + if (evaluated.result.subtype === "null") break; + const objectId = evaluated.result.objectId; + if (!objectId) throw new Error("The focused preview frame is unavailable."); + let frameId: string | undefined; + try { + const described = (await debuggee.sendCommand( + "DOM.describeNode", + { objectId }, + sessionId, + )) as { node: { frameId?: string } }; + frameId = described.node.frameId; + } finally { + await debuggee + .sendCommand("Runtime.releaseObject", { objectId }, sessionId) + .catch(() => {}); + } + if (!frameId) throw new Error("The focused preview frame is unavailable."); + const targets = (await debuggee.sendCommand("Target.getTargets")) as { + targetInfos: ReadonlyArray<{ targetId: string; type: string }>; + }; + if ( + targets.targetInfos.some( + (target) => target.type === "iframe" && target.targetId === frameId, + ) + ) { + const attached = (await debuggee.sendCommand("Target.attachToTarget", { + targetId: frameId, + flatten: true, + })) as { sessionId: string }; + sessionId = attached.sessionId; + sessions.push(sessionId); + contextId = undefined; + } else { + const world = (await debuggee.sendCommand( + "Page.createIsolatedWorld", + { frameId, worldName: "t3-preview-keyboard" }, + sessionId, + )) as { executionContextId: number }; + contextId = world.executionContextId; + } + } + if (!sessionId) throw new Error("The focused preview frame is unavailable."); + await debuggee.sendCommand( + "Emulation.setFocusEmulationEnabled", + { enabled: true }, + sessionId, + ); + checkCurrent(); + await debuggee.sendCommand("Input.dispatchKeyEvent", params, sessionId); + } finally { + if (sessionId) + await debuggee + .sendCommand("Emulation.setFocusEmulationEnabled", { enabled: false }, sessionId) + .catch(() => {}); + for (const id of sessions) + await debuggee.sendCommand("Target.detachFromTarget", { sessionId: id }).catch(() => {}); + } + return; + } + const modifiers: NonNullable = []; + for (const [bit, modifier] of [ + [1, "alt"], + [2, "control"], + [4, "meta"], + [8, "shift"], + ] as const) { + if (mask & bit) modifiers.push(modifier); + } + if (params["isKeypad"] === true) modifiers.push("iskeypad"); + if (params["autoRepeat"] === true) modifiers.push("isautorepeat"); + if (params["location"] === 1) modifiers.push("left"); + if (params["location"] === 2) modifiers.push("right"); + const keyCode = key.startsWith("Arrow") ? key.slice(5) : key === " " ? "Space" : key; + const prevented = await dispatch( + { + type: type === "rawKeyDown" ? "keyDown" : type, + keyCode, + modifiers, + }, + checkCurrent, + ); + if (type === "keyUp" || type === "char" || prevented) return; + const commands = params["commands"]; + if (Array.isArray(commands) && commands.length > 0) { + const frame = await focusedFrame(); + const clipboardData: Array<{ type: string; data: string }> = []; + if (commands.includes("paste")) { + for (const item of await readClipboard()) { + for (const type of item.types) { + if (type.startsWith("electron ")) continue; + const blob = await item.getType(type); + if (!("arrayBuffer" in blob)) continue; + clipboardData.push({ + type, + data: type.startsWith("text/") + ? await blob.text() + : Buffer.from(await blob.arrayBuffer()).toString("base64"), + }); + } + } + } + checkCurrent(); + await frame.executeJavaScript( + `(() => { + for (const command of ${JSON.stringify(commands)}) { + let element = document.activeElement; + while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement; + if (command === "paste") { + const transfer = new DataTransfer(); + for (const { type, data } of ${JSON.stringify(clipboardData)}) { + if (type === "text/html") { + const container = document.createElement("div"); + container.setHTML(data); + transfer.setData(type, container.innerHTML); + } else if (type.startsWith("text/")) transfer.setData(type, data); + else transfer.items.add(new File([Uint8Array.from(atob(data), c => c.charCodeAt(0))], "clipboard", { type })); + } + if (!element?.dispatchEvent(new ClipboardEvent("paste", { clipboardData: transfer, bubbles: true, cancelable: true, composed: true }))) continue; + const text = transfer.getData("text/plain"); + if (!element.dispatchEvent(new InputEvent("beforeinput", { inputType: "insertFromPaste", data: text, dataTransfer: transfer, bubbles: true, cancelable: true, composed: true }))) continue; + const html = element.isContentEditable ? transfer.getData("text/html") : ""; + document.execCommand(html ? "insertHTML" : "insertText", false, html || text); + } else if (command.startsWith("moveTo")) { + document.getSelection()?.modify( + command.endsWith("AndModifySelection") ? "extend" : "move", + command.includes("Beginning") ? "backward" : command.includes("Left") ? "left" : command.includes("Right") ? "right" : "forward", + command.includes("Document") ? "documentboundary" : "lineboundary", + ); + } else if (command === "deleteToBeginningOfLine") { + const selection = document.getSelection(); + if (!element?.dispatchEvent(new InputEvent("beforeinput", { inputType: "deleteSoftLineBackward", bubbles: true, cancelable: true, composed: true }))) continue; + const collapsed = typeof element?.selectionStart === "number" ? element.selectionStart === element.selectionEnd : selection?.isCollapsed; + if (collapsed) selection?.modify("extend", "backward", "lineboundary"); + document.execCommand("delete"); + } else { + const inputType = command === "undo" ? "historyUndo" : command === "redo" ? "historyRedo" : null; + if (inputType && !element?.dispatchEvent(new InputEvent("beforeinput", { inputType, bubbles: true, cancelable: true, composed: true }))) continue; + document.execCommand(command); + } + } + })()`, + true, + ); + } + const text = params["text"]; + if (type === "keyDown" && typeof text === "string" && text.length > 0) { + await dispatch({ type: "char", keyCode: text, modifiers }, checkCurrent); + } + }; + + return { + isDispatching: () => pending > 0, + cancel: () => { + generation++; + }, + send: (method: string, params: Record) => { + pending++; + const current = generation; + const checkCurrent = () => { + if (current !== generation) + throw new Error("The preview keyboard connection was released."); + }; + const result = tail.then(async () => { + checkCurrent(); + if (method === "Input.insertText") { + const text = params["text"]; + if (typeof text !== "string") throw new Error("Invalid preview text."); + if (text.length > 0) { + const frame = await focusedFrame(); + checkCurrent(); + const inserted: unknown = await frame.executeJavaScript(`(() => { + let element = document.activeElement; + while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement; + const text = ${JSON.stringify(text)}; + const nonText = ["button", "checkbox", "color", "file", "hidden", "image", "radio", "range", "reset", "submit"]; + const textControl = element?.tagName === "TEXTAREA" || (element?.tagName === "INPUT" && !nonText.includes(element.type)); + if (!(textControl || element?.isContentEditable) || element.disabled || element.readOnly) return false; + if (!element?.dispatchEvent(new InputEvent("beforeinput", { inputType: "insertText", data: text, bubbles: true, cancelable: true, composed: true }))) return true; + return document.execCommand("insertText", false, text); + })()`); + if (inserted !== true) throw new Error("The preview page did not accept the text."); + } + } else { + await press(params, checkCurrent); + } + return {}; + }); + tail = result.catch(() => {}); + return result.finally(() => { + pending--; + }); + }, + }; +} diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 84c6273aad62..8d2f3d3b069e 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -685,6 +685,83 @@ describe("PreviewManager", () => { ), ); + effectIt.effect( + "agent keyboard input does not run desktop shortcuts, while human input still does", + () => + Effect.gen(function* () { + const manager = yield* PreviewManager.PreviewManager; + const preview = makeFaviconWebContents(); + const frame = { + processId: 1, + frames: [], + detached: false, + executeJavaScript: async (source: string) => + source.includes("indexOf(element.contentWindow)") + ? -1 + : source.endsWith("?.promise") + ? { prevented: false } + : undefined, + }; + const input = { + type: "keyDown", + key: "r", + code: "KeyR", + meta: true, + control: false, + shift: false, + alt: false, + }; + Object.assign(preview.webContents, { + mainFrame: frame, + sendInputEvent: () => + preview.listeners.get("before-input-event")!( + { preventDefault: vi.fn() } as never, + input as never, + ), + }); + fromId.mockReturnValue(preview.webContents); + getFocusedWebContents.mockReturnValue(preview.webContents as never); + yield* manager.createTab("tab_agent_keys"); + yield* manager.registerWebview("tab_agent_keys", 42); + const host = yield* DesktopBrowserHost.DesktopBrowserHost; + const key = { threadId: "thread-keys", tabId: "server-keys" }; + const replies = yield* Queue.unbounded(); + yield* host.events.pipe( + Stream.runForEach((line) => Queue.offer(replies, new TextDecoder().decode(line))), + Effect.forkScoped, + ); + host.attach(key, { + webContents: preview.webContents as Electron.WebContents, + debugger: (preview.webContents as Electron.WebContents).debugger, + }); + expect(JSON.parse(yield* Queue.take(replies)).type).toBe("attached"); + yield* host.handleCommandLine( + JSON.stringify({ + type: "cdp", + ...key, + message: JSON.stringify({ + id: 1, + method: "Input.dispatchKeyEvent", + params: { type: "rawKeyDown", key: "r", modifiers: 4 }, + sessionId: "t3-preview-page", + }), + }), + ); + let reply: string; + do { + reply = yield* Queue.take(replies); + } while (JSON.parse(reply).type !== "cdp"); + expect(JSON.parse(JSON.parse(reply).message)).toMatchObject({ id: 1, result: {} }); + expect(preview.reload).not.toHaveBeenCalled(); + expect(host.isDispatchingKeyboard(preview.webContents as Electron.WebContents)).toBe(false); + const preventDefault = vi.fn(); + preview.listeners.get("before-input-event")!({ preventDefault } as never, input as never); + expect(preventDefault).toHaveBeenCalledOnce(); + yield* Effect.yieldNow; + expect(preview.reload).toHaveBeenCalledOnce(); + }).pipe(Effect.provide(managerLayer()), Effect.scoped), + ); + effectIt.effect.each([ ["mod+shift+t", "view.reopenClosed"], ["ctrl+alt+u", "sidebar.toggle"], diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 123943f70f9a..5c47cfbd9923 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -1527,6 +1527,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); }; const beforeInput = (event: Electron.Event, input: Electron.Input): void => { + if (browserHost.isDispatchingKeyboard(wc)) { + wc.setIgnoreMenuShortcuts(true); + return; + } syncMenuShortcuts(wc, input); const host = wc.hostWebContents; const forwarded =