Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ jobs:
# The Azure archive took 16 minutes for one font package; bound setup
# separately so a slow mirror cannot consume the shard's test budget.
env: &e2e_tool_packages
ORCA_E2E_APT_PACKAGES: build-essential fonts-noto-cjk openssh-client python3 ripgrep xvfb zsh openbox x11-utils
ORCA_E2E_APT_PACKAGES: build-essential fonts-noto-cjk openssh-client python3 ripgrep xvfb zsh openbox x11-utils xdotool
run: *install_e2e_tools

- uses: ./.github/actions/install-node-dependencies
Expand Down
8 changes: 6 additions & 2 deletions src/main/browser/agent-browser-bridge-interaction-commands.ts
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,8 @@ export abstract class AgentBrowserBridgeInteractionCommands extends AgentBrowser
try {
releaseDebugger = acquireElectronDebugger(wc).release
const send: GuestCdpSender = (method, params) => wc.debugger.sendCommand(method, params)
const guardFocus = (): Promise<() => void> =>
this.browserManager.guardUserFocusForAgentInput(target.webContentsId)
await dispatchPageKeyEvent(
wc,
{
Expand All @@ -213,7 +215,8 @@ export abstract class AgentBrowserBridgeInteractionCommands extends AgentBrowser
...event,
...(parsed.text === null ? {} : { text: parsed.text, unmodifiedText: parsed.text })
},
send
send,
guardFocus
)
await dispatchPageKeyEvent(
wc,
Expand All @@ -223,7 +226,8 @@ export abstract class AgentBrowserBridgeInteractionCommands extends AgentBrowser
// Why: the self bit is keydown-only -- Blink reports shiftKey false on the Shift keyup.
modifiers: parsed.modifiers & ~parsed.selfModifier
},
send
send,
guardFocus
)
return { pressed: key }
} catch (error) {
Expand Down
6 changes: 5 additions & 1 deletion src/main/browser/agent-browser-bridge-lifecycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,11 @@ export abstract class AgentBrowserBridgeLifecycle extends AgentBrowserBridgeRawP
// Why: the daemon persists sessions (incl. CDP port) across restarts; close the stale one first or it ignores --cdp and hits the dead port.
await this.closeStaleAgentBrowserSession(sessionName)

const proxy = new CdpWsProxy(wc, () => this.browserManager.holdPaintForCapture(webContentsId))
const proxy = new CdpWsProxy(
wc,
() => this.browserManager.holdPaintForCapture(webContentsId),
() => this.browserManager.guardUserFocusForAgentInput(webContentsId)
)
const cdpEndpoint = await proxy.start()

this.sessions.set(sessionName, {
Expand Down
4 changes: 3 additions & 1 deletion src/main/browser/agent-browser-bridge-mouse-commands.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,9 @@ export abstract class AgentBrowserBridgeMouseCommands extends AgentBrowserBridge
const buttons = cdpPointerButtonMask(cdpButton)
const cdpModifiers = cdpMouseModifierMask(modifiers)
const lease = acquireElectronDebugger(wc)
let releaseFocusGuard = (): void => {}
try {
wc.focus()
releaseFocusGuard = await this.focusPageForPress(wc, target.webContentsId)
const point =
cdpButton === 'left'
? // Why: DOM activation can't carry Cmd/Ctrl/Alt/Shift, so modifier clicks use the adjusted point and let CDP dispatch the event.
Expand Down Expand Up @@ -75,6 +76,7 @@ export abstract class AgentBrowserBridgeMouseCommands extends AgentBrowserBridge
}
}
} finally {
releaseFocusGuard()
lease.release()
}
},
Expand Down
27 changes: 27 additions & 0 deletions src/main/browser/agent-browser-bridge-mouse-input.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,33 @@ describe('AgentBrowserBridge', () => {
bridge.setActiveTab(100)
})

it('guards the user’s focus instead of focusing an embedded guest on click', async () => {
const release = vi.fn()
const guardUserFocusForAgentInput = vi.fn(async () => release)
bridge = new AgentBrowserBridge(
mockBrowserManager(undefined, undefined, { guardUserFocusForAgentInput })
)
bridge.setActiveTab(100)
const wc = mockWebContents(100)
wc.getType.mockReturnValue('webview')
wc.debugger.sendCommand.mockImplementation(async (method: string) => {
if (method === 'Input.dispatchMouseEvent') {
expect(release).not.toHaveBeenCalled()
}
return method === 'Runtime.evaluate' ? { result: { value: null } } : {}
})
webContentsFromIdMock.mockReturnValue(wc)

await bridge.mouseClick(10, 20, 'left', undefined, 'tab-1')

expect(wc.focus).not.toHaveBeenCalled()
expect(guardUserFocusForAgentInput).toHaveBeenCalledWith(100)
expect(release).toHaveBeenCalledTimes(1)
expect(
wc.debugger.sendCommand.mock.calls.filter((call) => call[0] === 'Input.dispatchMouseEvent')
).toHaveLength(2)
})

it('uses the runtime mobile tap path when a nearby DOM target is handled', async () => {
const wc = mockWebContents(100)
wc.debugger.sendCommand.mockImplementation(async (method: string) => {
Expand Down
31 changes: 24 additions & 7 deletions src/main/browser/agent-browser-bridge-pointer-commands.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import {
} from './cdp-pointer-input'
import { acquireElectronDebugger } from './electron-debugger-lease'
import { AgentBrowserBridgeInputCommands } from './agent-browser-bridge-input-commands'
import { isEmbeddedGuest } from './guest-direct-key-input'
import { enablePageFocusEmulation } from './agent-input-focus-guard'

type CdpPointerEventParams = {
type: 'mouseMoved' | 'mousePressed' | 'mouseReleased' | 'mouseWheel'
Expand Down Expand Up @@ -38,7 +40,7 @@ export abstract class AgentBrowserBridgePointerCommands extends AgentBrowserBrid
describe: string,
build: (state: CdpPointerState) => {
params: CdpPointerEventParams
focus?: boolean
press?: boolean
result: T
}
): Promise<T> {
Expand All @@ -55,11 +57,12 @@ export abstract class AgentBrowserBridgePointerCommands extends AgentBrowserBrid
// keeping the mutation would leave a phantom held button on every later event.
const preDispatch = { ...state }
let releaseDebugger = (): void => {}
let releaseFocusGuard = (): void => {}
try {
releaseDebugger = acquireElectronDebugger(wc).release
const { params, focus, result } = build(state)
if (focus) {
wc.focus()
const { params, press, result } = build(state)
if (press) {
releaseFocusGuard = await this.focusPageForPress(wc, target.webContentsId)
}
await wc.debugger.sendCommand('Input.dispatchMouseEvent', params)
return result
Expand All @@ -79,10 +82,26 @@ export abstract class AgentBrowserBridgePointerCommands extends AgentBrowserBrid
`Failed to ${describe} in browser page ${target.browserPageId}: ${error instanceof Error ? error.message : String(error)}`
)
} finally {
releaseFocusGuard()
releaseDebugger()
}
}

// Why: a press on an embedded guest takes Chromium focus whatever we do; its pane gives the
// user's focus back (#20595) and emulation keeps the page from seeing that as a blur.
// Offscreen pages keep the native focus their caret and IME need.
protected async focusPageForPress(
wc: Electron.WebContents,
webContentsId: number
): Promise<() => void> {
if (!isEmbeddedGuest(wc)) {
wc.focus()
return () => {}
}
await enablePageFocusEmulation((method, params) => wc.debugger.sendCommand(method, params))
return await this.browserManager.guardUserFocusForAgentInput(webContentsId)
}

async mouseMove(
x: number,
y: number,
Expand Down Expand Up @@ -121,9 +140,7 @@ export abstract class AgentBrowserBridgePointerCommands extends AgentBrowserBrid
const cdpButton = normalizeCdpPointerButton(button)
pressCdpPointerButton(state, cdpButton)
return {
// Why: mirrors mouseClick — a press that does not focus the guest leaves
// keyboard input going to whatever held focus before.
focus: true,
press: true,
params: {
type: 'mousePressed',
x: state.x,
Expand Down
19 changes: 18 additions & 1 deletion src/main/browser/agent-browser-bridge-pointer-input.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -425,9 +425,26 @@ describe('AgentBrowserBridge coordinate pointer input', () => {
expect(wc.debugger.detach).not.toHaveBeenCalled()
})

it('focuses the guest on press, as mouseClick does', async () => {
it('focuses an offscreen page on press, as mouseClick does', async () => {
await bridge.mouseDown('left')

expect(wc.focus).toHaveBeenCalled()
})

it('guards the user’s focus instead of focusing an embedded guest on press', async () => {
const release = vi.fn()
const guardUserFocusForAgentInput = vi.fn(async () => release)
bridge = new AgentBrowserBridge(
mockBrowserManager(undefined, undefined, { guardUserFocusForAgentInput })
)
bridge.setActiveTab(100)
wc.getType.mockReturnValue('webview')

await bridge.mouseDown('left')
await bridge.mouseUp('left')

expect(wc.focus).not.toHaveBeenCalled()
expect(guardUserFocusForAgentInput).toHaveBeenCalledTimes(1)
expect(release).toHaveBeenCalledTimes(1)
})
})
2 changes: 2 additions & 0 deletions src/main/browser/agent-browser-bridge-test-harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ export type MockWebContents = {
// Why: defaults to an offscreen page; 'webview' opts a test into guest-direct key input.
getType: Mock<() => string>
sendInputEvent: Mock<(event: Electron.KeyboardInputEvent) => void>
setIgnoreMenuShortcuts: Mock<(ignore: boolean) => void>
debugger: MockWebContentsDebugger
}

Expand All @@ -88,6 +89,7 @@ export function mockWebContents(
focus: vi.fn(),
getType: vi.fn(() => 'window'),
sendInputEvent: vi.fn(),
setIgnoreMenuShortcuts: vi.fn(),
debugger: {
isAttached: vi.fn(() => true),
attach: vi.fn(),
Expand Down
115 changes: 115 additions & 0 deletions src/main/browser/agent-input-focus-guard.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'

const { ipcListeners } = vi.hoisted(() => ({
ipcListeners: new Map<string, ((event: unknown, reply: unknown) => void)[]>()
}))

vi.mock('electron', () => ({
ipcMain: {
on: vi.fn((channel: string, listener: (event: unknown, reply: unknown) => void) => {
ipcListeners.set(channel, [...(ipcListeners.get(channel) ?? []), listener])
}),
removeListener: vi.fn((channel: string, listener: (event: unknown, reply: unknown) => void) => {
ipcListeners.set(
channel,
(ipcListeners.get(channel) ?? []).filter((entry) => entry !== listener)
)
})
}
}))

import {
AGENT_INPUT_FOCUS_GUARD_CHANNEL,
AGENT_INPUT_FOCUS_GUARD_READY_CHANNEL,
guardPaneFocusForAgentInput,
isFocusTakingCdpInput
} from './agent-input-focus-guard'

function fakeRenderer(id: number, answer: boolean) {
const sent: { channel: string; data: { token: string; active: boolean } }[] = []
const renderer = {
id,
isDestroyed: () => false,
send: vi.fn((channel: string, data: { token: string; active: boolean }) => {
sent.push({ channel, data })
if (answer && data.active) {
for (const listener of ipcListeners.get(AGENT_INPUT_FOCUS_GUARD_READY_CHANNEL) ?? []) {
listener({ sender: { id } }, { token: data.token })
}
}
})
}
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the guard only reads id, isDestroyed and send.
return { sent, renderer: renderer as unknown as Electron.WebContents }
}

describe('guardPaneFocusForAgentInput', () => {
beforeEach(() => {
ipcListeners.clear()
vi.useRealTimers()
})

it('resolves when the pane is ready and ends the guard once', async () => {
const { sent, renderer } = fakeRenderer(7, true)

const release = await guardPaneFocusForAgentInput(renderer, 'page-1')
release()
release()

expect(sent.map(({ channel, data }) => [channel, data.active])).toEqual([
[AGENT_INPUT_FOCUS_GUARD_CHANNEL, true],
[AGENT_INPUT_FOCUS_GUARD_CHANNEL, false]
])
expect(sent[0].data.token).toBe(sent[1].data.token)
expect(ipcListeners.get(AGENT_INPUT_FOCUS_GUARD_READY_CHANNEL)).toEqual([])
})

it('refuses the input and ends the guard when the pane never answers', async () => {
vi.useFakeTimers()
const { sent, renderer } = fakeRenderer(7, false)

const guarded = guardPaneFocusForAgentInput(renderer, 'page-1')
const outcome = expect(guarded).rejects.toMatchObject({
code: 'browser_focus_guard_unavailable'
})
await vi.advanceTimersByTimeAsync(150)

await outcome
expect(sent.map(({ data }) => data.active)).toEqual([true, false])
expect(ipcListeners.get(AGENT_INPUT_FOCUS_GUARD_READY_CHANNEL)).toEqual([])
})

it('ignores a ready reply from another renderer', async () => {
vi.useFakeTimers()
const { renderer } = fakeRenderer(7, false)
let settled: 'pending' | 'ready' | 'refused' = 'pending'

void guardPaneFocusForAgentInput(renderer, 'page-1').then(
() => {
settled = 'ready'
},
() => {
settled = 'refused'
}
)
const [listener] = ipcListeners.get(AGENT_INPUT_FOCUS_GUARD_READY_CHANNEL) ?? []
const { calls } = vi.mocked(renderer.send).mock
listener({ sender: { id: 8 } }, { token: calls[0][1].token })
await Promise.resolve()

expect(settled).toBe('pending')
await vi.advanceTimersByTimeAsync(150)
expect(settled).toBe('refused')
})
})

describe('isFocusTakingCdpInput', () => {
it('matches presses only', () => {
expect(isFocusTakingCdpInput('Input.dispatchMouseEvent', { type: 'mousePressed' })).toBe(true)
expect(isFocusTakingCdpInput('Input.dispatchMouseEvent', { type: 'mouseReleased' })).toBe(false)
expect(isFocusTakingCdpInput('Input.dispatchMouseEvent', { type: 'mouseMoved' })).toBe(false)
expect(isFocusTakingCdpInput('Input.dispatchTouchEvent', { type: 'touchStart' })).toBe(true)
expect(isFocusTakingCdpInput('Input.synthesizeTapGesture', {})).toBe(true)
expect(isFocusTakingCdpInput('Input.dispatchKeyEvent', { type: 'keyDown' })).toBe(false)
})
})
Loading
Loading