diff --git a/apps/desktop/src/snapShot/DesktopSnapShot.test.ts b/apps/desktop/src/snapShot/DesktopSnapShot.test.ts index c724bb396cd8..205a72a55265 100644 --- a/apps/desktop/src/snapShot/DesktopSnapShot.test.ts +++ b/apps/desktop/src/snapShot/DesktopSnapShot.test.ts @@ -29,6 +29,7 @@ beforeEach(() => { transitionCapturePageMock.mockReset().mockResolvedValue(undefined); transitionSnapshotMock.mockReset().mockResolvedValue(undefined); prepareCaptureRevealMock.mockReset(); + accessibilityListMock.mockReset().mockResolvedValue([]); }); const { @@ -39,6 +40,7 @@ const { accessibilityProcessCoolMock, accessibilityProcessReadMock, accessibilityByPidMock, + accessibilityListMock, accessibilityForegroundMock, accessibilityTrustedMock, allWindowsMock, @@ -87,6 +89,7 @@ const { } >(), accessibilityByPidMock: vi.fn(), + accessibilityListMock: vi.fn(), accessibilityForegroundMock: vi.fn(), accessibilityTrustedMock: vi.fn((_prompt = false) => true), allWindowsMock: vi.fn( @@ -430,6 +433,7 @@ import * as SnapShotAccessibility from "./SnapShotAccessibility.ts"; // Tests hand it this stand-in so the mocks above drive window lookups. const accessibilityApp = { byPid: accessibilityByPidMock, + list: accessibilityListMock, foreground: accessibilityForegroundMock, } as unknown as Parameters[0]; const readAccessibleWindowContext = ( @@ -2654,6 +2658,140 @@ it.each([ } }); +it("reads Flatpak app text from the AT-SPI proxy when the compositor PID misses", async () => { + vi.stubEnv("XDG_SESSION_TYPE", "wayland"); + const bounds = { x: 0, y: 0, width: 1_920, height: 1_048 }; + const tree = vi.fn(async () => ({ + name: "Issue — Zen Browser", + children: [{ name: "New Tab", children: [] }], + })); + const proxyWindow = { + role: "window", + name: "Issue — Zen Browser", + bounds, + tree, + children: async () => [], + }; + accessibilityByPidMock + .mockReset() + .mockRejectedValue( + new Error("XA11Y_SELECTOR_NOT_MATCHED: No element matched selector: application[pid=207651]"), + ); + accessibilityListMock.mockReset().mockResolvedValue([ + { + pid: 1, + children: async () => [ + { + role: "window", + name: "Files", + bounds, + tree: async () => ({ name: "Files", children: [] }), + }, + ], + }, + { pid: 207646, children: async () => [proxyWindow] }, + ]); + try { + assert.strictEqual( + await readAccessibleWindowText( + { + title: "Issue — Zen Browser", + bounds: { x: 0, y: 0, width: 1_920, height: 1_048 }, + owner: { processId: 207651 }, + }, + "linux", + "Issue — Zen Browser", + ), + "Issue — Zen Browser\nNew Tab", + ); + assert.deepEqual(accessibilityByPidMock.mock.calls, [[207651, { timeout: 0 }]]); + assert.lengthOf(accessibilityListMock.mock.calls, 1); + assert.lengthOf(tree.mock.calls, 1); + } finally { + accessibilityListMock.mockReset().mockResolvedValue([]); + vi.unstubAllEnvs(); + } +}); + +it("reads GTK4 app text from one unnamed PID-scoped window of the captured size", async () => { + vi.stubEnv("XDG_SESSION_TYPE", "wayland"); + const bounds = { x: 0, y: 0, width: 1_920, height: 1_048 }; + const tree = vi.fn(async () => ({ + name: "System Monitor", + children: [{ name: "CPU", children: [] }], + })); + accessibilityByPidMock.mockReset().mockResolvedValue({ + children: async () => [ + { + role: "group", + name: null, + bounds, + tree, + children: async () => [ + { role: "page_tab_list", name: "Processes", bounds, children: async () => [] }, + ], + }, + ], + }); + accessibilityListMock.mockReset().mockResolvedValue([]); + try { + assert.strictEqual( + await readAccessibleWindowText( + { + title: "System Monitor", + bounds: { x: 12, y: 48, width: 1_920, height: 1_048 }, + owner: { processId: 210600 }, + }, + "linux", + "System Monitor", + ), + "System Monitor\nCPU", + ); + assert.deepEqual(accessibilityByPidMock.mock.calls, [[210600, { timeout: 0 }]]); + assert.lengthOf(accessibilityListMock.mock.calls, 0); + assert.lengthOf(tree.mock.calls, 1); + } finally { + accessibilityListMock.mockReset().mockResolvedValue([]); + vi.unstubAllEnvs(); + } +}); + +it("does not guess GTK4 app text when two unnamed PID windows share the captured size", async () => { + vi.stubEnv("XDG_SESSION_TYPE", "wayland"); + const bounds = { x: 0, y: 0, width: 1_920, height: 1_048 }; + const tree = vi.fn(async () => ({ value: "Wrong window", children: [] })); + accessibilityByPidMock.mockReset().mockResolvedValue({ + children: async () => [ + { role: "group", name: null, bounds, tree }, + { role: "group", name: null, bounds: { ...bounds, x: 12 }, tree }, + ], + }); + accessibilityListMock.mockReset().mockResolvedValue([ + { + pid: 99, + children: async () => [{ role: "window", name: "System Monitor", bounds, tree }], + }, + ]); + try { + assert.isUndefined( + await readAccessibleWindowText( + { + title: "System Monitor", + bounds: { x: 12, y: 48, width: 1_920, height: 1_048 }, + owner: { processId: 210600 }, + }, + "linux", + "System Monitor", + ), + ); + assert.lengthOf(accessibilityListMock.mock.calls, 0); + assert.lengthOf(tree.mock.calls, 0); + } finally { + accessibilityListMock.mockReset().mockResolvedValue([]); + vi.unstubAllEnvs(); + } +}); + it.each([20, 1_350, 2_999])( "includes accessibility text as soon as a %d ms read completes", async (duration) => { diff --git a/apps/desktop/src/snapShot/SnapShotAccessibility.ts b/apps/desktop/src/snapShot/SnapShotAccessibility.ts index bb6cb95952b3..b0a0fe7b959c 100644 --- a/apps/desktop/src/snapShot/SnapShotAccessibility.ts +++ b/apps/desktop/src/snapShot/SnapShotAccessibility.ts @@ -97,6 +97,23 @@ function accessibilityReadSnapshot( ) as CapturedWindowAccessibilityContext; } +type AccessibilityElement = Awaited["children"]>>[number]; + +async function windowsForPid( + App: AccessibilityApp, + processId: number, +): Promise { + return await App.byPid(processId, { timeout: 0 }) + .then((app) => app.children()) + .catch(() => []); +} + +/** Flatpak apps reach AT-SPI through xdg-dbus-proxy, so the compositor PID misses. */ +async function windowsFromAppList(App: AccessibilityApp): Promise { + const apps = await App.list().catch(() => []); + return (await Promise.all(apps.map((app) => app.children().catch(() => [])))).flat(); +} + async function readCapturedWindowAccessibility( App: AccessibilityApp, request: SnapShotAccessibilityRequest, @@ -105,17 +122,24 @@ async function readCapturedWindowAccessibility( ): Promise { const { active, platform, sourceTitle, imageSize } = request; const foreground = platform === "win32" ? await App.foreground({ timeout: 0 }) : undefined; - const windows = + const pidWindows = foreground !== undefined ? foreground.pid === active.owner.processId ? [foreground.asElement()] : [] - : await (await App.byPid(active.owner.processId, { timeout: 0 })).children(); + : await windowsForPid(App, active.owner.processId); const matchMode = isWaylandSession(platform, process.env) ? "wayland" : "screen-bounds"; + const captured = { + title: active.title, + sourceTitle, + bounds: active.bounds, + clientBounds: active.clientBounds, + }; const window = findAccessibleWindow( - windows, - { title: active.title, sourceTitle, bounds: active.bounds, clientBounds: active.clientBounds }, + pidWindows.length > 0 || foreground !== undefined ? pidWindows : await windowsFromAppList(App), + captured, matchMode, + { allowUntitledUniqueBounds: foreground === undefined && pidWindows.length > 0 }, ); if (!window) { onStarted(); diff --git a/apps/desktop/src/snapShot/snapShot.test.ts b/apps/desktop/src/snapShot/snapShot.test.ts index 6960718759aa..634c4653a5a6 100644 --- a/apps/desktop/src/snapShot/snapShot.test.ts +++ b/apps/desktop/src/snapShot/snapShot.test.ts @@ -464,6 +464,45 @@ describe("findAccessibleWindow", () => { expect(findAccessibleWindow(windows, { ...captured, title: "" }, "wayland")).toBeUndefined(); }); + it("accepts one PID-scoped untitled window whose bounds match", () => { + const windows = [{ name: null, bounds: captured.bounds }]; + + expect( + findAccessibleWindow(windows, captured, "wayland", { allowUntitledUniqueBounds: true }), + ).toBe(windows[0]); + expect(findAccessibleWindow(windows, captured, "wayland")).toBeUndefined(); + }); + + it("rejects ambiguous PID-scoped untitled windows even when bounds match", () => { + const windows = [ + { name: null, bounds: captured.bounds }, + { name: "", bounds: { ...captured.bounds, x: 0, y: 0 } }, + ]; + + expect( + findAccessibleWindow(windows, captured, "wayland", { allowUntitledUniqueBounds: true }), + ).toBeUndefined(); + }); + + it("does not fall back to a differently titled PID-scoped window with matching bounds", () => { + const windows = [{ name: "Preferences", bounds: captured.bounds }]; + + expect( + findAccessibleWindow(windows, captured, "wayland", { allowUntitledUniqueBounds: true }), + ).toBeUndefined(); + }); + + it("does not use unique bounds when a titled match is already ambiguous", () => { + const windows = [ + { name: "Editor", bounds: captured.bounds }, + { name: "Editor", bounds: { ...captured.bounds, x: 0, y: 0 } }, + ]; + + expect( + findAccessibleWindow(windows, captured, "wayland", { allowUntitledUniqueBounds: true }), + ).toBeUndefined(); + }); + it.each(["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"])( "ignores a leading Wayland title spinner frame %s", (frame) => { diff --git a/apps/desktop/src/snapShot/snapShot.ts b/apps/desktop/src/snapShot/snapShot.ts index 35c240c6e09a..2cfa406d1781 100644 --- a/apps/desktop/src/snapShot/snapShot.ts +++ b/apps/desktop/src/snapShot/snapShot.ts @@ -480,6 +480,7 @@ export function findAccessibleWindow< readonly clientBounds?: WindowBounds | undefined; }, matchMode: "screen-bounds" | "wayland" = "screen-bounds", + options: { readonly allowUntitledUniqueBounds?: boolean } = {}, ): T | undefined { const normalizeTitle = (value: string) => { const title = value.trim(); @@ -489,7 +490,6 @@ export function findAccessibleWindow< const titles = new Set( [captured.title, captured.sourceTitle ?? ""].map(normalizeTitle).filter(Boolean), ); - if (titles.size === 0) return undefined; // Wayland accessibility providers can expose window size without a screen position. const boundsKeys = matchMode === "wayland" @@ -499,19 +499,31 @@ export function findAccessibleWindow< matchMode === "wayland" && captured.clientBounds ? [captured.bounds, captured.clientBounds] : [captured.bounds]; - const matches = windows.filter((window) => { + const matchesBounds = (window: T) => { const bounds = window.bounds; return ( - titles.has(normalizeTitle(window.name ?? "")) && bounds !== null && candidateBounds.some((candidate) => boundsKeys.every((key) => Math.abs(bounds[key] - candidate[key]) <= 2), ) ); - }); - if (matches.length === 1) return matches[0]; - const activeMatches = matches.filter((window) => safeProperty(() => window.active) === true); - return activeMatches.length === 1 ? activeMatches[0] : undefined; + }; + if (titles.size > 0) { + const matches = windows.filter( + (window) => titles.has(normalizeTitle(window.name ?? "")) && matchesBounds(window), + ); + if (matches.length === 1) return matches[0]; + const activeMatches = matches.filter((window) => safeProperty(() => window.active) === true); + if (activeMatches.length === 1) return activeMatches[0]; + if (matches.length > 1) return undefined; + } + // GTK4/libadwaita often exposes the frame as an unnamed group. A PID-scoped + // lookup can accept the one window whose bounds match; size-only guesses cannot. + if (!options.allowUntitledUniqueBounds) return undefined; + const boundsMatches = windows.filter( + (window) => normalizeTitle(window.name ?? "") === "" && matchesBounds(window), + ); + return boundsMatches.length === 1 ? boundsMatches[0] : undefined; } const ELECTRON_KEY_NAMES: Readonly> = {