fix(apple): refuse captures whose window or panel identity is unknown

apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift

@@ -10,20 +10,15 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro
     throw HelperError.commandFailed("app window screenshots require macOS 14 or newer")
   }
   let pid = app.processIdentifier
-  let frontWindowID = frontWindowNumber(pid: pid)
+  let front = try frontWindow(pid: pid)
   let content = try awaitShareableContent()
-  let candidates = content.windows.filter {
-    $0.owningApplication?.processID == pid && $0.windowLayer == 0 && $0.frame.width > 0
-      && $0.frame.height > 0
-  }
-  guard
-    let window = candidates.first(where: { Int($0.windowID) == frontWindowID })
-      ?? candidates.max(by: { $0.frame.width * $0.frame.height < $1.frame.width * $1.frame.height })
-  else {
-    throw HelperError.commandFailed(
-      "screenshot could not find a window for the app",
-      details: ["reason": "window-not-found", "bundleId": app.bundleIdentifier ?? ""]
-    )
+  let candidates = content.windows.filter { $0.owningApplication?.processID == pid }
+  let window: SCWindow
+  switch frontShareableWindow(candidates, front: front, windowID: { Int($0.windowID) }) {
+  case .success(let shareable):
+    window = shareable
+  case .failure(let refusal):
+    throw refusal.error(bundleId: app.bundleIdentifier)
   }
 
   let filter = SCContentFilter(desktopIndependentWindow: window)
@@ -52,6 +47,58 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro
   try writePNG(capturedImage, to: outPath)
 }
 
+/// Why an app screenshot refuses, and the error the host reports for it: reason, remedy, and the
+/// front window it refused. The capture is the app's front window or nothing; another window of
+/// the app would show state the user is not looking at.
+enum AppWindowScreenshotRefusal: Error {
+  case windowNotFound
+  case frontWindowNotShareable(frontWindowId: Int)
+  case frontWindowCaptureExcluded(frontWindowId: Int)
+
+  func error(bundleId: String?) -> HelperError {
+    var details = ["bundleId": bundleId ?? ""]
+    let message: String
+    switch self {
+    case .windowNotFound:
+      details["reason"] = "window-not-found"
+      details["hint"] =
+        "Open or unminimize a window of the app on the current Space, then retry the screenshot."
+      message = "screenshot could not find an on-screen window for the app"
+    case .frontWindowCaptureExcluded(let frontWindowId):
+      details["reason"] = "front-window-capture-excluded"
+      details["frontWindowId"] = String(frontWindowId)
+      details["hint"] =
+        "The app excludes the front window from screen capture. Run `snapshot` to read the window's UI tree; retrying the excluded window will not help."
+      message =
+        "screenshot could not capture the app's front window: the app excludes it from capture"
+    case .frontWindowNotShareable(let frontWindowId):
+      details["reason"] = "front-window-not-shareable"
+      details["frontWindowId"] = String(frontWindowId)
+      details["hint"] =
+        "The front window is unavailable to screen capture. If it is opening or closing, retry after it settles."
+      message =
+        "screenshot could not capture the app's front window: ScreenCaptureKit does not offer it"
+    }
+    return .commandFailed(message, details: details)
+  }
+}
+
+/// The shareable candidate that is the app's front window, or the reason the screenshot refuses.
+func frontShareableWindow<Window>(
+  _ candidates: [Window],
+  front: OnScreenWindow?,
+  windowID: (Window) -> Int
+) -> Result<Window, AppWindowScreenshotRefusal> {
+  guard let front else { return .failure(.windowNotFound) }
+  guard front.sharingState != 0 else {
+    return .failure(.frontWindowCaptureExcluded(frontWindowId: front.number))
+  }
+  guard let shareable = candidates.first(where: { windowID($0) == front.number }) else {
+    return .failure(.frontWindowNotShareable(frontWindowId: front.number))
+  }
+  return .success(shareable)
+}
+
 @available(macOS 14.0, *)
 private func awaitShareableContent() throws -> SCShareableContent {
   let semaphore = DispatchSemaphore(value: 0)

apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift

@@ -206,7 +206,7 @@ func scrollInBackground(
   guard isVertical || direction == "left" || direction == "right" else {
     throw HelperError.invalidArgs("scroll requires --direction <up|down|left|right>")
   }
-  guard let frame = frontWindowFrame(pid: app.processIdentifier) else {
+  guard let frame = try frontWindowFrame(pid: app.processIdentifier) else {
     throw HelperError.commandFailed(
       "scroll could not resolve an on-screen window",
       details: ["reason": "window-not-found", "bundleId": app.bundleIdentifier ?? ""]
@@ -306,10 +306,7 @@ private func perform(_ target: PressTarget) -> BackgroundDeliveryMechanism? {
 /// own on-screen windows, its menus and sheets included, and never on a menu bar element: the app
 /// owns its menu bar's window too, so the window check alone does not exclude it.
 private func requireAppWindowPoint(_ point: CGPoint, app: NSRunningApplication) throws {
-  guard
-    let info = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID)
-      as? [[String: Any]]
-  else {
+  guard let info = appWindowInfo() else {
     throw refusal(.noAccessibleTarget, "the app's on-screen windows could not be read", app: app)
   }
   let inAppWindow = info.contains { entry in
@@ -363,13 +360,14 @@ private func pressSearchChain(from hit: AXUIElement) -> [AXUIElement] {
 
 /// The one window a point action may act in: the window the app's hit test landed in, else the
 /// app's front on-screen window. A fallback target never comes from a window behind it, and the
-/// response names the window so the host can check it.
+/// response names the window so the host can check it. The caller's `requireAppWindowPoint` has
+/// already read the window list, so a list that turned unreadable here is treated as no window.
 private func actionWindow(app: NSRunningApplication, hit: AXUIElement?) -> AXUIElement? {
   if let hit {
     if role(of: hit) == "AXWindow" { return hit }
     if let window = elementAttribute(hit, attribute: kAXWindowAttribute as String) { return window }
   }
-  guard let front = frontWindowFrame(pid: app.processIdentifier) else { return nil }
+  guard let front = try? frontWindowFrame(pid: app.processIdentifier) else { return nil }
   return windows(of: AXUIElementCreateApplication(app.processIdentifier)).first { window in
     guard let rect = rectAttribute(window) else { return false }
     return abs(rect.x - front.minX) < 1 && abs(rect.y - front.minY) < 1
@@ -482,39 +480,57 @@ private func numberAttribute(_ element: AXUIElement, attribute: String) -> Doubl
   return number.doubleValue
 }
 
-private struct OnScreenWindow {
+struct OnScreenWindow {
   let number: Int
   let bounds: CGRect
+  let sharingState: Int?
 }
 
-/// The pid's normal-layer windows, front to back. Bounds and owners need no Screen Recording
-/// permission; only titles do.
-private func onScreenWindows(pid: pid_t) -> [OnScreenWindow] {
-  guard
-    let info = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID)
-      as? [[String: Any]]
-  else {
-    return []
-  }
-  return info.compactMap { entry in
+/// Only normal-layer windows qualify; app-modal alerts at level 8 are excluded.
+let frontWindowLayer = 0
+
+/// The pid's visible, nonempty normal-layer windows, front to back.
+func onScreenWindows(pid: pid_t, windowInfo: [[String: Any]]) -> [OnScreenWindow] {
+  windowInfo.compactMap { entry in
     guard (entry[kCGWindowOwnerPID as String] as? Int32) == pid,
-      (entry[kCGWindowLayer as String] as? Int) == 0,
+      (entry[kCGWindowLayer as String] as? Int) == frontWindowLayer,
+      (entry[kCGWindowIsOnscreen as String] as? Bool) == true,
+      let alpha = entry[kCGWindowAlpha as String] as? Double, alpha > 0,
       let number = entry[kCGWindowNumber as String] as? Int,
       let boundsDict = entry[kCGWindowBounds as String] as? NSDictionary,
-      let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary)
+      let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary),
+      bounds.width > 0, bounds.height > 0
     else {
       return nil
     }
-    return OnScreenWindow(number: number, bounds: bounds)
+    return OnScreenWindow(
+      number: number, bounds: bounds, sharingState: entry[kCGWindowSharingState as String] as? Int)
   }
 }
 
-func frontWindowNumber(pid: pid_t) -> Int? {
-  onScreenWindows(pid: pid).first?.number
+func appWindowInfo() -> [[String: Any]]? {
+  CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID)
+    as? [[String: Any]]
+}
+
+/// The app's front on-screen window: frontmost among its eligible windows, or nil when it has
+/// none. A window list that cannot be read is a refusal, not an empty list: the caller must not
+/// report "no window" for a WindowServer it could not ask.
+func frontWindow(pid: pid_t, windowInfo: [[String: Any]]? = appWindowInfo()) throws -> OnScreenWindow? {
+  guard let windowInfo else {
+    throw HelperError.commandFailed(
+      "the app's on-screen windows could not be read",
+      details: [
+        "reason": "window-list-unreadable",
+        "hint": "The app's on-screen window list could not be read. Retry the command.",
+      ]
+    )
+  }
+  return onScreenWindows(pid: pid, windowInfo: windowInfo).first
 }
 
-private func frontWindowFrame(pid: pid_t) -> CGRect? {
-  onScreenWindows(pid: pid).first?.bounds
+func frontWindowFrame(pid: pid_t, windowInfo: [[String: Any]]? = appWindowInfo()) throws -> CGRect? {
+  try frontWindow(pid: pid, windowInfo: windowInfo)?.bounds
 }
 
 private let returnVirtualKey: CGKeyCode = 36

apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift

@@ -0,0 +1,188 @@
+import CoreGraphics
+import XCTest
+
+@testable import AgentDeviceMacOSHelper
+
+private struct Candidate: Equatable {
+  let id: Int
+  let area: Double
+}
+
+/// An app screenshot captures the front window or refuses; it never substitutes another window.
+final class AppWindowScreenshotTests: XCTestCase {
+  func testCaptureIdentityAndScrollAndClickFallbackFrameSelectTheSameEligibleFrontWindow() throws {
+    let windows = [
+      windowInfo(id: 1, pid: 99),
+      windowInfo(id: 2, layer: 1),
+      windowInfo(id: 3, onScreen: false),
+      windowInfo(id: 4, alpha: 0),
+      windowInfo(id: 5, width: 0),
+      windowInfo(id: 6, height: 0),
+      windowInfo(id: 7, alpha: 0.5, width: 20, height: 20),
+      windowInfo(id: 8),
+    ]
+
+    let front = try frontWindow(pid: 42, windowInfo: windows)
+    let selected = frontShareableWindow([7, 8], front: front, windowID: { $0 })
+
+    // Scroll and the click actionWindow fallback both consume frontWindowFrame.
+    XCTAssertEqual(
+      try frontWindowFrame(pid: 42, windowInfo: windows),
+      CGRect(x: 10, y: 10, width: 20, height: 20))
+    XCTAssertEqual(front?.number, 7)
+    XCTAssertEqual(try selected.get(), 7)
+  }
+
+  func testSkipsAppModalAlertInFrontOfNormalLayerWindow() throws {
+    let windows = [windowInfo(id: 8, layer: 8), windowInfo(id: 3)]
+    let front = try frontWindow(pid: 42, windowInfo: windows)
+    let selected = frontShareableWindow([8, 3], front: front, windowID: { $0 })
+
+    XCTAssertEqual(front?.number, 3)
+    XCTAssertEqual(try selected.get(), 3)
+    XCTAssertEqual(try frontWindowFrame(pid: 42, windowInfo: windows), front?.bounds)
+  }
+
+  func testRefusesWhenOnlyInvisibleOrEmptyWindowsRemain() throws {
+    let windows = [
+      windowInfo(id: 1, onScreen: false),
+      windowInfo(id: 2, alpha: 0),
+      windowInfo(id: 3, width: 0),
+      windowInfo(id: 4, height: 0),
+    ]
+    let front = try frontWindow(pid: 42, windowInfo: windows)
+    let selected = frontShareableWindow([5], front: front, windowID: { $0 })
+
+    XCTAssertNil(try frontWindowFrame(pid: 42, windowInfo: windows))
+    XCTAssertNil(front)
+    XCTAssertThrowsError(try selected.get()) { error in
+      assertRefusal(error, reason: "window-not-found", hint: "Open or unminimize")
+    }
+  }
+
+  func testUnreadableWindowListRefusesBeforeCaptureOrScrollCanReportNoWindow() {
+    XCTAssertThrowsError(try frontWindow(pid: 42, windowInfo: nil)) { error in
+      assertUnreadableList(error)
+    }
+    XCTAssertThrowsError(try frontWindowFrame(pid: 42, windowInfo: nil)) { error in
+      assertUnreadableList(error)
+    }
+    XCTAssertNil(try frontWindow(pid: 42, windowInfo: []))
+    XCTAssertNil(try frontWindowFrame(pid: 42, windowInfo: []))
+  }
+
+  func testSelectsTheFrontWindowEvenWhenAnotherIsLarger() {
+    let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 3, area: 100)]
+
+    let selected = frontShareableWindow(candidates, front: onScreenWindow(id: 3), windowID: \.id)
+
+    XCTAssertEqual(try selected.get(), Candidate(id: 3, area: 100))
+  }
+
+  func testRefusesWhenTheFrontWindowIsNotShareable() {
+    let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 8, area: 100)]
+
+    let selected = frontShareableWindow(candidates, front: onScreenWindow(id: 3), windowID: \.id)
+
+    XCTAssertThrowsError(try selected.get()) { error in
+      assertRefusal(
+        error,
+        reason: "front-window-not-shareable",
+        hint: "retry after it settles",
+        frontWindowId: "3")
+    }
+  }
+
+  func testCaptureExclusionSurvivesWindowSelectionAndNeverSubstitutesAnotherWindow() throws {
+    let windows = [windowInfo(id: 3, sharingState: 0), windowInfo(id: 7)]
+    let front = try frontWindow(pid: 42, windowInfo: windows)
+    XCTAssertEqual(front?.number, 3)
+    XCTAssertEqual(front?.sharingState, 0)
+
+    for candidates in [[3, 7], [7]] {
+      let selected = frontShareableWindow(candidates, front: front, windowID: { $0 })
+      XCTAssertThrowsError(try selected.get()) { error in
+        assertRefusal(
+          error,
+          reason: "front-window-capture-excluded",
+          hint: "Run `snapshot` to read the window",
+          frontWindowId: "3")
+      }
+    }
+
+    var entry = windowInfo(id: 3)
+    entry.removeValue(forKey: kCGWindowSharingState as String)
+    let unknown = try frontWindow(pid: 42, windowInfo: [entry])
+    let selected = frontShareableWindow([3], front: unknown, windowID: { $0 })
+    XCTAssertEqual(try selected.get(), 3)
+  }
+
+  func testRefusesWhenTheAppHasNoOnScreenWindow() {
+    let selected = frontShareableWindow(
+      [Candidate(id: 7, area: 10_000)], front: nil, windowID: \.id)
+
+    XCTAssertThrowsError(try selected.get()) { error in
+      assertRefusal(error, reason: "window-not-found", hint: "Open or unminimize")
+    }
+  }
+}
+
+/// The refusal carries the reason and the remedy the agent runs next, alongside the front window
+/// it refused.
+private func assertRefusal(
+  _ error: Error,
+  reason: String,
+  hint: String,
+  frontWindowId: String? = nil,
+  file: StaticString = #filePath,
+  line: UInt = #line
+) {
+  guard let refusal = error as? AppWindowScreenshotRefusal else {
+    return XCTFail("expected an AppWindowScreenshotRefusal, got \(error)", file: file, line: line)
+  }
+  guard
+    case HelperError.commandFailed(_, let details) = refusal.error(
+      bundleId: "com.apple.TextEdit")
+  else {
+    return XCTFail("expected a commandFailed refusal error", file: file, line: line)
+  }
+  XCTAssertEqual(details["reason"], reason, file: file, line: line)
+  XCTAssertEqual(details["bundleId"], "com.apple.TextEdit", file: file, line: line)
+  XCTAssertEqual(details["frontWindowId"], frontWindowId, file: file, line: line)
+  XCTAssertTrue(
+    (details["hint"] ?? "").contains(hint),
+    "hint \(details["hint"] ?? "<nil>") lacks \(hint)",
+    file: file, line: line)
+}
+
+private func assertUnreadableList(
+  _ error: Error, file: StaticString = #filePath, line: UInt = #line
+) {
+  guard case HelperError.commandFailed(_, let details) = error else {
+    return XCTFail("expected a typed window-list refusal, got \(error)", file: file, line: line)
+  }
+  XCTAssertEqual(details["reason"], "window-list-unreadable", file: file, line: line)
+  XCTAssertEqual(
+    details["hint"],
+    "The app's on-screen window list could not be read. Retry the command.",
+    file: file, line: line)
+}
+
+private func onScreenWindow(id: Int) -> OnScreenWindow {
+  OnScreenWindow(number: id, bounds: .zero, sharingState: 1)
+}
+
+private func windowInfo(
+  id: Int, pid: Int32 = 42, layer: Int = 0, onScreen: Bool = true,
+  alpha: Double = 1, sharingState: Int = 1, width: Double = 100, height: Double = 100
+) -> [String: Any] {
+  [
+    kCGWindowNumber as String: id,
+    kCGWindowOwnerPID as String: pid,
+    kCGWindowLayer as String: layer,
+    kCGWindowIsOnscreen as String: onScreen,
+    kCGWindowAlpha as String: alpha,
+    kCGWindowSharingState as String: sharingState,
+    kCGWindowBounds as String: CGRect(x: 10, y: 10, width: width, height: height).dictionaryRepresentation,
+  ]
+}