diff --git a/AxePlaygroundApp/AxePlayground/ContentView.swift b/AxePlaygroundApp/AxePlayground/ContentView.swift index c72ed08b7..c6e998003 100644 --- a/AxePlaygroundApp/AxePlayground/ContentView.swift +++ b/AxePlaygroundApp/AxePlayground/ContentView.swift @@ -53,6 +53,8 @@ struct ContentView: View { TouchControlView() case "gesture-presets": GesturePresetsView() + case "landscape-coordinate-test": + LandscapeCoordinateTestView() // Input & Text case "text-input": @@ -84,7 +86,8 @@ struct MainMenuView: View { ("tap-test", "Tap Test", "Displays coordinates of CLI taps"), ("touch-control", "Touch Control", "Touch down/up events"), ("swipe-test", "Swipe Test", "Shows CLI swipe paths"), - ("gesture-presets", "Gesture Presets", "Multi-touch gesture display") + ("gesture-presets", "Gesture Presets", "Multi-touch gesture display"), + ("landscape-coordinate-test", "Landscape Coordinates", "Verifies orientation-aware taps") ]), ("Input & Text", [ ("text-input", "Text Input", "Text typed by CLI commands"), diff --git a/AxePlaygroundApp/AxePlayground/Views/LandscapeCoordinateTestView.swift b/AxePlaygroundApp/AxePlayground/Views/LandscapeCoordinateTestView.swift new file mode 100644 index 000000000..d334a00dc --- /dev/null +++ b/AxePlaygroundApp/AxePlayground/Views/LandscapeCoordinateTestView.swift @@ -0,0 +1,247 @@ +import SwiftUI + +struct LandscapeCoordinateTestView: View { + @State private var tapHitCount = 0 + @State private var tapMissCount = 0 + @State private var swipeHitCount = 0 + @State private var swipeMissCount = 0 + @State private var lastTapHit = "none" + @State private var lastTapMiss = "none" + @State private var lastSwipeStart = "none" + @State private var lastSwipeEnd = "none" + @State private var dragStart: CGPoint? + + private let tapTargetSize = CGSize(width: 120, height: 80) + private let swipeTargetSize = CGSize(width: 90, height: 70) + + var body: some View { + GeometryReader { geometry in + let isLandscape = geometry.size.width > geometry.size.height + let tapTargetCenter = CGPoint( + x: isLandscape ? geometry.size.width * 0.78 : geometry.size.width * 0.22, + y: isLandscape ? geometry.size.height * 0.32 : geometry.size.height * 0.72 + ) + let swipeStartCenter = CGPoint( + x: isLandscape ? geometry.size.width * 0.20 : geometry.size.width * 0.24, + y: isLandscape ? geometry.size.height * 0.76 : geometry.size.height * 0.34 + ) + let swipeEndCenter = CGPoint( + x: isLandscape ? geometry.size.width * 0.84 : geometry.size.width * 0.76, + y: isLandscape ? geometry.size.height * 0.76 : geometry.size.height * 0.34 + ) + let tapTargetRect = CGRect(center: tapTargetCenter, size: tapTargetSize) + let swipeStartRect = CGRect(center: swipeStartCenter, size: swipeTargetSize) + let swipeEndRect = CGRect(center: swipeEndCenter, size: swipeTargetSize) + let globalFrame = geometry.frame(in: .global) + + ZStack { + Color(.systemBackground) + .ignoresSafeArea() + + VStack(spacing: 8) { + Text("Landscape Coordinate Test") + .font(.title2) + .fontWeight(.bold) + .accessibilityIdentifier("landscape-coordinate-title") + + Text(isLandscape ? "Layout: landscape" : "Layout: portrait") + .font(.headline) + .accessibilityIdentifier("landscape-coordinate-layout") + .accessibilityValue(isLandscape ? "landscape" : "portrait") + + Text("Landscape Hit Count: \(tapHitCount)") + .font(.headline) + .foregroundColor(.blue) + .accessibilityIdentifier("landscape-coordinate-hit-count") + .accessibilityValue("\(tapHitCount)") + + Text("Landscape Miss Count: \(tapMissCount)") + .font(.headline) + .foregroundColor(.red) + .accessibilityIdentifier("landscape-coordinate-miss-count") + .accessibilityValue("\(tapMissCount)") + + Text("Landscape Swipe Hit Count: \(swipeHitCount)") + .font(.headline) + .foregroundColor(.blue) + .accessibilityIdentifier("landscape-coordinate-swipe-hit-count") + .accessibilityValue("\(swipeHitCount)") + + Text("Landscape Swipe Miss Count: \(swipeMissCount)") + .font(.headline) + .foregroundColor(.red) + .accessibilityIdentifier("landscape-coordinate-swipe-miss-count") + .accessibilityValue("\(swipeMissCount)") + + Text("Last Tap Hit: \(lastTapHit)") + .font(.subheadline) + .foregroundColor(.secondary) + .accessibilityIdentifier("landscape-coordinate-last-tap-hit") + .accessibilityValue(lastTapHit) + + Text("Last Tap Miss: \(lastTapMiss)") + .font(.subheadline) + .foregroundColor(.secondary) + .accessibilityIdentifier("landscape-coordinate-last-tap-miss") + .accessibilityValue(lastTapMiss) + + Text("Last Swipe Start: \(lastSwipeStart)") + .font(.subheadline) + .foregroundColor(.secondary) + .accessibilityIdentifier("landscape-coordinate-last-swipe-start") + .accessibilityValue(lastSwipeStart) + + Text("Last Swipe End: \(lastSwipeEnd)") + .font(.subheadline) + .foregroundColor(.secondary) + .accessibilityIdentifier("landscape-coordinate-last-swipe-end") + .accessibilityValue(lastSwipeEnd) + } + .padding() + .background(Color.white.opacity(0.92)) + .cornerRadius(12) + .shadow(radius: 4) + .position(x: geometry.size.width * 0.5, y: 130) + .allowsHitTesting(false) + + targetLabel( + isLandscape ? "Landscape Target" : "Portrait Target", + color: isLandscape ? .green : .orange, + size: tapTargetSize + ) + .position(tapTargetCenter) + .accessibilityIdentifier("landscape-coordinate-target") + .accessibilityLabel(isLandscape ? "Landscape Target" : "Portrait Target") + .accessibilityValue(isLandscape ? "landscape-target" : "portrait-target") + .allowsHitTesting(false) + + targetLabel( + isLandscape ? "Landscape Swipe Start" : "Portrait Swipe Start", + color: .purple, + size: swipeTargetSize + ) + .position(swipeStartCenter) + .accessibilityIdentifier("landscape-coordinate-swipe-start") + .accessibilityLabel(isLandscape ? "Landscape Swipe Start" : "Portrait Swipe Start") + .accessibilityValue(isLandscape ? "landscape-swipe-start" : "portrait-swipe-start") + .allowsHitTesting(false) + + targetLabel( + isLandscape ? "Landscape Swipe End" : "Portrait Swipe End", + color: .indigo, + size: swipeTargetSize + ) + .position(swipeEndCenter) + .accessibilityIdentifier("landscape-coordinate-swipe-end") + .accessibilityLabel(isLandscape ? "Landscape Swipe End" : "Portrait Swipe End") + .accessibilityValue(isLandscape ? "landscape-swipe-end" : "portrait-swipe-end") + .allowsHitTesting(false) + + Circle() + .stroke(Color.red, lineWidth: 3) + .frame(width: 44, height: 44) + .position(x: geometry.size.width * 0.22, y: geometry.size.height * 0.72) + .opacity(isLandscape ? 0.25 : 0) + .allowsHitTesting(false) + .accessibilityHidden(true) + } + .contentShape(Rectangle()) + .gesture( + DragGesture(minimumDistance: 0, coordinateSpace: .local) + .onChanged { value in + if dragStart == nil { + dragStart = value.startLocation + } + } + .onEnded { value in + let start = dragStart ?? value.startLocation + let end = value.location + dragStart = nil + + let distance = hypot(end.x - start.x, end.y - start.y) + if distance < 20 { + recordTap(at: end, globalFrame: globalFrame, targetRect: tapTargetRect) + } else { + recordSwipe( + start: start, + end: end, + globalFrame: globalFrame, + startRect: swipeStartRect, + endRect: swipeEndRect + ) + } + } + ) + } + .navigationTitle("Landscape Coordinates") + .navigationBarTitleDisplayMode(.inline) + .accessibilityIdentifier("landscape-coordinate-screen") + } + + private func targetLabel(_ title: String, color: Color, size: CGSize) -> some View { + Text(title) + .font(.headline) + .multilineTextAlignment(.center) + .frame(width: size.width, height: size.height) + .background(color) + .foregroundColor(.white) + .cornerRadius(12) + } + + private func recordTap(at location: CGPoint, globalFrame: CGRect, targetRect: CGRect) { + let globalLocation = location.offsetBy(dx: globalFrame.minX, dy: globalFrame.minY) + if targetRect.contains(location) { + tapHitCount += 1 + lastTapHit = formatted(globalLocation) + } else { + tapMissCount += 1 + lastTapMiss = formatted(globalLocation) + } + } + + private func recordSwipe( + start: CGPoint, + end: CGPoint, + globalFrame: CGRect, + startRect: CGRect, + endRect: CGRect + ) { + let globalStart = start.offsetBy(dx: globalFrame.minX, dy: globalFrame.minY) + let globalEnd = end.offsetBy(dx: globalFrame.minX, dy: globalFrame.minY) + lastSwipeStart = formatted(globalStart) + lastSwipeEnd = formatted(globalEnd) + + if startRect.contains(start) && endRect.contains(end) { + swipeHitCount += 1 + } else { + swipeMissCount += 1 + } + } + + private func formatted(_ point: CGPoint) -> String { + "x:\(Int(point.x.rounded())),y:\(Int(point.y.rounded()))" + } +} + +private extension CGRect { + init(center: CGPoint, size: CGSize) { + self.init( + x: center.x - size.width / 2, + y: center.y - size.height / 2, + width: size.width, + height: size.height + ) + } +} + +private extension CGPoint { + func offsetBy(dx: CGFloat, dy: CGFloat) -> CGPoint { + CGPoint(x: x + dx, y: y + dy) + } +} + +#Preview { + NavigationStack { + LandscapeCoordinateTestView() + } +} diff --git a/BATCHING.md b/BATCHING.md index 26cf6448b..8ed1ac6cd 100644 --- a/BATCHING.md +++ b/BATCHING.md @@ -178,12 +178,15 @@ Each step command keeps its normal options and validation. Examples: - `tap --id BackButton` - `tap -x 200 -y 400 --pre-delay 0.2 --post-delay 0.2` +- `swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100` - `gesture scroll-down --duration 1.0` - `touch -x 150 -y 300 --down --up --delay 1.0` One important rule: - Do not include `--udid` inside a step. Use the batch-level `--udid` only. +Coordinate-based `tap`, `swipe`, and `touch` steps use the same orientation-aware coordinate mapping as the standalone commands. Use coordinates from `describe-ui` directly; AXe detects rotated landscape simulator orientation and letterboxed landscape-only app layouts automatically. + ## Verification strategy Batch is execution-focused. It does not do built-in assertions. diff --git a/CHANGELOG.md b/CHANGELOG.md index 49321535d..6b4a71358 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to the AXe iOS testing framework will be documented in this The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed + +- Fixed `tap`, `touch`, `swipe`, and matching batch steps dispatching logical UI coordinates directly to FBSimulatorHIDEvent without landscape rotation or letterbox correction, causing interactions to land in the wrong location in rotated landscape simulators and portrait-hardware landscape-only apps. AXe now detects the simulator UI orientation automatically instead of requiring callers to pass landscape flags ([#5](https://github.com/cameroncooke/AXe/issues/5) by [@Nitewriter](https://github.com/Nitewriter)) + ## [v1.6.0] - 2026-04-05 ### Added diff --git a/Package.swift b/Package.swift index 556893c19..a9065c7e7 100644 --- a/Package.swift +++ b/Package.swift @@ -7,6 +7,10 @@ let package = Package( .macOS(.v14) ], products: [ + .library( + name: "AXeCore", + targets: ["AXeCore"] + ), .executable( name: "axe", targets: ["AXe"] @@ -16,9 +20,14 @@ let package = Package( .package(url: "https://github.com/apple/swift-argument-parser", from: "1.5.0"), ], targets: [ + .target( + name: "AXeCore", + path: "Sources/AXeCore" + ), .executableTarget( name: "AXe", dependencies: [ + "AXeCore", .product(name: "ArgumentParser", package: "swift-argument-parser"), "FBSimulatorControl", "FBDeviceControl", @@ -46,7 +55,7 @@ let package = Package( ), .testTarget( name: "AXeTests", - dependencies: ["AXe"], + dependencies: ["AXe", "AXeCore"], path: "Tests", resources: [ .copy("README.md") diff --git a/README.md b/README.md index 0b29a7e15..cbfaa7b00 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,12 @@ axe tap --label "Safari" --udid SIMULATOR_UDID axe swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100 --udid SIMULATOR_UDID axe swipe --start-x 50 --start-y 500 --end-x 350 --end-y 500 --duration 2.0 --delta 25 --udid SIMULATOR_UDID +# Orientation-aware coordinates +# AXe automatically maps logical UI coordinates for rotated landscape and letterboxed landscape-only apps. +# Use the coordinates from describe-ui directly; AXe detects the simulator orientation. +axe tap -x 100 -y 200 --udid SIMULATOR_UDID +axe swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100 --udid SIMULATOR_UDID + # Advanced touch control axe touch -x 150 -y 250 --down --udid SIMULATOR_UDID axe touch -x 150 -y 250 --up --udid SIMULATOR_UDID diff --git a/Skills/CLI/axe/SKILL.md b/Skills/CLI/axe/SKILL.md index 3ea4e129d..5c1b9cd41 100644 --- a/Skills/CLI/axe/SKILL.md +++ b/Skills/CLI/axe/SKILL.md @@ -33,6 +33,7 @@ HID commands (`tap`, `swipe`, `type`, `key`, etc.) are fire-and-forget — AXe c ## Step 4: Apply timing and input best practices - Use `--pre-delay` / `--post-delay` on tap, swipe, and gesture commands for fixed delays around actions. - Use `--duration` to control how long a swipe, gesture, button press, or key press lasts. +- Coordinate-based `tap`, `swipe`, and `touch` accept coordinates from `describe-ui` directly; AXe detects rotated landscape simulator orientation and letterboxed landscape-only app layouts automatically. - For text with shell-sensitive characters, prefer `--stdin` or `--file` over inline quotes. - Use single quotes for inline text arguments to avoid shell expansion issues. diff --git a/Sources/AXe/Commands/Swipe.swift b/Sources/AXe/Commands/Swipe.swift index 57bb68f60..231de3947 100644 --- a/Sources/AXe/Commands/Swipe.swift +++ b/Sources/AXe/Commands/Swipe.swift @@ -35,6 +35,7 @@ struct Swipe: AsyncParsableCommand { @Option(name: .customLong("udid"), help: "The UDID of the simulator.") var simulatorUDID: String + func validate() throws { // Validate coordinates are non-negative guard startX >= 0, startY >= 0, endX >= 0, endY >= 0 else { @@ -87,6 +88,14 @@ struct Swipe: AsyncParsableCommand { logger.info().log("Performing swipe from (\(startX), \(startY)) to (\(endX), \(endY))") logger.info().log("Duration: \(swipeDuration)s, Delta: \(swipeDelta)px") + let physicalPoints = try await OrientationAwareCoordinates.translateBatch( + points: [(x: startX, y: startY), (x: endX, y: endY)], + for: simulatorUDID, + logger: logger + ) + let physicalStart = physicalPoints[0] + let physicalEnd = physicalPoints[1] + // Broadcast swipe notification for external listeners NotificationCenter.default.post( name: .hidSwipePerformed, @@ -112,10 +121,10 @@ struct Swipe: AsyncParsableCommand { // Create main swipe HID event let swipeEvent = FBSimulatorHIDEvent.swipe( - startX, - yStart: startY, - xEnd: endX, - yEnd: endY, + physicalStart.x, + yStart: physicalStart.y, + xEnd: physicalEnd.x, + yEnd: physicalEnd.y, delta: swipeDelta, duration: swipeDuration ) diff --git a/Sources/AXe/Commands/Tap.swift b/Sources/AXe/Commands/Tap.swift index f96946e8d..c4ec728d0 100644 --- a/Sources/AXe/Commands/Tap.swift +++ b/Sources/AXe/Commands/Tap.swift @@ -41,6 +41,7 @@ struct Tap: AsyncParsableCommand { @Option(name: .customLong("udid"), help: "The UDID of the simulator.") var simulatorUDID: String + func validate() throws { if pointX != nil || pointY != nil { guard let pointX, let pointY else { @@ -68,6 +69,7 @@ struct Tap: AsyncParsableCommand { } } + if let preDelay = preDelay { guard preDelay >= 0 && preDelay <= 10.0 else { throw ValidationError("Pre-delay must be between 0 and 10 seconds.") @@ -97,11 +99,11 @@ struct Tap: AsyncParsableCommand { try await performGlobalSetup(logger: logger) - let resolvedPoint: (x: Double, y: Double) + let logicalPoint: (x: Double, y: Double) let resolvedDescription: String if let pointX, let pointY { - resolvedPoint = (x: pointX, y: pointY) + logicalPoint = (x: pointX, y: pointY) resolvedDescription = "(\(pointX), \(pointY))" } else { let query: AccessibilityQuery @@ -116,7 +118,7 @@ struct Tap: AsyncParsableCommand { } do { - resolvedPoint = try await AccessibilityPoller.resolveWithPolling( + logicalPoint = try await AccessibilityPoller.resolveWithPolling( query: query, simulatorUDID: simulatorUDID, waitTimeout: waitTimeout, @@ -129,11 +131,17 @@ struct Tap: AsyncParsableCommand { throw error } - resolvedDescription = "center of matched element at (\(resolvedPoint.x), \(resolvedPoint.y))" + resolvedDescription = "center of matched element at (\(logicalPoint.x), \(logicalPoint.y))" } logger.info().log("Tapping at \(resolvedDescription)") + let resolvedPoint = try await OrientationAwareCoordinates.translate( + point: logicalPoint, + for: simulatorUDID, + logger: logger + ) + var events: [FBSimulatorHIDEvent] = [] if let preDelay = preDelay, preDelay > 0 { logger.info().log("Pre-delay: \(preDelay)s") @@ -152,6 +160,6 @@ struct Tap: AsyncParsableCommand { try await HIDInteractor.performHIDEvent(finalEvent, for: simulatorUDID, logger: logger) logger.info().log("Tap completed successfully") - print("✓ Tap at (\(resolvedPoint.x), \(resolvedPoint.y)) completed successfully") + print("✓ Tap at \(resolvedDescription) completed successfully") } } diff --git a/Sources/AXe/Commands/Touch.swift b/Sources/AXe/Commands/Touch.swift index 0c8aded32..f8f222b11 100644 --- a/Sources/AXe/Commands/Touch.swift +++ b/Sources/AXe/Commands/Touch.swift @@ -36,6 +36,7 @@ struct Touch: AsyncParsableCommand { @Option(name: .customLong("udid"), help: "The UDID of the simulator.") var simulatorUDID: String + func validate() throws { // Validate coordinates are non-negative guard pointX >= 0, pointY >= 0 else { @@ -71,6 +72,12 @@ struct Touch: AsyncParsableCommand { logger.info().log("Performing touch events at (\(pointX), \(pointY))") + let physicalPoint = try await OrientationAwareCoordinates.translate( + point: (x: pointX, y: pointY), + for: simulatorUDID, + logger: logger + ) + if touchDown && touchUp { // Send down and up as separate HID submissions so iOS recognizers // observe a real hold duration for long-press gestures. @@ -79,7 +86,7 @@ struct Touch: AsyncParsableCommand { logger.info().log("Touch down") try await HIDInteractor .performHIDEvent( - FBSimulatorHIDEvent.touchDownAt(x: pointX, y: pointY), + FBSimulatorHIDEvent.touchDownAt(x: physicalPoint.x, y: physicalPoint.y), for: simulatorUDID, logger: logger ) @@ -93,7 +100,7 @@ struct Touch: AsyncParsableCommand { logger.info().log("Touch up") try await HIDInteractor .performHIDEvent( - FBSimulatorHIDEvent.touchUpAt(x: pointX, y: pointY), + FBSimulatorHIDEvent.touchUpAt(x: physicalPoint.x, y: physicalPoint.y), for: simulatorUDID, logger: logger ) @@ -101,7 +108,7 @@ struct Touch: AsyncParsableCommand { logger.info().log("Touch down") try await HIDInteractor .performHIDEvent( - FBSimulatorHIDEvent.touchDownAt(x: pointX, y: pointY), + FBSimulatorHIDEvent.touchDownAt(x: physicalPoint.x, y: physicalPoint.y), for: simulatorUDID, logger: logger ) @@ -109,7 +116,7 @@ struct Touch: AsyncParsableCommand { logger.info().log("Touch up") try await HIDInteractor .performHIDEvent( - FBSimulatorHIDEvent.touchUpAt(x: pointX, y: pointY), + FBSimulatorHIDEvent.touchUpAt(x: physicalPoint.x, y: physicalPoint.y), for: simulatorUDID, logger: logger ) diff --git a/Sources/AXe/Resources/skills/axe/SKILL.md b/Sources/AXe/Resources/skills/axe/SKILL.md index d5d1bde61..db2c2bc99 100644 --- a/Sources/AXe/Resources/skills/axe/SKILL.md +++ b/Sources/AXe/Resources/skills/axe/SKILL.md @@ -33,6 +33,7 @@ HID commands (`tap`, `swipe`, `type`, `key`, etc.) are fire-and-forget — AXe c ## Step 4: Apply timing and input best practices - Use `--pre-delay` / `--post-delay` on tap, swipe, and gesture commands for fixed delays around actions. - Use `--duration` to control how long a swipe, gesture, button press, or key press lasts. +- Coordinate-based `tap`, `swipe`, and `touch` accept coordinates from `describe-ui` directly; AXe detects rotated landscape simulator orientation and letterboxed landscape-only app layouts automatically. - For text with shell-sensitive characters, prefer `--stdin` or `--file` over inline quotes. - Use single quotes for inline text arguments to avoid shell expansion issues. diff --git a/Sources/AXe/Utilities/Batch/Command+BatchConvertible.swift b/Sources/AXe/Utilities/Batch/Command+BatchConvertible.swift index 5e8fe47d8..1a016c5ad 100644 --- a/Sources/AXe/Utilities/Batch/Command+BatchConvertible.swift +++ b/Sources/AXe/Utilities/Batch/Command+BatchConvertible.swift @@ -23,6 +23,39 @@ private func buildDelayedEvent( return events.count == 1 ? events[0] : FBSimulatorHIDEvent(events: events) } +private func resolveBatchTapPoint( + query: AccessibilityQuery, + context: BatchContext, + elementType: String?, + logger: AxeLogger +) async throws -> (point: (x: Double, y: Double), roots: [AccessibilityElement]) { + let roots = try await context.accessibilityRoots(logger: logger) + do { + let point = try AccessibilityTargetResolver.resolveCenterPoint(roots: roots, query: query, elementType: elementType) + return (point, roots) + } catch let error as ElementResolutionError where error.isNotFound && context.waitTimeout > 0 { + let clock = ContinuousClock() + let deadline = clock.now + .seconds(context.waitTimeout) + var lastError = error + + while clock.now < deadline { + logger.info().log("Element not found, retrying in \(context.pollInterval)s…") + try await Task.sleep(for: .seconds(context.pollInterval)) + + let freshRoots = try await context.accessibilityRoots(logger: logger, forceRefresh: true) + do { + let point = try AccessibilityTargetResolver.resolveCenterPoint(roots: freshRoots, query: query, elementType: elementType) + return (point, freshRoots) + } catch let retryError as ElementResolutionError where retryError.isNotFound { + lastError = retryError + continue + } + } + + throw lastError + } +} + func parseCommaSeparatedIntsStrict(_ rawValue: String, fieldName: String) throws -> [Int] { let rawTokens = rawValue .split(separator: ",", omittingEmptySubsequences: false) @@ -41,9 +74,11 @@ func parseCommaSeparatedIntsStrict(_ rawValue: String, fieldName: String) throws extension Tap: BatchConvertible { func toBatchPrimitives(context: BatchContext, logger: AxeLogger) async throws -> [BatchPrimitive] { let resolvedPoint: (x: Double, y: Double) + let resolvedRoots: [AccessibilityElement]? if let pointX, let pointY { resolvedPoint = (pointX, pointY) + resolvedRoots = nil } else { let query: AccessibilityQuery if let elementID { @@ -56,17 +91,33 @@ extension Tap: BatchConvertible { throw CLIError(errorDescription: "Unexpected state: no coordinates and no element query.") } - resolvedPoint = try await AccessibilityPoller.resolveWithPolling( + let resolved = try await resolveBatchTapPoint( query: query, - simulatorUDID: context.simulatorUDID, - waitTimeout: context.waitTimeout, - pollInterval: context.pollInterval, + context: context, elementType: elementType, logger: logger ) + resolvedPoint = resolved.point + resolvedRoots = resolved.roots } - let tapEvent = FBSimulatorHIDEvent.tapAt(x: resolvedPoint.x, y: resolvedPoint.y) + let physicalPoint: (x: Double, y: Double) + if let resolvedRoots { + physicalPoint = try await OrientationAwareCoordinates.translate( + point: resolvedPoint, + roots: resolvedRoots, + for: context.simulatorUDID, + logger: logger + ) + } else { + physicalPoint = try await OrientationAwareCoordinates.translate( + point: resolvedPoint, + for: context.simulatorUDID, + logger: logger + ) + } + + let tapEvent = FBSimulatorHIDEvent.tapAt(x: physicalPoint.x, y: physicalPoint.y) return [.hidMergeable(buildDelayedEvent(preDelay: preDelay, mainEvent: tapEvent, postDelay: postDelay))] } } @@ -75,11 +126,19 @@ extension Swipe: BatchConvertible { func toBatchPrimitives(context: BatchContext, logger: AxeLogger) async throws -> [BatchPrimitive] { let swipeDuration = duration ?? 1.0 let swipeDelta = delta ?? 50.0 + let physicalPoints = try await OrientationAwareCoordinates.translateBatch( + points: [(x: startX, y: startY), (x: endX, y: endY)], + for: context.simulatorUDID, + logger: logger + ) + let physicalStart = physicalPoints[0] + let physicalEnd = physicalPoints[1] + let swipeEvent = FBSimulatorHIDEvent.swipe( - startX, - yStart: startY, - xEnd: endX, - yEnd: endY, + physicalStart.x, + yStart: physicalStart.y, + xEnd: physicalEnd.x, + yEnd: physicalEnd.y, delta: swipeDelta, duration: swipeDuration ) @@ -110,8 +169,14 @@ extension Gesture: BatchConvertible { extension Touch: BatchConvertible { func toBatchPrimitives(context: BatchContext, logger: AxeLogger) async throws -> [BatchPrimitive] { - let touchDownEvent = FBSimulatorHIDEvent.touchDownAt(x: pointX, y: pointY) - let touchUpEvent = FBSimulatorHIDEvent.touchUpAt(x: pointX, y: pointY) + let physicalPoint = try await OrientationAwareCoordinates.translate( + point: (x: pointX, y: pointY), + for: context.simulatorUDID, + logger: logger + ) + + let touchDownEvent = FBSimulatorHIDEvent.touchDownAt(x: physicalPoint.x, y: physicalPoint.y) + let touchUpEvent = FBSimulatorHIDEvent.touchUpAt(x: physicalPoint.x, y: physicalPoint.y) if touchDown && touchUp { let holdDelay = delay ?? 0.1 diff --git a/Sources/AXe/Utilities/OrientationAwareCoordinates.swift b/Sources/AXe/Utilities/OrientationAwareCoordinates.swift new file mode 100644 index 000000000..bfdca6653 --- /dev/null +++ b/Sources/AXe/Utilities/OrientationAwareCoordinates.swift @@ -0,0 +1,588 @@ +import Darwin +import Foundation +import AXeCore +import FBControlCore +import FBSimulatorControl + +// MARK: - Simulator Orientation + +/// Logical UI orientation of a simulator as reported by its accessibility tree. +/// +/// Matches UIInterfaceOrientation semantics: +/// - `portrait` — home button at the bottom (default) +/// - `portraitUpsideDown` — home button at the top (180°) +/// - `landscape` — home button to the right (90° CW from portrait) +/// - `landscapeFlipped` — home button to the left (90° CCW from portrait) +enum SimulatorOrientation: String, CaseIterable { + case portrait + case portraitUpsideDown + case landscape + case landscapeFlipped + + /// True when the logical screen width is wider than tall. + var isLandscape: Bool { + coreOrientation.isLandscape + } + + var coreOrientation: OrientationCoordinateMath.Orientation { + switch self { + case .portrait: + return .portrait + case .portraitUpsideDown: + return .portraitUpsideDown + case .landscape: + return .landscape + case .landscapeFlipped: + return .landscapeFlipped + } + } +} + +// MARK: - Coordinate Mapping + +/// Describes how logical coordinates from the accessibility tree map to the physical +/// portrait coordinate space that FBSimulatorHIDEvent expects. +/// +/// The AX application frame alone cannot distinguish two landscape cases: +/// - **Rotation:** hardware is in landscape (device rotated). Rotation math applies. +/// - **Letterbox:** hardware is portrait, but the app declares landscape-only +/// orientations. iOS scales + centers the landscape UI inside the portrait +/// viewport. Scale + offset math applies. +/// +/// Detection: read SimulatorKit's private screen `uiOrientation` when available, +/// then fall back to screenshot pixel aspect ratio to distinguish portrait-hardware +/// letterboxing from a rotated simulator. +enum CoordinateMapping { + /// Portrait device, portrait app — pass coordinates through unchanged. + case passthrough + + /// Hardware is rotated. Apply rotation math. + /// + /// `portraitSize` = `(width: shortSide, height: longSide)` in logical points. + case rotation(SimulatorOrientation, portraitSize: (width: Double, height: Double)) + + /// Hardware is portrait, app is landscape-only. Apply letterbox scale + offset. + /// + /// Physical point = `(offsetX + lx * scale, offsetY + ly * scale)`. + case letterbox(scale: Double, offsetX: Double, offsetY: Double) +} + +// MARK: - Orientation-Aware Coordinate Translation + +/// Translates logical UI coordinates (as reported by the accessibility tree and +/// consumed by `axe tap`, `axe touch`, and `axe swipe`) into the physical portrait +/// coordinate space that FBSimulatorHIDEvent expects. +/// +/// ## Root cause +/// The iOS Simulator's HID layer always operates in physical portrait space. +/// The accessibility tree, however, reports element positions in the *logical* +/// orientation the app currently presents. When the device is rotated, logical +/// coordinates must be mapped back to physical portrait before dispatching HID +/// events, otherwise taps land in the wrong location. +/// +/// ## Two landscape cases +/// Both a rotated device and a portrait device running a landscape-only app produce +/// a landscape-shaped AX application frame. AXe first asks the simulator's private +/// screen properties for the current UI orientation so it can distinguish landscape +/// left from landscape right automatically. If that private probe is unavailable, +/// screenshot dimensions are used only to distinguish rotated hardware from a +/// portrait device running a letterboxed landscape-only app. If the screenshot +/// confirms rotated hardware but private orientation is unavailable, AXe fails +/// clearly instead of guessing landscape direction. +/// +/// ## Portrait dimensions +/// The short side of the device in points is the portrait width; the long side +/// is the portrait height. These are derived from the application frame: in +/// portrait they are `frame.width` and `frame.height`; in landscape they are +/// swapped. +@MainActor +struct OrientationAwareCoordinates { + + // MARK: - Coordinate Mapping Detection + + /// Determines which coordinate mapping to apply for the given simulator. + /// + /// Detection flow: + /// 1. Fetch AX application frame. + /// 2. If an explicit override was supplied → rotation case. + /// 3. Otherwise read SimulatorKit private UI orientation. + /// 4. If private orientation is unavailable and the AX frame is portrait-shaped → passthrough. + /// 5. If unavailable and landscape-shaped, probe screenshot dimensions. + /// - Screenshot portrait-shaped (taller than wide) → letterbox case. + /// - Screenshot landscape-shaped (wider than tall) → rotated device case. + /// + /// - Parameters: + /// - simulatorUDID: Target simulator UDID. + /// - orientationOverride: Forces a specific rotated-device `SimulatorOrientation`. + /// Intended for internal tests or future explicit callers; normal CLI commands + /// rely on automatic private orientation detection. + /// - logger: AXe logger instance. + /// - Returns: The `CoordinateMapping` that applies to the current simulator state. + static func detectMapping( + for simulatorUDID: String, + orientationOverride: SimulatorOrientation? = nil, + logger: AxeLogger + ) async throws -> CoordinateMapping { + let roots = try await AccessibilityFetcher.fetchAccessibilityElements( + for: simulatorUDID, + logger: logger + ) + + return try await detectMapping( + from: roots, + simulatorUDID: simulatorUDID, + orientationOverride: orientationOverride, + logger: logger + ) + } + + static func detectMapping( + from roots: [AccessibilityElement], + simulatorUDID: String, + orientationOverride: SimulatorOrientation? = nil, + logger: AxeLogger + ) async throws -> CoordinateMapping { + guard let appFrame = applicationFrame(from: roots) else { + throw CLIError(errorDescription: "Unable to determine coordinate mapping because the accessibility application frame is unavailable.") + } + + if let orientationOverride { + logger.info().log("Using explicit orientation override: \(orientationOverride.rawValue)") + guard let portraitSize = portraitDimensions(from: roots, orientation: orientationOverride) else { + throw CLIError(errorDescription: "Unable to determine coordinate mapping because portrait dimensions are unavailable.") + } + return .rotation(orientationOverride, portraitSize: portraitSize) + } + + // Prefer the simulator's private UI orientation state when available; this + // distinguishes landscape-left from landscape-right and catches upside-down portrait. + let simulatorOrientation = await SimulatorOrientationReader.currentOrientation( + simulatorUDID: simulatorUDID, + logger: logger + ) + + if let simulatorOrientation { + logger.info().log("Detected simulator UI orientation: \(simulatorOrientation.rawValue)") + guard let portraitSize = portraitDimensions(from: roots, orientation: simulatorOrientation) else { + throw CLIError(errorDescription: "Unable to determine coordinate mapping because portrait dimensions are unavailable.") + } + + switch simulatorOrientation { + case .portrait: + guard appFrame.width > appFrame.height else { + return .passthrough + } + logger.info().log( + "Simulator reports portrait but AX frame \(Int(appFrame.width))×\(Int(appFrame.height)) is landscape; probing for letterbox mapping" + ) + case .portraitUpsideDown, .landscape, .landscapeFlipped: + return .rotation(simulatorOrientation, portraitSize: portraitSize) + } + } + + guard appFrame.width > appFrame.height else { + logger.info().log( + "AX frame \(Int(appFrame.width))×\(Int(appFrame.height)) is portrait; passthrough" + ) + return .passthrough + } + + // Fall back to screenshot aspect ratio to distinguish rotated hardware from a + // portrait device running a landscape-only app. + logger.info().log( + "AX frame \(Int(appFrame.width))×\(Int(appFrame.height)) is landscape; probing screenshot dimensions" + ) + + let screenshotDims = await screenshotPixelDimensions(for: simulatorUDID, logger: logger) + + if let dims = screenshotDims, dims.height > dims.width { + // Screenshot is portrait-shaped despite a landscape AX frame → + // hardware is portrait, app is landscape-only: letterbox case. + logger.info().log( + "Screenshot \(dims.width)×\(dims.height)px is portrait-shaped; using letterbox mapping" + ) + + // Physical viewport in points: short side = portrait width, long side = portrait height. + // Derived from the landscape AX frame by swapping axes (same as portraitDimensions). + let physW = appFrame.height // portrait width = landscape logical height + let physH = appFrame.width // portrait height = landscape logical width + let logW = appFrame.width // logical landscape width + let logH = appFrame.height // logical landscape height + + // Uniform scale to fit logical content into physical viewport. + let scale = min(physW / logW, physH / logH) + let offsetX = (physW - logW * scale) / 2 + let offsetY = (physH - logH * scale) / 2 + + logger.info().log( + String(format: "Letterbox: scale=%.4f offsetX=%.1f offsetY=%.1f", scale, offsetX, offsetY) + ) + return .letterbox(scale: scale, offsetX: offsetX, offsetY: offsetY) + } + + guard let screenshotDims else { + throw CLIError(errorDescription: "Unable to determine coordinate mapping because screenshot probing failed for a landscape accessibility frame.") + } + + // Screenshot is landscape-shaped — hardware is rotated. + logger.info().log( + "Screenshot \(screenshotDims.width)×\(screenshotDims.height)px is landscape-shaped; using rotation mapping" + ) + + throw CLIError(errorDescription: "Unable to determine rotated simulator orientation. AXe can read landscape coordinates only when SimulatorKit reports the current UI orientation; the screenshot confirms the simulator is rotated, but the private orientation probe was unavailable.") + } + + // MARK: - Screenshot Dimension Probe + + /// Reads the pixel dimensions of the current simulator screenshot from a + /// temporary PNG generated by `xcrun simctl io screenshot`. + /// + /// Only the PNG header is read from the temporary file, which is removed + /// before this function returns. + /// + /// Returns `nil` if the process fails or the output is not a valid PNG. + static func screenshotPixelDimensions( + for simulatorUDID: String, + logger: AxeLogger + ) async -> (width: Int, height: Int)? { + let tempURL = FileManager.default.temporaryDirectory + .appendingPathComponent("axe-screenshot-probe-\(UUID().uuidString)") + .appendingPathExtension("png") + defer { try? FileManager.default.removeItem(at: tempURL) } + + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/xcrun") + process.arguments = ["simctl", "io", simulatorUDID, "screenshot", "--type", "png", tempURL.path] + process.standardOutput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + + do { + try process.run() + } catch { + logger.info().log("Screenshot probe: could not launch xcrun simctl: \(error)") + return nil + } + + guard await waitForProcessExit(process, timeout: 3.0, logger: logger), process.terminationStatus == 0 else { + logger.info().log("Screenshot probe: simctl screenshot failed or timed out") + return nil + } + + let headerData: Data + do { + let fileHandle = try FileHandle(forReadingFrom: tempURL) + defer { try? fileHandle.close() } + headerData = try fileHandle.read(upToCount: 24) ?? Data() + } catch { + logger.info().log("Screenshot probe: could not read temp PNG: \(error)") + return nil + } + + guard headerData.count >= 24 else { + logger.info().log("Screenshot probe: PNG header too short (\(headerData.count) bytes)") + return nil + } + + // Verify PNG signature: 8 bytes \x89PNG\r\n\x1a\n + let pngSignature: [UInt8] = [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A] + guard headerData.prefix(8).elementsEqual(pngSignature) else { + logger.info().log("Screenshot probe: output is not a valid PNG") + return nil + } + + // IHDR width and height at bytes 16-19 and 20-23 (big-endian uint32). + let widthBytes = headerData[16..<20] + let heightBytes = headerData[20..<24] + + let width = widthBytes.reduce(0) { Int($0) << 8 | Int($1) } + let height = heightBytes.reduce(0) { Int($0) << 8 | Int($1) } + + logger.info().log("Screenshot probe: \(width)×\(height)px") + return (width: width, height: height) + } + + private static func waitForProcessExit( + _ process: Process, + timeout: TimeInterval, + logger: AxeLogger + ) async -> Bool { + let deadline = Date().addingTimeInterval(timeout) + while process.isRunning && Date() < deadline { + try? await Task.sleep(nanoseconds: 10_000_000) + } + + guard process.isRunning else { + process.waitUntilExit() + return true + } + + logger.info().log("Screenshot probe: timed out waiting for simctl screenshot") + process.terminate() + + let terminateDeadline = Date().addingTimeInterval(0.5) + while process.isRunning && Date() < terminateDeadline { + try? await Task.sleep(nanoseconds: 10_000_000) + } + + if process.isRunning { + logger.info().log("Screenshot probe: force-killing unresponsive simctl screenshot process") + kill(process.processIdentifier, SIGKILL) + } + process.waitUntilExit() + return false + } + + // MARK: - Coordinate Translation + + /// Translates a logical point (as reported by the accessibility tree) to the + /// physical portrait point expected by FBSimulatorHIDEvent using rotation math. + /// + /// - Parameters: + /// - point: Logical (x, y) in the current UI orientation. + /// - orientation: The current logical orientation of the simulator. + /// - portraitSize: The device's portrait dimensions `(width: shortSide, height: longSide)`. + /// - Returns: Physical (x, y) in portrait space. + static func translateToPhysical( + point: (x: Double, y: Double), + orientation: SimulatorOrientation, + portraitSize: (width: Double, height: Double) + ) -> (x: Double, y: Double) { + OrientationCoordinateMath.translateToPhysical( + x: point.x, + y: point.y, + orientation: orientation.coreOrientation, + portraitWidth: portraitSize.width, + portraitHeight: portraitSize.height + ) + } + + /// Translates a logical point to a physical point using letterbox scale + offset math. + /// + /// iOS places a landscape-only app inside the portrait viewport by scaling it + /// uniformly and centering it. The physical point is: + /// + /// `px = offsetX + lx * scale` + /// `py = offsetY + ly * scale` + /// + /// - Parameters: + /// - point: Logical (x, y) as reported by the AX tree. + /// - scale: Uniform scale factor (`min(physW / logW, physH / logH)`). + /// - offsetX: Horizontal letterbox offset in points (`(physW - logW * scale) / 2`). + /// - offsetY: Vertical letterbox offset in points (`(physH - logH * scale) / 2`). + /// - Returns: Physical (x, y) in the portrait hardware viewport. + static func letterboxToPhysical( + point: (x: Double, y: Double), + scale: Double, + offsetX: Double, + offsetY: Double + ) -> (x: Double, y: Double) { + OrientationCoordinateMath.letterboxToPhysical( + x: point.x, + y: point.y, + scale: scale, + offsetX: offsetX, + offsetY: offsetY + ) + } + + /// Derives the device's portrait dimensions from the accessibility tree. + /// + /// In portrait, the application frame is already in portrait coordinates. + /// In landscape, the width and height are swapped compared to portrait. + /// + /// - Parameters: + /// - roots: Root accessibility elements from `AccessibilityFetcher`. + /// - orientation: The current logical orientation. + /// - Returns: Portrait `(width, height)`, or `nil` if the application frame is unavailable. + static func portraitDimensions( + from roots: [AccessibilityElement], + orientation: SimulatorOrientation + ) -> (width: Double, height: Double)? { + guard let frame = applicationFrame(from: roots) else { return nil } + + if orientation.isLandscape { + // In landscape: logical width = portrait height, logical height = portrait width + return (width: frame.height, height: frame.width) + } + return (width: frame.width, height: frame.height) + } + + // MARK: - Full Pipeline + + /// Detects the appropriate coordinate mapping and translates a single logical + /// point to the physical portrait point expected by FBSimulatorHIDEvent. + /// + /// This is the single entry point used by `Tap`, `Touch`, and `Swipe`. + /// + /// Handles three cases automatically: + /// - Portrait device, portrait app → passthrough. + /// - Portrait device, landscape-only app → letterbox scale + offset. + /// - Rotated hardware → rotation math. + /// + /// - Parameters: + /// - point: Logical (x, y) as supplied by the caller. + /// - simulatorUDID: Target simulator UDID. + /// - orientationOverride: Optional explicit rotated-device orientation. Pass + /// `.landscapeFlipped` when the device is rotated counter-clockwise. Leave nil for + /// automatic letterbox detection in portrait-hardware landscape-only apps. + /// - logger: AXe logger instance. + /// - Returns: Physical (x, y) ready for FBSimulatorHIDEvent. + static func translate( + point: (x: Double, y: Double), + for simulatorUDID: String, + orientationOverride: SimulatorOrientation? = nil, + logger: AxeLogger + ) async throws -> (x: Double, y: Double) { + let mapping = try await detectMapping( + for: simulatorUDID, + orientationOverride: orientationOverride, + logger: logger + ) + + return applyMapping(mapping, to: point, logger: logger) + } + + static func translate( + point: (x: Double, y: Double), + roots: [AccessibilityElement], + for simulatorUDID: String, + orientationOverride: SimulatorOrientation? = nil, + logger: AxeLogger + ) async throws -> (x: Double, y: Double) { + let mapping = try await detectMapping( + from: roots, + simulatorUDID: simulatorUDID, + orientationOverride: orientationOverride, + logger: logger + ) + + return applyMapping(mapping, to: point, logger: logger) + } + + /// Translates multiple logical points using a single detection round-trip. + /// + /// Prefer this over calling `translate(point:for:orientationOverride:logger:)` in a loop when + /// translating several points for the same simulator state (e.g. swipe start and end). + /// + /// - Parameters: + /// - points: Logical (x, y) pairs in the current UI orientation. + /// - simulatorUDID: Target simulator UDID. + /// - orientationOverride: Optional explicit orientation for the rotated device case. + /// - logger: AXe logger instance. + /// - Returns: Physical (x, y) pairs ready for FBSimulatorHIDEvent, in the same order as input. + static func translateBatch( + points: [(x: Double, y: Double)], + for simulatorUDID: String, + orientationOverride: SimulatorOrientation? = nil, + logger: AxeLogger + ) async throws -> [(x: Double, y: Double)] { + let mapping = try await detectMapping( + for: simulatorUDID, + orientationOverride: orientationOverride, + logger: logger + ) + + return points.map { applyMapping(mapping, to: $0, logger: logger) } + } + + static func translateBatch( + points: [(x: Double, y: Double)], + roots: [AccessibilityElement], + for simulatorUDID: String, + orientationOverride: SimulatorOrientation? = nil, + logger: AxeLogger + ) async throws -> [(x: Double, y: Double)] { + let mapping = try await detectMapping( + from: roots, + simulatorUDID: simulatorUDID, + orientationOverride: orientationOverride, + logger: logger + ) + + return points.map { applyMapping(mapping, to: $0, logger: logger) } + } + + // MARK: - Private Helpers + + /// Applies a `CoordinateMapping` to a single logical point, logging the translation. + private static func applyMapping( + _ mapping: CoordinateMapping, + to point: (x: Double, y: Double), + logger: AxeLogger + ) -> (x: Double, y: Double) { + let physical: (x: Double, y: Double) + + switch mapping { + case .passthrough: + return point + + case .rotation(let orientation, let portraitSize): + physical = translateToPhysical(point: point, orientation: orientation, portraitSize: portraitSize) + logger.info().log( + "Translated logical (\(Int(point.x)), \(Int(point.y))) → physical (\(Int(physical.x)), \(Int(physical.y))) [rotation: \(orientation.rawValue)]" + ) + + case .letterbox(let scale, let offsetX, let offsetY): + physical = letterboxToPhysical(point: point, scale: scale, offsetX: offsetX, offsetY: offsetY) + logger.info().log( + String(format: "Translated logical (%d, %d) → physical (%.1f, %.1f) [letterbox: scale=%.4f offsetX=%.1f offsetY=%.1f]", + Int(point.x), Int(point.y), physical.x, physical.y, scale, offsetX, offsetY) + ) + } + + return physical + } + + private static func applicationFrame( + from roots: [AccessibilityElement] + ) -> AccessibilityElement.Frame? { + roots.first { $0.type == "Application" }?.frame + ?? roots.first?.frame + } + + // MARK: - Orientation Detection (legacy — preserved for callers that need it directly) + + /// Determines the current logical orientation by inspecting the accessibility tree. + /// + /// Prefer `detectMapping(for:orientationOverride:logger:)` for full tap/touch/swipe + /// coordinate translation. This method only reads the AX frame and cannot distinguish + /// a rotated device from a letterboxed landscape-only app. + /// + /// - Parameters: + /// - simulatorUDID: Target simulator UDID. + /// - orientationOverride: When non-nil, skip detection and use this value. + /// - logger: AXe logger instance. + /// - Returns: The detected (or overridden) orientation. + static func detectOrientation( + for simulatorUDID: String, + orientationOverride: SimulatorOrientation? = nil, + logger: AxeLogger + ) async throws -> SimulatorOrientation { + if let override = orientationOverride { + logger.info().log("Using explicit orientation override: \(override.rawValue)") + return override + } + + let roots = try await AccessibilityFetcher.fetchAccessibilityElements( + for: simulatorUDID, + logger: logger + ) + + guard let appFrame = applicationFrame(from: roots) else { + logger.info().log("Could not read application frame; assuming portrait orientation") + return .portrait + } + + if appFrame.width > appFrame.height { + logger.info().log( + "Detected landscape orientation from application frame \(Int(appFrame.width))x\(Int(appFrame.height))" + ) + return .landscape + } + + logger.info().log( + "Detected portrait orientation from application frame \(Int(appFrame.width))x\(Int(appFrame.height))" + ) + return .portrait + } +} diff --git a/Sources/AXe/Utilities/SimulatorOrientationReader.swift b/Sources/AXe/Utilities/SimulatorOrientationReader.swift new file mode 100644 index 000000000..df188c0f9 --- /dev/null +++ b/Sources/AXe/Utilities/SimulatorOrientationReader.swift @@ -0,0 +1,127 @@ +import Foundation +import FBSimulatorControl + +@MainActor +struct SimulatorOrientationReader { + static func currentOrientation( + simulatorUDID: String, + logger: AxeLogger + ) async -> SimulatorOrientation? { + do { + let frameworkLoader = FBSimulatorControlFrameworkLoader.xcodeFrameworks + try frameworkLoader.loadPrivateFrameworks(logger) + + let simulatorSet = try await getSimulatorSet( + deviceSetPath: nil, + logger: logger, + reporter: EmptyEventReporter.shared + ) + + guard let simulator = simulatorSet.allSimulators.first(where: { $0.udid == simulatorUDID }) else { + logger.info().log("Orientation probe: simulator \(simulatorUDID) not found") + return nil + } + + guard let device = sendObject(simulator, selector: "device") else { + logger.info().log("Orientation probe: simulator device unavailable") + return nil + } + + return readOrientation(from: device, logger: logger) + } catch { + logger.info().log("Orientation probe failed: \(error)") + return nil + } + } + + private static func readOrientation(from device: AnyObject, logger: AxeLogger) -> SimulatorOrientation? { + guard let screenClass = NSClassFromString("SimulatorKit.SimDeviceScreen") as? NSObject.Type else { + logger.info().log("Orientation probe: SimulatorKit.SimDeviceScreen unavailable") + return nil + } + + let initSelector = NSSelectorFromString("initWithDevice:screenID:") + guard let allocated = allocate(screenClass) else { + logger.info().log("Orientation probe: SimDeviceScreen allocation failed") + return nil + } + guard let initMethod = allocated.method(for: initSelector) else { + logger.info().log("Orientation probe: initWithDevice:screenID: unavailable") + return nil + } + + typealias InitFunction = @convention(c) (AnyObject, Selector, AnyObject, Int) -> AnyObject + let initFunction = unsafeBitCast(initMethod, to: InitFunction.self) + let screenDevice = initFunction(allocated, initSelector, device, 1) + + guard let screen = sendObject(screenDevice, selector: "screen") else { + logger.info().log("Orientation probe: screen unavailable") + return nil + } + + guard let properties = sendObject(screen, selector: "screenProperties") else { + logger.info().log("Orientation probe: screenProperties unavailable") + return nil + } + + guard let rawOrientation = sendUInt32(properties, selector: "uiOrientation") else { + logger.info().log("Orientation probe: uiOrientation unavailable") + return nil + } + + logger.info().log("Orientation probe: uiOrientation=\(rawOrientation)") + return mapUIOrientation(rawOrientation) + } + + private static func allocate(_ objectClass: NSObject.Type) -> AnyObject? { + let selector = NSSelectorFromString("alloc") + guard let method = class_getClassMethod(objectClass, selector) else { + return nil + } + + typealias Function = @convention(c) (AnyClass, Selector) -> AnyObject? + let function = unsafeBitCast(method_getImplementation(method), to: Function.self) + return function(objectClass, selector) + } + + private static func sendObject(_ target: AnyObject, selector selectorName: String) -> AnyObject? { + let selector = NSSelectorFromString(selectorName) + guard let object = target as? NSObject, + object.responds(to: selector), + let method = object.method(for: selector) else { + return nil + } + + typealias Function = @convention(c) (AnyObject, Selector) -> AnyObject? + let function = unsafeBitCast(method, to: Function.self) + return function(target, selector) + } + + private static func sendUInt32(_ target: AnyObject, selector selectorName: String) -> UInt32? { + let selector = NSSelectorFromString(selectorName) + guard let object = target as? NSObject, + object.responds(to: selector), + let method = object.method(for: selector) else { + return nil + } + + typealias Function = @convention(c) (AnyObject, Selector) -> UInt32 + let function = unsafeBitCast(method, to: Function.self) + return function(target, selector) + } + + private static func mapUIOrientation(_ rawOrientation: UInt32) -> SimulatorOrientation? { + switch rawOrientation { + case 1: + return .portrait + case 2: + return .portraitUpsideDown + case 3: + return .landscape + case 4: + return .landscapeFlipped + default: + return nil + } + } +} diff --git a/Sources/AXeCore/OrientationCoordinateMath.swift b/Sources/AXeCore/OrientationCoordinateMath.swift new file mode 100644 index 000000000..a0b27f8bc --- /dev/null +++ b/Sources/AXeCore/OrientationCoordinateMath.swift @@ -0,0 +1,61 @@ +import Foundation + +public enum OrientationCoordinateMath { + public enum Orientation: String, CaseIterable, Sendable { + case portrait + case portraitUpsideDown + case landscape + case landscapeFlipped + + public var isLandscape: Bool { + self == .landscape || self == .landscapeFlipped + } + } + + public static func translateToPhysical( + x: Double, + y: Double, + orientation: Orientation, + portraitWidth: Double, + portraitHeight: Double + ) -> (x: Double, y: Double) { + switch orientation { + case .portrait: + return (x, y) + + case .portraitUpsideDown: + return (x: portraitWidth - x, y: portraitHeight - y) + + case .landscape: + return (x: y, y: portraitHeight - x) + + case .landscapeFlipped: + return (x: portraitWidth - y, y: x) + } + } + + public static func letterboxToPhysical( + x: Double, + y: Double, + scale: Double, + offsetX: Double, + offsetY: Double + ) -> (x: Double, y: Double) { + return ( + x: offsetX + x * scale, + y: offsetY + y * scale + ) + } + + public static func letterboxParameters( + logicalWidth: Double, + logicalHeight: Double, + physicalWidth: Double, + physicalHeight: Double + ) -> (scale: Double, offsetX: Double, offsetY: Double) { + let scale = min(physicalWidth / logicalWidth, physicalHeight / logicalHeight) + let offsetX = (physicalWidth - logicalWidth * scale) / 2 + let offsetY = (physicalHeight - logicalHeight * scale) / 2 + return (scale, offsetX, offsetY) + } +} diff --git a/TEST_RUNNER.md b/TEST_RUNNER.md index 94b7a71c9..06d460d9a 100644 --- a/TEST_RUNNER.md +++ b/TEST_RUNNER.md @@ -12,7 +12,7 @@ The `test-runner.sh` script provides a comprehensive, automated solution for bui ✅ **Comprehensive Testing** - Runs individual test suites or entire test plan - Sequential execution to prevent state conflicts -- Proper environment variable setup (`SIMULATOR_UDID`, `AXE_E2E=1`) +- Proper environment variable setup (`SIMULATOR_UDID`, `AXE_E2E=1`, optional `AXE_LANDSCAPE_E2E=1`) ✅ **Flexible Options** - Build-only mode for CI/CD pipelines @@ -60,6 +60,9 @@ chmod +x test-runner.sh # Run all tests with verbose output ./test-runner.sh --verbose + +# Run landscape orientation precision tests when Simulator menu automation is available +AXE_LANDSCAPE_E2E=1 ./test-runner.sh ``` ### Build-Only Workflows @@ -83,6 +86,7 @@ chmod +x test-runner.sh | `ButtonTests` | Hardware button press simulation | | `GestureTests` | Predefined gesture patterns | | `ListSimulatorsTests` | Simulator discovery and listing | +| `BatchTests` | Batch workflow execution | ## Command Line Options @@ -109,6 +113,7 @@ To change these settings, edit the configuration section at the top of `test-run The script automatically sets up the following environment: - `SIMULATOR_UDID`: Set to the configured iPhone 16 simulator - `AXE_E2E=1`: Enables simulator E2E tests +- `AXE_LANDSCAPE_E2E=0`: Landscape orientation precision tests are skipped by default. Set `AXE_LANDSCAPE_E2E=1` to run them; the runner preflights Simulator menu automation and skips them if the menu is unavailable. - Sequential test execution (prevents state conflicts) - Clean app state between test suites diff --git a/Tests/BatchTests.swift b/Tests/BatchTests.swift index 507c71744..928fd56c3 100644 --- a/Tests/BatchTests.swift +++ b/Tests/BatchTests.swift @@ -59,6 +59,48 @@ struct BatchTests { } } + @Test("Batch landscape coordinates preserve translated tap and swipe points", .enabled(if: isLandscapeE2EEnabled)) + func landscapeCoordinatesPreserveTranslatedTapAndSwipePoints() async throws { + try await TestHelpers.launchPlaygroundApp(to: "landscape-coordinate-test") + do { + try await TestHelpers.resetLandscapeCoordinateFixtureToPortrait() + try await TestHelpers.setSimulatorOrientationLandscapeRight() + + let initialState = try await TestHelpers.waitForLandscapeCoordinateFixtureLayout(timeout: 6) + + guard let tapTarget = UIStateParser.findElementByLabel(in: initialState, label: "Landscape Target"), + let tapFrame = tapTarget.frame, + let swipeStart = UIStateParser.findElementByLabel(in: initialState, label: "Landscape Swipe Start"), + let swipeStartFrame = swipeStart.frame, + let swipeEnd = UIStateParser.findElementByLabel(in: initialState, label: "Landscape Swipe End"), + let swipeEndFrame = swipeEnd.frame else { + throw TestError.elementNotFound("landscape-coordinate-batch-targets") + } + + let tapX = Int(tapFrame.x + tapFrame.width / 2) + let tapY = Int(tapFrame.y + tapFrame.height / 2) + let startX = Int(swipeStartFrame.x + swipeStartFrame.width / 2) + let startY = Int(swipeStartFrame.y + swipeStartFrame.height / 2) + let endX = Int(swipeEndFrame.x + swipeEndFrame.width / 2) + let endY = Int(swipeEndFrame.y + swipeEndFrame.height / 2) + + try await TestHelpers.runAxeCommand( + "batch --step \"tap -x \(tapX) -y \(tapY)\" --step \"swipe --start-x \(startX) --start-y \(startY) --end-x \(endX) --end-y \(endY) --duration 0.3 --delta 10\"", + simulatorUDID: defaultSimulatorUDID + ) + + _ = try await waitForLabel(containing: "Landscape Hit Count:", timeout: 3) { $0 == "Landscape Hit Count: 1" } + _ = try await waitForLabel(containing: "Landscape Swipe Hit Count:", timeout: 3) { $0 == "Landscape Swipe Hit Count: 1" } + try await assertLastNamedCoordinate(containing: "Last Tap Hit:", expectedX: tapX, expectedY: tapY, timeout: 3) + try await assertLastNamedCoordinate(containing: "Last Swipe Start:", expectedX: startX, expectedY: startY, timeout: 3) + try await assertLastNamedCoordinate(containing: "Last Swipe End:", expectedX: endX, expectedY: endY, timeout: 3) + try await TestHelpers.resetLandscapeCoordinateFixtureToPortrait() + } catch { + try? await TestHelpers.resetLandscapeCoordinateFixtureToPortrait() + throw error + } + } + @Test("Batch continue-on-error runs later steps and reports failures") func continueOnError() async throws { try await TestHelpers.launchPlaygroundApp(to: "batch-test") @@ -208,4 +250,31 @@ struct BatchTests { let digits = label.filter { $0.isNumber } return Int(digits) } + + private func assertLastNamedCoordinate( + containing text: String, + expectedX: Int, + expectedY: Int, + timeout: TimeInterval + ) async throws { + let deadline = Date().addingTimeInterval(timeout) + var lastValue: String? + + while Date() < deadline { + let uiState = try await TestHelpers.getUIState() + if let label = UIStateParser.findElementContainingLabel(in: uiState, containing: text)?.label { + lastValue = label + if let point = CoordinateParser.parseNamedCoordinates(from: label), + abs(point.x - expectedX) <= 1, + abs(point.y - expectedY) <= 1 { + return + } + } + try await Task.sleep(nanoseconds: 200_000_000) + } + + throw TestError.unexpectedState( + "Timed out waiting for \(text) near x:\(expectedX),y:\(expectedY). Last value: \(lastValue ?? "none")" + ) + } } diff --git a/Tests/OrientationAwareCoordinatesTests.swift b/Tests/OrientationAwareCoordinatesTests.swift new file mode 100644 index 000000000..bfb1ca0f9 --- /dev/null +++ b/Tests/OrientationAwareCoordinatesTests.swift @@ -0,0 +1,463 @@ +import Testing +import Foundation +import AXeCore + +// MARK: - Orientation Math Tests +// +// The iOS Simulator HID layer always operates in physical portrait space. +// The accessibility tree reports positions in the *logical* UI orientation. +// These tests verify the coordinate math that maps logical coordinates to physical ones. +// +// Two landscape cases are covered: +// (a) Rotated hardware — `translateToPhysical` rotation math. +// (b) Portrait hardware + landscape-only app — `letterboxToPhysical` scale + offset math. +// +// The pure coordinate math lives in AXeCore so production and tests exercise the +// same implementation rather than mirrored copies. +// +// Device under test in the parameterized tests: iPad Pro 13-inch (M5) +// portrait width = 1032 pts (short side) +// portrait height = 1376 pts (long side) +// landscape frame = 1376 × 1032 (width > height → isLandscape) +// screenshot (portrait physical): 2064 × 2752 px → 1032 × 1376 pts at 2x scale + +// MARK: - Test Helpers + +typealias TestOrientation = OrientationCoordinateMath.Orientation + +private func translateToPhysical( + lx: Double, ly: Double, + orientation: TestOrientation, + portraitWidth pw: Double, + portraitHeight ph: Double +) -> (x: Double, y: Double) { + OrientationCoordinateMath.translateToPhysical( + x: lx, + y: ly, + orientation: orientation, + portraitWidth: pw, + portraitHeight: ph + ) +} + +private func letterboxToPhysical( + lx: Double, ly: Double, + scale: Double, + offsetX: Double, + offsetY: Double +) -> (x: Double, y: Double) { + OrientationCoordinateMath.letterboxToPhysical( + x: lx, + y: ly, + scale: scale, + offsetX: offsetX, + offsetY: offsetY + ) +} + +private func letterboxParameters( + logicalW: Double, logicalH: Double, + physicalW: Double, physicalH: Double +) -> (scale: Double, offsetX: Double, offsetY: Double) { + OrientationCoordinateMath.letterboxParameters( + logicalWidth: logicalW, + logicalHeight: logicalH, + physicalWidth: physicalW, + physicalHeight: physicalH + ) +} +// MARK: - Tests + +private let iPadPro13PortraitWidth = 1032.0 +private let iPadPro13PortraitHeight = 1376.0 + +@Suite("Orientation-aware coordinate translation math") +struct OrientationAwareCoordinatesTests { + + // MARK: Portrait — identity + + @Test("Portrait: any point passes through unchanged") + func portraitIsIdentity() { + let (x, y) = translateToPhysical( + lx: 200, ly: 400, + orientation: .portrait, + portraitWidth: iPadPro13PortraitWidth, + portraitHeight: iPadPro13PortraitHeight + ) + #expect(x == 200) + #expect(y == 400) + } + + @Test("Portrait: origin is identity") + func portraitOriginIsIdentity() { + let (x, y) = translateToPhysical( + lx: 0, ly: 0, + orientation: .portrait, + portraitWidth: iPadPro13PortraitWidth, + portraitHeight: iPadPro13PortraitHeight + ) + #expect(x == 0) + #expect(y == 0) + } + + // MARK: Portrait Upside-Down — 180° mirror + + @Test("PortraitUpsideDown: mirrors both axes") + func portraitUpsideDownMirrors() { + // (200, 300) → (pw - 200, ph - 300) = (832, 1076) + let (x, y) = translateToPhysical( + lx: 200, ly: 300, + orientation: .portraitUpsideDown, + portraitWidth: iPadPro13PortraitWidth, + portraitHeight: iPadPro13PortraitHeight + ) + #expect(x == iPadPro13PortraitWidth - 200) + #expect(y == iPadPro13PortraitHeight - 300) + } + + @Test("PortraitUpsideDown: center maps to center") + func portraitUpsideDownCenterIsFixed() { + let cx = iPadPro13PortraitWidth / 2 + let cy = iPadPro13PortraitHeight / 2 + let (x, y) = translateToPhysical( + lx: cx, ly: cy, + orientation: .portraitUpsideDown, + portraitWidth: iPadPro13PortraitWidth, + portraitHeight: iPadPro13PortraitHeight + ) + #expect(x == cx) + #expect(y == cy) + } + + // MARK: Landscape — 90° CW (home button right) + + @Test("Landscape: formula is px=ly, py=ph-lx") + func landscapeFormula() { + // Verifies the known point from paul-foreflight's Python workaround: + // logical (1174, 428) → physical (428, 1376 - 1174) = (428, 202) + let (x, y) = translateToPhysical( + lx: 1174, ly: 428, + orientation: .landscape, + portraitWidth: iPadPro13PortraitWidth, + portraitHeight: iPadPro13PortraitHeight + ) + #expect(x == 428) + #expect(y == iPadPro13PortraitHeight - 1174) + } + + @Test("Landscape: parameterized corner mapping", arguments: [ + // (logicalX, logicalY, expectedPhysicalX, expectedPhysicalY) + // Formula: px = ly, py = ph - lx + (0.0, 0.0, 0.0, 1376.0), // top-left → bottom-left physical + (1376.0, 0.0, 0.0, 0.0), // top-right → top-left physical + (0.0, 1032.0, 1032.0, 1376.0), // bottom-left → bottom-right physical + (1376.0, 1032.0, 1032.0, 0.0), // bottom-right→ top-right physical + ]) + func landscapeCorners( + _ testCase: (lx: Double, ly: Double, px: Double, py: Double) + ) { + let (x, y) = translateToPhysical( + lx: testCase.lx, ly: testCase.ly, + orientation: .landscape, + portraitWidth: iPadPro13PortraitWidth, + portraitHeight: iPadPro13PortraitHeight + ) + #expect(x == testCase.px, + "x mismatch for logical (\(testCase.lx), \(testCase.ly)): got \(x), expected \(testCase.px)") + #expect(y == testCase.py, + "y mismatch for logical (\(testCase.lx), \(testCase.ly)): got \(y), expected \(testCase.py)") + } + + @Test("Landscape: output stays within portrait bounds for any in-frame input") + func landscapeOutputInPortraitBounds() { + let samples: [(Double, Double)] = [ + (0, 0), (688, 516), (1376, 1032), (100, 900), (1200, 50) + ] + for (lx, ly) in samples { + let (x, y) = translateToPhysical( + lx: lx, ly: ly, + orientation: .landscape, + portraitWidth: iPadPro13PortraitWidth, + portraitHeight: iPadPro13PortraitHeight + ) + #expect(x >= 0 && x <= iPadPro13PortraitWidth, + "physical x \(x) out of portrait width bounds for logical (\(lx), \(ly))") + #expect(y >= 0 && y <= iPadPro13PortraitHeight, + "physical y \(y) out of portrait height bounds for logical (\(lx), \(ly))") + } + } + + // MARK: Landscape Flipped — 90° CCW (home button left) + + @Test("LandscapeFlipped: formula is px=pw-ly, py=lx") + func landscapeFlippedFormula() { + // logical (500, 200) → physical (1032 - 200, 500) = (832, 500) + let (x, y) = translateToPhysical( + lx: 500, ly: 200, + orientation: .landscapeFlipped, + portraitWidth: iPadPro13PortraitWidth, + portraitHeight: iPadPro13PortraitHeight + ) + #expect(x == iPadPro13PortraitWidth - 200) + #expect(y == 500) + } + + @Test("LandscapeFlipped: parameterized corner mapping", arguments: [ + // (logicalX, logicalY, expectedPhysicalX, expectedPhysicalY) + // Formula: px = pw - ly, py = lx + (0.0, 0.0, 1032.0, 0.0), // top-left → top-right physical + (1376.0, 0.0, 1032.0, 1376.0), // top-right → bottom-right physical + (0.0, 1032.0, 0.0, 0.0), // bottom-left → top-left physical + (1376.0, 1032.0, 0.0, 1376.0), // bottom-right→ bottom-left physical + ]) + func landscapeFlippedCorners( + _ testCase: (lx: Double, ly: Double, px: Double, py: Double) + ) { + let (x, y) = translateToPhysical( + lx: testCase.lx, ly: testCase.ly, + orientation: .landscapeFlipped, + portraitWidth: iPadPro13PortraitWidth, + portraitHeight: iPadPro13PortraitHeight + ) + #expect(x == testCase.px, + "x mismatch for logical (\(testCase.lx), \(testCase.ly)): got \(x), expected \(testCase.px)") + #expect(y == testCase.py, + "y mismatch for logical (\(testCase.lx), \(testCase.ly)): got \(y), expected \(testCase.py)") + } + + @Test("LandscapeFlipped: output stays within portrait bounds for any in-frame input") + func landscapeFlippedOutputInPortraitBounds() { + let samples: [(Double, Double)] = [ + (0, 0), (688, 516), (1376, 1032), (100, 900), (1200, 50) + ] + for (lx, ly) in samples { + let (x, y) = translateToPhysical( + lx: lx, ly: ly, + orientation: .landscapeFlipped, + portraitWidth: iPadPro13PortraitWidth, + portraitHeight: iPadPro13PortraitHeight + ) + #expect(x >= 0 && x <= iPadPro13PortraitWidth, + "physical x \(x) out of portrait width bounds for logical (\(lx), \(ly))") + #expect(y >= 0 && y <= iPadPro13PortraitHeight, + "physical y \(y) out of portrait height bounds for logical (\(lx), \(ly))") + } + } + + // MARK: Determinism + + @Test("Same input always produces same output (determinism)") + func translationIsDeterministic() { + let input = (lx: 500.0, ly: 300.0) + let orientations: [TestOrientation] = [.portrait, .portraitUpsideDown, .landscape, .landscapeFlipped] + + for orientation in orientations { + let first = translateToPhysical( + lx: input.lx, ly: input.ly, + orientation: orientation, + portraitWidth: iPadPro13PortraitWidth, + portraitHeight: iPadPro13PortraitHeight + ) + let second = translateToPhysical( + lx: input.lx, ly: input.ly, + orientation: orientation, + portraitWidth: iPadPro13PortraitWidth, + portraitHeight: iPadPro13PortraitHeight + ) + #expect(first.x == second.x, "Non-deterministic x for \(orientation)") + #expect(first.y == second.y, "Non-deterministic y for \(orientation)") + } + } + + // MARK: Portrait Dimension Detection + + @Test("Portrait frame: width is short side, height is long side") + func portraitDimensionsFromPortraitFrame() { + // Simulates what OrientationAwareCoordinates.portraitDimensions does: + // portrait → frame is already portrait dimensions + let frameWidth = 1032.0 + let frameHeight = 1376.0 + // In portrait, width < height → portrait dimensions = (frameWidth, frameHeight) + let isLandscape = frameWidth > frameHeight + let portraitWidth = isLandscape ? frameHeight : frameWidth + let portraitHeight = isLandscape ? frameWidth : frameHeight + #expect(portraitWidth == 1032) + #expect(portraitHeight == 1376) + } + + @Test("Landscape frame: width and height are swapped to get portrait dimensions") + func portraitDimensionsFromLandscapeFrame() { + // In landscape, the AX frame reports (1376, 1032) — width > height. + // Portrait dimensions are the inverse: (1032, 1376). + let frameWidth = 1376.0 + let frameHeight = 1032.0 + let isLandscape = frameWidth > frameHeight + let portraitWidth = isLandscape ? frameHeight : frameWidth + let portraitHeight = isLandscape ? frameWidth : frameHeight + #expect(portraitWidth == 1032) + #expect(portraitHeight == 1376) + } +} + +// MARK: - Letterbox Case Tests +// +// Setup: portrait hardware (iPad Pro 13" M5), landscape-only app. +// AX application frame (logical): 1376 × 1032 pts (landscape-shaped) +// Physical portrait viewport: 1032 × 1376 pts +// Screenshot pixel dims: 2064 × 2752 px (portrait-shaped — detection trigger) +// +// Scale: min(1032/1376, 1376/1032) = min(0.75, 1.333) = 0.75 +// offsetX: (1032 - 1376*0.75) / 2 = (1032 - 1032) / 2 = 0 +// offsetY: (1376 - 1032*0.75) / 2 = (1376 - 774) / 2 = 301 +// +// Empirical verification: logical (86, 269) → physical (64.5, 502.75) +// Confirmed on JAMDOG iPad Pro 13" M5 — tap switched fixture from 4-player to 8-player Lobby. + +@Suite("Letterbox coordinate translation math — portrait HW, landscape-only app") +struct LetterboxCoordinatesTests { + + private let logicalW = 1376.0 // landscape AX frame width + private let logicalH = 1032.0 // landscape AX frame height + private let physicalW = 1032.0 // portrait hardware viewport width + private let physicalH = 1376.0 // portrait hardware viewport height + + private var params: (scale: Double, offsetX: Double, offsetY: Double) { + letterboxParameters( + logicalW: logicalW, logicalH: logicalH, + physicalW: physicalW, physicalH: physicalH + ) + } + + // MARK: Parameter derivation + + @Test("Letterbox: scale is 0.75 for iPad Pro 13\" M5") + func letterboxScale() { + let (scale, _, _) = params + #expect(abs(scale - 0.75) < 1e-9, + "Expected scale 0.75, got \(scale)") + } + + @Test("Letterbox: offsetX is 0 (landscape fills portrait width exactly at 0.75 scale)") + func letterboxOffsetX() { + let (_, offsetX, _) = params + #expect(abs(offsetX) < 1e-9, + "Expected offsetX 0, got \(offsetX)") + } + + @Test("Letterbox: offsetY is 301 (top+bottom pillarbox of ~301 pts each)") + func letterboxOffsetY() { + let (_, _, offsetY) = params + #expect(abs(offsetY - 301.0) < 1e-9, + "Expected offsetY 301, got \(offsetY)") + } + + // MARK: Empirical regression + + @Test("Letterbox: logical (86, 269) → physical (64.5, 502.75) — JAMDOG empirical fixture") + func letterboxEmpiricalPoint() { + // Verified on JAMDOG iPad Pro 13" M5 / iOS 26.4.1: + // BeatSmart landscape-only app, device never rotated. + // Tap (86, 269) switched Lobby from 4-player to 8-player fixture. + let (scale, offsetX, offsetY) = params + let (px, py) = letterboxToPhysical( + lx: 86, ly: 269, + scale: scale, offsetX: offsetX, offsetY: offsetY + ) + #expect(abs(px - 64.5) < 1e-9, "physical x: expected 64.5, got \(px)") + #expect(abs(py - 502.75) < 1e-9, "physical y: expected 502.75, got \(py)") + } + + // MARK: Corner mapping + + @Test("Letterbox: parameterized corner mapping", arguments: [ + // (logicalX, logicalY, expectedPhysicalX, expectedPhysicalY) + // px = offsetX + lx * scale = 0 + lx * 0.75 + // py = offsetY + ly * scale = 301 + ly * 0.75 + (0.0, 0.0, 0.0, 301.0), // top-left + (1376.0, 0.0, 1032.0, 301.0), // top-right + (0.0, 1032.0, 0.0, 1075.0), // bottom-left (301 + 1032*0.75 = 301+774 = 1075) + (1376.0, 1032.0, 1032.0, 1075.0), // bottom-right + ]) + func letterboxCorners( + _ testCase: (lx: Double, ly: Double, px: Double, py: Double) + ) { + let (scale, offsetX, offsetY) = params + let (x, y) = letterboxToPhysical( + lx: testCase.lx, ly: testCase.ly, + scale: scale, offsetX: offsetX, offsetY: offsetY + ) + #expect(abs(x - testCase.px) < 1e-9, + "x mismatch for logical (\(testCase.lx), \(testCase.ly)): got \(x), expected \(testCase.px)") + #expect(abs(y - testCase.py) < 1e-9, + "y mismatch for logical (\(testCase.lx), \(testCase.ly)): got \(y), expected \(testCase.py)") + } + + // MARK: Bounds + + @Test("Letterbox: output stays within physical portrait bounds for any in-frame input") + func letterboxOutputInPortraitBounds() { + let (scale, offsetX, offsetY) = params + let samples: [(Double, Double)] = [ + (0, 0), (688, 516), (1376, 1032), (100, 900), (1200, 50), + (86, 269) // JAMDOG empirical point + ] + for (lx, ly) in samples { + let (x, y) = letterboxToPhysical( + lx: lx, ly: ly, + scale: scale, offsetX: offsetX, offsetY: offsetY + ) + #expect(x >= 0 && x <= physicalW, + "physical x \(x) out of portrait width \(physicalW) bounds for logical (\(lx), \(ly))") + #expect(y >= 0 && y <= physicalH, + "physical y \(y) out of portrait height \(physicalH) bounds for logical (\(lx), \(ly))") + } + } + + // MARK: Determinism + + @Test("Letterbox: same input always produces same output") + func letterboxIsDeterministic() { + let (scale, offsetX, offsetY) = params + let input = (lx: 86.0, ly: 269.0) + let first = letterboxToPhysical(lx: input.lx, ly: input.ly, scale: scale, offsetX: offsetX, offsetY: offsetY) + let second = letterboxToPhysical(lx: input.lx, ly: input.ly, scale: scale, offsetX: offsetX, offsetY: offsetY) + #expect(first.x == second.x, "Non-deterministic x") + #expect(first.y == second.y, "Non-deterministic y") + } + + // MARK: Detection signal + + @Test("Letterbox detection: portrait screenshot + landscape AX frame = letterbox case") + func letterboxDetectionSignal() { + // The production detection logic triggers letterbox when: + // screenshot.height > screenshot.width (portrait-shaped PNG) + // AND axFrame.width > axFrame.height (landscape-shaped AX frame) + let screenshotWidth = 2064 // pixel dims from JAMDOG iPad Pro 13" M5 + let screenshotHeight = 2752 + let axFrameWidth = 1376.0 + let axFrameHeight = 1032.0 + + let screenshotIsPortrait = screenshotHeight > screenshotWidth + let axFrameIsLandscape = axFrameWidth > axFrameHeight + + #expect(screenshotIsPortrait, "Screenshot should be portrait-shaped for letterbox detection") + #expect(axFrameIsLandscape, "AX frame should be landscape-shaped for letterbox detection") + #expect(screenshotIsPortrait && axFrameIsLandscape, "Both conditions must hold to trigger letterbox mapping") + } + + @Test("Rotation detection: landscape screenshot + landscape AX frame = rotation case") + func rotationDetectionSignal() { + // Rotated hardware: both screenshot and AX frame are landscape-shaped. + let screenshotWidth = 2752 // landscape pixel dims (long edge horizontal) + let screenshotHeight = 2064 + let axFrameWidth = 1376.0 + let axFrameHeight = 1032.0 + + let screenshotIsLandscape = screenshotWidth > screenshotHeight + let axFrameIsLandscape = axFrameWidth > axFrameHeight + + #expect(screenshotIsLandscape, "Screenshot should be landscape-shaped for rotation detection") + #expect(axFrameIsLandscape, "AX frame should be landscape-shaped for rotation detection") + #expect(screenshotIsLandscape && axFrameIsLandscape, "Both landscape → rotation mapping (not letterbox)") + } +} diff --git a/Tests/SwipeTests.swift b/Tests/SwipeTests.swift index af936aa40..2d6d9a39b 100644 --- a/Tests/SwipeTests.swift +++ b/Tests/SwipeTests.swift @@ -24,6 +24,80 @@ struct SwipeTests { #expect(endElement?.label == "End: (300, 400)") } + @Test("Landscape coordinate swipe preserves translated start and end", .enabled(if: isLandscapeE2EEnabled)) + func landscapeCoordinateSwipePreservesTranslatedStartAndEnd() async throws { + try await TestHelpers.launchPlaygroundApp(to: "landscape-coordinate-test") + do { + try await TestHelpers.resetLandscapeCoordinateFixtureToPortrait() + try await TestHelpers.setSimulatorOrientationLandscapeLeft() + + let initialState = try await TestHelpers.waitForLandscapeCoordinateFixtureLayout(timeout: 6) + + guard let start = UIStateParser.findElementByLabel(in: initialState, label: "Landscape Swipe Start"), + let startFrame = start.frame, + let end = UIStateParser.findElementByLabel(in: initialState, label: "Landscape Swipe End"), + let endFrame = end.frame else { + throw TestError.elementNotFound("landscape-coordinate-swipe-targets") + } + + let startX = Int(startFrame.x + startFrame.width / 2) + let startY = Int(startFrame.y + startFrame.height / 2) + let endX = Int(endFrame.x + endFrame.width / 2) + let endY = Int(endFrame.y + endFrame.height / 2) + + try await TestHelpers.runAxeCommand( + "swipe --start-x \(startX) --start-y \(startY) --end-x \(endX) --end-y \(endY) --duration 0.3 --delta 10", + simulatorUDID: defaultSimulatorUDID + ) + + let hitCount = try await waitForLandscapeSwipeHitCount(expected: "Landscape Swipe Hit Count: 1", timeout: 3) + #expect(hitCount == "Landscape Swipe Hit Count: 1") + try await assertLastNamedCoordinate(containing: "Last Swipe Start:", expectedX: startX, expectedY: startY, timeout: 3) + try await assertLastNamedCoordinate(containing: "Last Swipe End:", expectedX: endX, expectedY: endY, timeout: 3) + try await TestHelpers.resetLandscapeCoordinateFixtureToPortrait() + } catch { + try? await TestHelpers.resetLandscapeCoordinateFixtureToPortrait() + throw error + } + } + + @Test("Landscape-right coordinate swipe preserves translated start and end", .enabled(if: isLandscapeE2EEnabled)) + func landscapeRightCoordinateSwipePreservesTranslatedStartAndEnd() async throws { + try await TestHelpers.launchPlaygroundApp(to: "landscape-coordinate-test") + do { + try await TestHelpers.resetLandscapeCoordinateFixtureToPortrait() + try await TestHelpers.setSimulatorOrientationLandscapeRight() + + let initialState = try await TestHelpers.waitForLandscapeCoordinateFixtureLayout(timeout: 6) + + guard let start = UIStateParser.findElementByLabel(in: initialState, label: "Landscape Swipe Start"), + let startFrame = start.frame, + let end = UIStateParser.findElementByLabel(in: initialState, label: "Landscape Swipe End"), + let endFrame = end.frame else { + throw TestError.elementNotFound("landscape-coordinate-swipe-targets") + } + + let startX = Int(startFrame.x + startFrame.width / 2) + let startY = Int(startFrame.y + startFrame.height / 2) + let endX = Int(endFrame.x + endFrame.width / 2) + let endY = Int(endFrame.y + endFrame.height / 2) + + try await TestHelpers.runAxeCommand( + "swipe --start-x \(startX) --start-y \(startY) --end-x \(endX) --end-y \(endY) --duration 0.3 --delta 10", + simulatorUDID: defaultSimulatorUDID + ) + + let hitCount = try await waitForLandscapeSwipeHitCount(expected: "Landscape Swipe Hit Count: 1", timeout: 3) + #expect(hitCount == "Landscape Swipe Hit Count: 1") + try await assertLastNamedCoordinate(containing: "Last Swipe Start:", expectedX: startX, expectedY: startY, timeout: 3) + try await assertLastNamedCoordinate(containing: "Last Swipe End:", expectedX: endX, expectedY: endY, timeout: 3) + try await TestHelpers.resetLandscapeCoordinateFixtureToPortrait() + } catch { + try? await TestHelpers.resetLandscapeCoordinateFixtureToPortrait() + throw error + } + } + @Test("Swipe direction detection", arguments: [ (start: (100, 400), end: (300, 400), direction: "Right"), (start: (300, 400), end: (100, 400), direction: "Left"), @@ -91,6 +165,52 @@ struct SwipeTests { #expect(swipeCountElement?.label == "Count: \(swipeCount)", "Swipe count should be \(swipeCount)") } + private func waitForLandscapeSwipeHitCount(expected: String, timeout: TimeInterval) async throws -> String { + let deadline = Date().addingTimeInterval(timeout) + var lastValue: String? + + while Date() < deadline { + let uiState = try await TestHelpers.getUIState() + if let element = UIStateParser.findElementContainingLabel(in: uiState, containing: "Landscape Swipe Hit Count:"), + let label = element.label { + lastValue = label + if label == expected { + return label + } + } + try await Task.sleep(nanoseconds: 200_000_000) + } + + throw TestError.unexpectedState("Timed out waiting for \(expected). Last value: \(lastValue ?? "none")") + } + + private func assertLastNamedCoordinate( + containing text: String, + expectedX: Int, + expectedY: Int, + timeout: TimeInterval + ) async throws { + let deadline = Date().addingTimeInterval(timeout) + var lastValue: String? + + while Date() < deadline { + let uiState = try await TestHelpers.getUIState() + if let label = UIStateParser.findElementContainingLabel(in: uiState, containing: text)?.label { + lastValue = label + if let point = CoordinateParser.parseNamedCoordinates(from: label), + abs(point.x - expectedX) <= 1, + abs(point.y - expectedY) <= 1 { + return + } + } + try await Task.sleep(nanoseconds: 200_000_000) + } + + throw TestError.unexpectedState( + "Timed out waiting for \(text) near x:\(expectedX),y:\(expectedY). Last value: \(lastValue ?? "none")" + ) + } + @Test("Draw complex shapes") func swipeSegmentedStarTenPaths() async throws { // Arrange diff --git a/Tests/TapTests.swift b/Tests/TapTests.swift index fbcdd98b1..def9cb4a4 100644 --- a/Tests/TapTests.swift +++ b/Tests/TapTests.swift @@ -20,6 +20,64 @@ struct TapTests { #expect(tapLocationElement?.label == "Tap Location: (200, 400)", "Tap location should be (200, 400)") } + @Test("Landscape coordinate tap hits shifted target", .enabled(if: isLandscapeE2EEnabled)) + func landscapeCoordinateTapHitsShiftedTarget() async throws { + try await TestHelpers.launchPlaygroundApp(to: "landscape-coordinate-test") + do { + try await TestHelpers.resetLandscapeCoordinateFixtureToPortrait() + try await TestHelpers.setSimulatorOrientationLandscapeLeft() + + let initialState = try await TestHelpers.waitForLandscapeCoordinateFixtureLayout(timeout: 6) + + guard let target = UIStateParser.findElementByLabel(in: initialState, label: "Landscape Target"), + let frame = target.frame else { + throw TestError.elementNotFound("landscape-coordinate-target") + } + + let logicalX = Int(frame.x + frame.width / 2) + let logicalY = Int(frame.y + frame.height / 2) + + try await TestHelpers.runAxeCommand("tap -x \(logicalX) -y \(logicalY)", simulatorUDID: defaultSimulatorUDID) + + let hitCount = try await waitForLandscapeHitCount(expected: "Landscape Hit Count: 1", timeout: 3) + #expect(hitCount == "Landscape Hit Count: 1") + try await assertLastNamedCoordinate(containing: "Last Tap Hit:", expectedX: logicalX, expectedY: logicalY, timeout: 3) + try await TestHelpers.resetLandscapeCoordinateFixtureToPortrait() + } catch { + try? await TestHelpers.resetLandscapeCoordinateFixtureToPortrait() + throw error + } + } + + @Test("Landscape-right coordinate tap preserves translated point", .enabled(if: isLandscapeE2EEnabled)) + func landscapeRightCoordinateTapPreservesTranslatedPoint() async throws { + try await TestHelpers.launchPlaygroundApp(to: "landscape-coordinate-test") + do { + try await TestHelpers.resetLandscapeCoordinateFixtureToPortrait() + try await TestHelpers.setSimulatorOrientationLandscapeRight() + + let initialState = try await TestHelpers.waitForLandscapeCoordinateFixtureLayout(timeout: 6) + + guard let target = UIStateParser.findElementByLabel(in: initialState, label: "Landscape Target"), + let frame = target.frame else { + throw TestError.elementNotFound("landscape-coordinate-target") + } + + let logicalX = Int(frame.x + frame.width / 2) + let logicalY = Int(frame.y + frame.height / 2) + + try await TestHelpers.runAxeCommand("tap -x \(logicalX) -y \(logicalY)", simulatorUDID: defaultSimulatorUDID) + + let hitCount = try await waitForLandscapeHitCount(expected: "Landscape Hit Count: 1", timeout: 3) + #expect(hitCount == "Landscape Hit Count: 1") + try await assertLastNamedCoordinate(containing: "Last Tap Hit:", expectedX: logicalX, expectedY: logicalY, timeout: 3) + try await TestHelpers.resetLandscapeCoordinateFixtureToPortrait() + } catch { + try? await TestHelpers.resetLandscapeCoordinateFixtureToPortrait() + throw error + } + } + @Test("Tap by AXUniqueId navigates back to home") func tapByIDNavigatesBack() async throws { // Arrange @@ -93,6 +151,52 @@ struct TapTests { #expect(tapCountElement?.label == "Tap Count: 1", "Tap should still register with delays") } + private func waitForLandscapeHitCount(expected: String, timeout: TimeInterval) async throws -> String { + let deadline = Date().addingTimeInterval(timeout) + var lastValue: String? + + while Date() < deadline { + let uiState = try await TestHelpers.getUIState() + if let element = UIStateParser.findElementContainingLabel(in: uiState, containing: "Landscape Hit Count:"), + let label = element.label { + lastValue = label + if label == expected { + return label + } + } + try await Task.sleep(nanoseconds: 200_000_000) + } + + throw TestError.unexpectedState("Timed out waiting for \(expected). Last value: \(lastValue ?? "none")") + } + + private func assertLastNamedCoordinate( + containing text: String, + expectedX: Int, + expectedY: Int, + timeout: TimeInterval + ) async throws { + let deadline = Date().addingTimeInterval(timeout) + var lastValue: String? + + while Date() < deadline { + let uiState = try await TestHelpers.getUIState() + if let label = UIStateParser.findElementContainingLabel(in: uiState, containing: text)?.label { + lastValue = label + if let point = CoordinateParser.parseNamedCoordinates(from: label), + abs(point.x - expectedX) <= 1, + abs(point.y - expectedY) <= 1 { + return + } + } + try await Task.sleep(nanoseconds: 200_000_000) + } + + throw TestError.unexpectedState( + "Timed out waiting for \(text) near x:\(expectedX),y:\(expectedY). Last value: \(lastValue ?? "none")" + ) + } + @Test("At least one tap registers at screen edges") func tapAtEdgesRegistersAtLeastOne() async throws { // Arrange diff --git a/Tests/TestUtilities.swift b/Tests/TestUtilities.swift index 7f9c455f9..3c248a01e 100644 --- a/Tests/TestUtilities.swift +++ b/Tests/TestUtilities.swift @@ -9,6 +9,10 @@ let isE2EEnabled = { let raw = ProcessInfo.processInfo.environment["AXE_E2E"]?.lowercased() ?? "" return raw == "1" || raw == "true" || raw == "yes" }() +let isLandscapeE2EEnabled = { + let raw = ProcessInfo.processInfo.environment["AXE_LANDSCAPE_E2E"]?.lowercased() ?? "" + return isE2EEnabled && (raw == "1" || raw == "true" || raw == "yes") +}() struct CommandOutput { let output: String @@ -301,6 +305,52 @@ struct TestHelpers { throw TestError.unexpectedState("axe binary not found at \(axePath). Please run 'swift build'.") } + static func setSimulatorOrientationPortrait() async throws { + try await setSimulatorOrientation(menuItem: "Portrait") + } + + static func setSimulatorOrientationLandscapeLeft() async throws { + try await setSimulatorOrientation(menuItem: "Landscape Left") + } + + static func setSimulatorOrientationLandscapeRight() async throws { + try await setSimulatorOrientation(menuItem: "Landscape Right") + } + + static func rotateSimulatorLeft() async throws { + try await selectSimulatorDeviceMenuItem("Rotate Left") + } + + private static func setSimulatorOrientation(menuItem: String) async throws { + let script = """ + tell application "Simulator" to activate + delay 0.5 + tell application "System Events" + tell process "Simulator" + click menu item "\(menuItem)" of menu "Orientation" of menu item "Orientation" of menu "Device" of menu bar 1 + end tell + end tell + """ + let escapedScript = script.replacingOccurrences(of: "'", with: "'\\''") + _ = try await CommandRunner.run("osascript -e '\(escapedScript)'", timeout: 10) + try await Task.sleep(nanoseconds: 1_500_000_000) + } + + private static func selectSimulatorDeviceMenuItem(_ menuItem: String) async throws { + let script = """ + tell application "Simulator" to activate + delay 0.5 + tell application "System Events" + tell process "Simulator" + click menu item "\(menuItem)" of menu "Device" of menu bar 1 + end tell + end tell + """ + let escapedScript = script.replacingOccurrences(of: "'", with: "'\\''") + _ = try await CommandRunner.run("osascript -e '\(escapedScript)'", timeout: 10) + try await Task.sleep(nanoseconds: 1_500_000_000) + } + static func launchPlaygroundApp(to screen: String, simulatorUDID: String? = nil) async throws { let udid: String if let simulatorUDID { @@ -334,6 +384,39 @@ struct TestHelpers { return try UIStateParser.parseDescribeUIOutput(result.output) } + + static func waitForLandscapeCoordinateFixtureLayout(timeout: TimeInterval) async throws -> UIElement { + let deadline = Date().addingTimeInterval(timeout) + + while Date() < deadline { + let uiState = try await getUIState() + if UIStateParser.findElement(in: uiState, matching: { element in + element.label == "Layout: landscape" && element.value == "landscape" + }) != nil { + return uiState + } + try await Task.sleep(nanoseconds: 200_000_000) + } + + throw TestError.unexpectedState("Timed out waiting for landscape fixture layout") + } + + static func resetLandscapeCoordinateFixtureToPortrait() async throws { + try await setSimulatorOrientationPortrait() + + let deadline = Date().addingTimeInterval(3) + while Date() < deadline { + let uiState = try await getUIState() + if UIStateParser.findElement(in: uiState, matching: { element in + element.label == "Layout: portrait" && element.value == "portrait" + }) != nil { + return + } + try await Task.sleep(nanoseconds: 200_000_000) + } + + throw TestError.unexpectedState("Unable to reset simulator to portrait fixture layout") + } @discardableResult static func runAxeCommand( @@ -449,4 +532,21 @@ struct CoordinateParser { return (x, y) } + + static func parseNamedCoordinates(from string: String) -> (x: Int, y: Int)? { + let pattern = #"x:(\d+),y:(\d+)"# + guard let regex = try? NSRegularExpression(pattern: pattern), + let match = regex.firstMatch(in: string, range: NSRange(string.startIndex..., in: string)) else { + return nil + } + + guard let xRange = Range(match.range(at: 1), in: string), + let yRange = Range(match.range(at: 2), in: string), + let x = Int(string[xRange]), + let y = Int(string[yRange]) else { + return nil + } + + return (x, y) + } } diff --git a/Tests/TouchTests.swift b/Tests/TouchTests.swift index c00c00d00..3f9434ca9 100644 --- a/Tests/TouchTests.swift +++ b/Tests/TouchTests.swift @@ -27,6 +27,16 @@ struct TouchTests { #expect(touchUpElement?.label == "Last touch up: (200, 400)", "Touch up coordinates should be recorded") } + @Test("Landscape coordinate touch preserves translated point", .enabled(if: isLandscapeE2EEnabled)) + func landscapeCoordinateTouchPreservesTranslatedPoint() async throws { + try await runLandscapeTouchPrecisionTest(orientation: .left) + } + + @Test("Landscape-right coordinate touch preserves translated point", .enabled(if: isLandscapeE2EEnabled)) + func landscapeRightCoordinateTouchPreservesTranslatedPoint() async throws { + try await runLandscapeTouchPrecisionTest(orientation: .right) + } + @Test("Touch move") func touchMove() async throws { // Arrange @@ -134,6 +144,93 @@ struct TouchTests { #expect(touchUpElement?.label == "Last touch up: (200, 300)", "Touch up should register with delays") } + private enum LandscapeTestOrientation { + case left + case right + } + + private func runLandscapeTouchPrecisionTest(orientation: LandscapeTestOrientation) async throws { + try await TestHelpers.launchPlaygroundApp(to: "landscape-coordinate-test") + do { + try await TestHelpers.resetLandscapeCoordinateFixtureToPortrait() + switch orientation { + case .left: + try await TestHelpers.setSimulatorOrientationLandscapeLeft() + case .right: + try await TestHelpers.setSimulatorOrientationLandscapeRight() + } + + let initialState = try await TestHelpers.waitForLandscapeCoordinateFixtureLayout(timeout: 6) + + guard let target = UIStateParser.findElementByLabel(in: initialState, label: "Landscape Target"), + let frame = target.frame else { + throw TestError.elementNotFound("landscape-coordinate-target") + } + + let logicalX = Int(frame.x + frame.width / 2) + let logicalY = Int(frame.y + frame.height / 2) + + try await TestHelpers.runAxeCommand( + "touch -x \(logicalX) -y \(logicalY) --down --up", + simulatorUDID: defaultSimulatorUDID + ) + + let hitCount = try await waitForLandscapeHitCount(expected: "Landscape Hit Count: 1", timeout: 3) + #expect(hitCount == "Landscape Hit Count: 1") + try await assertLastNamedCoordinate(containing: "Last Tap Hit:", expectedX: logicalX, expectedY: logicalY, timeout: 3) + try await TestHelpers.resetLandscapeCoordinateFixtureToPortrait() + } catch { + try? await TestHelpers.resetLandscapeCoordinateFixtureToPortrait() + throw error + } + } + + private func waitForLandscapeHitCount(expected: String, timeout: TimeInterval) async throws -> String { + let deadline = Date().addingTimeInterval(timeout) + var lastValue: String? + + while Date() < deadline { + let uiState = try await TestHelpers.getUIState() + if let element = UIStateParser.findElementContainingLabel(in: uiState, containing: "Landscape Hit Count:"), + let label = element.label { + lastValue = label + if label == expected { + return label + } + } + try await Task.sleep(nanoseconds: 200_000_000) + } + + throw TestError.unexpectedState("Timed out waiting for \(expected). Last value: \(lastValue ?? "none")") + } + + private func assertLastNamedCoordinate( + containing text: String, + expectedX: Int, + expectedY: Int, + timeout: TimeInterval + ) async throws { + let deadline = Date().addingTimeInterval(timeout) + var lastValue: String? + + while Date() < deadline { + let uiState = try await TestHelpers.getUIState() + if let label = UIStateParser.findElementContainingLabel(in: uiState, containing: text)?.label { + lastValue = label + if let point = CoordinateParser.parseNamedCoordinates(from: label), + abs(point.x - expectedX) <= 1, + abs(point.y - expectedY) <= 1 { + return + } + } + try await Task.sleep(nanoseconds: 200_000_000) + } + + throw TestError.unexpectedState( + "Timed out waiting for \(text) near x:\(expectedX),y:\(expectedY). Last value: \(lastValue ?? "none")" + ) + } + @Test("Touch delay triggers long press recognition") func touchDelayTriggersLongPressRecognition() async throws { // Arrange diff --git a/test-runner.sh b/test-runner.sh index 533910ec6..690b13f13 100755 --- a/test-runner.sh +++ b/test-runner.sh @@ -54,6 +54,9 @@ show_usage() { echo " -s, --sequential Run suites one-by-one (single simulator-safe flow)" echo " -v, --verbose Verbose output" echo "" + echo "Environment:" + echo " AXE_LANDSCAPE_E2E=1 Run gated landscape orientation precision tests when Simulator menu automation is available" + echo "" echo "Test Filters (optional):" echo " SwipeTests Run only swipe tests" echo " TapTests Run only tap tests" @@ -106,7 +109,7 @@ while [[ $# -gt 0 ]]; do VERBOSE=true shift ;; - SwipeTests|TapTests|KeyTests|TouchTests|TypeTests|ButtonTests|GestureTests|ListSimulatorsTests) + BatchTests|SwipeTests|TapTests|KeyTests|TouchTests|TypeTests|ButtonTests|GestureTests|ListSimulatorsTests) TEST_FILTER="$1" shift ;; @@ -259,6 +262,17 @@ build_playground_app() { print_info "App path: $APP_PATH" } +landscape_orientation_menu_available() { + local enabled_items + enabled_items=$(osascript \ + -e 'tell application "Simulator" to activate' \ + -e 'delay 0.5' \ + -e 'tell application "System Events" to tell process "Simulator" to get enabled of menu items of menu "Orientation" of menu item "Orientation" of menu "Device" of menu bar 1' \ + 2>/dev/null || true) + + [[ "$enabled_items" == true,\ true,\ true,\ true* ]] +} + # Function to run tests run_tests() { print_header "Running Tests" @@ -266,8 +280,24 @@ run_tests() { # Set up environment export SIMULATOR_UDID="$SIMULATOR_UDID" export AXE_E2E=1 + local requested_landscape_e2e="${AXE_LANDSCAPE_E2E:-0}" + local normalized_landscape_e2e + normalized_landscape_e2e=$(printf '%s' "$requested_landscape_e2e" | tr '[:upper:]' '[:lower:]') + case "$normalized_landscape_e2e" in + 1|true|yes) + if landscape_orientation_menu_available; then + export AXE_LANDSCAPE_E2E=1 + else + print_warning "Skipping landscape orientation precision tests because Simulator menu automation is unavailable. Open Simulator and grant macOS Accessibility permissions, then rerun with AXE_LANDSCAPE_E2E=1." + export AXE_LANDSCAPE_E2E=0 + fi + ;; + *) + export AXE_LANDSCAPE_E2E=0 + ;; + esac - print_info "Environment: SIMULATOR_UDID=$SIMULATOR_UDID, AXE_E2E=$AXE_E2E" + print_info "Environment: SIMULATOR_UDID=$SIMULATOR_UDID, AXE_E2E=$AXE_E2E, AXE_LANDSCAPE_E2E=$AXE_LANDSCAPE_E2E" run_swift_test() { local filter="$1"