fix(commands): orientation-aware coords for both rotated devices and letterboxed landscape-only apps (#5)

AxePlaygroundApp/AxePlayground/ContentView.swift

@@ -53,6 +53,8 @@ struct ContentView: View {
             TouchControlView()
         case "gesture-presets":
             GesturePresetsView()
+        case "landscape-coordinate-test":
+            LandscapeCoordinateTestView()
 
         // Input & Text
         case "text-input":
@@ -84,7 +86,8 @@ struct MainMenuView: View {
             ("tap-test", "Tap Test", "Displays coordinates of CLI taps"),
             ("touch-control", "Touch Control", "Touch down/up events"),
             ("swipe-test", "Swipe Test", "Shows CLI swipe paths"),
-            ("gesture-presets", "Gesture Presets", "Multi-touch gesture display")
+            ("gesture-presets", "Gesture Presets", "Multi-touch gesture display"),
+            ("landscape-coordinate-test", "Landscape Coordinates", "Verifies orientation-aware taps")
         ]),
         ("Input & Text", [
             ("text-input", "Text Input", "Text typed by CLI commands"),

AxePlaygroundApp/AxePlayground/Views/LandscapeCoordinateTestView.swift

@@ -0,0 +1,247 @@
+import SwiftUI
+
+struct LandscapeCoordinateTestView: View {
+    @State private var tapHitCount = 0
+    @State private var tapMissCount = 0
+    @State private var swipeHitCount = 0
+    @State private var swipeMissCount = 0
+    @State private var lastTapHit = "none"
+    @State private var lastTapMiss = "none"
+    @State private var lastSwipeStart = "none"
+    @State private var lastSwipeEnd = "none"
+    @State private var dragStart: CGPoint?
+
+    private let tapTargetSize = CGSize(width: 120, height: 80)
+    private let swipeTargetSize = CGSize(width: 90, height: 70)
+
+    var body: some View {
+        GeometryReader { geometry in
+            let isLandscape = geometry.size.width > geometry.size.height
+            let tapTargetCenter = CGPoint(
+                x: isLandscape ? geometry.size.width * 0.78 : geometry.size.width * 0.22,
+                y: isLandscape ? geometry.size.height * 0.32 : geometry.size.height * 0.72
+            )
+            let swipeStartCenter = CGPoint(
+                x: isLandscape ? geometry.size.width * 0.20 : geometry.size.width * 0.24,
+                y: isLandscape ? geometry.size.height * 0.76 : geometry.size.height * 0.34
+            )
+            let swipeEndCenter = CGPoint(
+                x: isLandscape ? geometry.size.width * 0.84 : geometry.size.width * 0.76,
+                y: isLandscape ? geometry.size.height * 0.76 : geometry.size.height * 0.34
+            )
+            let tapTargetRect = CGRect(center: tapTargetCenter, size: tapTargetSize)
+            let swipeStartRect = CGRect(center: swipeStartCenter, size: swipeTargetSize)
+            let swipeEndRect = CGRect(center: swipeEndCenter, size: swipeTargetSize)
+            let globalFrame = geometry.frame(in: .global)
+
+            ZStack {
+                Color(.systemBackground)
+                    .ignoresSafeArea()
+
+                VStack(spacing: 8) {
+                    Text("Landscape Coordinate Test")
+                        .font(.title2)
+                        .fontWeight(.bold)
+                        .accessibilityIdentifier("landscape-coordinate-title")
+
+                    Text(isLandscape ? "Layout: landscape" : "Layout: portrait")
+                        .font(.headline)
+                        .accessibilityIdentifier("landscape-coordinate-layout")
+                        .accessibilityValue(isLandscape ? "landscape" : "portrait")
+
+                    Text("Landscape Hit Count: \(tapHitCount)")
+                        .font(.headline)
+                        .foregroundColor(.blue)
+                        .accessibilityIdentifier("landscape-coordinate-hit-count")
+                        .accessibilityValue("\(tapHitCount)")
+
+                    Text("Landscape Miss Count: \(tapMissCount)")
+                        .font(.headline)
+                        .foregroundColor(.red)
+                        .accessibilityIdentifier("landscape-coordinate-miss-count")
+                        .accessibilityValue("\(tapMissCount)")
+
+                    Text("Landscape Swipe Hit Count: \(swipeHitCount)")
+                        .font(.headline)
+                        .foregroundColor(.blue)
+                        .accessibilityIdentifier("landscape-coordinate-swipe-hit-count")
+                        .accessibilityValue("\(swipeHitCount)")
+
+                    Text("Landscape Swipe Miss Count: \(swipeMissCount)")
+                        .font(.headline)
+                        .foregroundColor(.red)
+                        .accessibilityIdentifier("landscape-coordinate-swipe-miss-count")
+                        .accessibilityValue("\(swipeMissCount)")
+
+                    Text("Last Tap Hit: \(lastTapHit)")
+                        .font(.subheadline)
+                        .foregroundColor(.secondary)
+                        .accessibilityIdentifier("landscape-coordinate-last-tap-hit")
+                        .accessibilityValue(lastTapHit)
+
+                    Text("Last Tap Miss: \(lastTapMiss)")
+                        .font(.subheadline)
+                        .foregroundColor(.secondary)
+                        .accessibilityIdentifier("landscape-coordinate-last-tap-miss")
+                        .accessibilityValue(lastTapMiss)
+
+                    Text("Last Swipe Start: \(lastSwipeStart)")
+                        .font(.subheadline)
+                        .foregroundColor(.secondary)
+                        .accessibilityIdentifier("landscape-coordinate-last-swipe-start")
+                        .accessibilityValue(lastSwipeStart)
+
+                    Text("Last Swipe End: \(lastSwipeEnd)")
+                        .font(.subheadline)
+                        .foregroundColor(.secondary)
+                        .accessibilityIdentifier("landscape-coordinate-last-swipe-end")
+                        .accessibilityValue(lastSwipeEnd)
+                }
+                .padding()
+                .background(Color.white.opacity(0.92))
+                .cornerRadius(12)
+                .shadow(radius: 4)
+                .position(x: geometry.size.width * 0.5, y: 130)
+                .allowsHitTesting(false)
+
+                targetLabel(
+                    isLandscape ? "Landscape Target" : "Portrait Target",
+                    color: isLandscape ? .green : .orange,
+                    size: tapTargetSize
+                )
+                .position(tapTargetCenter)
+                .accessibilityIdentifier("landscape-coordinate-target")
+                .accessibilityLabel(isLandscape ? "Landscape Target" : "Portrait Target")
+                .accessibilityValue(isLandscape ? "landscape-target" : "portrait-target")
+                .allowsHitTesting(false)
+
+                targetLabel(
+                    isLandscape ? "Landscape Swipe Start" : "Portrait Swipe Start",
+                    color: .purple,
+                    size: swipeTargetSize
+                )
+                .position(swipeStartCenter)
+                .accessibilityIdentifier("landscape-coordinate-swipe-start")
+                .accessibilityLabel(isLandscape ? "Landscape Swipe Start" : "Portrait Swipe Start")
+                .accessibilityValue(isLandscape ? "landscape-swipe-start" : "portrait-swipe-start")
+                .allowsHitTesting(false)
+
+                targetLabel(
+                    isLandscape ? "Landscape Swipe End" : "Portrait Swipe End",
+                    color: .indigo,
+                    size: swipeTargetSize
+                )
+                .position(swipeEndCenter)
+                .accessibilityIdentifier("landscape-coordinate-swipe-end")
+                .accessibilityLabel(isLandscape ? "Landscape Swipe End" : "Portrait Swipe End")
+                .accessibilityValue(isLandscape ? "landscape-swipe-end" : "portrait-swipe-end")
+                .allowsHitTesting(false)
+
+                Circle()
+                    .stroke(Color.red, lineWidth: 3)
+                    .frame(width: 44, height: 44)
+                    .position(x: geometry.size.width * 0.22, y: geometry.size.height * 0.72)
+                    .opacity(isLandscape ? 0.25 : 0)
+                    .allowsHitTesting(false)
+                    .accessibilityHidden(true)
+            }
+            .contentShape(Rectangle())
+            .gesture(
+                DragGesture(minimumDistance: 0, coordinateSpace: .local)
+                    .onChanged { value in
+                        if dragStart == nil {
+                            dragStart = value.startLocation
+                        }
+                    }
+                    .onEnded { value in
+                        let start = dragStart ?? value.startLocation
+                        let end = value.location
+                        dragStart = nil
+
+                        let distance = hypot(end.x - start.x, end.y - start.y)
+                        if distance < 20 {
+                            recordTap(at: end, globalFrame: globalFrame, targetRect: tapTargetRect)
+                        } else {
+                            recordSwipe(
+                                start: start,
+                                end: end,
+                                globalFrame: globalFrame,
+                                startRect: swipeStartRect,
+                                endRect: swipeEndRect
+                            )
+                        }
+                    }
+            )
+        }
+        .navigationTitle("Landscape Coordinates")
+        .navigationBarTitleDisplayMode(.inline)
+        .accessibilityIdentifier("landscape-coordinate-screen")
+    }
+
+    private func targetLabel(_ title: String, color: Color, size: CGSize) -> some View {
+        Text(title)
+            .font(.headline)
+            .multilineTextAlignment(.center)
+            .frame(width: size.width, height: size.height)
+            .background(color)
+            .foregroundColor(.white)
+            .cornerRadius(12)
+    }
+
+    private func recordTap(at location: CGPoint, globalFrame: CGRect, targetRect: CGRect) {
+        let globalLocation = location.offsetBy(dx: globalFrame.minX, dy: globalFrame.minY)
+        if targetRect.contains(location) {
+            tapHitCount += 1
+            lastTapHit = formatted(globalLocation)
+        } else {
+            tapMissCount += 1
+            lastTapMiss = formatted(globalLocation)
+        }
+    }
+
+    private func recordSwipe(
+        start: CGPoint,
+        end: CGPoint,
+        globalFrame: CGRect,
+        startRect: CGRect,
+        endRect: CGRect
+    ) {
+        let globalStart = start.offsetBy(dx: globalFrame.minX, dy: globalFrame.minY)
+        let globalEnd = end.offsetBy(dx: globalFrame.minX, dy: globalFrame.minY)
+        lastSwipeStart = formatted(globalStart)
+        lastSwipeEnd = formatted(globalEnd)
+
+        if startRect.contains(start) && endRect.contains(end) {
+            swipeHitCount += 1
+        } else {
+            swipeMissCount += 1
+        }
+    }
+
+    private func formatted(_ point: CGPoint) -> String {
+        "x:\(Int(point.x.rounded())),y:\(Int(point.y.rounded()))"
+    }
+}
+
+private extension CGRect {
+    init(center: CGPoint, size: CGSize) {
+        self.init(
+            x: center.x - size.width / 2,
+            y: center.y - size.height / 2,
+            width: size.width,
+            height: size.height
+        )
+    }
+}
+
+private extension CGPoint {
+    func offsetBy(dx: CGFloat, dy: CGFloat) -> CGPoint {
+        CGPoint(x: x + dx, y: y + dy)
+    }
+}
+
+#Preview {
+    NavigationStack {
+        LandscapeCoordinateTestView()
+    }
+}

BATCHING.md

@@ -178,12 +178,15 @@ Each step command keeps its normal options and validation.
 Examples:
 - `tap --id BackButton`
 - `tap -x 200 -y 400 --pre-delay 0.2 --post-delay 0.2`
+- `swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100`
 - `gesture scroll-down --duration 1.0`
 - `touch -x 150 -y 300 --down --up --delay 1.0`
 
 One important rule:
 - Do not include `--udid` inside a step. Use the batch-level `--udid` only.
 
+Coordinate-based `tap`, `swipe`, and `touch` steps use the same orientation-aware coordinate mapping as the standalone commands. Use coordinates from `describe-ui` directly; AXe detects rotated landscape simulator orientation and letterboxed landscape-only app layouts automatically.
+
 ## Verification strategy
 
 Batch is execution-focused. It does not do built-in assertions.

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to the AXe iOS testing framework will be documented in this
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Fixed
+
+- Fixed `tap`, `touch`, `swipe`, and matching batch steps dispatching logical UI coordinates directly to FBSimulatorHIDEvent without landscape rotation or letterbox correction, causing interactions to land in the wrong location in rotated landscape simulators and portrait-hardware landscape-only apps. AXe now detects the simulator UI orientation automatically instead of requiring callers to pass landscape flags ([#5](https://github.com/cameroncooke/AXe/issues/5) by [@Nitewriter](https://github.com/Nitewriter))
+
 ## [v1.6.0] - 2026-04-05
 
 ### Added

Package.swift

@@ -7,6 +7,10 @@ let package = Package(
         .macOS(.v14)
     ],
     products: [
+        .library(
+            name: "AXeCore",
+            targets: ["AXeCore"]
+        ),
         .executable(
             name: "axe",
             targets: ["AXe"]
@@ -16,9 +20,14 @@ let package = Package(
         .package(url: "https://github.com/apple/swift-argument-parser", from: "1.5.0"),
     ],
     targets: [
+        .target(
+            name: "AXeCore",
+            path: "Sources/AXeCore"
+        ),
         .executableTarget(
             name: "AXe",
             dependencies: [
+                "AXeCore",
                 .product(name: "ArgumentParser", package: "swift-argument-parser"),
                 "FBSimulatorControl",
                 "FBDeviceControl",
@@ -46,7 +55,7 @@ let package = Package(
         ),
         .testTarget(
             name: "AXeTests",
-            dependencies: ["AXe"],
+            dependencies: ["AXe", "AXeCore"],
             path: "Tests",
             resources: [
                 .copy("README.md")

README.md

@@ -194,6 +194,12 @@ axe tap --label "Safari" --udid SIMULATOR_UDID
 axe swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100 --udid SIMULATOR_UDID
 axe swipe --start-x 50 --start-y 500 --end-x 350 --end-y 500 --duration 2.0 --delta 25 --udid SIMULATOR_UDID
 
+# Orientation-aware coordinates
+# AXe automatically maps logical UI coordinates for rotated landscape and letterboxed landscape-only apps.
+# Use the coordinates from describe-ui directly; AXe detects the simulator orientation.
+axe tap -x 100 -y 200 --udid SIMULATOR_UDID
+axe swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100 --udid SIMULATOR_UDID
+
 # Advanced touch control
 axe touch -x 150 -y 250 --down --udid SIMULATOR_UDID
 axe touch -x 150 -y 250 --up --udid SIMULATOR_UDID

Skills/CLI/axe/SKILL.md

@@ -33,6 +33,7 @@ HID commands (`tap`, `swipe`, `type`, `key`, etc.) are fire-and-forget — AXe c
 ## Step 4: Apply timing and input best practices
 - Use `--pre-delay` / `--post-delay` on tap, swipe, and gesture commands for fixed delays around actions.
 - Use `--duration` to control how long a swipe, gesture, button press, or key press lasts.
+- Coordinate-based `tap`, `swipe`, and `touch` accept coordinates from `describe-ui` directly; AXe detects rotated landscape simulator orientation and letterboxed landscape-only app layouts automatically.
 - For text with shell-sensitive characters, prefer `--stdin` or `--file` over inline quotes.
 - Use single quotes for inline text arguments to avoid shell expansion issues.
 

Sources/AXe/Commands/Swipe.swift

@@ -35,6 +35,7 @@ struct Swipe: AsyncParsableCommand {
     @Option(name: .customLong("udid"), help: "The UDID of the simulator.")
     var simulatorUDID: String
 
+
     func validate() throws {
         // Validate coordinates are non-negative
         guard startX >= 0, startY >= 0, endX >= 0, endY >= 0 else {
@@ -87,6 +88,14 @@ struct Swipe: AsyncParsableCommand {
         logger.info().log("Performing swipe from (\(startX), \(startY)) to (\(endX), \(endY))")
         logger.info().log("Duration: \(swipeDuration)s, Delta: \(swipeDelta)px")
 
+        let physicalPoints = try await OrientationAwareCoordinates.translateBatch(
+            points: [(x: startX, y: startY), (x: endX, y: endY)],
+            for: simulatorUDID,
+            logger: logger
+        )
+        let physicalStart = physicalPoints[0]
+        let physicalEnd = physicalPoints[1]
+
         // Broadcast swipe notification for external listeners
         NotificationCenter.default.post(
             name: .hidSwipePerformed,
@@ -112,10 +121,10 @@ struct Swipe: AsyncParsableCommand {
 
         // Create main swipe HID event
         let swipeEvent = FBSimulatorHIDEvent.swipe(
-            startX,
-            yStart: startY,
-            xEnd: endX,
-            yEnd: endY,
+            physicalStart.x,
+            yStart: physicalStart.y,
+            xEnd: physicalEnd.x,
+            yEnd: physicalEnd.y,
             delta: swipeDelta,
             duration: swipeDuration
         )