diff --git a/mobile/README.md b/mobile/README.md index 64f1081b73cf..035c7d8df037 100644 --- a/mobile/README.md +++ b/mobile/README.md @@ -200,6 +200,26 @@ Read on every request, so behaviour can be flipped mid-session without a restart 2. In Orca, go to Settings > Mobile and scan the QR code with this app 3. The QR encodes the connection endpoint, device token, and TLS fingerprint +## Samsung DeX / desktop mode + +Manifest changes require a native Android rebuild; Expo Go cannot validate them. + +```bash +pnpm exec expo prebuild --platform android --clean +cd android +./gradlew assembleDebug +``` + +Use the manual DeX, multi-window, hardware-keyboard, and DPI checklist in +[`docs/dex-mode-layout-design.md`](docs/dex-mode-layout-design.md#62-manual-dex-checklist). +The native Activity display-metrics hook works around +[React Native #55659](https://github.com/facebook/react-native/issues/55659), which otherwise uses +the phone display's density for windows on a DeX monitor. A density-only change at identical pixel +bounds may wait for the next geometry-driven layout; DeX enter, exit, resize, and maximize also +change the window bounds. +RN Android does not deliver every arrow/Ctrl/F-key event; the terminal accessory row remains the +fallback until the planned native hardware-key module lands. + ## Project Structure ``` diff --git a/mobile/app.json b/mobile/app.json index d5a420cc74b9..173cad2be37e 100644 --- a/mobile/app.json +++ b/mobile/app.json @@ -80,6 +80,7 @@ "plugins": [ "expo-router", "./plugins/android-respect-rotation-lock.js", + "./plugins/android-desktop-mode.js", [ "expo-splash-screen", { diff --git a/mobile/app/_layout.tsx b/mobile/app/_layout.tsx index 9080cdedcf95..b96d1d525d59 100644 --- a/mobile/app/_layout.tsx +++ b/mobile/app/_layout.tsx @@ -1,5 +1,5 @@ import { useCallback, useEffect, useRef } from 'react' -import { View, StyleSheet } from 'react-native' +import { StyleSheet } from 'react-native' import { Stack, useRouter } from 'expo-router' import { StatusBar } from 'expo-status-bar' import * as SplashScreen from 'expo-splash-screen' @@ -13,6 +13,7 @@ import { useOpenNotificationRoute } from '../src/notifications/use-open-notifica import { loadHostCatalog } from '../src/transport/host-store' import { extractPairingCodeFromUrl } from '../src/transport/pairing' import { recoverMobileRelayPairing } from '../src/transport/mobile-relay-pairing-recovery' +import { WindowBoundsProvider } from '../src/layout/window-bounds' // Why: keeps the native splash screen visible until the React tree is mounted // and ready to render. Without this the user sees a blank white/black frame @@ -159,7 +160,7 @@ export default function RootLayout() { return ( - + - + ) } diff --git a/mobile/app/h/[hostId]/session/[worktreeId].tsx b/mobile/app/h/[hostId]/session/[worktreeId].tsx index a34c8643ab0b..2e9a9758d0a8 100644 --- a/mobile/app/h/[hostId]/session/[worktreeId].tsx +++ b/mobile/app/h/[hostId]/session/[worktreeId].tsx @@ -4,7 +4,6 @@ import { AppState, Linking, type AppStateStatus, - BackHandler, FlatList, Image, View, @@ -133,6 +132,9 @@ import { isTerminalSendRpcAccepted } from '../../../../src/terminal/terminal-sen import { sendMobileTerminalQueryReply } from '../../../../src/terminal/mobile-terminal-query-reply' import { TERMINAL_QUERY_REPLY_INPUT_RUNTIME_CAPABILITY } from '../../../../../src/shared/protocol-version' import { useTerminalLiveInputCommit } from '../../../../src/terminal/use-terminal-live-input-commit' +import { useHardwareKeyboardAttached } from '../../../../src/platform/hardware-keyboard' +import { useSessionBackHandler } from '../../../../src/session/use-session-back-handler' +import { useTerminalFrameLayout } from '../../../../src/session/use-terminal-frame-layout' import { resolveMobileTerminalInputGate } from '../../../../src/terminal/terminal-input-connection-gate' import { buildTerminalSendParams, @@ -897,6 +899,8 @@ export default function SessionScreen() { ) // Why: Expo SDK 55 edge-to-edge doesn't resize the window on IME open, so track keyboard height ourselves and lift the input without resizing the desktop PTY. const [keyboardHeight, setKeyboardHeight] = useState(0) + const { hardwareKeyboard, notifyHardwareKeyEvent, notifyImeShown, notifyImeHidden } = + useHardwareKeyboardAttached() // Why: server-authoritative display mode per terminal, populated from subscribe responses. const [terminalModes, setTerminalModes] = useState>(new Map()) const [terminalKeyboardMetrics, setTerminalKeyboardMetrics] = useState< @@ -977,6 +981,7 @@ export default function SessionScreen() { const sendingRef = useRef(false) // Why: exact terminal-frame height for measureFitDimensions; window.innerHeight can overstate the visible area. const terminalFrameHeightRef = useRef(0) + const terminalFrameWidthRef = useRef(0) // Why: sidebar resizes change the terminal frame width without a window-dim change; track it so the refit hook re-fits (see terminal-viewport-refit.ts). const [terminalFrameWidth, setTerminalFrameWidth] = useState(0) const activeSessionTab = sessionTabs.find((tab) => tab.id === activeSessionTabId) ?? null @@ -997,7 +1002,8 @@ export default function SessionScreen() { liveInputTerminalHandles, liveInputTerminalHandlesRef, sendLiveTerminalInputRef, - setLiveInputCapture + setLiveInputCapture, + onKeyPressObserved: notifyHardwareKeyEvent }) const { canCompose, canSend } = resolveMobileTerminalInputGate({ connState, @@ -1010,6 +1016,7 @@ export default function SessionScreen() { canSend, inputRef: liveInputRef, keyboardHeight, + hardwareKeyboard, lifecycleIdentity: client, lifecycleKey: JSON.stringify([hostId, worktreeId, connState]), liveInputEnabled, @@ -1322,7 +1329,8 @@ export default function SessionScreen() { return } const dims = await getTerminalRef(handle)?.measureFitDimensions( - terminalFrameHeightRef.current || undefined + terminalFrameHeightRef.current || undefined, + terminalFrameWidthRef.current || undefined ) terminalDiagnosticsRef.current.viewportMeasured(handle, dims, terminalFrameHeightRef.current) if (dims) { @@ -2172,43 +2180,15 @@ export default function SessionScreen() { [markdownDocs, showToast] ) - const getDirtyMarkdownDrafts = useCallback(() => { - const drafts: DirtyMarkdownDraft[] = [] - for (const [tabId, doc] of markdownDocs) { - if (doc.status === 'ready' && doc.isDirty) { - const tab = sessionTabs.find((candidate) => candidate.id === tabId) - drafts.push({ tabId, title: tab?.title || 'Markdown', content: doc.localContent }) - } - } - return drafts - }, [markdownDocs, sessionTabs]) - - const leaveSession = useCallback(() => { - if (router.canGoBack()) { - router.back() - return - } - // Why: Android back can fire at the root route; replace avoids React Navigation's dev-only GO_BACK warning. - router.replace(`/h/${hostId}`) - }, [hostId, router]) - - const requestLeaveSession = useCallback(() => { - const dirtyDrafts = getDirtyMarkdownDrafts() - if (dirtyDrafts.length === 0) { - leaveSession() - return - } - Keyboard.dismiss() - setLeaveDrafts(dirtyDrafts) - }, [getDirtyMarkdownDrafts, leaveSession]) - - useEffect(() => { - const subscription = BackHandler.addEventListener('hardwareBackPress', () => { - requestLeaveSession() - return true - }) - return () => subscription.remove() - }, [requestLeaveSession]) + const { leaveSession, requestLeaveSession } = useSessionBackHandler({ + handleLiveInputKeyPress, + hardwareKeyboard, + hostId, + liveInputRef, + markdownDocs, + sessionTabs, + setLeaveDrafts + }) const discardMarkdownLocalContent = useCallback( (tab: Extract) => { @@ -2559,7 +2539,12 @@ export default function SessionScreen() { }, [connState, scheduleDelayedAction, subscribeToTerminal, unsubscribeTerminal]) // Why: non-subscribe layout refits (tab strip, fold, rotation) live in a dedicated hook — see terminal-viewport-refit.ts. - const { notifyTerminalFrameHeight, notifyKeyboardVisibility } = useTerminalViewportRefit({ + const { + notifyTerminalFrameHeight, + notifyKeyboardVisibility, + notifyHardwareKeyboard, + notifyWebViewViewport + } = useTerminalViewportRefit({ activeHandleRef, terminalRefs, terminalFrameHeightRef, @@ -2580,10 +2565,13 @@ export default function SessionScreen() { useEffect(() => { const onShow = (e: KeyboardEvent) => { notifyKeyboardVisibility(true) - setKeyboardHeight(e.endCoordinates?.height ?? 0) + const height = e.endCoordinates?.height ?? 0 + notifyImeShown(height) + setKeyboardHeight(height) } const onHide = () => { notifyKeyboardVisibility(false) + notifyImeHidden() setKeyboardHeight(0) } const showEvent = Platform.OS === 'ios' ? 'keyboardWillShow' : 'keyboardDidShow' @@ -2594,7 +2582,22 @@ export default function SessionScreen() { showSub.remove() hideSub.remove() } - }, [notifyKeyboardVisibility]) + }, [notifyImeHidden, notifyImeShown, notifyKeyboardVisibility]) + + useEffect(() => { + if (hardwareKeyboard) { + notifyHardwareKeyboard() + } + }, [hardwareKeyboard, notifyHardwareKeyboard]) + + const handleTerminalFrameLayout = useTerminalFrameLayout({ + terminalRefs, + heightRef: terminalFrameHeightRef, + widthRef: terminalFrameWidthRef, + onHeightChange: notifyTerminalFrameHeight, + onWidthChange: (width) => + setTerminalFrameWidth((current) => (current === width ? current : width)) + }) const scrollActiveTabIntoView = useCallback((tabId: string | null, animated: boolean) => { if (!tabId) { @@ -2953,6 +2956,9 @@ export default function SessionScreen() { terminalDiagnosticsRef.current.webViewRef(handle, ref != null) if (ref) { terminalRefs.current.set(handle, ref) + if (terminalFrameWidthRef.current > 0 && terminalFrameHeightRef.current > 0) { + ref.setViewport(terminalFrameWidthRef.current, terminalFrameHeightRef.current) + } } else { terminalRefs.current.delete(handle) terminalGestureInputBucketsRef.current.delete(handle) @@ -4268,7 +4274,7 @@ export default function SessionScreen() { // Why: iOS keyboard height includes the home-indicator inset; Android IME height does not. const keyboardLift = - keyboardHeight > 0 + !hardwareKeyboard && keyboardHeight > 0 ? Platform.OS === 'ios' ? Math.max(0, keyboardHeight - insets.bottom) : keyboardHeight @@ -4717,17 +4723,7 @@ export default function SessionScreen() { )} ) : ( - { - terminalFrameHeightRef.current = e.nativeEvent.layout.height - // Why: notify height imperatively so dock settling re-fits the PTY without rerendering SessionScreen. - const nextWidth = Math.round(e.nativeEvent.layout.width) - const nextHeight = Math.round(e.nativeEvent.layout.height) - setTerminalFrameWidth((prev) => (prev === nextWidth ? prev : nextWidth)) - notifyTerminalFrameHeight(nextHeight) - }} - > + {terminals.map((terminal) => ( { + if (handle === activeHandleRef.current) { + notifyWebViewViewport(width, height) + } + }} onFileTap={handleFileTap} onOpenUrl={handleTerminalOpenUrl} /> diff --git a/mobile/app/h/_layout.tsx b/mobile/app/h/_layout.tsx index 7f77c33b6489..00dc001e7079 100644 --- a/mobile/app/h/_layout.tsx +++ b/mobile/app/h/_layout.tsx @@ -60,7 +60,7 @@ function HostStack({ animation }: { animation: 'none' | 'default' }) { export default function HostGroupLayout() { // Wide layout = tablet/foldable canvas (see responsive-layout-metrics). - const { isWideLayout, width: windowWidth } = useResponsiveLayout() + const { windowClass, width: windowWidth } = useResponsiveLayout() const { hostId, action } = useGlobalSearchParams<{ hostId?: string; action?: string }>() const pathname = usePathname() const [sidebarOpen, setSidebarOpen] = useState(true) @@ -94,7 +94,7 @@ export default function HostGroupLayout() { }, [windowWidth]) const hideSidebar = useCallback(() => setSidebarOpen(false), []) - const showSidebar = isWideLayout && !!hostId + const showSidebar = windowClass !== 'compact' && !!hostId const detailHasContent = !!hostId && pathname !== `/h/${hostId}` const canCollapseSidebar = showSidebar && detailHasContent diff --git a/mobile/app/mobile-onboarding.tsx b/mobile/app/mobile-onboarding.tsx index 50957a465fce..0bc108f57958 100644 --- a/mobile/app/mobile-onboarding.tsx +++ b/mobile/app/mobile-onboarding.tsx @@ -1,12 +1,5 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' -import { - AccessibilityInfo, - Animated, - BackHandler, - Text, - useWindowDimensions, - View -} from 'react-native' +import { AccessibilityInfo, Animated, BackHandler, Text, View } from 'react-native' import { useFocusEffect, useLocalSearchParams, useRouter } from 'expo-router' import { SafeAreaView } from 'react-native-safe-area-context' import { OrcaLogo } from '../src/components/OrcaLogo' @@ -23,6 +16,7 @@ import { type MobileSessionView } from '../src/storage/session-view-preferences' import { savePushNotificationsEnabled } from '../src/storage/preferences' +import { useWindowBounds } from '../src/layout/window-bounds' const SLIDE_DURATION_MS = 280 @@ -52,7 +46,7 @@ function MobileOnboardingFlow({ }) { const router = useRouter() const steps = useMemo(() => parseMobileOnboardingSteps(rawSteps), [rawSteps]) - const { width } = useWindowDimensions() + const { width } = useWindowBounds() const [activeIndex, setActiveIndex] = useState(0) const [busyChoice, setBusyChoice] = useState(null) const [error, setError] = useState(null) diff --git a/mobile/docs/dex-mode-layout-design.md b/mobile/docs/dex-mode-layout-design.md new file mode 100644 index 000000000000..54d611bef442 --- /dev/null +++ b/mobile/docs/dex-mode-layout-design.md @@ -0,0 +1,958 @@ +# Samsung DeX / desktop-mode layout fix — design + +Status: design only (no code changes). Scope: `mobile/` only. +Target: Orca mobile Android (Expo SDK 55, React Native 0.83, expo-router, `newArchEnabled: true`). + +Samsung DeX (and Android 15+ desktop windowing) changes five things at once that a +phone build never sees together: a **freeform, user-resizable window** (resized +without Activity recreation only if the manifest allows it), a **density (DPI) +change** when the display switches, **`smallestScreenWidthDp` crossings** as the +window is dragged, **no soft keyboard** (physical keyboard + mouse), and a window +whose size is _not_ the display size. Almost every layout primitive in the app +assumes the opposite of at least one of those. + +All line numbers below refer to the tree at the time of writing. + +--- + +## 1. Symptom model + +The report is "screen renders broken / garbled". Nobody has attached a +screenshot, so the design covers every symptom class that the code can +produce. Each class maps to one or more root causes in §2; the fix in §3–§4 +addresses all of them. + +| # | Symptom class (what the user sees) | Root causes | +| --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| S1 | App "restarts": jumps back to the home route, splash flashes, terminals reconnect when DeX is entered/exited, a monitor is plugged in, or the window is dragged past a size breakpoint | R1 (Activity recreation: `density` / `smallestScreenSize` not in `configChanges`), R2 (no Samsung keep-alive metadata) | +| S2 | Terminal is a narrow phone-width strip inside a wide window, or text is cut off / overlaps / wraps at the old width; PTY columns don't follow the window | R5 (WebView measure uses stale `window.innerWidth`), R6 (refit only keyed off height + RN width, 150 ms debounce races the WebView re-layout), R7 (`applyTextScale` resizes xterm itself to `innerWidth`) | +| S3 | Drawers/sheets wider or taller than the window, centered content off-center, sidebar/dock decisions wrong for the window size, onboarding pages mis-paged | R3 (`useWindowDimensions` reports display size, not the freeform window, on Android) | +| S4 | Blurry, doubled or smeared glyphs after moving the window to another display / toggling DeX; canvas and DOM layers disagree | R4 (DPR change not observed by the fit pipeline; WebGL canvas scaled by CSS `scale()`) | +| S5 | With a hardware keyboard: `Esc` kicks the user out of the session; arrow/ctrl keys dead; caret focus flickers; input bar parks above an empty band reserved for a keyboard that never opens | R8 (`hardwareBackPress` == Esc), R9 (focus helper assumes `keyboardHeight==0` means "IME dismissed"), R10 (IME-height lift is the only avoidance path) | +| S6 | App opens in a fixed phone-sized window that can't be resized, or rotates/relayouts oddly when resized | R11 (`android:screenOrientation="fullUser"` set unconditionally by the rotation-lock plugin; no `resizeableActivity`) | +| S7 | Extra top padding or content under a title bar; bottom gap | R12 (edge-to-edge + safe-area assumptions; insets change per window) | +| S8 | Phone-sized DeX window gets tablet layout (sidebar + dock) and everything is squished, or a tablet-sized window stays in phone layout | R3, R13 (breakpoints only look at width/height, not at density or window class) | + +--- + +## 2. Root-cause analysis (with evidence) + +### R1 — Activity recreation on density / smallest-width changes (primary) + +- `mobile/app.json` has no `expo-build-properties` Android manifest overrides + beyond `usesCleartextTraffic` (`app.json:104-113`) and no `android.manifest` + plugin. The only manifest plugin is `plugins/android-respect-rotation-lock.js`, + which touches exactly one attribute (`android:screenOrientation`, lines 8 and + 13). +- The Expo SDK 55 bare template emits + `android:configChanges="keyboard|keyboardHidden|orientation|screenSize|screenLayout|uiMode"` + on `MainActivity` (verify after prebuild in + `android/app/src/main/AndroidManifest.xml`). **`density` and + `smallestScreenSize` are absent.** Entering/leaving DeX changes density; + dragging a freeform window across 600 dp / 720 dp width crosses + `smallestScreenWidthDp`. Either change **destroys and recreates + `MainActivity`**. +- Under RN 0.83 bridgeless, the `ReactHost` survives but the + `ReactSurface`/root view is rebuilt, so the whole React tree remounts: + `app/_layout.tsx` `RootLayout` re-runs (the splash hide hook at + `_layout.tsx:156-158` fires again), expo-router's navigation container is + recreated with no persisted state (user lands on `index`), `RpcClientProvider` + remounts, and every `TerminalWebView` is recreated (xterm reload; all the + `webReadyHandlesRef`/`initializedHandlesRef` bookkeeping in the session route, + `[worktreeId].tsx:949-955`, starts from zero). +- This alone explains S1 and contributes to S2 (the first measure after a + remount runs against a WebView that has not laid out yet). + +### R2 — No Samsung DeX keep-alive metadata + +Samsung documents two `` entries that suppress the DeX-specific +restarts (`com.samsung.android.keepalive.density` — don't restart on the +density change when DeX toggles; `com.samsung.android.multidisplay.keep_process_alive` +— keep the process when the app moves between phone and DeX displays). Neither +is emitted by any plugin in `mobile/plugins/` or `app.json`. + +### R3 — `useWindowDimensions()` is the display, not the freeform window + +On Android, RN's `DeviceInfoModule`/`DisplayMetricsHolder` derives `window` +metrics from the application context's display metrics (display size minus +system bars), **not** from the Activity's window bounds. In a freeform DeX +window those diverge (a 1920×1080 desktop vs. an 800×600 app window). Yoga +layout (`flex`, `onLayout`) is driven by the real root view size and is +correct; everything that reads `useWindowDimensions` is not. Consumers: + +| File:line | What it decides from window dims | +| ---------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `src/layout/responsive-layout.ts:10-11` → `responsive-layout-metrics.ts:29-31` | `isWideLayout`, `isTabletLayout` (breakpoints 700/600) | +| `app/h/_layout.tsx:63, 91-94, 22-28` | sidebar show/hide, sidebar width clamp (`MIN_DETAIL_WIDTH = 320`) | +| `app/h/[hostId]/session/[worktreeId].tsx:757-766` | `canDockPanel` (combined with the correct `sessionContentRowWidth` from `onLayout`, `:4313-4316`) | +| `src/components/RightDrawer.tsx:90-93` → `right-drawer-panel-width.ts:9-22` | panel width = `windowWidth - 48` on narrow, `min(420, windowWidth)` on wide | +| `src/components/mounted-bottom-drawer.tsx:76-92` → `bottom-drawer-fill-height.ts:7-19` | fill-sheet height = `screenHeight - insets.top - 16 - keyboard` | +| `src/files/MobileFilePreviewScreen.tsx:49, 279-280` | image preview width/height | +| `app/mobile-onboarding.tsx:55, 141, 170, 177` | pager page width / slide translate | +| `src/terminal/terminal-viewport-refit.ts:206-221` | refit trigger on dims change (it will simply not fire on freeform resize; the frame-width/height triggers at `:234-243` and `:245-261` do) | +| `app/index.tsx:230`, `app/h/[hostId]/index.tsx:141, 1403`, `src/components/MobileDiffReviewScreenView.tsx:23-31` | content max-width / presentation mode | + +In DeX a phone-width window on a 1080p display therefore gets +`isWideLayout = true`, a 1872 px-wide `RightDrawer`, a fill sheet taller than +the window, and a sidebar + docked panel crammed into ~800 px (S3, S8). + +### R4 — Device-pixel-ratio changes are never observed + +- `src/terminal/terminal-webview-html.ts` has a `resize` listener + (`:1862-1872`) but no `matchMedia('(resolution: …dppx)')` / DPR listener. + The WebGL addon (`@xterm/addon-webgl`, loaded in the engine) rasterises the + glyph atlas at the DPR current at `term.open()`. Moving the window to a + display with a different DPR (phone 2.6–3.0× → monitor 1.0–1.5×), or DeX + toggling density, leaves the canvas at the wrong backing-store scale until + something triggers a renderer resize. +- The fit pipeline then applies a CSS `scale()` on top + (`updateTransform`, `:364-368`; `computeFitScale`, `:353-360`), which + rasterises an already mis-scaled canvas → smeared/doubled glyphs (S4). +- On the RN side `PixelRatio.get()` is read inside a `useMemo` keyed only on + `[layout, browserViewMode]` (`src/browser/use-mobile-browser-stream.ts`), so + the browser screencast request keeps the old DPR after a density change. + +### R5 — `measureFitDimensions` uses `window.innerWidth`, only height comes from RN + +- RN passes the exact frame **height** into the measure + (`src/terminal/TerminalWebView.tsx:299-328`, message at `:319`), and the + engine prefers it (`terminal-webview-html.ts:895-903`), but **width is always + `window.innerWidth`** (`:895`). During a freeform drag the Android WebView + re-lays out asynchronously; a measure issued 150 ms after the RN `onLayout` + can read the _previous_ `innerWidth`, compute the old column count, and + push it to the PTY via `terminal.updateViewport` (`terminal-viewport-refit.ts:145-149`). +- `computeFitScale` (`:353-360`) and `clampPan` (`:409-424`) have the same + dependency, so the CSS scale can be computed against a stale viewport. + +### R6 — Refit trigger set is incomplete / racy for freeform resize + +- `useTerminalViewportRefit` refits on: tab-strip toggle, window dims + (`:206-221`), text scale, `terminalFrameWidth` (`:234-243`) and frame + height (`:245-261`). The frame width/height path is the only one that fires + in a freeform resize, and it is debounced 150 ms (`:86, :174`) from the _RN_ + layout, not from the WebView's own layout. There is no "WebView reported a + new `innerWidth`" signal back to RN, so the race in R5 has no correction + step until the next unrelated refit. +- The width/height effect deliberately skips height-only changes while the + keyboard is visible (`:216`). With a hardware keyboard `keyboardVisible` is + always false, so that guard is inert — fine — but there is also no + coalescing for the 20–60 layout events/sec that a drag produces beyond the + 150 ms debounce; each commit triggers a server `updateViewport` + local + `reflow` (`:155-160`). + +### R7 — `applyTextScale` resizes xterm to `innerWidth/innerHeight` + +`terminal-webview-html.ts:276-297` computes `cols = floor(innerWidth/cellW)`, +`rows = floor(innerHeight/cellH)` and calls `term.resize()` directly — while +the documented model (`:51`) is "init at desktop cols/rows and fit via CSS +scale". On a wide DeX window `innerWidth` is the _unscaled_ WebView width, so a +text-size change produces a local grid that disagrees with the PTY grid until +the next server refit (contributes to S2 when the user changes text size in +DeX). + +### R8 — `Esc` on a hardware keyboard is `hardwareBackPress` + +`app/h/[hostId]/session/[worktreeId].tsx:2160-2165` installs a +`BackHandler` that always calls `requestLeaveSession()`. Android maps the +physical `Escape` key to `KEYCODE_ESCAPE`, which the framework delivers as a +back press when no view consumes it. `RightDrawer.tsx:114-122` does the same. +The live-input `TextInput` only sees `Escape` via `onKeyPress` +(`src/terminal/terminal-live-input.ts:36-38`) when it is focused _and_ RN +reports the key; for many hardware-keyboard events RN Android's `onKeyPress` +only fires for a subset of keys (printable, Backspace, Enter), so arrows/F-keys/ +Ctrl-combos never reach `handleLiveInputKeyPress` +(`src/terminal/use-terminal-live-input-commit.ts:150-175`) (S5). + +### R9 — Focus helper treats `keyboardHeight === 0` as "IME dismissed" + +`src/terminal/terminal-live-input.ts:231-248`: if `keyboardHeight <= 0` and the +input is focused, it blurs and schedules a refocus to "reopen the keyboard". +With a hardware keyboard attached there is never a `keyboardDidShow` +(`[worktreeId].tsx:2524-2541`), so `keyboardHeight` stays 0 and **every +terminal tap blurs + refocuses the input** — caret flicker, lost +composition, and on DeX the on-screen keyboard toggle may pop up (S5). + +### R10 — Keyboard avoidance is IME-height only + +`keyboardLift` (`[worktreeId].tsx:4206-4216`) and the pane `translateY` +(`src/session/TerminalPaneView.tsx:62-71`) only ever move for a reported IME +height. That is correct for hardware keyboards (lift 0) _but_ +`terminal-webview-html.ts` `emitKeyboardAvoidanceMetrics` and the reduce logic in +`terminal-viewport-refit-state.ts:52-92` keep a `keyboardVisible` flag from +the same events; nothing resets it if a soft keyboard was open when the user +docked the phone (DeX hides the IME without a `keyboardDidHide` in some +Samsung builds), leaving `pending` refits deferred forever (S2 after docking +mid-typing). + +### R11 — Orientation plugin forces `fullUser`; no `resizeableActivity` + +`plugins/android-respect-rotation-lock.js:8,13` writes +`android:screenOrientation="fullUser"` unconditionally. DeX treats activities +with a _restricting_ orientation as "fixed-ratio" and opens them in a +phone-proportioned window; `fullUser` is not `unspecified`, and on some One UI +versions it is enough to disable free resize (S6). `android:resizeableActivity` +is not declared anywhere (Android defaults it to `true` for targetSdk ≥ 24, +but Samsung's DeX compatibility heuristics and Android 16's large-screen +compatibility mode both key off the explicit attribute). + +### R12 — Edge-to-edge + safe-area in a windowed Activity + +Expo SDK 55 builds are edge-to-edge by default; `app/_layout.tsx:163` draws a +light status bar and every screen pads by `useSafeAreaInsets()` (see the 20+ +call sites in `app/**` and `src/**`). In a DeX window the system supplies +**zero** top inset and a decor caption bar; the `SafeAreaView edges={['top']}` +wrappers (`app/index.tsx:674`, `[worktreeId].tsx:4356`, …) are fine, but any +code that _adds_ a constant on top of the inset (e.g. fill-sheet `topGap`, +`bottom-drawer-fill-height.ts:13`) or subtracts `insets.bottom` from a keyboard +height (`[worktreeId].tsx:4209`) will be computing against a different inset +set after the window moves displays; `react-native-safe-area-context` does +re-emit, so this is a secondary cause (S7) and mostly self-heals once R1 is +fixed. + +### R13 — Breakpoints ignore window class / density + +`responsive-layout-metrics.ts:4,8` use dp widths (700 / 600) which is the right +unit, but the inputs come from R3. Additionally there is no hysteresis: a +freeform drag oscillating around 700 dp flips `isWideLayout` on every frame, +which remounts the sidebar (`app/h/_layout.tsx:146-160`) and clears the +docked panel (`[worktreeId].tsx:768-772`). + +### Checked and ruled out + +- No module-scope `Dimensions.get()` anywhere in `app/` or `src/` (grep) — + the "fixed dims read at module load" class does not apply. +- No `expo-screen-orientation` usage; no `orientation` screen option + (`_layout.tsx:171-175` comment is accurate). +- `textZoom={100}` and `scalesPageToFit={false}` (`TerminalWebView.tsx:378-381`) + are correct for DeX. +- Reanimated/gesture-handler drawers measure via `useWindowDimensions`, not a + one-shot measure — they are covered by R3, not a separate cause. +- The `react-native@0.83.9.patch` in `mobile/patches/` is iOS TextInput + composition only; unrelated. + +--- + +## 3. Android manifest / config-plugin changes + +All of this is delivered by **one new config plugin**, +`mobile/plugins/android-desktop-mode.js`, registered in `app.json` `plugins` +_after_ the rotation-lock plugin so it can override it. Keep the rotation-lock +plugin file (its iOS-parity intent is still valid) but narrow it (see 3.4). + +### 3.1 `MainActivity` attributes + +```xml + + android:launchMode="singleTask" + android:screenOrientation="fullUser" /> +``` + +- `density` + `smallestScreenSize` stop the recreation (R1). RN's + `ReactActivityDelegate.onConfigurationChanged` → `ReactHost.onConfigurationChanged` + refreshes native layout metrics, but bridgeless RN does not re-emit + `didUpdateDimensions`; the Activity display-metrics hook does so explicitly. +- `fontScale` is included because DeX exposes a separate font-size setting; + RN handles it via the same path. +- `resizeableActivity="true"` explicitly (R11). +- Do **not** set `android:maxAspectRatio` / `android:minAspectRatio` (they + would reintroduce letterboxing in DeX). + +### 3.2 `` metadata + +```xml + + + + + +``` + +(`` goes inside ``; `AndroidConfig.Manifest` has no +helper for it — the plugin pushes a raw `{ $: {...} }` node onto +`application.property`.) These are harmless on non-Samsung devices. + +### 3.3 `expo-build-properties` + +No SDK bumps are needed; keep the SDK 55 defaults (see §7). Add nothing to +`expo-build-properties` for this work — the manifest plugin owns it. (If +`android.enableEdgeToEdge`-style keys are ever added, keep edge-to-edge +**on**; turning it off does not help DeX and targetSdk 36 ignores it.) + +### 3.4 Rotation-lock plugin fix + +`plugins/android-respect-rotation-lock.js` keeps writing `fullUser` (it is the +right phone behaviour). Two additions, both in the new desktop-mode plugin so +the file stays single-purpose: + +1. Manifest: nothing else — `fullUser` is acceptable to DeX **once + `resizeableActivity="true"` is explicit**. Empirically verify (§6 manual + checklist, item M1). If DeX still opens a fixed-ratio window on the test + device, fall back to plan B. +2. Plan B (only if M1 fails): a tiny Expo module-less native hook is + overkill; instead use `expo-build-properties`' Kotlin `MainActivity` + dangerous mod to add `onCreate`: + `if (resources.configuration.isDesktopMode()) requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED` + where `isDesktopMode()` checks `uiMode & UI_MODE_TYPE_MASK == UI_MODE_TYPE_DESK` + or the Samsung `semDesktopModeEnabled` field via reflection. This is a + `withMainActivity` mod in the same plugin, guarded behind a plugin option + `{ desktopUnspecifiedOrientation: true }` so it can be toggled from + `app.json` without code changes. + +--- + +## 4. React Native layout changes + +### 4.1 Window-bounds provider (fixes R3, R13) + +New `src/layout/window-bounds.tsx`: + +- `WindowBoundsProvider` renders a `flex: 1` `View` with `onLayout` and + publishes `{ width, height }` through context. Mount it once in + `app/_layout.tsx` directly inside `RpcClientProvider`, replacing the + current `styles.root` `View` (keep its `onLayout` splash hook — compose both). +- `useWindowBounds()` returns the measured root size; until the first layout + it falls back to `useWindowDimensions()` so first render is unchanged. +- `useResponsiveLayout()` (`src/layout/responsive-layout.ts`) switches from + `useWindowDimensions()` to `useWindowBounds()`. All eight consumers of + `isWideLayout` are fixed by this one change. +- Add hysteresis in `responsive-layout-metrics.ts`: `getResponsiveLayoutMetrics(width, height, previous?)` + keeps the previous `isWideLayout` unless the width moved ≥ 24 dp past the + breakpoint in the other direction. Pure function; unit-tested. +- Replace `useWindowDimensions` with `useWindowBounds` in: + `RightDrawer.tsx:90`, `mounted-bottom-drawer.tsx:77`, + `MobileFilePreviewScreen.tsx:49`, `mobile-onboarding.tsx:55`, + `app/h/_layout.tsx:63` (via `useResponsiveLayout` already), + `terminal-viewport-refit.ts:207`. +- `terminal-viewport-refit.ts:206-221`: keep the window-dims effect (it covers + fold/rotate on phones) but source it from `useWindowBounds`, and merge it + with the frame-width/height effects into a single `layoutEpoch` counter so a + freeform drag produces one debounced refit, not three. + +### 4.2 Terminal WebView refit on resize (fixes R5, R6, R7) + +`src/terminal/TerminalWebView.tsx` + +- `measureFitDimensions(containerHeight?, containerWidth?)` — add width. The + frame `onLayout` in `[worktreeId].tsx:4656-4665` already has it; thread + `terminalFrameWidth` into `useTerminalViewportRefit` (it's in the options, + `:36`) and pass both to `ref.measureFitDimensions(height, width)` at `:125` + and `measureViewportOnce` at `[worktreeId].tsx:1302`. +- Forward the RN frame size to the engine on every change: + `postMessage({ type: 'set-viewport', width, height, dpr: PixelRatio.get() })` + from a new `setViewport(width, height)` handle method, called from the frame + `onLayout`. The engine stores it as `rnViewport` and prefers it over + `window.innerWidth/innerHeight` everywhere (`computeFitScale`, `clampPan`, + `measureFitDimensions`, `applyTextScale`, `updateScrollIndicator`). +- Keep the 379-line ceiling in `.oxlintrc.json:30-35` by moving the + imperative-handle body into `src/terminal/terminal-webview-handle.ts` + (pure factory taking refs); **never** bump `max-lines`. + +`src/terminal/terminal-webview-html.ts` (ceiling 1784 lines (`.oxlintrc.json:36-41`) — put new JS in +injected modules like `terminal-webview-reflow-injected.ts`): + +- New `terminal-webview-viewport-injected.ts` exporting + `TERMINAL_VIEWPORT_JS`: `rnViewport` state, `getViewportWidth()/Height()` + helpers, `set-viewport` handler, and a **DPR observer**: + `matchMedia('(resolution: ' + devicePixelRatio + 'dppx)')` re-armed on each + change → on change, call `term._core._renderService?.handleDevicePixelRatioChange?.()` + if present else toggle the WebGL addon (dispose + re-load via the existing + `terminal-webview-webgl-recovery-injected.ts` path), then `applyFitScale('dpr')` + and `notify({ type: 'viewport-changed', innerWidth, innerHeight, dpr })`. +- `window.addEventListener('resize')` (`:1862-1872`) additionally posts + `viewport-changed`. RN (`terminal-webview-notification-dispatch.ts`) routes it + to `useTerminalViewportRefit` as a `notifyWebViewViewport(w,h)` event, which + marks the viewport stale and schedules a refit **only after both** the RN + frame and the WebView agree on width (±1 px) — this closes the race in R5. +- `applyTextScale` (`:276-297`): stop calling `term.resize()` locally; set + `fontSize`, then `applyFitScale('text-scale')` and let the RN refit (already + triggered by `textScale` at `terminal-viewport-refit.ts:224-232`) resize the + PTY and reflow. +- Engine `measureFitDimensions`: accept `containerWidthPx`; when the passed + width differs from `innerWidth` by > 1 px, retry (same rAF loop, `:863-886`) + up to 30 frames before trusting the RN value — the WebView is still + re-laying out. + +`src/terminal/terminal-viewport-refit.ts` + +- Debounce: 150 ms while a drag is in progress (layout events < 100 ms apart) + → extend to 250 ms trailing; cap server `updateViewport` to ≤ 4/s. +- Reset `frameHeightRefitStateRef.keyboardVisible` to `false` when a + `hardware-keyboard` signal (4.3) arrives or when window width changes by + > 10 % (an IME cannot do that) — fixes the stuck `pending` state (R10). + +### 4.3 Hardware keyboard / no-IME handling (fixes R8, R9, R10) + +- New `src/platform/hardware-keyboard.ts`: `useHardwareKeyboardAttached()` + — Android: `Keyboard.isVisible()` false **and** the last `keyboardDidShow` + height < 120 dp or never fired, combined with `Platform.constants` + `uiMode`/`keyboard` from a tiny Expo config-plugin-free read via + `expo-constants`? None of those expose `Configuration.keyboard`. Use the + pragmatic heuristic: `hardwareKeyboard = !keyboardVisible && lastImeHeight === 0 && hasReceivedHardwareKeyEvent`, + where `hasReceivedHardwareKeyEvent` flips true the first time the live input + sees an `onKeyPress` with `key.length > 1` (arrows, F-keys) or any key event + while no IME is visible. Pure reducer + test. +- `terminal-live-input.ts:231-248`: take `hardwareKeyboard` into the options; + when true, never blur/refocus — just `focus()`. +- `[worktreeId].tsx:2160-2165` and `RightDrawer.tsx:114-122`: when the live + input is focused and `hardwareKeyboard` is true, the `BackHandler` returns + `true` **after sending `escape`** via the existing special-key path + (`terminal-live-input.ts:36` id `'escape'`), not `requestLeaveSession()`. + Leaving the session stays on the header back button. Extract to + `src/session/session-back-press.ts` (`resolveSessionBackPress({ liveInputFocused, hardwareKeyboard }) → 'send-escape' | 'leave'`). +- Key coverage: RN Android `TextInput.onKeyPress` does not deliver arrows/ + Ctrl. Two-tier plan: + 1. **Now:** document the gap; accessory bar keys keep working. Known + limitation: hardware Escape reaches JS only as `hardwareBackPress` + (no `KeyEvent`), so it is routed to the PTY only once + `hasReceivedHardwareKeyEvent` is true — an Escape pressed before any + other hardware key since the last IME show still leaves the session. + 2. **Follow-up (separate task):** a `react-native-key-command`-style native + listener is not available in Expo Go; implement + `mobile/packages/expo-hardware-keys` (Expo module, `dispatchKeyEvent` + override in `MainActivity` via `withMainActivity`) that emits raw + `KeyEvent`s to JS. Out of scope for t2's first pass. +- `keyboardLift` (`[worktreeId].tsx:4206-4216`): unchanged for IME; with + `hardwareKeyboard` force 0 even if a stale `keyboardHeight` survives. + +### 4.4 Breakpoints / wide layout in a window + +- `responsive-layout-metrics.ts`: add `windowClass: 'compact' | 'medium' | 'expanded'` + (Material window-size classes at 600 / 840 dp) and expose it; keep + `isWideLayout`/`isTabletLayout` semantics unchanged for existing callers. +- `app/h/_layout.tsx`: sidebar rule becomes `windowClass !== 'compact'` + (currently `isWideLayout`, which requires _both_ sides ≥ 600 — in a + 1280×720 DeX window that is fine, but an 800×600 window is `medium` and + keeps the sidebar: `clampSidebarToWindow` (`HOST_SIDEBAR_MIN_WIDTH` 280 + + `MIN_DETAIL_WIDTH` 320 = 600) guarantees both panes fit at every + non-compact width, so only the hysteresis matters — `windowClass` carries + the same ±24 dp margin as `isWideLayout`). +- `session-panel-host.ts:33-42`: unchanged; `availableWidth` already comes + from `onLayout`. + +### 4.5 Density changes + +- `use-mobile-browser-stream.ts`: include the + `Dimensions.addEventListener('change')`-driven `usePixelRatio()` hook + (`src/platform/use-pixel-ratio.ts`) in the screencast request `useMemo` deps. +- Nothing else reads density on the RN side. + +--- + +## 5. File-by-file implementation plan (for t2) + +Ordered so each step is shippable; tests listed per step (all `vitest`, node +env, no RN render harness — keep logic in pure modules exactly like the +existing `*-state.ts` / `*-metrics.ts` files). + +| # | File | Change | Test | +| --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | `mobile/plugins/android-desktop-mode.js` (new) | `withAndroidManifest`: set `configChanges`, `resizeableActivity`, `supportsPictureInPicture="false"`; push the 3 `` + 1 `` into ``; option `desktopUnspecifiedOrientation` → `withMainActivity` Kotlin snippet (plan B, default off) | `mobile/plugins/android-desktop-mode.test.js`: run the plugin against a minimal manifest object (copy the pattern from `plugins/android-respect-rotation-lock.js`), assert attributes and idempotency (running twice yields one meta-data each). Add `plugins/**/*.test.js` to `vitest.config.ts` `include`. | +| 2 | `mobile/app.json` | add `"./plugins/android-desktop-mode.js"` **after** the rotation-lock plugin | covered by 1 (order assertion: desktop plugin must not clobber `screenOrientation`) | +| 3 | `src/layout/window-bounds.tsx` (new), `app/_layout.tsx:162` | provider + hook; wrap root | `src/layout/window-bounds-state.test.ts` for the pure reducer (`resolveWindowBounds({ measured, fallback })`) | +| 4 | `src/layout/responsive-layout-metrics.ts`, `responsive-layout.ts` | hysteresis + `windowClass`; consume `useWindowBounds` | extend `responsive-layout-metrics.test.ts`: oscillation around 700 dp keeps previous class; 800×600 → compact-ish sidebar rule; 1280×720 → expanded | +| 5 | `RightDrawer.tsx:90`, `mounted-bottom-drawer.tsx:77`, `MobileFilePreviewScreen.tsx:49`, `app/mobile-onboarding.tsx:55` | swap to `useWindowBounds` | existing `right-drawer-panel-width.test.ts`, `bottom-drawer-fill-height.test.ts` already cover the pure parts; add a case "windowWidth < displayWidth" to each | +| 6 | `src/terminal/terminal-webview-viewport-injected.ts` (new), `terminal-webview-html.ts` (`:895`, `:353-360`, `:409-424`, `:276-297`, `:863-920`, `:1862-1872`), `terminal-webview-messages.ts` (`set-viewport` command, `viewport-changed` notification) | RN-viewport-first geometry; DPR observer; stop local `term.resize` in `applyTextScale`; width-retry in measure | `terminal-webview-viewport.test.ts` using the existing `terminal-webview-mouse-test-harness.ts` pattern (happy-dom): inject `TERMINAL_VIEWPORT_JS`, feed `set-viewport`, assert `computeFitScale` uses it; simulate `matchMedia` change → `viewport-changed` notify. Extend `terminal-webview-text-zoom.test.ts`: `set-font-scale` no longer calls `term.resize`. | +| 7 | `src/terminal/TerminalWebView.tsx`, `src/terminal/terminal-webview-handle.ts` (new), `terminal-webview-contract.ts` | `measureFitDimensions(height, width)`, `setViewport(w,h)`; move handle factory out to stay under 379 lines | `terminal-webview-contract.test.ts` (shape); `terminal-webview-query-reply.test.ts` unaffected | +| 8 | `src/terminal/terminal-webview-notification-dispatch.ts` | route `viewport-changed` → `onViewportChanged` | extend `terminal-webview-notification-dispatch.test.ts` | +| 9 | `src/terminal/terminal-viewport-refit-state.ts`, `terminal-viewport-refit.ts` | `layoutEpoch` merge; `webview-viewport` event; width-agreement gate; drag-aware debounce; keyboardVisible reset rule | extend `terminal-viewport-refit.test.ts`: (a) RN width 800 + WebView width 720 → no refit; both 800 → refit; (b) 30 layout events in 500 ms → ≤ 2 refits; (c) width change > 10 % clears `keyboardVisible` | +| 10 | `app/h/[hostId]/session/[worktreeId].tsx:1302, :4656-4665, :2506-2522` | pass width to measure; call `setViewport`; wire `onViewportChanged` | covered by 9; keep file under its 5015 ceiling by moving the frame `onLayout` handler into `src/session/use-terminal-frame-layout.ts` | +| 11 | `src/platform/hardware-keyboard.ts` (new), `src/terminal/terminal-live-input.ts:231-248`, `use-terminal-live-input-focus.ts` | heuristic reducer; no blur/refocus when hardware keyboard | `hardware-keyboard.test.ts`; extend `terminal-live-input.test.ts` ("hardware keyboard → focus() only") | +| 12 | `src/session/session-back-press.ts` (new), `[worktreeId].tsx:2160-2165`, `RightDrawer.tsx:114-122` | Esc → send escape when live input focused + hardware keyboard | `session-back-press.test.ts` | +| 13 | `src/platform/use-pixel-ratio.ts` (new), `src/browser/use-mobile-browser-stream.ts` | DPR-reactive screencast request | extend `browser-screencast-request.test.ts` with a DPR change | +| 14 | `mobile/README.md` | add "Samsung DeX / desktop mode" section: build, manual checklist pointer | — | + +Constraints for t2: no `max-lines` bumps or disables (`AGENTS.md`); no new +`helpers/utils` files; all new modules named after the concept they hold; +`pnpm exec tsc --noEmit && pnpm lint && pnpm test` in `mobile/`, then +`pnpm typecheck:node` at the repo root. + +--- + +## 6. Test plan + +### 6.1 Unit (vitest, `mobile/`) + +Listed per step in §5. Highlights that must exist before merge: + +- Plugin: manifest attributes + metadata + idempotency + ordering with the + rotation plugin. +- `responsive-layout-metrics`: hysteresis; window classes; freeform sizes + (800×600, 1024×768, 1280×720, 1920×1080 at density 1.0 and 1.5). +- `terminal-viewport-refit-state`: width-agreement gate; drag coalescing; + keyboardVisible reset. +- Engine viewport module: RN viewport precedence; DPR observer; text-scale no + longer resizes the grid. +- Hardware-keyboard reducer; back-press resolver; live-input focus. + +### 6.2 Manual DeX checklist + +Device: Galaxy S23/S24 or Tab S9 (One UI 6+), DeX via HDMI/USB-C dock and via +"DeX on PC"; pair to a desktop Orca on the same LAN; open a worktree session +with one Claude Code TUI and one plain shell tab. + +| ID | Step | Pass criteria | +| --- | -------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| M1 | Launch app in DeX | Opens as a freeform window with resize handles (not a fixed phone-ratio window). If it is fixed, enable plan B (§3.4) and retest. | +| M2 | Drag-resize the window continuously 600→1400 px wide | No remount (route, scroll positions, terminal scrollback preserved); PTY columns follow within ~300 ms of release; ≤ 4 `updateViewport` per second (watch `[fit]` logs); sidebar toggles once, not repeatedly | +| M3 | Maximize / restore (double-click title bar) | Same as M2 in one step; no blank terminal, no stale scale (text fills width, no clipped right edge) | +| M4 | Enter DeX with the app foregrounded on the phone, then exit DeX | App stays on the same route; no splash; terminals reconnect without a full remount; glyphs crisp after each switch (DPR changed) | +| M5 | Hot-plug a monitor with a different DPI / move window phone↔monitor (Multi-display) | No restart; crisp glyphs; browser tab screencast scale correct | +| M6 | Rotate the phone while docked (DeX continues) and undocked | Phone: rotation honours the OS rotation lock (existing behaviour); DeX: window unaffected | +| M7 | Attach Bluetooth/USB keyboard; tap terminal | Caret focuses once (no flicker); typing reaches PTY; `Esc` sends ESC to the TUI and does **not** leave the session; header back still leaves; detaching keyboard and tapping reopens the IME | +| M8 | Open a fill-sheet (New Workspace) and the RightDrawer in an 800×600 window | Sheet ≤ window height with the status-bar gap; drawer width = window − 48 (narrow) or 420 (wide), never wider than the window | +| M9 | Change terminal text size in Settings while in DeX | Grid and PTY agree (no half-width rendering); one refit | +| M10 | Multi-window: place Orca side-by-side with another app, then resize the split | Same criteria as M2 | +| M11 | Android 15/16 tablet desktop windowing (non-Samsung) | M1–M3, M8 pass (same manifest path) | +| M12 | Phone regression: portrait/landscape, fold/unfold (Z Fold), IME open/close, pinch zoom | Unchanged from today | + +Capture `adb logcat | grep -E 'ReactNative|\[fit\]'` for M2–M5; an +`ActivityManager: Recreating` / `onDestroy` line during M4/M5 means R1 is not +fixed (check the prebuilt manifest). + +--- + +## 7. APK build plan (this repo) + +Mirrors `.github/workflows/mobile-android-release.yml:23-77`. + +```bash +cd mobile +pnpm install --frozen-lockfile # also builds the xterm/mermaid WebView engines (postinstall) +node scripts/prepare-android-release.mjs # optional: validates app.json version/versionCode +npx expo prebuild --platform android --no-install --clean +# → android/ is gitignored (.gitignore:/android/); --clean regenerates it so the new plugin's +# manifest edits are applied. Verify: +grep -n 'configChanges\|resizeableActivity\|keepalive\|keep_process_alive' android/app/src/main/AndroidManifest.xml +cd android +./gradlew assembleDebug # install with: adb install -r app/build/outputs/apk/debug/app-debug.apk +./gradlew assembleRelease # output: app/build/outputs/apk/release/*.apk +``` + +Toolchain (Expo SDK 55 defaults emitted by prebuild; `expo-build-properties` +in `app.json:104-113` overrides nothing relevant — confirm after prebuild in +`android/build.gradle` / `gradle.properties`): + +- JDK 17 (CI uses Temurin 17, workflow `:62-66`), Gradle wrapper from the + template, AGP 8.x. +- `compileSdkVersion` 36, `targetSdkVersion` 36, `minSdkVersion` 24, + `buildToolsVersion` 36.0.0, NDK 27.x, Kotlin 2.x. Android 16 (API 36) + is what makes the `resizeableActivity` / large-screen properties matter. +- Hermes on, New Architecture on (`app.json:10`). + +Dev-client loop for DeX testing: `pnpm exec expo run:android --device` to the +docked phone (DeX forwards ADB over the dock), then `pnpm start --dev-client`. +Expo Go cannot be used for the manifest changes; every manifest/plugin change +needs a fresh `expo prebuild --clean` + `run:android`. + +--- + +## 8. Open questions (non-blocking) + +1. Which symptom class the reporter actually saw — a screenshot or the `[fit]` + log would let t2 prioritise §5 steps 6–10 (terminal) vs 3–5 (chrome). The + manifest work (steps 1–2) is required regardless. +2. Whether One UI treats `fullUser` as resize-restricting (M1). Plan B is + specified but gated off by default. +3. Hardware-key coverage beyond what RN's `TextInput` delivers needs a native + module (§4.3); scheduled as a follow-up, not part of this fix. + +--- + +## Appendix A — Maximized-window root-surface bug (round 2) + +Report (build `0.0.44` / versionCode 13, `dist/orca-mobile-dex-fix.apk`): on a +Galaxy phone in DeX with the Orca window **maximized** on the external +monitor, the app paints a phone-sized block in the top-left corner (sidebar + +session pane + composer, ≈⅓ of the window width, ≈¼ of its height); the rest +of the window is the black root background. Nothing is stretched — it is a +correctly laid-out UI at the wrong size. + +### A.1 TL;DR + +The React root **does** fill the window. What is small is the +`react-native-screens` `Screen` that expo-router's `` puts directly +under the root. `Screen` measures itself in **pixels** on the Android side and +pushes that size into Fabric as **dp** using `PixelUtil.toDIPFromPixel`, and +since RN 0.83 that helper divides by +`DisplayMetricsHolder.getScreenDisplayMetrics().density` — the density of the +**phone's built-in display** (default display, read through the _Application_ +context) — while every other part of the pipeline (root layout constraints, +Yoga → pixel mount) uses the **Activity's** density, i.e. the DeX monitor's. +With a phone at ~2.6–3.0× and DeX at ~1.0–1.5×, the Screen's Yoga size comes +out `density_dex / density_phone` ≈ ⅓–½ of the window in each dimension, +anchored at (0,0). It is an upstream RN regression +(facebook/react-native#55659, introduced by commit `1ad2ec099a`, PR #53523, +"replace getWindowDisplayMetrics with getScreenDisplayMetrics", Aug 2025, +first shipped in 0.83; still present on `main` and reported on 0.85), not a +consequence of the round-1 manifest changes — round 1 merely made the window +resizable so the mismatch became visible at monitor size. + +### A.2 Mechanism, step by step (file/line evidence, all under `mobile/node_modules`) + +Notation: `d_act` = density of the Activity's display (DeX monitor); +`d_app` = density `DisplayMetricsHolder` holds (phone panel). + +1. **Root constraints use `d_act` (correct).** + `react-native/ReactAndroid/src/main/java/com/facebook/react/runtime/ReactSurfaceView.kt:50-86` + `onMeasure` takes the EXACTLY specs from the DecorView (the full window in + px) and calls `surface.updateLayoutSpecs(...)`. + `runtime/ReactSurfaceImpl.kt:178-194` forwards them with + `context.resources.displayMetrics.density` — `context` is the **Activity**, + so this is `d_act`. + `fabric/SurfaceHandlerBinding.kt:37-59` divides the px specs by that density + → Yoga root = `W_px/d_act × H_px/d_act` dp, and stores + `pointScaleFactor = d_act` in the surface's `LayoutContext` + (`ReactAndroid/src/main/jni/react/fabric/SurfaceHandlerBinding.cpp:42-66`). +2. **Mount uses `d_act` (correct).** Every `UpdateLayout` mount item scales the + Yoga frame by `layoutMetrics.pointScaleFactor` + (`jni/react/fabric/FabricMountingManager.cpp:388-396`), so the + `ScreenStack` Android view (flex:1 child of the root) is laid out to the + full window in px. So far the tree fills the window. +3. **`Screen` reports its size back in dp using `d_app` (wrong).** + `react-native-screens/android/src/main/java/com/swmansion/rnscreens/ScreenContainer.kt:47-60, 333-341` + lays every fragment view to its own full size; `ScreenStackFragment.kt:178-203` + hosts the `Screen` in a MATCH_PARENT `CoordinatorLayout`; then + `Screen.kt:348-376` (`onLayout`, `changed == true` on every resize) calls + `dispatchShadowStateUpdate(width_px, height_px, …)` → + `updateScreenSizeFabric` → + `android/src/fabric/java/com/swmansion/rnscreens/FabricEnabledViewGroup.kt:34-69` + `updateState`, which converts with `PixelUtil.toDIPFromPixel(width_px)` and + sends `{frameWidth, frameHeight}` as Fabric state. +4. **`PixelUtil` divides by the phone density.** + `react-native/.../uimanager/PixelUtil.kt:61-67`: + `return value / DisplayMetricsHolder.getScreenDisplayMetrics().density` + (0.81.2 used `getWindowDisplayMetrics()`; diff of commit `1ad2ec099a`). + `uimanager/DisplayMetricsHolder.kt:68-84` fills `screenDisplayMetrics` from + `context.getSystemService(WINDOW_SERVICE).defaultDisplay.getRealMetrics()`, + and the `context` is always the **ReactApplicationContext** (Application + context): `runtime/ReactInstance.kt:251` at instance creation and + `runtime/ReactHostImpl.kt:722-735` on every `onConfigurationChanged` + (`DisplayMetricsHolder.initDisplayMetrics(currentReactContext)`). For a + non-Activity context `defaultDisplay` is `Display.DEFAULT_DISPLAY` (the + phone panel) regardless of which display the Activity is on, so + `screenDisplayMetrics.density == d_phone` in DeX, and re-initialising on + config change does not help — it re-reads the same display. +5. **Fabric forces the Screen's Yoga size from that state.** + `react-native-screens/common/cpp/react/renderer/components/rnscreens/RNSScreenComponentDescriptor.h:35-119` + `adopt()`: when `frameSize` is non-zero, + `layoutableShadowNode.setSize({frameSize.width, frameSize.height})`. So the + Screen node becomes `W_px/d_app × H_px/d_app` dp instead of + `W_px/d_act × H_px/d_act`; step 2 then multiplies by `d_act`, giving a + view of `W_px·(d_act/d_app) × H_px·(d_act/d_app)` px at origin (0,0) — concretely + the `ScreenContentWrapper` (the Screen's Yoga subtree; the `Screen` Android + view itself is sized by the `CoordinatorLayout` and stays full-size). Every + descendant (host sidebar, session pane, composer — all `flex:1` / + `onLayout`-driven) lays out inside that block. The visible + "phone-sized" block is exactly this; the black area is the root/`ScreenStack` + background. +6. **Steady state, not a race.** `FabricEnabledViewGroup.updateState` only + re-sends when the _dp_ value moves by ≥0.9, and `Screen.onLayout` only fires + on px change, so nothing later corrects it; each resize (maximize, restore, + drag) re-applies the same wrong ratio. It also affects every other consumer + of `PixelUtil` with the same `d_app/d_act` error: text size + (`PixelUtil.toPixelFromSP`, `PixelUtil.kt:38-53` → glyphs rendered + `d_phone/d_dex`× too large for their Yoga frames = the "garbled / clipped + text" of the original report), `ScreenStackHeaderConfig` header height, + `react-native-safe-area-context` insets (`SerializationUtils.kt`), + `RightDrawer`/modal sizing through `Modal`, etc. + +### A.3 Why the round-1 changes did not cause it, and what they did change + +- `configChanges=density` / `smallestScreenSize` / `screenSize` are **not** the + trigger. With recreation (pre-round-1 manifest) the new Activity's + `ReactSurfaceView` still used `d_act` and `DisplayMetricsHolder` still held + the Application/default-display value, so the ratio would be identical; + recreation only added the remount/splash/route-reset symptoms (R1). Keep the + round-1 `configChanges` and the Samsung keep-alive metadata. +- What round 1 changed is that the window is now resizable + (`resizeableActivity=true`), so the user can maximize and the fixed-ratio + block is obvious against a 1080p/4K window. The same ratio applies at any + window size in DeX (a non-maximized window shows the same proportionally + small block); at phone-window sizes it was previously read as "broken / + garbled". +- `ReactHostImpl.onConfigurationChanged` (`ReactHostImpl.kt:722-735`) does + refresh `DisplayMetricsHolder` (flag `enableFontScaleChangesUpdatingLayout` + defaults to `true`, `internal/featureflags/ReactNativeFeatureFlagsDefaults.kt:70`) + and calls `requestLayout()` on the surfaces — but from the Application + context, so the refreshed value is still the phone's. +- Correction to §3.1 of this document: RN bridgeless does **not** re-emit + `didUpdateDimensions` on configuration change. The only emitters are + `ReactRootView.onMeasure` (old architecture only, `ReactRootView.java:1009-1015`) + and `DeviceInfoModule.onHostResume` on a **fontScale** change + (`modules/deviceinfo/DeviceInfoModule.kt:45-51`). JS `useWindowDimensions()` + / `PixelRatio.get()` therefore stay stale for the life of the process on a + window resize or display move. This is harmless for layout because round 1 + moved every consumer to `WindowBoundsProvider` (`onLayout`-driven), but + `src/platform/use-pixel-ratio.ts` (`Dimensions` `change` listener) will never + fire on Android and the browser screencast DPR stays the phone's (see A.6 #6). + +### A.4 Reproduction (emulator, 2026-08-23) + +Reproduced on a stock Android 15 emulator — no Samsung code involved — using +a simulated secondary display whose density differs from the phone's. Setup: +AVD `dex35` (pixel_7 profile, `system-images;android-35;google_apis;arm64-v8a`, +built-in display 1080×2400 @ 420 dpi, density 2.625), overlay display +`1920x1080/160` (density 1.0), `dist/orca-mobile-dex-fix.apk` (0.0.44 / 13) +installed unchanged. Expected ratio `160/420 = 0.381`. + +| Case | Activity config (`dumpsys activity top`) | `ReactSurfaceView` | `ScreenContentWrapper` (Screen's Yoga-sized content) | +| ------------------------------------------------------------------------------------ | -------------------------------------------------------- | ------------------------------ | ------------------------------------------------------------------------- | +| Freeform window on the 160 dpi display (`force_desktop_mode_on_external_displays=1`) | `w412dp h732dp 160dpi`, `mBounds=Rect(754,146-1166,878)` | `0,0-412,732` (fills window) | **`0,0-157,279`** = 412×0.381 × 732×0.381 | +| **Fullscreen (= maximized) on the 160 dpi display** | `w1920dp h1080dp 160dpi`, `mBounds=Rect(0,0-1920,1080)` | `0,0-1920,1080` (fills window) | **`0,0-731,411`** = 1920×0.381 × 1080×0.381 — the reported top-left block | +| Control: phone display (`--display 0`, 420 dpi) | `w411dp h914dp 420dpi` | `0,0-1080,2400` | `0,0-1080,2400` (correct) | + +Also observed: `ScreenStackHeaderConfig` is `731×22` in the maximized case +(same ratio), the `Screen` Android view itself stays full-size (it is laid +out by the `CoordinatorLayout`, not Yoga), there is **no** size-compat mode +(`mBounds == mAppBounds == mMaxBounds`, windowing mode `fullscreen`), and +`logcat -s ReactNative ReactNativeJS` shows nothing beyond `Running "main"` +— the pipeline believes everything is fine, which is why there is no error +to grep for. The hierarchy lines come from +`adb shell dumpsys activity top` (view hierarchy with px bounds); +`screencap -d ` does not work for overlay displays on this image, +so `dumpsys` is the evidence path t4 should use too. + +Implication for the symptom description: the block is _not_ "phone-sized"; +it is `window × (d_dex / d_phone)`, so it scales with the window. In the +user's photo (≈⅓ width) the phone is ~2.6–3.0× and the DeX monitor ~1.0×. + +### A.5 Fix strategy for t4 + +Goal: make every `PixelUtil` conversion use the density of the display the +**Activity** is on, and keep it that way across configuration changes; then +make sure already-mounted `Screen`s re-send their size once the density is +right. Two layers, both needed: + +**Layer 1 (primary, root fix, proven in A.8) — make `DisplayMetricsHolder` +follow the Activity's display.** Step 2 below is the fix; step 1 is optional +hardening. Note that the `PixelUtil` patch _alone_ is **not** sufficient: +`windowDisplayMetrics` is also filled from the Application context +(`resources.displayMetrics` of the global config = phone density on the +emulator and on DeX), so reverting upstream's change without step 2 still +yields the wrong density on a secondary display. + +1. (Optional) Extend the existing pnpm patch `mobile/patches/react-native@0.83.9.patch` + (`package.json` `pnpm.patchedDependencies`) with a hunk for + `ReactAndroid/src/main/java/com/facebook/react/uimanager/PixelUtil.kt` + that reverts commit `1ad2ec099a` for the three conversions: + `toPixelFromDIP` (`:22-26`), `toPixelFromSP` (`:43`), `toDIPFromPixel` + (`:66`) → `DisplayMetricsHolder.getWindowDisplayMetrics()`. This is the + exact 0.81 behaviour and the change the upstream issue asks for; it is + Android-only and affects no wire/RPC surface. (`getScreenDisplayMetrics` + stays in use for `DeviceInfo` constants / `Dimensions.screen`, which is + what it is meant for.) +2. Keep `windowDisplayMetrics` pointed at the Activity's resources. Add to + `plugins/android-desktop-mode.js` a `withMainActivity` mod (available from + `expo/config-plugins`; prefer it over the existing `withDangerousMod` + string-replace for the orientation hook — both can stay in the same plugin) + that inserts into `MainActivity.kt` (exact Kotlin that was verified in A.8 is + reproduced there): + - an `override fun onConfigurationChanged(newConfig: Configuration)` that + calls `super.onConfigurationChanged(newConfig)` **first** (so + `ReactHostImpl.onConfigurationChanged` runs its Application-context + `initDisplayMetrics` and our override wins afterwards), then + `DisplayMetricsHolder.setWindowDisplayMetrics(resources.displayMetrics)` + and `DisplayMetricsHolder.setScreenDisplayMetrics(realMetricsOfThisDisplay())` + where `realMetricsOfThisDisplay()` is `DisplayMetrics().also { (getSystemService(WINDOW_SERVICE) as WindowManager).defaultDisplay.getRealMetrics(it); it.scaledDensity = resources.displayMetrics.scaledDensity }` + — called on the **Activity** so `defaultDisplay` is the Activity's + display (API 30+ `display` property is equivalent). Both setters are + public `@JvmStatic` (`DisplayMetricsHolder.kt:37-52`). + - the same two calls in `onCreate` **before** `super.onCreate(null)`: + `ReactInstance.kt:251` uses `initDisplayMetricsIfNotInitialized`, which + is a no-op once `screenDisplayMetrics` is set, so a cold start in DeX + (or on any secondary display) initialises from the Activity's display + instead of the phone's. Also repeat them in `onResume` (cheap; covers + the window being moved to another display without a density config + change, and Samsung's `onConfigurationChanged → onRestart → onStart → +onResume` relaunch sequence). + - guard the Kotlin with an `// ORCA_DISPLAY_METRICS` marker for idempotency + exactly like the `ORCA_DESKTOP_ORIENTATION` block, and add the imports + (`android.content.res.Configuration`, `android.util.DisplayMetrics`, + `android.view.WindowManager`, `com.facebook.react.uimanager.DisplayMetricsHolder`). + Extend `plugins/android-desktop-mode.test.js` to assert the snippet is + inserted once (run the mod twice → one marker). +3. After the override in `onConfigurationChanged`, force a fresh layout pass: + `window.decorView.requestLayout()`. For the maximize/restore/drag case the + px size changes anyway so `Screen.onLayout(changed=true)` fires and + `updateState` recomputes with the corrected density. The residual case is + a density-only change with identical px size (practically only + DeX-enter/exit, where px also change) — acceptable; note it in the README. + +**Layer 2 (belt-and-braces, JS-visible) — keep the rest of the app honest +about density and dimensions.** + +4. JS `Dimensions`/`PixelRatio` never update on Android bridgeless (A.3). In + the same `MainActivity` hook, after the metrics override, emit the RN + dimensions event so JS sees the new scale: + `(reactHost as? ReactHostImpl)?.currentReactContext?.getNativeModule(DeviceInfoModule::class.java)?.emitUpdateDimensionsEvent()` + (`modules/deviceinfo/DeviceInfoModule.kt:57-77`, public; it diff-checks + and emits `didUpdateDimensions` → `Dimensions` `change` → `usePixelRatio`). + If `currentReactContext` is not reachable from Kotlin without reflection + in this RN version, fall back to dropping the `usePixelRatio` hook's + dependency on `Dimensions` and instead read the DPR inside the terminal + WebView (`window.devicePixelRatio`, already observed by the round-1 + `matchMedia` listener in `terminal-webview-viewport-injected.ts`) — the + browser screencast is the only RN-side consumer. +5. Leave `WindowBoundsProvider` (`src/layout/window-bounds.tsx`) as the source + of truth for window size — it is correct once Layer 1 lands because the + Yoga root was always right; only the Screen subtree was wrong. + +**Rejected directions (and why):** + +- Removing `density` from `configChanges` to force recreation: the recreated + Activity's `ReactSurfaceView` still uses the Activity density while + `DisplayMetricsHolder` stays on the Application context → same ratio, plus + R1 regressions. No. +- Overriding `onConfigurationChanged` to call `updateLayoutSpecs` / + `requestLayout` alone: the root constraints were never wrong; without + fixing the `PixelUtil` density the Screen re-sends the same wrong dp. No. +- `android:windowLayoutInDisplayCutoutMode`, `maxAspectRatio`, size-compat + toggles: the Activity is **not** in size-compat mode (the window is + resizable, the DecorView measures at full size; `dumpsys activity` shows no + `mSizeCompatBounds` in the repro). No. +- Reverting to old architecture (`newArchEnabled=false`): `ReactRootView` + would also re-emit dimensions, but it is a product-wide regression and + `react-native-screens` 4.x old-arch path has the same `PixelUtil` call + (`Screen.kt:420-433` → `UIManagerModule.updateNodeSize` in px, but header + and insets still go through `PixelUtil`). No. +- Waiting for upstream: #55659 was auto-closed by the bot and the commit is + still on `main`; a patch is the only near-term route. File/upvote upstream + anyway and link this appendix. + +### A.6 File-by-file plan for t4 + +| # | File | Change | +| --- | ----------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | `mobile/patches/react-native@0.83.9.patch` (optional) | add `PixelUtil.kt` hunk (3 call sites → `getWindowDisplayMetrics()`); regenerate with `pnpm patch react-native@0.83.9` so the lockfile hash updates; run `pnpm install`. Skip if #2 alone passes A.7 — it does on the emulator (A.8) | +| 2 | `mobile/plugins/android-desktop-mode.js` | new `withMainActivity` mod `withActivityDisplayMetrics` inserting the `ORCA_DISPLAY_METRICS` Kotlin block (`onCreate` pre-super, `onConfigurationChanged` post-super + `requestLayout` + `emitUpdateDimensionsEvent`, `onResume`); keep manifest mod unchanged | +| 3 | `mobile/plugins/android-desktop-mode.test.js` | assert insertion + idempotency of the new block; assert `onConfigurationChanged` calls `super` before `DisplayMetricsHolder` | +| 4 | `mobile/src/platform/use-pixel-ratio.ts` (+ test) | keep; add a comment that Android only fires via the native hook in #2. If #4 of A.5 falls back, replace with WebView-reported DPR | +| 5 | `mobile/app.json` | local verification builds only bumped to `0.0.44-dex2` / `versionCode` 14 so the tester could confirm the install; not part of the upstream change | +| 6 | `mobile/README.md` | DeX section: note the RN #55659 patch and the residual density-only case | +| 7 | this doc | keep Appendix A; fix the §3.1 sentence about `didUpdateDimensions` (A.3) | + +Build exactly as §7: `npx expo prebuild --platform android --no-install --clean` +(the Kotlin mod only applies on prebuild; verify +`grep -n ORCA_DISPLAY_METRICS android/app/src/main/java/com/stably/orca/mobile/MainActivity.kt`) +then `./gradlew assembleRelease`; copy to `dist/orca-mobile-dex2.apk`. + +### A.7 How t4 verifies on the emulator + +Toolchain now present on this machine: `JAVA_HOME=/opt/homebrew/opt/openjdk@17` +(there is no Temurin install; `/usr/libexec/java_home` finds nothing), +`ANDROID_HOME=/opt/homebrew/share/android-commandlinetools` with +`platform-tools`, `emulator`, `platforms;android-35`, +`system-images;android-35;google_apis;arm64-v8a`, AVD `dex35` (pixel_7, +420 dpi). `adb`/`emulator` are **not** on `PATH` — export +`PATH=$ANDROID_HOME/platform-tools:$ANDROID_HOME/emulator:$PATH`. + +The emulator cannot run Samsung DeX, but the bug only needs _an Activity on +a display whose density differs from the default display_, which stock +Android provides via a simulated secondary display + forced desktop mode: + +```bash +export PATH=$ANDROID_HOME/platform-tools:$ANDROID_HOME/emulator:$PATH +emulator -avd dex35 -no-window -no-audio -no-boot-anim -no-snapshot -gpu swiftshader_indirect & +adb wait-for-device; until [ "$(adb shell getprop sys.boot_completed | tr -d '\r')" = 1 ]; do sleep 2; done +adb install -r dist/orca-mobile-dex2.apk + +# Case 1 — "maximized": fullscreen on a 160 dpi secondary display (phone is 420 dpi). +adb shell settings put global force_desktop_mode_on_external_displays 0 +adb shell settings put global overlay_display_devices "1920x1080/160"; sleep 3 +D=$(adb shell dumpsys display | grep -o "displayId [0-9]*, displayGroupId" | grep -v "displayId 0," | head -1 | sed -E 's/displayId ([0-9]+).*/\1/') +adb shell am start --display $D -n com.stably.orca.mobile/.MainActivity; sleep 8 +adb shell dumpsys activity top | awk '/ACTIVITY com.stably.orca/{p=1} p' \ + | grep -E "mCurrentConfig|ReactSurfaceView|ScreenContentWrapper|ScreenStackHeaderConfig" + +# Case 2 — freeform window on that display (desktop mode must be set BEFORE the overlay is created). +adb shell am force-stop com.stably.orca.mobile +adb shell settings put global overlay_display_devices none +adb shell settings put global enable_freeform_support 1 +adb shell settings put global force_desktop_mode_on_external_displays 1 +adb shell settings put global overlay_display_devices "1920x1080/160"; sleep 3 +# re-derive $D (the id changes), start on it, dump as above + +# Case 3 — regression control on the phone display. +adb shell am force-stop com.stably.orca.mobile +adb shell am start --display 0 -n com.stably.orca.mobile/.MainActivity; sleep 8 # dump as above +``` + +Pass criteria (`dumpsys activity top` view hierarchy; `screencap -d` does +**not** work for overlay displays on this image): + +- Case 1: `ReactSurfaceView 0,0-1920,1080` **and** `ScreenContentWrapper 0,0-1920,1080` + (header `1920×22`). Before the fix the wrapper is `0,0-731,411`. +- Case 2: `mCurrentConfig … w412dp h732dp 160dpi`, `ReactSurfaceView 0,0-412,732` + and `ScreenContentWrapper 0,0-412,732` (before: `157×279`). +- Case 3: everything `0,0-1080,2400`, unchanged from today. +- `adb logcat -s ReactNative ReactNativeJS` shows no + `Cannot updateRootLayoutSpecs` / `[RNScreens] … differs` lines; text is + not clipped. + +API 35's `am` has no task-resize command, so a _live_ maximize/drag of a +running window needs the Android Studio emulator window (desktop mode on, +drag the caption / double-click to maximize) or a Galaxy in DeX; run §6.2 +M1–M3 there and confirm the wrapper tracks the window after each resize. + +### A.8 Proof-of-fix diagnostic (temporary, reverted) + +To de-risk the plan, the Activity-side override was added to the **generated** +`android/app/src/main/java/com/stably/orca/mobile/MainActivity.kt` (gitignored +`android/`; no source file changed, and the edit was reverted afterwards), +built with `./gradlew assembleRelease`, and run on the same emulator. No RN +patch, no JS change: + +```kotlin +import android.content.res.Configuration +import android.util.DisplayMetrics +import android.view.WindowManager +import com.facebook.react.uimanager.DisplayMetricsHolder + +// in onCreate, before super.onCreate(null): +applyActivityDisplayMetrics() + +private fun applyActivityDisplayMetrics() { + val window = resources.displayMetrics + val screen = DisplayMetrics() + @Suppress("DEPRECATION") + (getSystemService(WINDOW_SERVICE) as WindowManager).defaultDisplay.getRealMetrics(screen) + screen.scaledDensity = window.scaledDensity // DisplayMetricsHolder.initDisplayMetrics does the same + DisplayMetricsHolder.setWindowDisplayMetrics(window) + DisplayMetricsHolder.setScreenDisplayMetrics(screen) +} + +override fun onConfigurationChanged(newConfig: Configuration) { + super.onConfigurationChanged(newConfig) // ReactHostImpl re-inits from the app context first… + applyActivityDisplayMetrics() // …then the Activity's display wins + window.decorView.requestLayout() +} + +override fun onResume() { + super.onResume() + applyActivityDisplayMetrics() +} +``` + +Result (`dumpsys activity top`): + +| Case | `ReactSurfaceView` | `ScreenContentWrapper` | +| ------------------------------------------------ | ------------------ | ------------------------------------------------------ | +| Fullscreen on the 160 dpi display, diag build | `0,0-1920,1080` | **`0,0-1920,1080`** (was `731×411`) — header `1920×22` | +| Phone display 0, diag build (regression control) | `0,0-1080,2400` | `0,0-1080,2400` (unchanged) | + +A temporary `Log.i("OrcaDiag", …)` confirmed `window=1.0 screen=1.0 1920x1080` +on the secondary display. The `dist/orca-mobile-dex-fix.apk` artifact was not +modified; `android/app/build/outputs/apk/release/app-release.apk` on this +machine currently contains the diagnostic build and will be overwritten by +t4's build. + +What the diagnostic did **not** cover (t4 must): a live resize of an already +running window with the metadata path through `onConfigurationChanged` +(freeform drag / maximize without relaunch — the emulator's `am` on API 35 has +no task-resize command; use the Android Studio emulator UI with desktop mode, +or a Galaxy in DeX), DeX enter/exit with `keepalive.density`, and the JS +`didUpdateDimensions` emission (A.5 #4). diff --git a/mobile/plugins/android-desktop-mode.js b/mobile/plugins/android-desktop-mode.js new file mode 100644 index 000000000000..a51c59fcd932 --- /dev/null +++ b/mobile/plugins/android-desktop-mode.js @@ -0,0 +1,187 @@ +const { withAndroidManifest, withMainActivity } = require('expo/config-plugins') +const { AndroidConfig } = require('expo/config-plugins') + +const CONFIG_CHANGES = [ + 'keyboard', + 'keyboardHidden', + 'orientation', + 'screenSize', + 'smallestScreenSize', + 'screenLayout', + 'uiMode', + 'density', + 'locale', + 'layoutDirection', + 'fontScale', + 'navigation', + 'mcc', + 'mnc' +].join('|') + +const APPLICATION_METADATA = [ + ['com.samsung.android.keepalive.density', 'true'], + ['com.samsung.android.multidisplay.keep_process_alive', 'true'], + ['android.allow_multiple_resumed_activities', 'true'] +] + +const DISPLAY_METRICS_IMPORTS = [ + 'android.content.res.Configuration', + 'android.util.DisplayMetrics', + 'android.view.WindowManager', + 'com.facebook.react.uimanager.DisplayMetricsHolder' +] + +const DISPLAY_METRICS_METHODS = ` + // ORCA_DISPLAY_METRICS + @Suppress("DEPRECATION") + private fun applyActivityDisplayMetrics() { + val windowMetrics = resources.displayMetrics + val screenMetrics = DisplayMetrics() + (getSystemService(WINDOW_SERVICE) as WindowManager).defaultDisplay.getRealMetrics(screenMetrics) + screenMetrics.scaledDensity = windowMetrics.scaledDensity + DisplayMetricsHolder.setWindowDisplayMetrics(windowMetrics) + DisplayMetricsHolder.setScreenDisplayMetrics(screenMetrics) + } + + private fun emitActivityDimensions() { + reactHost?.currentReactContext?.emitDeviceEvent( + "didUpdateDimensions", + DisplayMetricsHolder.getDisplayMetricsWritableMap( + resources.configuration.fontScale.toDouble() + ) + ) + } + + override fun onConfigurationChanged(newConfig: Configuration) { + super.onConfigurationChanged(newConfig) + applyActivityDisplayMetrics() + window.decorView.requestLayout() + emitActivityDimensions() + } + + override fun onResume() { + super.onResume() + applyActivityDisplayMetrics() + emitActivityDimensions() + } +` + +function upsertNamedEntry(entries, name, value) { + const next = Array.isArray(entries) ? entries : [] + const existing = next.find((entry) => entry.$?.['android:name'] === name) + if (existing) { + existing.$['android:value'] = value + return next + } + next.push({ $: { 'android:name': name, 'android:value': value } }) + return next +} + +function applyAndroidDesktopModeManifest(manifest) { + const activity = AndroidConfig.Manifest.getMainActivityOrThrow(manifest) + activity.$['android:configChanges'] = CONFIG_CHANGES + activity.$['android:resizeableActivity'] = 'true' + activity.$['android:supportsPictureInPicture'] = 'false' + + const application = AndroidConfig.Manifest.getMainApplicationOrThrow(manifest) + for (const [name, value] of APPLICATION_METADATA) { + application['meta-data'] = upsertNamedEntry(application['meta-data'], name, value) + } + application.property = upsertNamedEntry( + application.property, + 'android.window.PROPERTY_COMPAT_ALLOW_RESIZEABLE_ACTIVITY_OVERRIDES', + 'true' + ) + return manifest +} + +function applyActivityDisplayMetrics(source) { + if (source.includes('ORCA_DISPLAY_METRICS')) { + return source + } + + const missingImports = DISPLAY_METRICS_IMPORTS.filter( + (importName) => !source.includes(`import ${importName}`) + ) + if (missingImports.length > 0) { + source = source.replace( + /^(package [^\n]+\n)/, + `$1${missingImports.map((importName) => `import ${importName}`).join('\n')}\n` + ) + } + + if (!source.includes('super.onCreate(null)')) { + throw new Error('MainActivity.kt is missing the expected super.onCreate(null) call') + } + source = source.replace( + 'super.onCreate(null)', + `applyActivityDisplayMetrics() + super.onCreate(null)` + ) + + const classEnd = source.lastIndexOf('\n}') + if (classEnd === -1) { + throw new Error('MainActivity.kt is missing its class closing brace') + } + return `${source.slice(0, classEnd)}\n${DISPLAY_METRICS_METHODS}${source.slice(classEnd)}` +} + +function withActivityDisplayMetrics(config) { + return withMainActivity(config, (cfg) => { + if (cfg.modResults.language !== 'kt') { + throw new Error('Android desktop display metrics require a Kotlin MainActivity') + } + cfg.modResults.contents = applyActivityDisplayMetrics(cfg.modResults.contents) + return cfg + }) +} + +const DESKTOP_ORIENTATION_SNIPPET = `super.onCreate(null) + // ORCA_DESKTOP_ORIENTATION + val configuration = resources.configuration + val samsungDesktopMode = try { + configuration.javaClass.getField("semDesktopModeEnabled").getInt(configuration) == 1 + } catch (_: Exception) { false } + if ((configuration.uiMode and android.content.res.Configuration.UI_MODE_TYPE_MASK) == + android.content.res.Configuration.UI_MODE_TYPE_DESK || samsungDesktopMode) { + requestedOrientation = android.content.pm.ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED + }` + +function applyDesktopUnspecifiedOrientation(source) { + if (source.includes('ORCA_DESKTOP_ORIENTATION')) { + return source + } + if (!source.includes('super.onCreate(null)')) { + throw new Error('MainActivity.kt is missing the expected super.onCreate(null) call') + } + return source.replace('super.onCreate(null)', DESKTOP_ORIENTATION_SNIPPET) +} + +function withDesktopUnspecifiedOrientation(config) { + return withMainActivity(config, (cfg) => { + if (cfg.modResults.language !== 'kt') { + throw new Error('Android desktop orientation hook requires a Kotlin MainActivity') + } + cfg.modResults.contents = applyDesktopUnspecifiedOrientation(cfg.modResults.contents) + return cfg + }) +} + +module.exports = function withAndroidDesktopMode(config, options = {}) { + config = withAndroidManifest(config, (cfg) => { + cfg.modResults = applyAndroidDesktopModeManifest(cfg.modResults) + return cfg + }) + + config = withActivityDisplayMetrics(config) + + if (options.desktopUnspecifiedOrientation) { + config = withDesktopUnspecifiedOrientation(config) + } + return config +} + +module.exports.applyAndroidDesktopModeManifest = applyAndroidDesktopModeManifest +module.exports.applyActivityDisplayMetrics = applyActivityDisplayMetrics +module.exports.applyDesktopUnspecifiedOrientation = applyDesktopUnspecifiedOrientation +module.exports.CONFIG_CHANGES = CONFIG_CHANGES diff --git a/mobile/plugins/android-desktop-mode.test.js b/mobile/plugins/android-desktop-mode.test.js new file mode 100644 index 000000000000..3c5f13d12bc6 --- /dev/null +++ b/mobile/plugins/android-desktop-mode.test.js @@ -0,0 +1,127 @@ +import { describe, expect, it } from 'vitest' +import { readFileSync } from 'node:fs' +import desktopModePlugin from './android-desktop-mode.js' + +const { + applyActivityDisplayMetrics, + applyAndroidDesktopModeManifest, + applyDesktopUnspecifiedOrientation, + CONFIG_CHANGES +} = desktopModePlugin + +const MAIN_ACTIVITY = `package com.stably.orca.mobile + +import android.os.Bundle +import com.facebook.react.ReactActivity + +class MainActivity : ReactActivity() { + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(null) + } +} +` + +function createManifest() { + return { + manifest: { + application: [ + { + $: { 'android:name': '.MainApplication' }, + activity: [ + { + $: { + 'android:name': '.MainActivity', + 'android:screenOrientation': 'fullUser' + } + } + ] + } + ] + } + } +} + +describe('android desktop mode plugin', () => { + it('runs after the rotation-lock plugin', () => { + const app = JSON.parse(readFileSync(new URL('../app.json', import.meta.url), 'utf8')) + const plugins = app.expo.plugins + expect(plugins.indexOf('./plugins/android-desktop-mode.js')).toBeGreaterThan( + plugins.indexOf('./plugins/android-respect-rotation-lock.js') + ) + }) + + it('sets desktop activity attributes without clobbering rotation behavior', () => { + const manifest = applyAndroidDesktopModeManifest(createManifest()) + const activity = manifest.manifest.application[0].activity[0] + expect(activity.$).toMatchObject({ + 'android:configChanges': CONFIG_CHANGES, + 'android:resizeableActivity': 'true', + 'android:supportsPictureInPicture': 'false', + 'android:screenOrientation': 'fullUser' + }) + }) + + it('adds application metadata and properties idempotently', () => { + const manifest = createManifest() + applyAndroidDesktopModeManifest(manifest) + applyAndroidDesktopModeManifest(manifest) + const application = manifest.manifest.application[0] + expect(application['meta-data']).toHaveLength(3) + expect(application.property).toHaveLength(1) + expect(application['meta-data'].map((entry) => entry.$['android:name'])).toEqual([ + 'com.samsung.android.keepalive.density', + 'com.samsung.android.multidisplay.keep_process_alive', + 'android.allow_multiple_resumed_activities' + ]) + }) + + it('adds activity display metrics lifecycle handling idempotently', () => { + const once = applyActivityDisplayMetrics(MAIN_ACTIVITY) + const twice = applyActivityDisplayMetrics(once) + + expect(twice).toBe(once) + expect(once.match(/ORCA_DISPLAY_METRICS/g)).toHaveLength(1) + expect(once).toContain('DisplayMetricsHolder.setWindowDisplayMetrics(windowMetrics)') + expect(once).toContain('DisplayMetricsHolder.setScreenDisplayMetrics(screenMetrics)') + expect(once).toContain('window.decorView.requestLayout()') + expect(once).toContain('emitActivityDimensions()') + expect(once.indexOf('applyActivityDisplayMetrics()')).toBeLessThan( + once.indexOf('super.onCreate(null)') + ) + }) + + it('applies activity metrics after React handles configuration changes', () => { + const activity = applyActivityDisplayMetrics(MAIN_ACTIVITY) + const configurationHandler = activity.slice( + activity.indexOf('override fun onConfigurationChanged') + ) + + expect(configurationHandler.indexOf('super.onConfigurationChanged(newConfig)')).toBeLessThan( + configurationHandler.indexOf('applyActivityDisplayMetrics()') + ) + }) + + it('adds the desktop orientation hook idempotently after the activity metrics hook', () => { + const metrics = applyActivityDisplayMetrics(MAIN_ACTIVITY) + const once = applyDesktopUnspecifiedOrientation(metrics) + const twice = applyDesktopUnspecifiedOrientation(once) + + expect(twice).toBe(once) + expect(once.match(/ORCA_DESKTOP_ORIENTATION/g)).toHaveLength(1) + expect(once.indexOf('applyActivityDisplayMetrics()')).toBeLessThan( + once.indexOf('super.onCreate(null)') + ) + expect(once.indexOf('super.onCreate(null)')).toBeLessThan( + once.indexOf('SCREEN_ORIENTATION_UNSPECIFIED') + ) + }) + + it('fails loudly when the onCreate anchor is missing', () => { + const noAnchor = MAIN_ACTIVITY.replace( + 'super.onCreate(null)', + 'super.onCreate(savedInstanceState)' + ) + expect(() => applyDesktopUnspecifiedOrientation(noAnchor)).toThrow(/super\.onCreate\(null\)/) + expect(() => applyActivityDisplayMetrics(noAnchor)).toThrow(/super\.onCreate\(null\)/) + }) +}) diff --git a/mobile/src/browser/browser-screencast-request.test.ts b/mobile/src/browser/browser-screencast-request.test.ts index d688220e6afc..120aea5157bd 100644 --- a/mobile/src/browser/browser-screencast-request.test.ts +++ b/mobile/src/browser/browser-screencast-request.test.ts @@ -43,4 +43,16 @@ describe('buildMobileBrowserScreencastRequest', () => { expect(buildMobileBrowserScreencastRequest(null, 2)).toBeNull() expect(buildMobileBrowserScreencastRequest({ width: 0, height: 640 }, 2)).toBeNull() }) + + it('updates requested pixels when display density changes', () => { + const layout = { width: 800, height: 600 } + expect(buildMobileBrowserScreencastRequest(layout, 1)).toMatchObject({ + maxWidth: 800, + maxHeight: 600 + }) + expect(buildMobileBrowserScreencastRequest(layout, 1.5)).toMatchObject({ + maxWidth: 1200, + maxHeight: 900 + }) + }) }) diff --git a/mobile/src/browser/mobile-browser-frameless-stream.test.tsx b/mobile/src/browser/mobile-browser-frameless-stream.test.tsx index bf486d3c5c00..74fc8f01e442 100644 --- a/mobile/src/browser/mobile-browser-frameless-stream.test.tsx +++ b/mobile/src/browser/mobile-browser-frameless-stream.test.tsx @@ -12,6 +12,7 @@ import { MobileBrowserPane, type MobileBrowserTab } from './MobileBrowserPane' vi.mock('react-native', () => ({ ActivityIndicator: 'ActivityIndicator', AppState: { currentState: 'active', addEventListener: () => ({ remove: () => {} }) }, + Dimensions: { addEventListener: () => ({ remove: () => {} }) }, Image: 'Image', PanResponder: { create: () => ({ panHandlers: {} }) }, PixelRatio: { get: () => 2 }, diff --git a/mobile/src/browser/use-mobile-browser-stream.ts b/mobile/src/browser/use-mobile-browser-stream.ts index 5b55dd03ff54..f2119b383961 100644 --- a/mobile/src/browser/use-mobile-browser-stream.ts +++ b/mobile/src/browser/use-mobile-browser-stream.ts @@ -1,5 +1,6 @@ import { useEffect, useMemo, type Dispatch, type SetStateAction } from 'react' -import { PixelRatio, type Image, type View } from 'react-native' +import type { Image, View } from 'react-native' +import { usePixelRatio } from '../platform/use-pixel-ratio' import type { RpcClient } from '../transport/rpc-client' import type { BrowserScreencastFrame, @@ -129,9 +130,10 @@ export function useMobileBrowserStream(args: MobileBrowserStreamArgs) { visibleFrameLayerRef }) + const pixelRatio = usePixelRatio() const streamRequest = useMemo( - () => buildMobileBrowserScreencastRequest(layout, PixelRatio.get(), browserViewMode), - [browserViewMode, layout] + () => buildMobileBrowserScreencastRequest(layout, pixelRatio, browserViewMode), + [browserViewMode, layout, pixelRatio] ) const frameGeometry = useMemo( diff --git a/mobile/src/components/RightDrawer.tsx b/mobile/src/components/RightDrawer.tsx index eeeeccb0a2d9..47f83320b862 100644 --- a/mobile/src/components/RightDrawer.tsx +++ b/mobile/src/components/RightDrawer.tsx @@ -1,13 +1,5 @@ import { type ReactNode, useCallback, useEffect, useState } from 'react' -import { - View, - Pressable, - StyleSheet, - Platform, - useWindowDimensions, - Keyboard, - BackHandler -} from 'react-native' +import { View, Pressable, StyleSheet, Platform, Keyboard, BackHandler } from 'react-native' import { useSafeAreaInsets } from 'react-native-safe-area-context' import { Gesture, GestureDetector, GestureHandlerRootView } from 'react-native-gesture-handler' import Animated, { @@ -26,6 +18,8 @@ import { colors, spacing } from '../theme/mobile-theme' import { resolveBottomDrawerMounted } from './bottom-drawer-mount-state' import { resolveRightDrawerPanelWidth } from './right-drawer-panel-width' import { useResponsiveLayout } from '../layout/responsive-layout' +import { useWindowBounds } from '../layout/window-bounds' +import { resolveSessionBackPress } from '../session/session-back-press' const DISMISS_THRESHOLD = 80 const SPRING_CONFIG = { damping: 28, stiffness: 400 } @@ -41,9 +35,21 @@ type Props = { children: ReactNode zIndex?: number widthPx?: number + hardwareKeyboard?: boolean + liveInputFocused?: boolean + onHardwareEscape?: () => void } -export function RightDrawer({ visible, onClose, children, zIndex, widthPx }: Props) { +export function RightDrawer({ + visible, + onClose, + children, + zIndex, + widthPx, + hardwareKeyboard, + liveInputFocused, + onHardwareEscape +}: Props) { const [mounted, setMounted] = useState(visible) const resolvedMounted = resolveBottomDrawerMounted(visible, mounted) @@ -66,6 +72,9 @@ export function RightDrawer({ visible, onClose, children, zIndex, widthPx }: Pro onHidden={() => setMounted(false)} zIndex={zIndex} widthPx={widthPx} + hardwareKeyboard={hardwareKeyboard} + liveInputFocused={liveInputFocused} + onHardwareEscape={onHardwareEscape} > {children} @@ -82,12 +91,15 @@ function MountedRightDrawer({ onHidden, children, zIndex = 1000, - widthPx + widthPx, + hardwareKeyboard = false, + liveInputFocused = false, + onHardwareEscape }: MountedRightDrawerProps) { const translateX = useSharedValue(0) const progress = useSharedValue(0) const scrollOffsetY = useSharedValue(0) - const { width: screenWidth } = useWindowDimensions() + const { width: screenWidth } = useWindowBounds() const insets = useSafeAreaInsets() const { isWideLayout } = useResponsiveLayout() const panelWidth = resolveRightDrawerPanelWidth(screenWidth, isWideLayout, widthPx) @@ -112,11 +124,18 @@ function MountedRightDrawer({ return } const sub = BackHandler.addEventListener('hardwareBackPress', () => { + if ( + resolveSessionBackPress({ hardwareKeyboard, liveInputFocused }) === 'send-escape' && + onHardwareEscape + ) { + onHardwareEscape() + return true + } onClose() return true }) return () => sub.remove() - }, [visible, onClose]) + }, [hardwareKeyboard, liveInputFocused, onClose, onHardwareEscape, visible]) const dismiss = useCallback(() => { onClose() diff --git a/mobile/src/components/bottom-drawer-fill-height.test.ts b/mobile/src/components/bottom-drawer-fill-height.test.ts index 84a0498d768f..d96ce46646d3 100644 --- a/mobile/src/components/bottom-drawer-fill-height.test.ts +++ b/mobile/src/components/bottom-drawer-fill-height.test.ts @@ -13,6 +13,19 @@ describe('resolveBottomDrawerFillHeight', () => { ).toBe(844 - 54 - 16) }) + it('sizes from a freeform window instead of the taller display', () => { + const displayHeight = 1080 + const windowHeight = 600 + const height = resolveBottomDrawerFillHeight({ + screenHeight: windowHeight, + topInset: 0, + keyboardInset: 0, + topGap: 16 + }) + expect(height).toBe(windowHeight - 16) + expect(height).toBeLessThan(displayHeight) + }) + it('shrinks by the keyboard inset so the sheet top stays under the status bar', () => { expect( resolveBottomDrawerFillHeight({ diff --git a/mobile/src/components/mounted-bottom-drawer.tsx b/mobile/src/components/mounted-bottom-drawer.tsx index cea320553f1f..613e507413ec 100644 --- a/mobile/src/components/mounted-bottom-drawer.tsx +++ b/mobile/src/components/mounted-bottom-drawer.tsx @@ -1,14 +1,5 @@ import { type ReactNode, useCallback, useEffect, useRef, useState } from 'react' -import { - View, - Pressable, - useWindowDimensions, - ScrollView, - Keyboard, - BackHandler, - Modal, - Platform -} from 'react-native' +import { View, Pressable, ScrollView, Keyboard, BackHandler, Modal, Platform } from 'react-native' import { useSafeAreaInsets } from 'react-native-safe-area-context' import { Gesture, GestureDetector, GestureHandlerRootView } from 'react-native-gesture-handler' import Animated, { @@ -28,6 +19,7 @@ import { BOTTOM_DRAWER_HIDE_DURATION_MS } from './bottom-drawer-constants' import { bottomDrawerStyles as styles } from './bottom-drawer-styles' import { useInsideBottomDrawerModalHost } from './bottom-drawer-modal-host' import { useResponsiveLayout } from '../layout/responsive-layout' +import { useWindowBounds } from '../layout/window-bounds' const DISMISS_THRESHOLD = 80 const SPRING_CONFIG = { damping: 28, stiffness: 400 } @@ -73,7 +65,7 @@ export function MountedBottomDrawer({ // reanimated translate) so height shrinks as the sheet lifts and the top // edge stays under the status bar. const [keyboardInset, setKeyboardInset] = useState(0) - const { height: screenHeight } = useWindowDimensions() + const { height: screenHeight } = useWindowBounds() const insets = useSafeAreaInsets() // Why: on wide/tablet canvases a full-width sheet looks stretched; cap it and // center it horizontally. Vertical bottom-anchoring (and all the drag/keyboard diff --git a/mobile/src/components/right-drawer-panel-width.test.ts b/mobile/src/components/right-drawer-panel-width.test.ts index ebba3ba31fec..3a374a3bce35 100644 --- a/mobile/src/components/right-drawer-panel-width.test.ts +++ b/mobile/src/components/right-drawer-panel-width.test.ts @@ -14,6 +14,14 @@ describe('resolveRightDrawerPanelWidth', () => { expect(resolveRightDrawerPanelWidth(400, false, undefined)).toBe(400 - NARROW_BACKDROP_GUTTER) }) + it('sizes from a freeform window instead of the wider display', () => { + const displayWidth = 1920 + const windowWidth = 800 + const width = resolveRightDrawerPanelWidth(windowWidth, false, undefined) + expect(width).toBe(windowWidth - NARROW_BACKDROP_GUTTER) + expect(width).toBeLessThan(displayWidth) + }) + it('honors an explicit widthPx but never exceeds the window width', () => { expect(resolveRightDrawerPanelWidth(1024, true, 320)).toBe(320) expect(resolveRightDrawerPanelWidth(280, false, 320)).toBe(280) diff --git a/mobile/src/files/MobileFilePreviewScreen.tsx b/mobile/src/files/MobileFilePreviewScreen.tsx index 3f56890243a2..6136f3113d2c 100644 --- a/mobile/src/files/MobileFilePreviewScreen.tsx +++ b/mobile/src/files/MobileFilePreviewScreen.tsx @@ -1,5 +1,5 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' -import { Alert, BackHandler, Pressable, Text, View, useWindowDimensions } from 'react-native' +import { Alert, BackHandler, Pressable, Text, View } from 'react-native' import { SafeAreaView } from 'react-native-safe-area-context' import { useRouter } from 'expo-router' import { ChevronLeft, Save } from 'lucide-react-native' @@ -26,6 +26,7 @@ import { shouldKeepDirtyDraftOnPreviewLoadResult } from './mobile-file-preview-editability' import { filePreviewStyles as styles } from './mobile-file-preview-styles' +import { useWindowBounds } from '../layout/window-bounds' type Props = { route: MobileFilePreviewRouteState @@ -46,7 +47,7 @@ export function MobileFilePreviewScreen({ route }: Props) { const draftContentRef = useRef(draftContent) const savedContentRef = useRef(savedContent) const draftSourceKeyRef = useRef(null) - const { width, height } = useWindowDimensions() + const { width, height } = useWindowBounds() const routePreviewSource = useMemo( () => (previewParams ? previewSourceFromRoute(previewParams) : null), [previewParams] diff --git a/mobile/src/layout/responsive-layout-metrics.test.ts b/mobile/src/layout/responsive-layout-metrics.test.ts index 263344d75020..aeef8561ac2b 100644 --- a/mobile/src/layout/responsive-layout-metrics.test.ts +++ b/mobile/src/layout/responsive-layout-metrics.test.ts @@ -36,4 +36,34 @@ describe('responsive layout metrics', () => { horizontalPadding: spacing.lg }) }) + + it('classifies freeform desktop windows by their bounds', () => { + expect(getResponsiveLayoutMetrics(800, 600).windowClass).toBe('medium') + expect(getResponsiveLayoutMetrics(1024, 768).windowClass).toBe('expanded') + expect(getResponsiveLayoutMetrics(1280, 720).windowClass).toBe('expanded') + expect(getResponsiveLayoutMetrics(1920 / 1.5, 1080 / 1.5).windowClass).toBe('expanded') + }) + + it('keeps wide layout stable while resizing around the breakpoint', () => { + const wide = getResponsiveLayoutMetrics(760, 700) + expect(getResponsiveLayoutMetrics(690, 700, wide).isWideLayout).toBe(true) + expect(getResponsiveLayoutMetrics(675, 700, wide).isWideLayout).toBe(false) + }) + + it('keeps window class stable while resizing around the 600 and 840 dp breakpoints', () => { + const medium = getResponsiveLayoutMetrics(700, 600) + expect(medium.windowClass).toBe('medium') + expect(getResponsiveLayoutMetrics(590, 600, medium).windowClass).toBe('medium') + expect(getResponsiveLayoutMetrics(575, 600, medium).windowClass).toBe('compact') + expect(getResponsiveLayoutMetrics(850, 600, medium).windowClass).toBe('medium') + expect(getResponsiveLayoutMetrics(864, 600, medium).windowClass).toBe('expanded') + + const compact = getResponsiveLayoutMetrics(500, 600) + expect(getResponsiveLayoutMetrics(610, 600, compact).windowClass).toBe('compact') + expect(getResponsiveLayoutMetrics(624, 600, compact).windowClass).toBe('medium') + + const expanded = getResponsiveLayoutMetrics(1000, 600) + expect(getResponsiveLayoutMetrics(830, 600, expanded).windowClass).toBe('expanded') + expect(getResponsiveLayoutMetrics(815, 600, expanded).windowClass).toBe('medium') + }) }) diff --git a/mobile/src/layout/responsive-layout-metrics.ts b/mobile/src/layout/responsive-layout-metrics.ts index 4fffbcea9894..1955642548f5 100644 --- a/mobile/src/layout/responsive-layout-metrics.ts +++ b/mobile/src/layout/responsive-layout-metrics.ts @@ -2,14 +2,39 @@ import { spacing } from '../theme/mobile-theme' // Use actual window size so narrow iPad splits keep phone-like layouts. const WIDE_LAYOUT_MIN_WIDTH = 700 +const WIDE_LAYOUT_HYSTERESIS = 24 // Why: width alone catches landscape phones; capped tablet layouts need room // in both dimensions so phone rotation does not switch UI classes. const TABLET_LAYOUT_MIN_SHORT_SIDE = 600 +// Material window-size classes; resizing a freeform window across a breakpoint +// must not toggle the sidebar on every pixel, so leaving a class costs the margin. +const WINDOW_CLASS_MEDIUM_MIN_WIDTH = 600 +const WINDOW_CLASS_EXPANDED_MIN_WIDTH = 840 +const WINDOW_CLASS_HYSTERESIS = 24 + const CONTENT_MAX_WIDTH = 720 const MODAL_MAX_WIDTH = 480 +export type WindowClass = 'compact' | 'medium' | 'expanded' + +const WINDOW_CLASS_ORDER: WindowClass[] = ['compact', 'medium', 'expanded'] + +function resolveWindowClass(width: number, previous?: WindowClass): WindowClass { + const previousIndex = previous ? WINDOW_CLASS_ORDER.indexOf(previous) : -1 + const threshold = (breakpoint: number, classIndex: number) => + previousIndex < 0 + ? breakpoint + : previousIndex >= classIndex + ? breakpoint - WINDOW_CLASS_HYSTERESIS + : breakpoint + WINDOW_CLASS_HYSTERESIS + if (width >= threshold(WINDOW_CLASS_EXPANDED_MIN_WIDTH, 2)) { + return 'expanded' + } + return width >= threshold(WINDOW_CLASS_MEDIUM_MIN_WIDTH, 1) ? 'medium' : 'compact' +} + export type ResponsiveLayoutMetrics = { width: number height: number @@ -18,6 +43,7 @@ export type ResponsiveLayoutMetrics = { isWideLayout: boolean /** Tablet-class canvas (both dimensions large); false in narrow splits. */ isTabletLayout: boolean + windowClass: WindowClass /** Max width for primary scrollable content on wide layouts. */ contentMaxWidth: number /** Max width for centered sheets/dialogs on wide layouts. */ @@ -26,9 +52,19 @@ export type ResponsiveLayoutMetrics = { horizontalPadding: number } -export function getResponsiveLayoutMetrics(width: number, height: number): ResponsiveLayoutMetrics { +export function getResponsiveLayoutMetrics( + width: number, + height: number, + previous?: ResponsiveLayoutMetrics +): ResponsiveLayoutMetrics { const isTabletLayout = Math.min(width, height) >= TABLET_LAYOUT_MIN_SHORT_SIDE - const isWideLayout = width >= WIDE_LAYOUT_MIN_WIDTH && isTabletLayout + const wideThreshold = previous + ? previous.isWideLayout + ? WIDE_LAYOUT_MIN_WIDTH - WIDE_LAYOUT_HYSTERESIS + : WIDE_LAYOUT_MIN_WIDTH + WIDE_LAYOUT_HYSTERESIS + : WIDE_LAYOUT_MIN_WIDTH + const isWideLayout = width >= wideThreshold && isTabletLayout + const windowClass = resolveWindowClass(width, previous?.windowClass) return { width, @@ -36,6 +72,7 @@ export function getResponsiveLayoutMetrics(width: number, height: number): Respo isLandscape: width > height, isWideLayout, isTabletLayout, + windowClass, contentMaxWidth: CONTENT_MAX_WIDTH, modalMaxWidth: MODAL_MAX_WIDTH, // Roomier gutters once content is capped so it isn't glued to the edges. diff --git a/mobile/src/layout/responsive-layout.ts b/mobile/src/layout/responsive-layout.ts index 9a93afbaa428..b37b2b00a3ad 100644 --- a/mobile/src/layout/responsive-layout.ts +++ b/mobile/src/layout/responsive-layout.ts @@ -1,12 +1,16 @@ -import { useWindowDimensions } from 'react-native' +import { useRef } from 'react' import { getResponsiveLayoutMetrics, type ResponsiveLayoutMetrics } from './responsive-layout-metrics' +import { useWindowBounds } from './window-bounds' export type ResponsiveLayout = ResponsiveLayoutMetrics export function useResponsiveLayout(): ResponsiveLayout { - const { width, height } = useWindowDimensions() - return getResponsiveLayoutMetrics(width, height) + const { width, height } = useWindowBounds() + const previousRef = useRef(undefined) + const metrics = getResponsiveLayoutMetrics(width, height, previousRef.current) + previousRef.current = metrics + return metrics } diff --git a/mobile/src/layout/window-bounds-state.test.ts b/mobile/src/layout/window-bounds-state.test.ts new file mode 100644 index 000000000000..e6b674d2e59a --- /dev/null +++ b/mobile/src/layout/window-bounds-state.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from 'vitest' +import { resolveWindowBounds } from './window-bounds-state' + +describe('resolveWindowBounds', () => { + it('prefers measured root bounds over display dimensions', () => { + expect( + resolveWindowBounds({ + measured: { width: 800, height: 600 }, + fallback: { width: 1920, height: 1080 } + }) + ).toEqual({ width: 800, height: 600 }) + }) + + it('uses display dimensions until root layout is measured', () => { + expect(resolveWindowBounds({ measured: null, fallback: { width: 390, height: 844 } })).toEqual({ + width: 390, + height: 844 + }) + }) +}) diff --git a/mobile/src/layout/window-bounds-state.ts b/mobile/src/layout/window-bounds-state.ts new file mode 100644 index 000000000000..b5bf32a66f96 --- /dev/null +++ b/mobile/src/layout/window-bounds-state.ts @@ -0,0 +1,11 @@ +export type WindowBounds = { width: number; height: number } + +export function resolveWindowBounds({ + measured, + fallback +}: { + measured: WindowBounds | null + fallback: WindowBounds +}): WindowBounds { + return measured && measured.width > 0 && measured.height > 0 ? measured : fallback +} diff --git a/mobile/src/layout/window-bounds.tsx b/mobile/src/layout/window-bounds.tsx new file mode 100644 index 000000000000..ab26b860dd64 --- /dev/null +++ b/mobile/src/layout/window-bounds.tsx @@ -0,0 +1,51 @@ +import { createContext, useCallback, useContext, useMemo, useState, type ReactNode } from 'react' +import { + StyleSheet, + View, + useWindowDimensions, + type LayoutChangeEvent, + type StyleProp, + type ViewStyle +} from 'react-native' +import { resolveWindowBounds, type WindowBounds } from './window-bounds-state' + +const WindowBoundsContext = createContext(null) + +export function WindowBoundsProvider({ + children, + onLayout, + style +}: { + children: ReactNode + onLayout?: (event: LayoutChangeEvent) => void + style?: StyleProp +}) { + const [measured, setMeasured] = useState(null) + const handleLayout = useCallback( + (event: LayoutChangeEvent) => { + const { width, height } = event.nativeEvent.layout + setMeasured((current) => + current?.width === width && current.height === height ? current : { width, height } + ) + onLayout?.(event) + }, + [onLayout] + ) + const value = useMemo(() => measured, [measured]) + + return ( + + + {children} + + + ) +} + +export function useWindowBounds(): WindowBounds { + const measured = useContext(WindowBoundsContext) + const fallback = useWindowDimensions() + return resolveWindowBounds({ measured, fallback }) +} + +const styles = StyleSheet.create({ root: { flex: 1 } }) diff --git a/mobile/src/platform/hardware-keyboard.test.ts b/mobile/src/platform/hardware-keyboard.test.ts new file mode 100644 index 000000000000..330ee2bc87c7 --- /dev/null +++ b/mobile/src/platform/hardware-keyboard.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest' +import { + reduceHardwareKeyboardState, + resolveHardwareKeyboardAttached, + type HardwareKeyboardState +} from './hardware-keyboard' + +describe('hardware keyboard state', () => { + it('attaches after a key event arrives without a visible IME', () => { + const state = reduceHardwareKeyboardState( + { keyboardVisible: false, lastImeHeight: 0, hasReceivedHardwareKeyEvent: false }, + { type: 'key-press' } + ) + expect(resolveHardwareKeyboardAttached(state)).toBe(true) + }) + + it('does not classify software-keyboard input as hardware input', () => { + let state: HardwareKeyboardState = { + keyboardVisible: false, + lastImeHeight: 0, + hasReceivedHardwareKeyEvent: false + } + state = reduceHardwareKeyboardState(state, { type: 'ime-shown', height: 300 }) + state = reduceHardwareKeyboardState(state, { type: 'key-press' }) + expect(resolveHardwareKeyboardAttached(state)).toBe(false) + }) +}) diff --git a/mobile/src/platform/hardware-keyboard.ts b/mobile/src/platform/hardware-keyboard.ts new file mode 100644 index 000000000000..951284ad8a5f --- /dev/null +++ b/mobile/src/platform/hardware-keyboard.ts @@ -0,0 +1,62 @@ +import { useCallback, useMemo, useReducer } from 'react' + +export type HardwareKeyboardState = { + keyboardVisible: boolean + lastImeHeight: number + hasReceivedHardwareKeyEvent: boolean +} + +export type HardwareKeyboardEvent = + | { type: 'ime-shown'; height: number } + | { type: 'ime-hidden' } + | { type: 'key-press' } + +const INITIAL_STATE: HardwareKeyboardState = { + keyboardVisible: false, + lastImeHeight: 0, + hasReceivedHardwareKeyEvent: false +} + +export function reduceHardwareKeyboardState( + state: HardwareKeyboardState, + event: HardwareKeyboardEvent +): HardwareKeyboardState { + if (event.type === 'ime-shown') { + return { + keyboardVisible: true, + lastImeHeight: Math.max(0, event.height), + hasReceivedHardwareKeyEvent: false + } + } + if (event.type === 'ime-hidden') { + return { ...state, keyboardVisible: false } + } + if (state.keyboardVisible) { + return state + } + return { ...state, lastImeHeight: 0, hasReceivedHardwareKeyEvent: true } +} + +export function resolveHardwareKeyboardAttached(state: HardwareKeyboardState): boolean { + return !state.keyboardVisible && state.lastImeHeight < 120 && state.hasReceivedHardwareKeyEvent +} + +export function useHardwareKeyboardAttached() { + const [state, dispatch] = useReducer(reduceHardwareKeyboardState, INITIAL_STATE) + const notifyHardwareKeyEvent = useCallback(() => dispatch({ type: 'key-press' }), []) + const notifyImeShown = useCallback( + (height: number) => dispatch({ type: 'ime-shown', height }), + [] + ) + const notifyImeHidden = useCallback(() => dispatch({ type: 'ime-hidden' }), []) + + return useMemo( + () => ({ + hardwareKeyboard: resolveHardwareKeyboardAttached(state), + notifyHardwareKeyEvent, + notifyImeShown, + notifyImeHidden + }), + [notifyHardwareKeyEvent, notifyImeHidden, notifyImeShown, state] + ) +} diff --git a/mobile/src/platform/use-pixel-ratio.ts b/mobile/src/platform/use-pixel-ratio.ts new file mode 100644 index 000000000000..fab0afcad814 --- /dev/null +++ b/mobile/src/platform/use-pixel-ratio.ts @@ -0,0 +1,16 @@ +import { useEffect, useState } from 'react' +import { Dimensions, PixelRatio } from 'react-native' + +export function usePixelRatio(): number { + const [pixelRatio, setPixelRatio] = useState(() => PixelRatio.get()) + + useEffect(() => { + // Android changes arrive through MainActivity's display-metrics hook. + const subscription = Dimensions.addEventListener('change', () => + setPixelRatio(PixelRatio.get()) + ) + return () => subscription.remove() + }, []) + + return pixelRatio +} diff --git a/mobile/src/session/TerminalPaneView.tsx b/mobile/src/session/TerminalPaneView.tsx index 06174cd0378d..5c7e07913bdd 100644 --- a/mobile/src/session/TerminalPaneView.tsx +++ b/mobile/src/session/TerminalPaneView.tsx @@ -28,6 +28,7 @@ type TerminalPaneViewProps = { onFileTap: (handle: string, pathText: string, line: number | null, column: number | null) => void onOpenUrl: (handle: string, url: string) => void onTextScaleChange: (scale: number) => void + onViewportChanged: (handle: string, width: number, height: number, dpr: number) => void } export function TerminalPaneView({ @@ -49,7 +50,8 @@ export function TerminalPaneView({ onTerminalTap, onFileTap, onOpenUrl, - onTextScaleChange + onTextScaleChange, + onViewportChanged }: TerminalPaneViewProps) { const setRef = useCallback( (ref: TerminalWebViewHandle | null) => { @@ -87,6 +89,7 @@ export function TerminalPaneView({ onFileTap={(pathText, line, column) => onFileTap(handle, pathText, line, column)} onOpenUrl={(url) => onOpenUrl(handle, url)} onTextScaleChange={onTextScaleChange} + onViewportChanged={(width, height, dpr) => onViewportChanged(handle, width, height, dpr)} /> ) diff --git a/mobile/src/session/session-back-press.test.ts b/mobile/src/session/session-back-press.test.ts new file mode 100644 index 000000000000..12d8ab790b5e --- /dev/null +++ b/mobile/src/session/session-back-press.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it, vi } from 'vitest' +import { createSessionBackPressHandler, resolveSessionBackPress } from './session-back-press' + +describe('session back press', () => { + it('sends escape for focused hardware-keyboard terminal input', () => { + expect(resolveSessionBackPress({ liveInputFocused: true, hardwareKeyboard: true })).toBe( + 'send-escape' + ) + }) + + it('leaves when terminal input is not focused', () => { + expect(resolveSessionBackPress({ liveInputFocused: false, hardwareKeyboard: true })).toBe( + 'leave' + ) + }) + + it('routes hardware-keyboard back to Escape', () => { + const sendEscape = vi.fn() + const requestLeave = vi.fn() + const handleBackPress = createSessionBackPressHandler({ + hardwareKeyboard: true, + isLiveInputFocused: () => true, + requestLeave, + sendEscape + }) + + expect(handleBackPress()).toBe(true) + expect(sendEscape).toHaveBeenCalledOnce() + expect(requestLeave).not.toHaveBeenCalled() + }) + + it('routes other back presses to session leave', () => { + const sendEscape = vi.fn() + const requestLeave = vi.fn() + const handleBackPress = createSessionBackPressHandler({ + hardwareKeyboard: false, + isLiveInputFocused: () => true, + requestLeave, + sendEscape + }) + + expect(handleBackPress()).toBe(true) + expect(requestLeave).toHaveBeenCalledOnce() + expect(sendEscape).not.toHaveBeenCalled() + }) +}) diff --git a/mobile/src/session/session-back-press.ts b/mobile/src/session/session-back-press.ts new file mode 100644 index 000000000000..a9034b883a07 --- /dev/null +++ b/mobile/src/session/session-back-press.ts @@ -0,0 +1,35 @@ +type SessionBackPressHandlerArgs = { + hardwareKeyboard: boolean + isLiveInputFocused: () => boolean + requestLeave: () => void + sendEscape: () => void +} + +export function resolveSessionBackPress({ + liveInputFocused, + hardwareKeyboard +}: { + liveInputFocused: boolean + hardwareKeyboard: boolean +}): 'send-escape' | 'leave' { + return liveInputFocused && hardwareKeyboard ? 'send-escape' : 'leave' +} + +export function createSessionBackPressHandler({ + hardwareKeyboard, + isLiveInputFocused, + requestLeave, + sendEscape +}: SessionBackPressHandlerArgs): () => boolean { + return () => { + if ( + resolveSessionBackPress({ liveInputFocused: isLiveInputFocused(), hardwareKeyboard }) === + 'send-escape' + ) { + sendEscape() + } else { + requestLeave() + } + return true + } +} diff --git a/mobile/src/session/use-session-back-handler.ts b/mobile/src/session/use-session-back-handler.ts new file mode 100644 index 000000000000..08bdec5c6fdb --- /dev/null +++ b/mobile/src/session/use-session-back-handler.ts @@ -0,0 +1,65 @@ +import { useCallback, useEffect, type RefObject } from 'react' +import { BackHandler, Keyboard, type TextInput } from 'react-native' +import { useRouter } from 'expo-router' +import type { + DirtyMarkdownDraft, + MarkdownDocState, + MobileSessionTab +} from './mobile-session-route-types' +import { createSessionBackPressHandler } from './session-back-press' + +export function useSessionBackHandler({ + handleLiveInputKeyPress, + hardwareKeyboard, + hostId, + liveInputRef, + markdownDocs, + sessionTabs, + setLeaveDrafts +}: { + handleLiveInputKeyPress: (event: { nativeEvent: { key: string } }) => void + hardwareKeyboard: boolean + hostId: string + liveInputRef: RefObject + markdownDocs: ReadonlyMap + sessionTabs: readonly MobileSessionTab[] + setLeaveDrafts: (drafts: DirtyMarkdownDraft[]) => void +}): { leaveSession: () => void; requestLeaveSession: () => void } { + const router = useRouter() + const leaveSession = useCallback(() => { + if (router.canGoBack()) { + router.back() + return + } + // Why: Android back can fire at the root route; replace avoids React Navigation's dev-only GO_BACK warning. + router.replace(`/h/${hostId}`) + }, [hostId, router]) + const requestLeaveSession = useCallback(() => { + const dirtyDrafts: DirtyMarkdownDraft[] = [] + for (const [tabId, doc] of markdownDocs) { + if (doc.status === 'ready' && doc.isDirty) { + const tab = sessionTabs.find((candidate) => candidate.id === tabId) + dirtyDrafts.push({ tabId, title: tab?.title || 'Markdown', content: doc.localContent }) + } + } + if (dirtyDrafts.length === 0) { + leaveSession() + return + } + Keyboard.dismiss() + setLeaveDrafts(dirtyDrafts) + }, [leaveSession, markdownDocs, sessionTabs, setLeaveDrafts]) + + useEffect(() => { + const handleBackPress = createSessionBackPressHandler({ + hardwareKeyboard, + isLiveInputFocused: () => liveInputRef.current?.isFocused() === true, + requestLeave: requestLeaveSession, + sendEscape: () => handleLiveInputKeyPress({ nativeEvent: { key: 'Escape' } }) + }) + const subscription = BackHandler.addEventListener('hardwareBackPress', handleBackPress) + return () => subscription.remove() + }, [handleLiveInputKeyPress, hardwareKeyboard, liveInputRef, requestLeaveSession]) + + return { leaveSession, requestLeaveSession } +} diff --git a/mobile/src/session/use-terminal-frame-layout.ts b/mobile/src/session/use-terminal-frame-layout.ts new file mode 100644 index 000000000000..f39c77f772ce --- /dev/null +++ b/mobile/src/session/use-terminal-frame-layout.ts @@ -0,0 +1,32 @@ +import { useCallback, type RefObject } from 'react' +import type { LayoutChangeEvent } from 'react-native' +import type { TerminalWebViewHandle } from '../terminal/terminal-webview-contract' + +export function useTerminalFrameLayout({ + terminalRefs, + heightRef, + widthRef, + onHeightChange, + onWidthChange +}: { + terminalRefs: RefObject> + heightRef: RefObject + widthRef: RefObject + onHeightChange: (height: number) => void + onWidthChange: (width: number) => void +}) { + return useCallback( + (event: LayoutChangeEvent) => { + const width = Math.round(event.nativeEvent.layout.width) + const height = Math.round(event.nativeEvent.layout.height) + widthRef.current = width + heightRef.current = height + for (const terminalRef of terminalRefs.current.values()) { + terminalRef.setViewport(width, height) + } + onWidthChange(width) + onHeightChange(height) + }, + [heightRef, onHeightChange, onWidthChange, terminalRefs, widthRef] + ) +} diff --git a/mobile/src/terminal/TerminalWebView.tsx b/mobile/src/terminal/TerminalWebView.tsx index d09c06612402..7626e5ccad10 100644 --- a/mobile/src/terminal/TerminalWebView.tsx +++ b/mobile/src/terminal/TerminalWebView.tsx @@ -1,7 +1,6 @@ import { useRef, useCallback, forwardRef, useImperativeHandle, useEffect, useMemo } from 'react' -import { Platform, View } from 'react-native' +import { View } from 'react-native' import { WebView, type WebViewMessageEvent } from 'react-native-webview' -import type { TerminalOscLinkRange } from '../../../src/shared/terminal-osc-link-ranges' import type { TerminalWebViewHandle, TerminalWebViewProps } from './terminal-webview-contract' import { TerminalWebViewEngineErrorOverlay, @@ -15,6 +14,7 @@ import { createTerminalWebViewPendingMessages } from './terminal-webview-pending import { dispatchTerminalWebViewNotification } from './terminal-webview-notification-dispatch' import { routeTerminalQueryReply } from './terminal-webview-query-reply-routing' import { createTerminalWriteCoalescer } from './terminal-write-coalescer' +import { createTerminalWebViewHandle } from './terminal-webview-handle' type Props = TerminalWebViewProps @@ -38,7 +38,8 @@ export const TerminalWebView = forwardRef(function onTerminalTap, onFileTap, onOpenUrl, - onTextScaleChange + onTextScaleChange, + onViewportChanged }, ref ) { @@ -171,7 +172,8 @@ export const TerminalWebView = forwardRef(function onTerminalTap, onFileTap, onOpenUrl, - onTextScaleChange + onTextScaleChange, + onViewportChanged }) } }, @@ -189,7 +191,8 @@ export const TerminalWebView = forwardRef(function onTerminalTap, onFileTap, onOpenUrl, - onTextScaleChange + onTextScaleChange, + onViewportChanged ] ) @@ -232,134 +235,20 @@ export const TerminalWebView = forwardRef(function useImperativeHandle( ref, - () => ({ - prepareForForegroundRecovery() { - if (Platform.OS !== 'ios') { - return - } - // Why: direct ping is the only command allowed through while readiness is - // invalid; init/write commands queue until this exact document answers. - isWebReadyRef.current = false - armWebReadyWatchdog() - pendingPingIdRef.current = sendToWebView({ type: 'ping' }) - }, - write(data: string) { - writeCoalescer.write(data) - }, - init( - cols: number, - rows: number, - initialData?: string, - preserveScroll?: boolean, - oscLinks?: TerminalOscLinkRange[] - ) { - // Why: arm a fresh ready promise BEFORE posting init. The WebView - // resolves it via the 'ready' notify at the end of its rAF chain. - // Resolve any prior in-flight ready first so awaiters from the - // previous generation don't sit on the 3s setTimeout fallback — - // each leaked timer + closure pinned an awaiting measure caller - // for the full 3s under rapid re-init (orientation change, - // multiple resubscribes), delaying cold-start fit chains. - const priorResolve = readyResolveRef.current - if (priorResolve) { - readyResolveRef.current = null - readyPromiseRef.current = null - priorResolve() - } - readyPromiseRef.current = new Promise((resolve) => { - readyResolveRef.current = resolve - }) - // Why: pending chunks are pre-snapshot data; the init snapshot supersedes - // them, and writing them after init would corrupt the fresh buffer. - writeCoalescer.clear() - postMessage({ - type: 'init', - cols, - rows, - initialData, - oscLinks, - terminalTheme, - fontScale: textScale, - preserveScroll - }) - }, - resize(cols: number, rows: number) { - // Why: resize/reflow must observe all prior writes or bytes reorder. - writeCoalescer.flushNow() - postMessage({ type: 'resize', cols, rows }) - }, - reflow(cols: number, rows: number) { - writeCoalescer.flushNow() - postMessage({ type: 'reflow', cols, rows }) - }, - clear() { - writeCoalescer.clear() - postMessage({ type: 'clear' }) - }, - measureFitDimensions( - containerHeight?: number - ): Promise<{ cols: number; rows: number } | null> { - if (!isWebReadyRef.current) { - return Promise.resolve(null) - } - return new Promise((resolve) => { - measureResolveRef.current?.(null) - let timeout: ReturnType | null = null - const finish = (result: { cols: number; rows: number } | null) => { - if (timeout) { - clearTimeout(timeout) - timeout = null - } - if (measureResolveRef.current === finish) { - measureResolveRef.current = null - } - resolve(result) - } - measureResolveRef.current = finish - sendToWebView({ type: 'measure', containerHeight }) - // Why: if the WebView doesn't respond within 2s (e.g., xterm - // failed to load), resolve null so the caller can disable - // Fit to Phone rather than hanging indefinitely. - timeout = setTimeout(() => { - if (measureResolveRef.current === finish) { - finish(null) - } - }, 2000) - }) - }, - resetZoom() { - postMessage({ type: 'reset-zoom' }) - }, - cancelSelect() { - postMessage({ type: 'cancel-select' }) - }, - doSelectAll() { - postMessage({ type: 'do-select-all' }) - }, - async awaitReady(): Promise { - // Why: returns the in-flight ready promise (set by init); resolves - // immediately if no init is pending. Capped at 3s so a stuck - // WebView doesn't hang the caller. - const p = readyPromiseRef.current - if (!p) { - return - } - await new Promise((resolve) => { - let settled = false - const timeout = setTimeout(() => { - settled = true - resolve() - }, 3000) - void p.finally(() => { - if (!settled) { - clearTimeout(timeout) - settled = true - resolve() - } - }) - }) - } - }), + () => + createTerminalWebViewHandle({ + armWebReadyWatchdog, + isWebReadyRef, + measureResolveRef, + pendingPingIdRef, + postMessage, + readyPromiseRef, + readyResolveRef, + sendToWebView, + terminalTheme, + textScale, + writeCoalescer + }), [armWebReadyWatchdog, postMessage, sendToWebView, terminalTheme, textScale, writeCoalescer] ) diff --git a/mobile/src/terminal/terminal-keyboard-avoidance-webview.test.ts b/mobile/src/terminal/terminal-keyboard-avoidance-webview.test.ts index 3a09ae3acd49..039e2c0855aa 100644 --- a/mobile/src/terminal/terminal-keyboard-avoidance-webview.test.ts +++ b/mobile/src/terminal/terminal-keyboard-avoidance-webview.test.ts @@ -198,13 +198,9 @@ describe('terminal keyboard-avoidance WebView metrics', () => { const resizeEnd = terminalHtmlSource.indexOf('\n // reflow()', resizeStart) const clearStart = terminalHtmlSource.indexOf("} else if (msg.type === 'clear') {") const clearEnd = terminalHtmlSource.indexOf("} else if (msg.type === 'measure')", clearStart) - const textScaleStart = terminalHtmlSource.indexOf(' function applyTextScale(scale)') - const textScaleEnd = terminalHtmlSource.indexOf('\n var panX', textScaleStart) - for (const block of [ terminalHtmlSource.slice(resizeStart, resizeEnd), terminalHtmlSource.slice(clearStart, clearEnd), - terminalHtmlSource.slice(textScaleStart, textScaleEnd), reflowSource ]) { expect(block.indexOf('emitKeyboardAvoidanceMetrics()')).toBeGreaterThan( diff --git a/mobile/src/terminal/terminal-live-input.test.ts b/mobile/src/terminal/terminal-live-input.test.ts index daca212d4a8c..21129285c2b1 100644 --- a/mobile/src/terminal/terminal-live-input.test.ts +++ b/mobile/src/terminal/terminal-live-input.test.ts @@ -214,6 +214,21 @@ describe('terminal live input', () => { expect(refocus).not.toHaveBeenCalled() }) + it('keeps hardware-keyboard focus without blur/refocus churn', () => { + const input = createFocusTarget(() => true) + const refocus = vi.fn() + + focusTerminalLiveInputTarget(input, { + hardwareKeyboard: true, + keyboardHeight: 0, + refocus + }) + + expect(input.blur).not.toHaveBeenCalled() + expect(input.focus).toHaveBeenCalledTimes(1) + expect(refocus).not.toHaveBeenCalled() + }) + it('focuses the capture input directly when it is not already focused', () => { const input = createFocusTarget(() => false) const refocus = vi.fn() diff --git a/mobile/src/terminal/terminal-live-input.ts b/mobile/src/terminal/terminal-live-input.ts index 321226c0c7d5..a075a5b8077b 100644 --- a/mobile/src/terminal/terminal-live-input.ts +++ b/mobile/src/terminal/terminal-live-input.ts @@ -74,6 +74,7 @@ export type TerminalLiveInputFocusTarget = { type FocusTerminalLiveInputTargetOptions = { readonly keyboardHeight: number + readonly hardwareKeyboard?: boolean readonly refocus: () => void } @@ -230,13 +231,13 @@ export function scheduleTerminalLiveInputFocus( export function focusTerminalLiveInputTarget( input: TerminalLiveInputFocusTarget | null, - { keyboardHeight, refocus }: FocusTerminalLiveInputTargetOptions + { hardwareKeyboard, keyboardHeight, refocus }: FocusTerminalLiveInputTargetOptions ): void { if (!input) { return } - if (keyboardHeight <= 0 && input.isFocused?.()) { + if (!hardwareKeyboard && keyboardHeight <= 0 && input.isFocused?.()) { // Why: Android can keep a hidden TextInput focused after the IME is dismissed; // focus() is then a no-op, so force a new focus session to reopen the keyboard. input.blur() diff --git a/mobile/src/terminal/terminal-viewport-refit-state.ts b/mobile/src/terminal/terminal-viewport-refit-state.ts index dfa4daadf4c5..f3a174f3e295 100644 --- a/mobile/src/terminal/terminal-viewport-refit-state.ts +++ b/mobile/src/terminal/terminal-viewport-refit-state.ts @@ -102,3 +102,21 @@ export function isTerminalViewportRefitTargetCurrent( state.currentRef === state.expectedRef ) } + +export function areTerminalViewportWidthsAligned( + nativeWidth: number, + webViewWidth: number | null +): boolean { + return nativeWidth <= 0 || webViewWidth === null || Math.abs(nativeWidth - webViewWidth) <= 1 +} + +export function resetTerminalKeyboardVisibilityForWidthChange( + state: TerminalFrameHeightRefitState, + previousWidth: number, + nextWidth: number +): TerminalFrameHeightRefitState { + if (previousWidth <= 0 || Math.abs(nextWidth - previousWidth) / previousWidth <= 0.1) { + return state + } + return { ...state, keyboardVisible: false } +} diff --git a/mobile/src/terminal/terminal-viewport-refit.test.ts b/mobile/src/terminal/terminal-viewport-refit.test.ts index b472fb7847e6..74c0c2cc8959 100644 --- a/mobile/src/terminal/terminal-viewport-refit.test.ts +++ b/mobile/src/terminal/terminal-viewport-refit.test.ts @@ -2,17 +2,21 @@ import { readFileSync } from 'node:fs' import { describe, expect, it } from 'vitest' import type { RpcResponse } from '../transport/types' import { + areTerminalViewportWidthsAligned, isTerminalUpdateViewportApplied, isTerminalUpdateViewportUpdated, isTerminalViewportRefitTargetCurrent, reduceTerminalFrameHeightRefit, + resetTerminalKeyboardVisibilityForWidthChange, resolveTerminalUpdateViewportCapability, type TerminalFrameHeightRefitEvent, type TerminalFrameHeightRefitState, type TerminalUpdateViewportCapability } from './terminal-viewport-refit-state' -const hookSource = readFileSync(new URL('./terminal-viewport-refit.ts', import.meta.url), 'utf8') +const hookSource = + readFileSync(new URL('./terminal-viewport-refit.ts', import.meta.url), 'utf8') + + readFileSync(new URL('./use-terminal-layout-refit-triggers.ts', import.meta.url), 'utf8') const sessionSource = readFileSync( new URL('../../app/h/[hostId]/session/[worktreeId].tsx', import.meta.url), 'utf8' @@ -22,21 +26,20 @@ describe('terminal viewport refit', () => { it('refits when the window dimensions change (fold/unfold, rotation)', () => { // Why: a PTY fitted on the folded cover screen must be re-measured when // the window grows, or the terminal renders in a fraction of the display. - expect(hookSource).toContain('useWindowDimensions()') + expect(hookSource).toContain('useWindowBounds()') const start = hookSource.indexOf('const { width: windowWidth, height: windowHeight }') expect(start).toBeGreaterThanOrEqual(0) const resizeEffect = hookSource.slice(start) expect(resizeEffect).toContain('viewportMeasuredRef.current = false') expect(resizeEffect).toContain('scheduleViewportRefit()') - expect(resizeEffect).toContain( - '[windowWidth, windowHeight, viewportMeasuredRef, scheduleViewportRefit]' - ) + expect(resizeEffect).toContain('windowHeight') + expect(resizeEffect).toContain('windowWidth') }) it('still refits when the tab strip toggles visibility', () => { - const start = hookSource.indexOf('const prevTabStripVisibleRef') + const start = hookSource.indexOf('const previousTabStripVisibleRef') expect(start).toBeGreaterThanOrEqual(0) - const tabEffect = hookSource.slice(start, hookSource.indexOf('useWindowDimensions()')) + const tabEffect = hookSource.slice(start, hookSource.indexOf('useWindowBounds()')) expect(tabEffect).toContain('viewportMeasuredRef.current = false') expect(tabEffect).toContain('scheduleViewportRefit()') }) @@ -44,13 +47,43 @@ describe('terminal viewport refit', () => { it('refits the PTY when terminal text scale changes', () => { // Why: mobile text size must change the real PTY grid, not just scale pixels // in the WebView, or wrapped CLI output diverges from what the shell sees. - const start = hookSource.indexOf('const prevTextScaleRef = useRef(textScale)') + const start = hookSource.indexOf('const previousTextScaleRef = useRef(textScale)') expect(start).toBeGreaterThanOrEqual(0) const textScaleEffect = hookSource.slice(start, start + 600) - expect(textScaleEffect).toContain('prevTextScaleRef.current === textScale') + expect(textScaleEffect).toContain('previousTextScaleRef.current === textScale') expect(textScaleEffect).toContain('viewportMeasuredRef.current = false') expect(textScaleEffect).toContain('scheduleViewportRefit()') - expect(textScaleEffect).toContain('[textScale, viewportMeasuredRef, scheduleViewportRefit]') + expect(textScaleEffect).toContain('textScale') + }) + + it('flushes a refit deferred behind the IME when a hardware keyboard is detected', () => { + // Why: notifyHardwareKeyboard used to clear keyboardVisible directly and + // strand `pending`; the later IME-hide event then matched state and never + // refit. Routing through the reducer as keyboard-visibility(false) flushes. + const start = hookSource.indexOf('const notifyHardwareKeyboard = useCallback(') + expect(start).toBeGreaterThanOrEqual(0) + expect(hookSource.slice(start, start + 300)).toContain( + "notifyFrameHeightEvent({ type: 'keyboard-visibility', visible: false })" + ) + + let state: TerminalFrameHeightRefitState = { + frameHeight: 600, + keyboardVisible: false, + pending: false + } + const dispatch = (event: TerminalFrameHeightRefitEvent) => { + const transition = reduceTerminalFrameHeightRefit(state, event) + state = transition.state + return transition.shouldRefit + } + expect(dispatch({ type: 'keyboard-visibility', visible: true })).toBe(false) + expect(dispatch({ type: 'frame-height', height: 540 })).toBe(false) + expect(state.pending).toBe(true) + // Hardware keyboard signal arrives as keyboard-visibility(false): flush now. + expect(dispatch({ type: 'keyboard-visibility', visible: false })).toBe(true) + expect(state).toEqual({ frameHeight: 540, keyboardVisible: false, pending: false }) + // The trailing IME-hide is a no-op instead of a stranded pending refit. + expect(dispatch({ type: 'keyboard-visibility', visible: false })).toBe(false) }) it('coalesces keyboard-visible frame-height churn into one refit after close', () => { @@ -84,7 +117,7 @@ describe('terminal viewport refit', () => { }) it('routes imperative height notifications through the keyboard-aware reducer', () => { - const start = hookSource.indexOf('const notifyFrameHeightRefitEvent = useCallback(') + const start = hookSource.indexOf('const notifyFrameHeightEvent = useCallback(') expect(start).toBeGreaterThanOrEqual(0) const notifier = hookSource.slice(start, start + 1_300) expect(notifier).toContain('reduceTerminalFrameHeightRefit(') @@ -152,11 +185,33 @@ describe('terminal viewport refit', () => { expect(sessionSource).toContain('tabStripVisible: terminals.length > 1') expect(sessionSource).toContain('textScale: terminalTextScale') expect(sessionSource).toContain('connState,') - expect(sessionSource).toContain('notifyTerminalFrameHeight(nextHeight)') + expect(sessionSource).toContain('onLayout={handleTerminalFrameLayout}') expect(sessionSource).toContain('notifyKeyboardVisibility(true)') expect(sessionSource).toContain('notifyKeyboardVisibility(false)') }) + it('waits for RN and WebView widths to agree', () => { + expect(areTerminalViewportWidthsAligned(800, 720)).toBe(false) + expect(areTerminalViewportWidthsAligned(800, 800)).toBe(true) + expect(hookSource).toContain('areTerminalViewportWidthsAligned(') + expect(hookSource).toContain('notifyWebViewViewport') + }) + + it('uses a trailing 250ms debounce during resize drags', () => { + expect(hookSource).toContain('}, 250)') + expect(hookSource).toContain('clearTimeout(refitTimerRef.current)') + }) + + it('clears stale keyboard visibility after a major width change', () => { + const state = resetTerminalKeyboardVisibilityForWidthChange( + { frameHeight: 500, keyboardVisible: true, pending: true }, + 700, + 900 + ) + expect(state.keyboardVisible).toBe(false) + expect(state.pending).toBe(true) + }) + it('does not rerender SessionScreen for frame-height-only layout changes', () => { // The imperative notifier keeps a dock-settling burst off React's render path. expect(sessionSource).not.toContain('setTerminalFrameHeight') @@ -164,10 +219,10 @@ describe('terminal viewport refit', () => { }) it('defers height-only window resizes while the keyboard is visible', () => { - const start = hookSource.indexOf('const prevWindowDimsRef') + const start = hookSource.indexOf('const previousWindowBoundsRef') const windowEffect = hookSource.slice(start, start + 1_100) expect(windowEffect).toContain( - 'prev.width === windowWidth && frameHeightRefitStateRef.current.keyboardVisible' + 'previous.width === windowWidth && frameHeightStateRef.current.keyboardVisible' ) }) diff --git a/mobile/src/terminal/terminal-viewport-refit.ts b/mobile/src/terminal/terminal-viewport-refit.ts index 8a9ab281455c..e9e75e8a02d1 100644 --- a/mobile/src/terminal/terminal-viewport-refit.ts +++ b/mobile/src/terminal/terminal-viewport-refit.ts @@ -1,5 +1,5 @@ import { useCallback, useEffect, useRef, type RefObject } from 'react' -import { AppState, Platform, useWindowDimensions, type AppStateStatus } from 'react-native' +import { AppState, Platform, type AppStateStatus } from 'react-native' import type { RpcClient } from '../transport/rpc-client' import type { ConnectionState } from '../transport/types' import type { TerminalWebViewHandle } from './TerminalWebView' @@ -8,12 +8,13 @@ import { isTerminalUpdateViewportApplied, isTerminalUpdateViewportUpdated, isTerminalViewportRefitTargetCurrent, + areTerminalViewportWidthsAligned, reduceTerminalFrameHeightRefit, resolveTerminalUpdateViewportCapability, - type TerminalFrameHeightRefitEvent, type TerminalFrameHeightRefitState, type TerminalUpdateViewportCapability } from './terminal-viewport-refit-state' +import { useTerminalLayoutRefitTriggers } from './use-terminal-layout-refit-triggers' export type TerminalViewportDims = { cols: number; rows: number } @@ -41,6 +42,8 @@ type TerminalViewportRefitOptions = { type TerminalViewportRefitNotifications = { notifyTerminalFrameHeight: (height: number) => void notifyKeyboardVisibility: (visible: boolean) => void + notifyHardwareKeyboard: () => void + notifyWebViewViewport: (width: number, height: number) => void } // Why: re-measure on layout changes outside the subscribe path (tab strip, fold/rotate/resize), or a PTY renders "cut in half" (#4579). @@ -75,6 +78,7 @@ export function useTerminalViewportRefit( keyboardVisible: false, pending: false }) + const webViewViewportRef = useRef<{ width: number; height: number } | null>(null) // Why: marks the armed timer as a height refit so its callback re-checks the keyboard; other refits always run unguarded. const heightOriginatedRefitRef = useRef(false) const scheduleViewportRefit = useCallback( @@ -110,6 +114,14 @@ export function useTerminalViewportRefit( if (!ref) { return } + if ( + !areTerminalViewportWidthsAligned( + terminalFrameWidth, + webViewViewportRef.current?.width ?? null + ) + ) { + return + } const isCurrentTarget = () => isTerminalViewportRefitTargetCurrent({ activeHandle: activeHandleRef.current, @@ -122,7 +134,10 @@ export function useTerminalViewportRefit( currentRunSeq: refitRunSeqRef.current }) void (async () => { - const dims = await ref.measureFitDimensions(terminalFrameHeightRef.current || undefined) + const dims = await ref.measureFitDimensions( + terminalFrameHeightRef.current || undefined, + terminalFrameWidth || undefined + ) if (!isCurrentTarget()) { return } @@ -171,7 +186,7 @@ export function useTerminalViewportRefit( initializedHandlesRef.current.delete(handle) subscribeToTerminal(handle) })() - }, 150) + }, 250) }, [ activeHandleRef, @@ -184,7 +199,8 @@ export function useTerminalViewportRefit( deviceTokenRef, initializedHandlesRef, unsubscribeTerminal, - subscribeToTerminal + subscribeToTerminal, + terminalFrameWidth ] ) const scheduleForcedViewportRefit = useCallback(() => { @@ -192,77 +208,15 @@ export function useTerminalViewportRefit( scheduleViewportRefit() }, [scheduleViewportRefit]) - // Why: the tab strip toggles at the 1↔2 terminal boundary (~40px area change), so the cached viewport goes stale. - const prevTabStripVisibleRef = useRef(tabStripVisible) - useEffect(() => { - if (prevTabStripVisibleRef.current === tabStripVisible) { - return - } - prevTabStripVisibleRef.current = tabStripVisible - viewportMeasuredRef.current = false - scheduleViewportRefit() - }, [tabStripVisible, viewportMeasuredRef, scheduleViewportRefit]) - - // Why: fold/unfold and rotation change window dims with no subscribe/tab change; refit or the grid stays stale (fit capped at 1). - const { width: windowWidth, height: windowHeight } = useWindowDimensions() - const prevWindowDimsRef = useRef({ width: windowWidth, height: windowHeight }) - useEffect(() => { - const prev = prevWindowDimsRef.current - if (prev.width === windowWidth && prev.height === windowHeight) { - return - } - prevWindowDimsRef.current = { width: windowWidth, height: windowHeight } - // Why: adjustResize can change only window height while the IME is open; the frame-height notifier corrects once it closes. - if (prev.width === windowWidth && frameHeightRefitStateRef.current.keyboardVisible) { - return - } - viewportMeasuredRef.current = false - scheduleViewportRefit() - }, [windowWidth, windowHeight, viewportMeasuredRef, scheduleViewportRefit]) - - // Why: on text-size change the refit's 150ms debounce lets the WebView apply the new fontSize before we re-measure cell metrics. - const prevTextScaleRef = useRef(textScale) - useEffect(() => { - if (prevTextScaleRef.current === textScale) { - return - } - prevTextScaleRef.current = textScale - viewportMeasuredRef.current = false - scheduleViewportRefit() - }, [textScale, viewportMeasuredRef, scheduleViewportRefit]) - - // Why: panel dock/undock or sidebar resize changes frame width with no window/tab change, so the cached viewport goes stale. - const prevFrameWidthRef = useRef(terminalFrameWidth) - useEffect(() => { - if (prevFrameWidthRef.current === terminalFrameWidth) { - return - } - prevFrameWidthRef.current = terminalFrameWidth - viewportMeasuredRef.current = false - scheduleViewportRefit() - }, [terminalFrameWidth, viewportMeasuredRef, scheduleViewportRefit]) - - const notifyFrameHeightRefitEvent = useCallback( - (event: TerminalFrameHeightRefitEvent) => { - const transition = reduceTerminalFrameHeightRefit(frameHeightRefitStateRef.current, event) - frameHeightRefitStateRef.current = transition.state - if (!transition.shouldRefit) { - return - } - viewportMeasuredRef.current = false - scheduleViewportRefit({ heightOriginated: true }) - }, - [viewportMeasuredRef, scheduleViewportRefit] - ) - // Why: notify imperatively so layout churn doesn't rerender the full session. - const notifyTerminalFrameHeight = useCallback( - (height: number) => notifyFrameHeightRefitEvent({ type: 'frame-height', height }), - [notifyFrameHeightRefitEvent] - ) - const notifyKeyboardVisibility = useCallback( - (visible: boolean) => notifyFrameHeightRefitEvent({ type: 'keyboard-visibility', visible }), - [notifyFrameHeightRefitEvent] - ) + const layoutNotifications = useTerminalLayoutRefitTriggers({ + frameHeightStateRef: frameHeightRefitStateRef, + scheduleViewportRefit, + tabStripVisible, + terminalFrameWidth, + textScale, + viewportMeasuredRef, + webViewViewportRef + }) useEffect(() => { if (Platform.OS !== 'ios') { @@ -311,5 +265,5 @@ export function useTerminalViewportRefit( } }, []) - return { notifyTerminalFrameHeight, notifyKeyboardVisibility } + return layoutNotifications } diff --git a/mobile/src/terminal/terminal-webview-contract.test.ts b/mobile/src/terminal/terminal-webview-contract.test.ts index 0a3527111e3e..ec97b63f9768 100644 --- a/mobile/src/terminal/terminal-webview-contract.test.ts +++ b/mobile/src/terminal/terminal-webview-contract.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from 'vitest' -import { parseTerminalKeyboardAvoidanceMetrics } from './terminal-webview-contract' +import { + parseTerminalKeyboardAvoidanceMetrics, + type TerminalWebViewHandle +} from './terminal-webview-contract' describe('parseTerminalKeyboardAvoidanceMetrics', () => { it('parses a full payload', () => { @@ -50,3 +53,19 @@ describe('parseTerminalKeyboardAvoidanceMetrics', () => { ).toEqual({ cursorY: 0, contentBottomRow: 0, rows: 0, altScreen: false }) }) }) + +describe('TerminalWebViewHandle viewport contract', () => { + it('accepts measured width and forwards explicit viewport bounds', async () => { + const viewportMethods = { + measureFitDimensions: async (height?: number, width?: number) => + height && width ? { cols: width, rows: height } : null, + setViewport: (_width: number, _height: number) => {} + } satisfies Pick + + await expect(viewportMethods.measureFitDimensions(600, 800)).resolves.toEqual({ + cols: 800, + rows: 600 + }) + expect(() => viewportMethods.setViewport(800, 600)).not.toThrow() + }) +}) diff --git a/mobile/src/terminal/terminal-webview-contract.ts b/mobile/src/terminal/terminal-webview-contract.ts index 45b9a38c05a5..866a7288f433 100644 --- a/mobile/src/terminal/terminal-webview-contract.ts +++ b/mobile/src/terminal/terminal-webview-contract.ts @@ -61,6 +61,7 @@ export type TerminalSelectionEvents = { // Why: pinch-to-zoom in the terminal snaps to a text-size preset and reports it // here so the app persists it and keeps Settings + other panes in sync. onTextScaleChange?: (scale: number) => void + onViewportChanged?: (width: number, height: number, dpr: number) => void } export type TerminalWebViewProps = { @@ -91,7 +92,11 @@ export type TerminalWebViewHandle = { // latest output. No-op on the alternate screen. reflow: (cols: number, rows: number) => void clear: () => void - measureFitDimensions: (containerHeight?: number) => Promise<{ cols: number; rows: number } | null> + measureFitDimensions: ( + containerHeight?: number, + containerWidth?: number + ) => Promise<{ cols: number; rows: number } | null> + setViewport: (width: number, height: number) => void resetZoom: () => void cancelSelect: () => void doSelectAll: () => void diff --git a/mobile/src/terminal/terminal-webview-handle.ts b/mobile/src/terminal/terminal-webview-handle.ts new file mode 100644 index 000000000000..db356014e54d --- /dev/null +++ b/mobile/src/terminal/terminal-webview-handle.ts @@ -0,0 +1,140 @@ +import type { RefObject } from 'react' +import { PixelRatio, Platform } from 'react-native' +import type { MobileTerminalTheme, TerminalWebViewHandle } from './terminal-webview-contract' +import type { TerminalWebViewCommand } from './terminal-webview-messages' +import type { createTerminalWriteCoalescer } from './terminal-write-coalescer' + +type ViewportDimensions = { cols: number; rows: number } + +export function createTerminalWebViewHandle({ + armWebReadyWatchdog, + isWebReadyRef, + measureResolveRef, + pendingPingIdRef, + postMessage, + readyPromiseRef, + readyResolveRef, + sendToWebView, + terminalTheme, + textScale, + writeCoalescer +}: { + armWebReadyWatchdog: () => void + isWebReadyRef: RefObject + measureResolveRef: RefObject<((result: ViewportDimensions | null) => void) | null> + pendingPingIdRef: RefObject + postMessage: (message: TerminalWebViewCommand) => void + readyPromiseRef: RefObject | null> + readyResolveRef: RefObject<(() => void) | null> + sendToWebView: (message: TerminalWebViewCommand) => number + terminalTheme?: MobileTerminalTheme + textScale: number + writeCoalescer: ReturnType +}): TerminalWebViewHandle { + return { + prepareForForegroundRecovery() { + if (Platform.OS !== 'ios') { + return + } + isWebReadyRef.current = false + armWebReadyWatchdog() + pendingPingIdRef.current = sendToWebView({ type: 'ping' }) + }, + write(data: string) { + writeCoalescer.write(data) + }, + init( + cols: number, + rows: number, + initialData?: string, + preserveScroll?: boolean, + oscLinks?: Parameters[4] + ) { + const priorResolve = readyResolveRef.current + if (priorResolve) { + readyResolveRef.current = null + readyPromiseRef.current = null + priorResolve() + } + readyPromiseRef.current = new Promise((resolve) => { + readyResolveRef.current = resolve + }) + writeCoalescer.clear() + postMessage({ + type: 'init', + cols, + rows, + initialData, + oscLinks, + terminalTheme, + fontScale: textScale, + preserveScroll + }) + }, + resize(cols: number, rows: number) { + writeCoalescer.flushNow() + postMessage({ type: 'resize', cols, rows }) + }, + reflow(cols: number, rows: number) { + writeCoalescer.flushNow() + postMessage({ type: 'reflow', cols, rows }) + }, + clear() { + writeCoalescer.clear() + postMessage({ type: 'clear' }) + }, + measureFitDimensions(containerHeight, containerWidth) { + if (!isWebReadyRef.current) { + return Promise.resolve(null) + } + return new Promise((resolve) => { + measureResolveRef.current?.(null) + let timeout: ReturnType | null = null + const finish = (result: ViewportDimensions | null) => { + if (timeout) { + clearTimeout(timeout) + } + timeout = null + if (measureResolveRef.current === finish) { + measureResolveRef.current = null + } + resolve(result) + } + measureResolveRef.current = finish + sendToWebView({ type: 'measure', containerHeight, containerWidth }) + timeout = setTimeout(() => { + if (measureResolveRef.current === finish) { + finish(null) + } + }, 2000) + }) + }, + setViewport(width, height) { + postMessage({ type: 'set-viewport', width, height, dpr: PixelRatio.get() }) + }, + resetZoom: () => postMessage({ type: 'reset-zoom' }), + cancelSelect: () => postMessage({ type: 'cancel-select' }), + doSelectAll: () => postMessage({ type: 'do-select-all' }), + async awaitReady() { + const pending = readyPromiseRef.current + if (!pending) { + return + } + await new Promise((resolve) => { + let settled = false + const timeout = setTimeout(() => { + settled = true + resolve() + }, 3000) + void pending.finally(() => { + if (settled) { + return + } + clearTimeout(timeout) + settled = true + resolve() + }) + }) + } + } +} diff --git a/mobile/src/terminal/terminal-webview-html.ts b/mobile/src/terminal/terminal-webview-html.ts index 8d79ecfab673..249be275df3b 100644 --- a/mobile/src/terminal/terminal-webview-html.ts +++ b/mobile/src/terminal/terminal-webview-html.ts @@ -15,6 +15,7 @@ import { TERMINAL_WEBGL_RECOVERY_JS } from './terminal-webview-webgl-recovery-in import { TERMINAL_MOUSE_CLICK_DRAG_JS } from './terminal-webview-mouse-click-drag-injected' import { TERMINAL_MOUSE_REPORT_CELL_JS } from './terminal-webview-mouse-report-cell-injected' import { TERMINAL_WHEEL_SCROLL_JS } from './terminal-webview-wheel-scroll-injected' +import { TERMINAL_VIEWPORT_JS } from './terminal-webview-viewport-injected' const DEFAULT_TERMINAL_THEME: RuntimeMobileTerminalTheme['theme'] = { background: colors.terminalBg, @@ -268,11 +269,8 @@ window.onerror = function(msg) { // fall to a non-monospace face; lead with the ui-monospace generic to avoid that. var TERMINAL_FONT_FALLBACKS = '"Menlo", "Monaco", "Cascadia Mono", "Consolas", "DejaVu Sans Mono", "Liberation Mono", "Symbols Nerd Font Mono", monospace'; var terminalFontFamily = (isIOSWebView() ? 'ui-monospace, ' : '"SF Mono", ') + TERMINAL_FONT_FALLBACKS; - // Why: change the real font size, then resize the grid to fit the viewport at - // the new cell metrics so the text shows at its true size immediately. RN's - // refit (measure → updateViewport) then makes the server reflow the PTY to the - // same column count so the shell rewraps. cell metrics update on the frame - // after fontSize changes, so the resize/fit is deferred one rAF. + // Why: RN owns PTY sizing; changing only the font avoids a transient local + // grid that disagrees with the server while the refit is in flight. function applyTextScale(scale) { currentTextScale = scale; if (!term) return; @@ -281,15 +279,6 @@ window.onerror = function(msg) { term.options.fontSize = px; requestAnimationFrame(function() { if (!term) return; - var cellW = getCellWidth(); - var cellH = getCellHeight(); - if (cellW > 0 && cellH > 0) { - var cols = Math.floor(window.innerWidth / cellW); - if (cols < MIN_FIT_COLS) return; - var rows = Math.max(8, Math.floor(window.innerHeight / cellH)); - term.resize(cols, rows); - emitKeyboardAvoidanceMetrics(); - } applyFitScale('text-scale'); }); } @@ -318,6 +307,7 @@ window.onerror = function(msg) { // once when the first live data chunk arrives so a wider line that pushes // scrollWidth past the previously-measured value gets re-scaled to fit. var firstDataPending = false; + ${TERMINAL_VIEWPORT_JS} // Diagnostic logger — bridges WebView console.log to RN via postMessage. // Tag with [fit] so it's easy to filter in the Expo/Metro logs. @@ -355,7 +345,7 @@ window.onerror = function(msg) { var cellW = getCellWidth(); var termWidth = cellW > 0 ? cellW * term.cols : (term.element ? term.element.scrollWidth : 0); if (termWidth <= 0) return 1; - var vpWidth = window.innerWidth; + var vpWidth = getViewportWidth(); return Math.min(1, vpWidth / termWidth); } @@ -375,7 +365,7 @@ window.onerror = function(msg) { scrollIndicator.classList.remove('visible'); return; } - var trackHeight = Math.max(0, window.innerHeight - 8); + var trackHeight = Math.max(0, getViewportHeight() - 8); var totalRows = maxViewportY + (term.rows || 0); if (trackHeight <= 0 || totalRows <= 0) return; var thumbHeight = Math.max(24, trackHeight * (term.rows || 0) / totalRows); @@ -411,8 +401,8 @@ ${TERMINAL_WEBVIEW_THEME_JS} var ts = getTotalScale(); var cw = term.element.scrollWidth * ts; var ch = term.element.scrollHeight * ts; - var vpW = window.innerWidth; - var vpH = window.innerHeight; + var vpW = getViewportWidth(); + var vpH = getViewportHeight(); if (cw > vpW) { panX = Math.min(0, Math.max(vpW - cw, panX)); } else { @@ -497,7 +487,7 @@ ${TERMINAL_WEBVIEW_THEME_JS} var cellW = getCellWidth(); var sw = term.element.scrollWidth; - var vpW = window.innerWidth; + var vpW = getViewportWidth(); var expectedW = cellW * term.cols; var suspect = currentScale === 1 && term.cols > 0 && expectedW > vpW + 1; // expected wider than viewport but no zoom @@ -860,7 +850,7 @@ ${TERMINAL_WEBGL_RECOVERY_JS} reportEngineError('terminal runtime error', err || msg); }; - function measureFitDimensions(containerHeightPx, retriesLeft) { + function measureFitDimensions(containerHeightPx, containerWidthPx, retriesLeft) { if (typeof retriesLeft !== 'number') retriesLeft = 30; // Why: init and measure are posted back-to-back from React, but // init has an async rAF chain. A measure that runs synchronously @@ -879,7 +869,7 @@ ${TERMINAL_WEBGL_RECOVERY_JS} if (notReady || cellWidth <= 0 || cellHeight <= 0) { if (retriesLeft > 0) { requestAnimationFrame(function() { - measureFitDimensions(containerHeightPx, retriesLeft - 1); + measureFitDimensions(containerHeightPx, containerWidthPx, retriesLeft - 1); }); return; } @@ -892,15 +882,24 @@ ${TERMINAL_WEBGL_RECOVERY_JS} notify({ type: 'measure-result', cols: null, rows: null }); return; } - var vpWidth = window.innerWidth; + if (typeof containerWidthPx === 'number' && containerWidthPx > 0 && + Math.abs(containerWidthPx - window.innerWidth) > 1 && retriesLeft > 0) { + requestAnimationFrame(function() { + measureFitDimensions(containerHeightPx, containerWidthPx, retriesLeft - 1); + }); + return; + } + var vpWidth = (typeof containerWidthPx === 'number' && containerWidthPx > 0) + ? containerWidthPx + : getViewportWidth(); // Why: prefer the container height passed from React Native over - // window.innerHeight. The RN layout system knows the exact pixel + // the WebView viewport. The RN layout system knows the exact pixel // height of the terminal frame after the accessory/input bars are // subtracted, whereas innerHeight can overstate the visible area // due to layout timing or safe-area insets. var vpHeight = (typeof containerHeightPx === 'number' && containerHeightPx > 0) ? containerHeightPx - : window.innerHeight; + : getViewportHeight(); var cols = Math.floor(vpWidth / cellWidth); if (cols < MIN_FIT_COLS) { flog('measure-skip-small-width', { @@ -942,6 +941,8 @@ ${TERMINAL_WEBGL_RECOVERY_JS} panY = 0; applyTextScale(msg.fontScale); } + } else if (msg.type === 'set-viewport') { + setRnViewport(msg.width, msg.height, msg.dpr); } else if (msg.type === 'resize') { resize(msg.cols, msg.rows); } else if (msg.type === 'reflow') { reflow(msg.cols, msg.rows); @@ -969,7 +970,7 @@ ${TERMINAL_WEBGL_RECOVERY_JS} cancelSelect(); } } else if (msg.type === 'measure') { - measureFitDimensions(msg.containerHeight); + measureFitDimensions(msg.containerHeight, msg.containerWidth); } else if (msg.type === 'reset-zoom') { applyFitScale('reset-zoom-msg'); } else if (msg.type === 'set-theme') { @@ -1520,8 +1521,8 @@ ${TERMINAL_WEBGL_RECOVERY_JS} handleStart.style.top = sPx.y + 'px'; handleEnd.style.left = ePx.x + 'px'; handleEnd.style.top = (ePx.y + cellH) + 'px'; - var startVisible = sPx.y >= 0 && sPx.y <= window.innerHeight; - var endVisible = ePx.y >= 0 && ePx.y <= window.innerHeight; + var startVisible = sPx.y >= 0 && sPx.y <= getViewportHeight(); + var endVisible = ePx.y >= 0 && ePx.y <= getViewportHeight(); handleStart.style.visibility = startVisible ? 'visible' : 'hidden'; handleEnd.style.visibility = endVisible ? 'visible' : 'hidden'; var menuCenterX, menuY, vTransform, marginTop; @@ -1529,14 +1530,14 @@ ${TERMINAL_WEBGL_RECOVERY_JS} menuCenterX = sPx.x; menuY = sPx.y; vTransform = 'translateY(-100%)'; marginTop = '-12px'; - } else if (endVisible && ePx.y + cellH + 56 < window.innerHeight) { + } else if (endVisible && ePx.y + cellH + 56 < getViewportHeight()) { menuCenterX = ePx.x; menuY = ePx.y + cellH; vTransform = 'translateY(0)'; marginTop = '12px'; } else { // selection covers full viewport — pin to visible center - menuCenterX = window.innerWidth / 2; - menuY = window.innerHeight / 2; + menuCenterX = getViewportWidth() / 2; + menuY = getViewportHeight() / 2; vTransform = 'translateY(-50%)'; marginTop = '0'; } @@ -1550,7 +1551,7 @@ ${TERMINAL_WEBGL_RECOVERY_JS} var EDGE_MARGIN = 8; var menuW = selMenu.offsetWidth || 0; var minLeft = EDGE_MARGIN; - var maxLeft = Math.max(EDGE_MARGIN, window.innerWidth - menuW - EDGE_MARGIN); + var maxLeft = Math.max(EDGE_MARGIN, getViewportWidth() - menuW - EDGE_MARGIN); var desiredLeft = menuCenterX - menuW / 2; var clampedLeft = Math.max(minLeft, Math.min(maxLeft, desiredLeft)); selMenu.style.left = clampedLeft + 'px'; @@ -1609,7 +1610,7 @@ ${TERMINAL_WEBGL_RECOVERY_JS} if (!syncSelectionHandleToViewportPoint(handle, clientX, clientY)) return; repositionOverlay(); if (clientY < EDGE_SCROLL_PX) startEdgeScroll(-1); - else if (clientY > window.innerHeight - EDGE_SCROLL_PX) startEdgeScroll(1); + else if (clientY > getViewportHeight() - EDGE_SCROLL_PX) startEdgeScroll(1); else stopEdgeScroll(); } @@ -1745,7 +1746,7 @@ ${TERMINAL_WEBGL_RECOVERY_JS} // scroll so scrollback stays reachable at any text size; calling the // never-defined contentWiderThanViewport() here threw and killed all // single-finger scrolling, scrollback included. - if (term.element && term.element.scrollWidth * getTotalScale() > window.innerWidth + 1) { + if (term.element && term.element.scrollWidth * getTotalScale() > getViewportWidth() + 1) { panX += x - ts.lastX; clampPan(); updateTransform(); @@ -1869,6 +1870,7 @@ ${TERMINAL_WEBGL_RECOVERY_JS} repositionOverlay(); clampPan(); updateTransform(); + notifyViewportChanged(); }); if (window.Terminal) { diff --git a/mobile/src/terminal/terminal-webview-messages.ts b/mobile/src/terminal/terminal-webview-messages.ts index 35c6e991b68d..18cec8bc3b95 100644 --- a/mobile/src/terminal/terminal-webview-messages.ts +++ b/mobile/src/terminal/terminal-webview-messages.ts @@ -18,10 +18,11 @@ export type TerminalWebViewCommand = preserveScroll?: boolean } | { type: 'set-font-scale'; id?: number; fontScale: number } + | { type: 'set-viewport'; id?: number; width: number; height: number; dpr: number } | { type: 'resize'; id?: number; cols: number; rows: number } | { type: 'reflow'; id?: number; cols: number; rows: number } | { type: 'clear'; id?: number } - | { type: 'measure'; id?: number; containerHeight?: number } + | { type: 'measure'; id?: number; containerHeight?: number; containerWidth?: number } | { type: 'reset-zoom'; id?: number } | { type: 'cancel-select'; id?: number } | { type: 'do-select-all'; id?: number } diff --git a/mobile/src/terminal/terminal-webview-notification-dispatch.test.ts b/mobile/src/terminal/terminal-webview-notification-dispatch.test.ts index bcd777a2e8f3..dba6cfb9ceba 100644 --- a/mobile/src/terminal/terminal-webview-notification-dispatch.test.ts +++ b/mobile/src/terminal/terminal-webview-notification-dispatch.test.ts @@ -36,4 +36,13 @@ describe('dispatchTerminalWebViewNotification', () => { altScreen: false }) }) + + it('routes measured WebView viewport changes', () => { + const onViewportChanged = vi.fn() + dispatchTerminalWebViewNotification( + { type: 'viewport-changed', innerWidth: 800, innerHeight: 600, dpr: 1.5 }, + { onViewportChanged, reportEngineError: vi.fn() } + ) + expect(onViewportChanged).toHaveBeenCalledWith(800, 600, 1.5) + }) }) diff --git a/mobile/src/terminal/terminal-webview-notification-dispatch.ts b/mobile/src/terminal/terminal-webview-notification-dispatch.ts index 77019d3f9965..cc1b4dfc6297 100644 --- a/mobile/src/terminal/terminal-webview-notification-dispatch.ts +++ b/mobile/src/terminal/terminal-webview-notification-dispatch.ts @@ -77,6 +77,13 @@ export function dispatchTerminalWebViewNotification( if (scale > 0) { handlers.onTextScaleChange?.(scale) } + } else if (msg.type === 'viewport-changed') { + const width = typeof msg.innerWidth === 'number' ? msg.innerWidth : 0 + const height = typeof msg.innerHeight === 'number' ? msg.innerHeight : 0 + const dpr = typeof msg.dpr === 'number' ? msg.dpr : 1 + if (width > 0 && height > 0) { + handlers.onViewportChanged?.(width, height, dpr) + } } else if (msg.type === 'mobile-clip-cancel-by-pinch') { // eslint-disable-next-line no-console console.warn('[mobile-clip] selection cancelled by pinch') diff --git a/mobile/src/terminal/terminal-webview-reflow.test.ts b/mobile/src/terminal/terminal-webview-reflow.test.ts index 5fb78bb33ec4..2e9ebf4091fc 100644 --- a/mobile/src/terminal/terminal-webview-reflow.test.ts +++ b/mobile/src/terminal/terminal-webview-reflow.test.ts @@ -10,7 +10,9 @@ const reflowSource = readFileSync( 'utf8' ) const htmlSource = readFileSync(new URL('./terminal-webview-html.ts', import.meta.url), 'utf8') -const handleSource = readFileSync(new URL('./TerminalWebView.tsx', import.meta.url), 'utf8') +const handleSource = + readFileSync(new URL('./TerminalWebView.tsx', import.meta.url), 'utf8') + + readFileSync(new URL('./terminal-webview-handle.ts', import.meta.url), 'utf8') function reflowFnBody(): string { const start = reflowSource.indexOf('function reflow(cols, rows) {') @@ -51,7 +53,7 @@ describe('terminal WebView reflow', () => { it('does not locally resize hidden WebViews to a one-column grid', () => { expect(htmlSource).toContain('var MIN_FIT_COLS = 20;') - expect(htmlSource).toContain('if (cols < MIN_FIT_COLS) return;') + expect(htmlSource).toContain('if (cols < MIN_FIT_COLS) {') expect(htmlSource).toContain("flog('measure-skip-small-width'") expect(htmlSource).toContain("notify({ type: 'measure-result', cols: null, rows: null });") }) diff --git a/mobile/src/terminal/terminal-webview-scroll-routing.test.ts b/mobile/src/terminal/terminal-webview-scroll-routing.test.ts index 32bc6dbd5294..81f0276c7fb2 100644 --- a/mobile/src/terminal/terminal-webview-scroll-routing.test.ts +++ b/mobile/src/terminal/terminal-webview-scroll-routing.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it } from 'vitest' // TerminalWebView.tsx. Concatenate both so assertions resolve regardless of file. const source = readFileSync(new URL('./TerminalWebView.tsx', import.meta.url), 'utf8') + + readFileSync(new URL('./terminal-webview-handle.ts', import.meta.url), 'utf8') + readFileSync(new URL('./terminal-webview-pending-messages.ts', import.meta.url), 'utf8') + readFileSync(new URL('./terminal-webview-url-tap.ts', import.meta.url), 'utf8') + readFileSync(new URL('./terminal-webview-tap-dispatch-injected.ts', import.meta.url), 'utf8') + @@ -130,13 +131,12 @@ describe('TerminalWebView scroll routing', () => { }) it('clears WebView await timers when the real response wins', () => { - const measureBlock = sliceBetween('measureFitDimensions(', 'resetZoom()') + const measureBlock = sliceBetween('measureFitDimensions(', 'resetZoom:') expect(measureBlock).toContain('clearTimeout(timeout)') expect(measureBlock).toContain('measureResolveRef.current === finish') - const readyBlock = sliceBetween('async awaitReady()', '})') - expect(readyBlock).toContain('clearTimeout(timeout)') - expect(readyBlock).toContain('void p.finally') + expect(source).toContain('async awaitReady()') + expect(source).toContain('void pending.finally') }) it('hides xterm scrollbars and drives the mobile scroll indicator from committed rows', () => { diff --git a/mobile/src/terminal/terminal-webview-text-zoom.test.ts b/mobile/src/terminal/terminal-webview-text-zoom.test.ts index cf16a93c0e6a..52ea11ab278c 100644 --- a/mobile/src/terminal/terminal-webview-text-zoom.test.ts +++ b/mobile/src/terminal/terminal-webview-text-zoom.test.ts @@ -60,6 +60,14 @@ output = terminalFontFamily; } describe('TerminalWebView text zoom', () => { + it('lets the RN refit own grid resizing after a text-scale change', () => { + const start = terminalHtmlSource.indexOf(' function applyTextScale(scale)') + const end = terminalHtmlSource.indexOf(' var panX =', start) + const applyTextScale = terminalHtmlSource.slice(start, end) + expect(applyTextScale).toContain("applyFitScale('text-scale')") + expect(applyTextScale).not.toContain('term.resize(') + }) + it('pins textZoom to 100 so Android system font scale cannot inflate glyphs past xterm cell metrics', () => { const start = terminalWebViewSource.indexOf(' 0 ? rnViewport.width : window.innerWidth; + } + + function getViewportHeight() { + return rnViewport && rnViewport.height > 0 ? rnViewport.height : window.innerHeight; + } + + function notifyViewportChanged() { + notify({ + type: 'viewport-changed', + innerWidth: window.innerWidth, + innerHeight: window.innerHeight, + dpr: window.devicePixelRatio || 1 + }); + } + + function handleDevicePixelRatioChange() { + try { + var renderService = term && term._core && term._core._renderService; + if (renderService && renderService.handleDevicePixelRatioChange) { + renderService.handleDevicePixelRatioChange(); + } else if (term) { + if (webglAddon) { + try { webglAddon.dispose(); } catch (e) {} + webglAddon = null; + } + attachWebglAddon(true); + } + } catch (e) {} + applyFitScale('dpr'); + notifyViewportChanged(); + observeDevicePixelRatio(); + } + + function observeDevicePixelRatio() { + if (!window.matchMedia) return; + if (dprMediaQuery) { + if (dprMediaQuery.removeEventListener) dprMediaQuery.removeEventListener('change', handleDevicePixelRatioChange); + else if (dprMediaQuery.removeListener) dprMediaQuery.removeListener(handleDevicePixelRatioChange); + } + dprMediaQuery = window.matchMedia('(resolution: ' + (window.devicePixelRatio || 1) + 'dppx)'); + if (dprMediaQuery.addEventListener) dprMediaQuery.addEventListener('change', handleDevicePixelRatioChange, { once: true }); + else if (dprMediaQuery.addListener) dprMediaQuery.addListener(handleDevicePixelRatioChange); + } + + function setRnViewport(width, height, dpr) { + if (!(width > 0) || !(height > 0)) return; + rnViewport = { width: width, height: height }; + if (typeof dpr === 'number' && dpr > 0 && Math.abs(dpr - rnDpr) > 0.001) { + rnDpr = dpr; + handleDevicePixelRatioChange(); + } + applyFitScale('rn-viewport'); + clampPan(); + updateTransform(); + notifyViewportChanged(); + } + + observeDevicePixelRatio(); +` diff --git a/mobile/src/terminal/terminal-webview-viewport.test.ts b/mobile/src/terminal/terminal-webview-viewport.test.ts new file mode 100644 index 000000000000..45e6995457d0 --- /dev/null +++ b/mobile/src/terminal/terminal-webview-viewport.test.ts @@ -0,0 +1,64 @@ +import { Script } from 'node:vm' +import { describe, expect, it, vi } from 'vitest' +import { TERMINAL_VIEWPORT_JS } from './terminal-webview-viewport-injected' + +function createViewportHarness() { + let mediaListener = () => {} + const notify = vi.fn() + const applyFitScale = vi.fn() + const context = { + applyFitScale, + attachWebglAddon: vi.fn(), + clampPan: vi.fn(), + notify, + term: null, + updateTransform: vi.fn(), + webglAddon: null, + window: { + devicePixelRatio: 1, + innerHeight: 1080, + innerWidth: 1920, + matchMedia: vi.fn(() => ({ + addEventListener: (_type: string, listener: () => void) => { + mediaListener = listener + }, + removeEventListener: vi.fn() + })) + } + } + new Script(`${TERMINAL_VIEWPORT_JS} +this.setViewport = setRnViewport; +this.getWidth = getViewportWidth; +this.getHeight = getViewportHeight; +`).runInNewContext(context) + return { + applyFitScale, + fireDprChange: () => mediaListener(), + setDpr: (value: number) => { + context.window.devicePixelRatio = value + }, + getHeight: (context as { getHeight: () => number }).getHeight, + getWidth: (context as { getWidth: () => number }).getWidth, + notify, + setViewport: (context as { setViewport: (width: number, height: number) => void }).setViewport + } +} + +describe('terminal WebView viewport bridge', () => { + it('prefers RN root bounds over display-sized WebView fallbacks', () => { + const harness = createViewportHarness() + harness.setViewport(800, 600) + expect(harness.getWidth()).toBe(800) + expect(harness.getHeight()).toBe(600) + }) + + it('re-fits and reports viewport state when DPR changes', () => { + const harness = createViewportHarness() + harness.setDpr(2) + harness.fireDprChange() + expect(harness.applyFitScale).toHaveBeenCalledWith('dpr') + expect(harness.notify).toHaveBeenCalledWith( + expect.objectContaining({ type: 'viewport-changed', dpr: 2 }) + ) + }) +}) diff --git a/mobile/src/terminal/terminal-write-coalescer-boundaries.test.ts b/mobile/src/terminal/terminal-write-coalescer-boundaries.test.ts index eb7f19b55001..c70f48d24dfd 100644 --- a/mobile/src/terminal/terminal-write-coalescer-boundaries.test.ts +++ b/mobile/src/terminal/terminal-write-coalescer-boundaries.test.ts @@ -7,7 +7,9 @@ import { TERMINAL_WRITE_FLUSH_WINDOW_MS } from './terminal-write-coalescer' -const webViewSource = readFileSync(new URL('./TerminalWebView.tsx', import.meta.url), 'utf8') +const webViewSource = + readFileSync(new URL('./TerminalWebView.tsx', import.meta.url), 'utf8') + + readFileSync(new URL('./terminal-webview-handle.ts', import.meta.url), 'utf8') // Simulates TerminalWebView's postMessage: ready → deliver, not ready → queue. // There is no React render harness in the node environment, so the boundary diff --git a/mobile/src/terminal/use-terminal-layout-refit-triggers.ts b/mobile/src/terminal/use-terminal-layout-refit-triggers.ts new file mode 100644 index 000000000000..e19da706acf3 --- /dev/null +++ b/mobile/src/terminal/use-terminal-layout-refit-triggers.ts @@ -0,0 +1,122 @@ +import { useCallback, useEffect, useRef, type RefObject } from 'react' +import { useWindowBounds } from '../layout/window-bounds' +import { + areTerminalViewportWidthsAligned, + reduceTerminalFrameHeightRefit, + resetTerminalKeyboardVisibilityForWidthChange, + type TerminalFrameHeightRefitEvent, + type TerminalFrameHeightRefitState +} from './terminal-viewport-refit-state' + +export function useTerminalLayoutRefitTriggers({ + frameHeightStateRef, + scheduleViewportRefit, + tabStripVisible, + terminalFrameWidth, + textScale, + viewportMeasuredRef, + webViewViewportRef +}: { + frameHeightStateRef: RefObject + scheduleViewportRefit: (options?: { heightOriginated?: boolean }) => void + tabStripVisible: boolean + terminalFrameWidth: number + textScale: number + viewportMeasuredRef: RefObject + webViewViewportRef: RefObject<{ width: number; height: number } | null> +}) { + const previousTabStripVisibleRef = useRef(tabStripVisible) + useEffect(() => { + if (previousTabStripVisibleRef.current === tabStripVisible) { + return + } + previousTabStripVisibleRef.current = tabStripVisible + viewportMeasuredRef.current = false + scheduleViewportRefit() + }, [scheduleViewportRefit, tabStripVisible, viewportMeasuredRef]) + + const { width: windowWidth, height: windowHeight } = useWindowBounds() + const previousWindowBoundsRef = useRef({ width: windowWidth, height: windowHeight }) + useEffect(() => { + const previous = previousWindowBoundsRef.current + if (previous.width === windowWidth && previous.height === windowHeight) { + return + } + previousWindowBoundsRef.current = { width: windowWidth, height: windowHeight } + frameHeightStateRef.current = resetTerminalKeyboardVisibilityForWidthChange( + frameHeightStateRef.current, + previous.width, + windowWidth + ) + // Why: a height-only change while the IME is up is the keyboard itself; width changes always refit. + if (previous.width === windowWidth && frameHeightStateRef.current.keyboardVisible) { + return + } + viewportMeasuredRef.current = false + scheduleViewportRefit() + }, [frameHeightStateRef, scheduleViewportRefit, viewportMeasuredRef, windowHeight, windowWidth]) + + const previousTextScaleRef = useRef(textScale) + useEffect(() => { + if (previousTextScaleRef.current === textScale) { + return + } + previousTextScaleRef.current = textScale + viewportMeasuredRef.current = false + scheduleViewportRefit() + }, [scheduleViewportRefit, textScale, viewportMeasuredRef]) + + const previousFrameWidthRef = useRef(terminalFrameWidth) + useEffect(() => { + if (previousFrameWidthRef.current === terminalFrameWidth) { + return + } + previousFrameWidthRef.current = terminalFrameWidth + viewportMeasuredRef.current = false + scheduleViewportRefit() + }, [scheduleViewportRefit, terminalFrameWidth, viewportMeasuredRef]) + + const notifyFrameHeightEvent = useCallback( + (event: TerminalFrameHeightRefitEvent) => { + const transition = reduceTerminalFrameHeightRefit(frameHeightStateRef.current, event) + frameHeightStateRef.current = transition.state + if (!transition.shouldRefit) { + return + } + viewportMeasuredRef.current = false + scheduleViewportRefit({ heightOriginated: true }) + }, + [frameHeightStateRef, scheduleViewportRefit, viewportMeasuredRef] + ) + const notifyTerminalFrameHeight = useCallback( + (height: number) => notifyFrameHeightEvent({ type: 'frame-height', height }), + [notifyFrameHeightEvent] + ) + const notifyKeyboardVisibility = useCallback( + (visible: boolean) => notifyFrameHeightEvent({ type: 'keyboard-visibility', visible }), + [notifyFrameHeightEvent] + ) + // Why: a hardware keyboard means no IME is covering the PTY; route through the reducer so a refit deferred while the IME was up flushes now. + const notifyHardwareKeyboard = useCallback( + () => notifyFrameHeightEvent({ type: 'keyboard-visibility', visible: false }), + [notifyFrameHeightEvent] + ) + const notifyWebViewViewport = useCallback( + (width: number, height: number) => { + webViewViewportRef.current = { width, height } + if (!areTerminalViewportWidthsAligned(terminalFrameWidth, width)) { + return + } + viewportMeasuredRef.current = false + scheduleViewportRefit() + }, + [scheduleViewportRefit, terminalFrameWidth, viewportMeasuredRef, webViewViewportRef] + ) + + return { + notifyHardwareKeyboard, + notifyKeyboardVisibility, + notifyTerminalFrameHeight, + notifyWebViewViewport + } +} diff --git a/mobile/src/terminal/use-terminal-live-input-commit.ts b/mobile/src/terminal/use-terminal-live-input-commit.ts index 952612f0a870..45e7e1194cac 100644 --- a/mobile/src/terminal/use-terminal-live-input-commit.ts +++ b/mobile/src/terminal/use-terminal-live-input-commit.ts @@ -38,6 +38,7 @@ type TerminalLiveInputCommitOptions = { readonly liveInputTerminalHandlesRef: RefObject> readonly sendLiveTerminalInputRef: RefObject readonly setLiveInputCapture: (text: string) => void + readonly onKeyPressObserved?: () => void } type TerminalLiveInputCommitHandlers = { @@ -61,7 +62,8 @@ export function useTerminalLiveInputCommit({ liveInputTerminalHandles, liveInputTerminalHandlesRef, sendLiveTerminalInputRef, - setLiveInputCapture + setLiveInputCapture, + onKeyPressObserved }: TerminalLiveInputCommitOptions): TerminalLiveInputCommitHandlers { const { applyLiveInputMirror, @@ -150,6 +152,7 @@ export function useTerminalLiveInputCommit({ const handleLiveInputKeyPress = useCallback( (event: TerminalLiveInputKeyPressEvent) => { + onKeyPressObserved?.() if (!activeHandle || !liveInputTerminalHandles.has(activeHandle)) { return } @@ -186,6 +189,7 @@ export function useTerminalLiveInputCommit({ clearPendingLiveInputCommit, flushPendingLiveInputText, liveInputTerminalHandles, + onKeyPressObserved, sendLiveTerminalInputRef, waitForPendingLiveInputFlush ] diff --git a/mobile/src/terminal/use-terminal-live-input-focus.ts b/mobile/src/terminal/use-terminal-live-input-focus.ts index b0ab456b721a..23522dddd4c7 100644 --- a/mobile/src/terminal/use-terminal-live-input-focus.ts +++ b/mobile/src/terminal/use-terminal-live-input-focus.ts @@ -10,6 +10,7 @@ import { type TerminalLiveInputFocusContext = { readonly canSend: boolean readonly keyboardHeight: number + readonly hardwareKeyboard?: boolean readonly liveInputEnabled: boolean } @@ -33,6 +34,7 @@ export function useTerminalLiveInputFocus): TerminalLiveInputFocusHandlers { const contextRef = useRef({ canSend, + hardwareKeyboard, keyboardHeight, liveInputEnabled }) useLayoutEffect(() => { - contextRef.current = { canSend, keyboardHeight, liveInputEnabled } - }, [canSend, keyboardHeight, liveInputEnabled]) + contextRef.current = { canSend, hardwareKeyboard, keyboardHeight, liveInputEnabled } + }, [canSend, hardwareKeyboard, keyboardHeight, liveInputEnabled]) const resetLiveInputFocus = useCallback(() => { clearTerminalLiveInputFocusTimer(timerRef) @@ -62,6 +65,7 @@ export function useTerminalLiveInputFocus scheduleTerminalLiveInputFocus(timerRef, focusLiveInput) }) }, [inputRef, timerRef]) diff --git a/mobile/vitest.config.ts b/mobile/vitest.config.ts index 981a1b55e740..019554504907 100644 --- a/mobile/vitest.config.ts +++ b/mobile/vitest.config.ts @@ -13,6 +13,6 @@ export default defineConfig({ onConsoleLog: (log) => !log.includes('react-test-renderer is deprecated'), // .tsx too: component tests exist (react-test-renderer + mocked react-native) and were // silently never collected, so render-level regressions shipped untested. - include: ['src/**/*.test.ts', 'src/**/*.test.tsx'] + include: ['src/**/*.test.ts', 'src/**/*.test.tsx', 'plugins/**/*.test.js'] } })