fix(devices): recover stalled video without losing simulator input

apps/mobile/src/features/devices/device-stream.browser.test.ts

@@ -38,6 +38,7 @@ async function setup() {
     static OPEN = 1;
     readyState = 1;
     onopen: (() => void) | null = null;
+    onmessage: ((event: { data: ArrayBuffer }) => void) | null = null;
     close = vi.fn();
     send = vi.fn();
     constructor() {
@@ -86,6 +87,8 @@ it("bridges shared first-frame readiness, image failure, and a successful fresh
   const { elements, messages, sockets, configuration } = await setup();
   sockets[0]!.onopen?.();
   expect(messages()).not.toContainEqual({ type: "status", status: "streaming" });
+  expect(messages()).not.toContainEqual({ type: "input", connected: true });
+  sockets[0]!.onmessage?.({ data: new Uint8Array([0x83]).buffer });
   expect(messages()).toContainEqual({ type: "input", connected: true });
   const image = elements.find((element) => element.tag === "img")!;
   image.naturalWidth = 400;

apps/server/src/device/DeviceToolchain.ts

@@ -26,9 +26,9 @@ import * as Semaphore from "effect/Semaphore";
 import * as ProcessRunner from "../processRunner.ts";
 
 const DEVICE_HUB_PACKAGE = "expo-device-hub";
-export const DEVICE_HUB_VERSION = "0.12.0";
+export const DEVICE_HUB_VERSION = "0.15.3";
 const AGENT_DEVICE_PACKAGE = "agent-device";
-export const AGENT_DEVICE_VERSION = "0.21.12";
+export const AGENT_DEVICE_VERSION = "0.21.23";
 
 const INSTALL_TIMEOUT = Duration.minutes(10);
 const installLock = Semaphore.makeUnsafe(1);

apps/web/src/components/device/DeviceStreamView.test.tsx

@@ -1,4 +1,4 @@
-import { act, useSyncExternalStore } from "react";
+import { act, useEffect, useSyncExternalStore } from "react";
 import { create, type ReactTestRenderer } from "react-test-renderer";
 import { EnvironmentId } from "@t3tools/contracts";
 import { afterEach, beforeEach, expect, it, vi } from "vite-plus/test";
@@ -20,6 +20,20 @@ vi.mock("~/state/device", () => ({
   useDeviceHubAccess: () => useSyncExternalStore(accessStore.subscribe, () => accessStore.value),
   refreshDeviceHubAccess: () => accessStore.refresh(),
 }));
+// Replace GPU allocation while keeping the real React and stream lifecycles.
+let viewerMounts = 0;
+let viewerUnmounts = 0;
+vi.mock("./DevicePhoneViewport", () => ({
+  DevicePhoneViewport: function Viewer() {
+    useEffect(() => {
+      viewerMounts++;
+      return () => {
+        viewerUnmounts++;
+      };
+    }, []);
+    return null;
+  },
+}));
 import { DeviceStreamView } from "./DeviceStreamView";
 
 class Image extends EventTarget {
@@ -34,6 +48,8 @@ let renderer: ReactTestRenderer | undefined;
 let primes = 0;
 beforeEach(() => {
   primes = 0;
+  viewerMounts = 0;
+  viewerUnmounts = 0;
 });
 afterEach(async () => {
   await act(async () => renderer?.unmount());
@@ -42,18 +58,57 @@ afterEach(async () => {
   vi.unstubAllGlobals();
 });
 
-async function setup() {
+async function setup(h264 = false) {
   vi.useFakeTimers();
+  vi.stubGlobal("window", globalThis);
   vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true);
-  vi.stubGlobal("fetch", () => {
+  let videoBody: ReadableStreamDefaultController<Uint8Array> | undefined;
+  let output: VideoFrameOutputCallback | undefined;
+  if (h264) {
+    vi.stubGlobal(
+      "VideoDecoder",
+      class {
+        static isConfigSupported = async () => ({ supported: true });
+        state = "unconfigured";
+        constructor(callbacks: VideoDecoderInit) {
+          output = callbacks.output;
+        }
+        configure() {
+          this.state = "configured";
+        }
+        close() {
+          this.state = "closed";
+        }
+      },
+    );
+    vi.stubGlobal("EncodedVideoChunk", vi.fn());
+  }
+  vi.stubGlobal("fetch", (url: string, init: RequestInit) => {
+    if (url.endsWith("stream.avcc")) {
+      return Promise.resolve(
+        new Response(
+          new ReadableStream<Uint8Array>({
+            start(controller) {
+              videoBody = controller;
+              init.signal?.addEventListener("abort", () => controller.error(new Error("aborted")));
+            },
+          }),
+        ),
+      );
+    }
     primes++;
     return Promise.resolve(new Response("prime"));
   });
+  const sockets: Array<{ onmessage?: (event: { data: ArrayBuffer }) => void }> = [];
   vi.stubGlobal(
     "WebSocket",
     class {
       static OPEN = 1;
       readyState = 1;
+      constructor() {
+        sockets.push(this);
+      }
+      onmessage?: (event: { data: ArrayBuffer }) => void;
       send() {}
       close() {}
     },
@@ -73,6 +128,7 @@ async function setup() {
       deviceId="test"
       platform="ios"
       visible={visible}
+      allowPhoneView={h264}
     />
   );
   await act(async () => {
@@ -84,13 +140,30 @@ async function setup() {
           return image;
         }
         return {
+          getContext: () => ({ drawImage() {} }),
           style: { setProperty() {} },
           getBoundingClientRect: () => ({ width: 400, height: 800 }),
         };
       },
     });
   });
-  return { images, view };
+  return {
+    images,
+    view,
+    configure() {
+      const json = new TextEncoder().encode(
+        JSON.stringify({ width: 400, height: 800, orientation: "portrait" }),
+      );
+      const packet = new Uint8Array(1 + json.length);
+      packet[0] = 0x82;
+      packet.set(json, 1);
+      sockets[0]?.onmessage?.({ data: packet.buffer });
+      videoBody?.enqueue(new Uint8Array([0, 0, 0, 5, 1, 1, 0x64, 0, 0x1f]));
+    },
+    frame() {
+      output?.({ displayWidth: 400, displayHeight: 800, close() {} } as VideoFrame);
+    },
+  };
 }
 
 it("removes MJPEG requests while hidden and reconnects when shown", async () => {
@@ -135,3 +208,25 @@ it("starts exactly one new stream per Reconnect press", async () => {
   await act(async () => renderer!.root.findByType("button").props.onClick());
   expect(primes).toBe(2);
 });
+
+it("keeps the 3D viewer mounted across iOS video recovery and releases it when hidden", async () => {
+  const { configure, frame, view } = await setup(true);
+  await act(async () => configure());
+  expect(viewerMounts).toBe(0);
+  await act(async () => frame());
+  expect(viewerMounts).toBe(1);
+  await act(async () => {
+    await vi.advanceTimersByTimeAsync(15_000);
+  });
+  expect(viewerUnmounts).toBe(0);
+  expect(viewerMounts).toBe(1);
+  await act(async () => {
+    await vi.advanceTimersByTimeAsync(1_000);
+    configure();
+  });
+  await act(async () => frame());
+  expect(viewerMounts).toBe(1);
+  await act(async () => renderer!.update(view(false)));
+  expect(viewerUnmounts).toBe(1);
+  expect(vi.getTimerCount()).toBe(0);
+});

apps/web/src/components/device/DeviceStreamView.tsx

@@ -94,6 +94,7 @@ export function DeviceStreamView(props: {
   const canvasRef = useRef<HTMLCanvasElement | null>(null);
   const clientRef = useRef<DeviceStreamClient | null>(null);
   const [status, setStatus] = useState<DeviceStreamStatus>("connecting");
+  const [hasFrame, setHasFrame] = useState(false);
   const [detail, setDetail] = useState<string | undefined>(undefined);
   const [showRestartNotice, setShowRestartNotice] = useState(false);
   const [screen, setScreen] = useState<DeviceScreenSize | null>(null);
@@ -123,6 +124,7 @@ export function DeviceStreamView(props: {
         onDuoUnavailable: onPhoneUnavailable,
         onStatus: (next, nextDetail) => {
           setStatus(next);
+          if (next === "streaming") setHasFrame(true);
           setDetail(nextDetail);
           if (next !== "connecting") setShowRestartNotice(false);
         },
@@ -150,6 +152,8 @@ export function DeviceStreamView(props: {
       },
     );
     clientRef.current = client;
+    setPhoneUnavailable(false);
+    setHasFrame(false);
     setMjpegUrl(null);
     setInputState({ connected: false });
     client.start();
@@ -188,27 +192,24 @@ export function DeviceStreamView(props: {
     return w / h;
   }, [props.platform, screen]);
 
-  // Android restarts its encoder when a fold changes the framebuffer size.
-  // Keep the last decoded frame and viewer mounted while the next keyframe arrives.
-  const retainingAndroidFrame =
-    props.platform === "android" &&
-    status === "connecting" &&
-    inputState.connected &&
-    screen !== null;
+  // Encoder restarts and video reconnects retain the decoded frame and viewer
+  // while input is still connected and the next keyframe is on its way.
+  const retainingFrame =
+    status === "connecting" && hasFrame && inputState.connected && screen !== null;
   const showPhone =
     props.allowPhoneView &&
-    (status === "streaming" || retainingAndroidFrame) &&
+    (status === "streaming" || retainingFrame) &&
     props.visible &&
     presentation === "phone" &&
     !phoneUnavailable &&
     !mjpegUrl &&
     !props.axOverlay &&
     (!isDuo || screen?.supportsHingeAngle === true);
   useEffect(() => {
-    if (!retainingAndroidFrame || !showPhone) return;
+    if (!retainingFrame || !showPhone) return;
     const timeout = window.setTimeout(() => setShowRestartNotice(true), 2_000);
     return () => window.clearTimeout(timeout);
-  }, [retainingAndroidFrame, showPhone]);
+  }, [retainingFrame, showPhone]);
   const controlsInset = props.renderControls && !showPhone ? CONTROLS_RAIL_WIDTH : 0;
 
   // The frame is the largest box at `aspect` that fits the container, so a
@@ -532,14 +533,14 @@ export function DeviceStreamView(props: {
             </span>
           </div>
         ) : null}
-        {retainingAndroidFrame && showPhone && showRestartNotice ? (
+        {retainingFrame && showPhone && showRestartNotice ? (
           <div className="pointer-events-none absolute inset-x-0 bottom-0 flex justify-center p-2">
             <span className="rounded-md bg-background/85 px-2 py-1 text-xs text-muted-foreground">
               Waiting for device video…
             </span>
           </div>
         ) : null}
-        {status !== "streaming" && !(retainingAndroidFrame && showPhone) ? (
+        {status !== "streaming" && !(retainingFrame && showPhone) ? (
           <div className="absolute inset-0">
             <DeviceLoadingView
               name={props.deviceName ?? "Device"}
@@ -556,6 +557,7 @@ export function DeviceStreamView(props: {
                     // An expired ticket surfaces as unauthorized on restart and
                     // refreshes access through the effect; no need to mint one here.
                     clientRef.current?.stop();
+                    setPhoneUnavailable(false);
                     clientRef.current?.start();
                   }}
                 >