fix(preview): keep one browser page across thread switches

apps/desktop/src/ipc/channels.ts

@@ -111,6 +111,7 @@ export const PREVIEW_RECORDING_SAVE_CHANNEL = "desktop:preview-recording-save";
 export const PREVIEW_RECORDING_FRAME_CHANNEL = "desktop:preview-recording-frame";
 export const PREVIEW_STATE_CHANGE_CHANNEL = "desktop:preview-state-change";
 export const PREVIEW_POINTER_EVENT_CHANNEL = "desktop:preview-pointer-event";
+export const PREVIEW_FRAME_REQUEST_CHANNEL = "desktop:preview-frame-request";
 export const PREVIEW_OPEN_LINK_CHANNEL = "desktop:preview-open-link";
 
 export const MAC_PERMISSION_HELPER_CHANNEL = "desktop:mac-permission-helper";

apps/desktop/src/ipc/methods/preview.ts

@@ -54,6 +54,9 @@ export const installPreviewEventForwarding = Effect.fn(
   yield* manager.subscribePointerEvents((event) =>
     electronWindow.sendAll(IpcChannels.PREVIEW_POINTER_EVENT_CHANNEL, event),
   );
+  yield* manager.subscribeFrameRequests((tabId) =>
+    electronWindow.sendAll(IpcChannels.PREVIEW_FRAME_REQUEST_CHANNEL, tabId),
+  );
   yield* manager.subscribeOpenLinks((event) =>
     electronWindow.sendAll(IpcChannels.PREVIEW_OPEN_LINK_CHANNEL, event),
   );

apps/desktop/src/preload.ts

@@ -429,6 +429,14 @@ contextBridge.exposeInMainWorld("desktopBridge", {
       return () =>
         ipcRenderer.removeListener(IpcChannels.PREVIEW_POINTER_EVENT_CHANNEL, wrappedListener);
     },
+    onFrameRequest: (listener) => {
+      const wrappedListener = (_event: Electron.IpcRendererEvent, tabId: unknown) => {
+        if (typeof tabId === "string") listener(tabId);
+      };
+      ipcRenderer.on(IpcChannels.PREVIEW_FRAME_REQUEST_CHANNEL, wrappedListener);
+      return () =>
+        ipcRenderer.removeListener(IpcChannels.PREVIEW_FRAME_REQUEST_CHANNEL, wrappedListener);
+    },
     onOpenLink: (listener) => {
       const wrappedListener = (_event: Electron.IpcRendererEvent, linkEvent: unknown) => {
         if (typeof linkEvent !== "object" || linkEvent === null) return;

apps/desktop/src/preview/DesktopBrowserHost.test.ts

@@ -148,4 +148,25 @@ describe("DesktopBrowserHost", () => {
       expect(host.humanStartedDownload(debuggee.tab.webContents)).toBe(false);
     }),
   );
+
+  it.effect("asks the tab to draw when the server waits for a capture", () =>
+    Effect.gen(function* () {
+      const host = yield* DesktopBrowserHost.make;
+      host.attach(key, makeDebuggee().tab);
+      const requests = yield* host.frameRequests.pipe(
+        Stream.take(1),
+        Stream.runCollect,
+        Effect.forkScoped,
+      );
+      yield* Effect.yieldNow;
+      const command = (id: number, method: string) =>
+        host.handleCommandLine(
+          encodeJson({ type: "cdp", ...key, message: encodeJson({ id, method, params: {} }) }),
+        );
+      // Reading the page needs no frame; a capture waits for the next one.
+      yield* command(1, "DOM.getDocument");
+      yield* command(2, "Page.captureScreenshot");
+      expect(yield* Fiber.join(requests)).toEqual([key]);
+    }).pipe(Effect.scoped),
+  );
 });

apps/desktop/src/preview/DesktopBrowserHost.ts

@@ -31,6 +31,8 @@ const lineEncoder = new TextEncoder();
 /** CDP commands through which an agent clicks, types, or navigates the page. */
 const AGENT_INPUT_COMMAND =
   /"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/;
+/** A capture waits for the page's next frame, which a hidden tab only draws on request. */
+const CAPTURE_COMMAND = /"method":"Page\.captureScreenshot"/;
 /** How long after an agent's input a download still counts as the agent's. */
 const AGENT_DOWNLOAD_WINDOW_MS = 5_000;
 
@@ -93,6 +95,8 @@ export class DesktopBrowserHost extends Context.Service<
       readonly x: number;
       readonly y: number;
     }>;
+    /** Server tabs whose page the server is waiting to capture, so they must draw. */
+    readonly frameRequests: Stream.Stream<DesktopBrowserTabKey>;
   }
 >()("@t3tools/desktop/preview/DesktopBrowserHost") {}
 
@@ -104,6 +108,7 @@ export const make = Effect.gen(function* () {
     readonly x: number;
     readonly y: number;
   }>(16);
+  const frameRequests = yield* PubSub.sliding<DesktopBrowserTabKey>(16);
   const runFork = Effect.runForkWith(yield* Effect.context<never>());
   const tabs = new Map<string, AttachedTab>();
   const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event));
@@ -204,6 +209,8 @@ export const make = Effect.gen(function* () {
         return;
       }
       if (AGENT_INPUT_COMMAND.test(command.value.message)) tab.agentInputAt = performance.now();
+      if (CAPTURE_COMMAND.test(command.value.message))
+        runFork(PubSub.publish(frameRequests, tab.key));
       relayFor(tab).receive(command.value.message);
     });
 
@@ -226,6 +233,7 @@ export const make = Effect.gen(function* () {
 
   return DesktopBrowserHost.of({
     pointers: Stream.fromPubSub(pointers),
+    frameRequests: Stream.fromPubSub(frameRequests),
     // Subscribes before announcing, so no attach falls between the two.
     events: Stream.unwrap(
       Effect.gen(function* () {

apps/desktop/src/preview/Manager.ts

@@ -415,6 +415,8 @@ type RecordingInputListener = (event: DesktopPreviewRecordingInputEvent) => Effe
 
 type PointerEventListener = (event: DesktopPreviewPointerEvent) => Effect.Effect<void>;
 
+type FrameRequestListener = (tabId: string) => Effect.Effect<void>;
+
 type OpenLinkListener = (event: DesktopPreviewOpenLinkEvent) => Effect.Effect<void>;
 
 /**
@@ -562,6 +564,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
   const attachedRef = yield* Ref.make<ReadonlyMap<number, ManagedListeners>>(new Map());
   const listenersRef = yield* Ref.make<ReadonlySet<Listener>>(new Set());
   const pointerEventListenersRef = yield* Ref.make<ReadonlySet<PointerEventListener>>(new Set());
+  const frameRequestListenersRef = yield* Ref.make<ReadonlySet<FrameRequestListener>>(new Set());
   const openLinkListenersRef = yield* Ref.make<ReadonlySet<OpenLinkListener>>(new Set());
   const recordingInputListenersRef = yield* Ref.make<ReadonlySet<RecordingInputListener>>(
     new Set(),
@@ -838,6 +841,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
       | "recording-frame"
       | "recording-input"
       | "pointer-event"
+      | "frame-request"
       | "open-link",
     tabId: string,
     delivery: () => Effect.Effect<void>,
@@ -2411,7 +2415,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
         return;
       }
       if (afterAttach.serverTab) {
-        yield* listenForAgentPointers;
+        yield* listenForAgentSignals;
         browserHost.attach(afterAttach.serverTab, { webContents: wc, debugger: control.debugger });
       }
       if (afterAttach.colorScheme !== "system") {
@@ -3343,17 +3347,36 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
       { discard: true },
     );
   });
-  // Listening starts with the first tab of the desktop's own server.
-  let pointersStarted = false;
-  const listenForAgentPointers = Effect.suspend(() => {
-    if (pointersStarted) return Effect.void;
-    pointersStarted = true;
-    return browserHost.pointers.pipe(
-      Stream.runForEach(emitAgentPointer),
-      Effect.forkIn(parentScope),
-      Effect.asVoid,
+  /** Tells the renderer that the server waits for a frame of this tab, so it must draw. */
+  const emitFrameRequest = Effect.fn("PreviewManager.emitFrameRequest")(function* (key: {
+    readonly threadId: string;
+    readonly tabId: string;
+  }) {
+    const tab = [...(yield* SynchronizedRef.get(tabsRef)).values()].find(
+      (candidate) =>
+        candidate.serverTab?.threadId === key.threadId && candidate.serverTab.tabId === key.tabId,
+    );
+    if (!tab) return;
+    const listeners = yield* Ref.get(frameRequestListenersRef);
+    yield* Effect.forEach(
+      listeners,
+      (listener) => deliverEvent("frame-request", tab.tabId, () => listener(tab.tabId)),
+      { discard: true },
     );
   });
+  // Listening starts with the first tab of the desktop's own server.
+  let agentSignalsStarted = false;
+  const listenForAgentSignals = Effect.suspend(() => {
+    if (agentSignalsStarted) return Effect.void;
+    agentSignalsStarted = true;
+    return Effect.all([
+      browserHost.pointers.pipe(Stream.runForEach(emitAgentPointer), Effect.forkIn(parentScope)),
+      browserHost.frameRequests.pipe(
+        Stream.runForEach(emitFrameRequest),
+        Effect.forkIn(parentScope),
+      ),
+    ]).pipe(Effect.asVoid);
+  });
 
   const emitOpenLink = Effect.fn("PreviewManager.emitOpenLink")(function* (
     event: DesktopPreviewOpenLinkEvent,
@@ -3386,6 +3409,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
       [
         Ref.set(listenersRef, new Set()),
         Ref.set(pointerEventListenersRef, new Set()),
+        Ref.set(frameRequestListenersRef, new Set()),
         Ref.set(openLinkListenersRef, new Set()),
         Ref.set(recordingFrameListenersRef, new Set()),
         Ref.set(recordingInputListenersRef, new Set()),
@@ -3432,6 +3456,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
     stopRecording,
     subscribePointerEvents: (listener: PointerEventListener) =>
       subscribe(pointerEventListenersRef, listener),
+    subscribeFrameRequests: (listener: FrameRequestListener) =>
+      subscribe(frameRequestListenersRef, listener),
     subscribeOpenLinks: (listener: OpenLinkListener) => subscribe(openLinkListenersRef, listener),
     subscribeRecordingInputs: (listener: RecordingInputListener) =>
       subscribe(recordingInputListenersRef, listener),
@@ -3674,6 +3700,9 @@ export class PreviewManager extends Context.Service<
     readonly subscribePointerEvents: (
       listener: PointerEventListener,
     ) => Effect.Effect<void, never, Scope.Scope>;
+    readonly subscribeFrameRequests: (
+      listener: FrameRequestListener,
+    ) => Effect.Effect<void, never, Scope.Scope>;
     readonly subscribeOpenLinks: (
       listener: OpenLinkListener,
     ) => Effect.Effect<void, never, Scope.Scope>;
@@ -3819,6 +3848,7 @@ export const make = Effect.gen(function* PreviewManagerMake() {
     saveRecording: operations.saveRecording,
     subscribeStateChanges: operations.subscribeStateChanges,
     subscribePointerEvents: operations.subscribePointerEvents,
+    subscribeFrameRequests: operations.subscribeFrameRequests,
     subscribeOpenLinks: operations.subscribeOpenLinks,
     subscribeRecordingFrames: operations.subscribeRecordingFrames,
     subscribeRecordingInputs: operations.subscribeRecordingInputs,

apps/server/src/preview/Manager.test.ts

@@ -100,6 +100,89 @@ it.layer(layer)("PreviewManager", (it) => {
     }),
   );
 
+  it.effect("publishes the server's page mode once and preserves a headless choice", () =>
+    Effect.gen(function* () {
+      const threadId = freshThreadId();
+      const manager = yield* PreviewManager.PreviewManager;
+      const collector = yield* collectEvents;
+      const opened = yield* manager.open({
+        threadId,
+        runtime: "server",
+        url: "http://localhost:5173/",
+        profileId: "work",
+        automationOwner: "agent",
+      });
+      const native = yield* manager.setServerPageMode({
+        threadId,
+        tabId: opened.tabId,
+        mode: "native",
+      });
+      const headless = yield* manager.setServerPageMode({
+        threadId,
+        tabId: opened.tabId,
+        mode: "headless",
+      });
+      expect(native.serverPageMode).toBe("native");
+      expect(headless).toMatchObject({
+        ...opened,
+        serverPageMode: "headless",
+        updatedAt: expect.any(String),
+      });
+      const before = yield* manager.list({ threadId });
+      for (const mode of ["headless", "native"] as const) {
+        expect(yield* manager.setServerPageMode({ threadId, tabId: opened.tabId, mode })).toEqual(
+          headless,
+        );
+      }
+      const after = yield* manager.list({ threadId });
+      expect(after).toEqual(before);
+      const events = yield* collector.drain;
+      expect(events.map((event) => event.type)).toEqual(["opened", "resized", "resized"]);
+      expect(events.at(-1)).toMatchObject({ snapshot: headless, revision: after.revision });
+      yield* manager.reportStatus({
+        threadId,
+        tabId: opened.tabId,
+        serverControlled: true,
+        navStatus: { _tag: "Success", url: "http://localhost:5173/next", title: "Next" },
+        canGoBack: true,
+        canGoForward: false,
+      });
+      yield* manager.resize({ threadId, tabId: opened.tabId, viewport: { _tag: "fill" } });
+      expect((yield* manager.list({ threadId })).sessions[0]?.serverPageMode).toBe("headless");
+    }),
+  );
+
+  it.effect(
+    "allows only server sessions to select a page mode, and never recreates a closed tab",
+    () =>
+      Effect.gen(function* () {
+        const threadId = freshThreadId();
+        const manager = yield* PreviewManager.PreviewManager;
+        const desktop = yield* manager.open({ threadId, serverPageMode: "headless" });
+        expect(desktop).not.toHaveProperty("serverPageMode");
+        expect(
+          (yield* manager
+            .setServerPageMode({ threadId, tabId: desktop.tabId, mode: "headless" })
+            .pipe(Effect.flip))._tag,
+        ).toBe("PreviewControlRequiredError");
+        const server = yield* manager.open({
+          threadId,
+          runtime: "server",
+          serverPageMode: "headless",
+        });
+        expect(server.serverPageMode).toBe("headless");
+        yield* manager.close({ threadId, tabId: server.tabId });
+        expect(
+          (yield* manager
+            .setServerPageMode({ threadId, tabId: server.tabId, mode: "headless" })
+            .pipe(Effect.flip))._tag,
+        ).toBe("PreviewSessionLookupError");
+        expect(
+          (yield* manager.list({ threadId })).sessions.map((session) => session.tabId),
+        ).toEqual([desktop.tabId]);
+      }),
+  );
+
   it.effect("lets any client size, theme, and zoom a server tab", () =>
     Effect.gen(function* () {
       const threadId = freshThreadId();

apps/server/src/preview/Manager.ts

@@ -48,6 +48,8 @@ export class PreviewManager extends Context.Service<
     readonly open: (
       input: PreviewOpenInput & {
         readonly automationOwner?: string;
+        /** Only the server browser selects how its page is presented. */
+        readonly serverPageMode?: PreviewSessionSnapshot["serverPageMode"];
         /** Runs before the `opened` event publishes, so subscribers find state keyed by the tab. */
         readonly beforePublish?: (snapshot: PreviewSessionSnapshot) => void;
       },
@@ -58,6 +60,13 @@ export class PreviewManager extends Context.Service<
     readonly reportStatus: (
       input: PreviewReportStatusInput & { readonly serverControlled?: boolean },
     ) => Effect.Effect<void, PreviewError>;
+    /** Publishes the backing page selected by the server; headless cannot be promoted later. */
+    readonly setServerPageMode: (
+      input: PreviewCloseInput & {
+        readonly tabId: string;
+        readonly mode: "native" | "headless";
+      },
+    ) => Effect.Effect<PreviewSessionSnapshot, PreviewError>;
     readonly requestReveal: (
       input: PreviewCloseInput & { readonly tabId: string; readonly force: boolean },
     ) => Effect.Effect<void, PreviewError>;
@@ -221,6 +230,9 @@ export const make = Effect.gen(function* PreviewManagerMake() {
         viewport: input.viewport ?? FILL_PREVIEW_VIEWPORT,
         ...(input.profileId === undefined ? {} : { profileId: input.profileId }),
         ...(runtime === undefined ? {} : { runtime }),
+        ...(runtime === "server" && input.serverPageMode !== undefined
+          ? { serverPageMode: input.serverPageMode }
+          : {}),
         ...(runtime === "server" && input.automationOwner !== undefined
           ? { automationOwner: input.automationOwner }
           : {}),
@@ -337,6 +349,40 @@ export const make = Effect.gen(function* PreviewManagerMake() {
     );
   });
 
+  const setServerPageMode: PreviewManager["Service"]["setServerPageMode"] = Effect.fn(
+    "PreviewManager.setServerPageMode",
+  )(function* (input) {
+    return yield* mutateExistingSession(
+      input.threadId,
+      input.tabId,
+      Effect.fn("PreviewManager.setSessionPageMode")(function* (session) {
+        if (session.snapshot.runtime !== "server")
+          return yield* new PreviewControlRequiredError({ tabId: input.tabId });
+        if (
+          session.snapshot.serverPageMode === input.mode ||
+          session.snapshot.serverPageMode === "headless"
+        )
+          return { next: session, emit: null, result: session.snapshot };
+        const snapshot: PreviewSessionSnapshot = {
+          ...session.snapshot,
+          serverPageMode: input.mode,
+          updatedAt: yield* currentIsoTimestamp,
+        };
+        return {
+          next: { ...session, snapshot },
+          emit: {
+            type: "resized",
+            threadId: session.threadId,
+            tabId: session.tabId,
+            createdAt: snapshot.updatedAt,
+            snapshot,
+          },
+          result: snapshot,
+        };
+      }),
+    );
+  });
+
   const resize: PreviewManager["Service"]["resize"] = Effect.fn("PreviewManager.resize")(
     function* (input) {
       return yield* mutateExistingSession(
@@ -456,7 +502,8 @@ export const make = Effect.gen(function* PreviewManagerMake() {
 
   const list: PreviewManager["Service"]["list"] = Effect.fn("PreviewManager.list")(
     function* (input) {
-      return yield* SynchronizedRef.get(stateRef).pipe(
+      // Events publish inside a mutation; their consumers must wait for its state commit.
+      return yield* SynchronizedRef.modify(stateRef, (state) => [state, state] as const).pipe(
         Effect.map((state): PreviewListResult => ({
           sessions: sessionsForThread(state, input.threadId)
             .map((s) => s.snapshot)
@@ -504,6 +551,7 @@ export const make = Effect.gen(function* PreviewManagerMake() {
     requestReveal,
     navigate,
     reportStatus,
+    setServerPageMode,
     resize,
     adjust,
     refresh,