From 8262db7d19bbef425649ad0760126739f0a1f88d Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:16:45 +0000 Subject: [PATCH 01/68] feat(preview): run the browser on the environment server Standalone servers (`t3 serve`, `npx t3`) now host preview tabs in headless Chromium, so phones, web clients, and agents can use the browser while no desktop app is connected. The desktop app's own environment keeps Electron tabs. - server: ServerBrowser owns tabs, serves the agent preview tools as the preferred automation host, and streams JPEG screencasts over /api/preview-stream with paced acks, motion-mode downscaling, and settle stills. Recording encodes in-browser to mp4. A pinned Chromium and font pack install on first use. - web: server tabs render through ServerBrowserSurface with touch input and a server-probed soft keyboard, float over chat, and pop into system picture-in-picture. - mobile: a browser screen, floating player, and iOS picture-in-picture built on the same stream. - fix: the details card no longer loops with a tall floating preview. --- apps/mobile/app.config.ts | 3 + apps/mobile/metro.config.js | 54 +- .../mobile/scripts/generate-device-stream.mts | 30 +- apps/mobile/src/Stack.tsx | 11 + .../browser/BrowserPreviewRouteScreen.tsx | 311 +++++ .../features/browser/PreviewStreamWebView.tsx | 265 ++++ .../features/browser/ThreadBrowserFloat.tsx | 138 ++ .../browser/browser-preview-button.tsx | 37 + .../src/features/browser/browserTabs.ts | 23 + .../browser/preview-stream-document.ts | 76 + .../browser/preview-stream.browser.ts | 523 +++++++ .../features/threads/ThreadDetailScreen.tsx | 36 + .../threads/floating-working-control.tsx | 36 +- apps/mobile/src/state/preview.ts | 187 +++ .../src/types/mobile-preview-stream.d.ts | 4 + apps/server/package.json | 1 + apps/server/src/auth/http.ts | 29 + apps/server/src/device/DeviceHubProxy.ts | 46 +- .../src/environment/ServerEnvironment.ts | 2 + .../server/src/mcp/PreviewAutomationBroker.ts | 18 +- apps/server/src/preview/Manager.ts | 11 + apps/server/src/preview/ServerBrowser.ts | 1224 +++++++++++++++++ apps/server/src/preview/ServerBrowserPage.ts | 422 ++++++ .../server/src/preview/ServerBrowserStream.ts | 131 ++ .../src/preview/ServerBrowserToolchain.ts | 600 ++++++++ .../src/preview/serverBrowserEnabled.ts | 13 + apps/server/src/server.ts | 10 + apps/web/src/browser/ElectronBrowserHost.tsx | 25 +- apps/web/src/browser/ServerBrowserSurface.tsx | 576 ++++++++ apps/web/src/browser/openFileInPreview.ts | 32 +- .../web/src/browser/serverPictureInPicture.ts | 191 +++ apps/web/src/components/ChatView.tsx | 55 +- .../src/components/preview/PreviewPanel.tsx | 4 +- .../src/components/preview/PreviewView.tsx | 101 +- .../preview/ThreadPreviewMiniPlayer.tsx | 197 ++- .../preview/addBrowserSurface.test.ts | 3 + .../src/components/preview/localServerTabs.ts | 37 + .../components/preview/openDiscoveredPort.ts | 6 +- .../preview/openPreviewSession.test.ts | 4 + .../components/preview/openPreviewSession.ts | 28 +- apps/web/src/routes/_chat.tsx | 7 +- apps/web/src/state/entities.ts | 17 + apps/web/src/state/previewStream.ts | 56 + docs/user/remote-access.md | 17 + packages/client-runtime/package.json | 4 + .../src/preview/serverBrowserStream.ts | 281 ++++ packages/contracts/src/environment.ts | 4 + packages/contracts/src/preview.ts | 12 + pnpm-lock.yaml | 3 + 49 files changed, 5704 insertions(+), 197 deletions(-) create mode 100644 apps/mobile/src/features/browser/BrowserPreviewRouteScreen.tsx create mode 100644 apps/mobile/src/features/browser/PreviewStreamWebView.tsx create mode 100644 apps/mobile/src/features/browser/ThreadBrowserFloat.tsx create mode 100644 apps/mobile/src/features/browser/browser-preview-button.tsx create mode 100644 apps/mobile/src/features/browser/browserTabs.ts create mode 100644 apps/mobile/src/features/browser/preview-stream-document.ts create mode 100644 apps/mobile/src/features/browser/preview-stream.browser.ts create mode 100644 apps/mobile/src/state/preview.ts create mode 100644 apps/mobile/src/types/mobile-preview-stream.d.ts create mode 100644 apps/server/src/preview/ServerBrowser.ts create mode 100644 apps/server/src/preview/ServerBrowserPage.ts create mode 100644 apps/server/src/preview/ServerBrowserStream.ts create mode 100644 apps/server/src/preview/ServerBrowserToolchain.ts create mode 100644 apps/server/src/preview/serverBrowserEnabled.ts create mode 100644 apps/web/src/browser/ServerBrowserSurface.tsx create mode 100644 apps/web/src/browser/serverPictureInPicture.ts create mode 100644 apps/web/src/components/preview/localServerTabs.ts create mode 100644 apps/web/src/state/previewStream.ts create mode 100644 packages/client-runtime/src/preview/serverBrowserStream.ts diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index 4eb77db7b651..f61798d53d63 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -270,6 +270,9 @@ const config: ExpoConfig = { NSLocalNetworkUsageDescription: "Allow T3 Code to connect to T3 Code servers on your local network or tailnet.", NSPhotoLibraryAddUsageDescription: "Allow T3 Code to save images to your photo library.", + // "Audio, AirPlay, and Picture in Picture": the browser screen's system + // picture in picture needs it to start and to stay up outside the app. + UIBackgroundModes: ["audio"], ITSAppUsesNonExemptEncryption: false, // The App Store screenshot harness rotates the iPad interface from // inside the app (CI denies osascript the Accessibility access that diff --git a/apps/mobile/metro.config.js b/apps/mobile/metro.config.js index 9025dc4f6b28..5a9068675280 100644 --- a/apps/mobile/metro.config.js +++ b/apps/mobile/metro.config.js @@ -18,6 +18,7 @@ const licenseGeneratorSource = path.join( const escapedWorkspaceRoot = workspaceRoot.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); const mobileShikiRoot = path.dirname(require.resolve("shiki/package.json", { paths: [__dirname] })); const generatedDeviceStreamRoot = path.join(__dirname, ".generated", "device-stream"); +const generatedPreviewStreamRoot = path.join(__dirname, ".generated", "preview-stream"); const resolveShikiDependencyRoot = (packageName) => { const entryPath = require.resolve(packageName, { paths: [mobileShikiRoot] }); let currentDir = path.dirname(entryPath); @@ -48,6 +49,7 @@ config.resolver = { ...config.resolver?.extraNodeModules, "@t3tools/mobile-third-party-licenses": generatedLicenseModuleRoot, "@t3tools/mobile-device-stream": generatedDeviceStreamRoot, + "@t3tools/mobile-preview-stream": generatedPreviewStreamRoot, shiki: mobileShikiRoot, "@shikijs/core": resolveShikiDependencyRoot("@shikijs/core"), "@shikijs/engine-javascript": resolveShikiDependencyRoot("@shikijs/engine-javascript"), @@ -95,35 +97,57 @@ async function generateMobileThirdPartyLicenses() { ]); } -async function prepareDeviceStream() { - const { generateDeviceStreamScript } = await import( +async function prepareStreamScripts() { + const { generateDeviceStreamScript, generatePreviewStreamScript } = await import( pathToFileURL(path.join(__dirname, "scripts", "generate-device-stream.mts")).href ); - await generateDeviceStreamScript(); + const generateAll = () => + Promise.all([generateDeviceStreamScript(), generatePreviewStreamScript()]); + await generateAll(); if (process.env.NODE_ENV !== "production") { let rebuild = Promise.resolve(); - for (const [directory, files] of [ - [path.join(__dirname, "src/features/devices"), ["device-stream.browser.ts"]], + for (const [directory, files, generate] of [ + [ + path.join(__dirname, "src/features/devices"), + ["device-stream.browser.ts"], + generateDeviceStreamScript, + ], + [ + path.join(__dirname, "src/features/browser"), + ["preview-stream.browser.ts"], + generatePreviewStreamScript, + ], + // The preview transport also imports `hubAccess.ts`. [ path.join(workspaceRoot, "packages/client-runtime/src/device"), ["stream.ts", "hubAccess.ts"], + generateAll, + ], + [ + path.join(workspaceRoot, "packages/client-runtime/src/preview"), + ["serverBrowserStream.ts"], + generatePreviewStreamScript, ], ]) { - // The generated module participates in Metro's normal Fast Refresh. + // The generated modules participate in Metro's normal Fast Refresh. fs.watch(directory, { persistent: false }, (_event, filename) => { if (filename && !files.includes(String(filename))) return; - rebuild = rebuild.then(generateDeviceStreamScript).catch((error) => { - console.error("Could not rebuild the device stream:", error); - }); + rebuild = rebuild + .then(generate) + .then(() => undefined) + .catch((error) => { + console.error("Could not rebuild a WebView stream script:", error); + }); }); } } } -module.exports = Promise.all([generateMobileThirdPartyLicenses(), prepareDeviceStream()]).then(() => - withUniwindConfig(config, { - cssEntryFile: "./global.css", - extraThemes, - polyfills: { rem: 14 }, - }), +module.exports = Promise.all([generateMobileThirdPartyLicenses(), prepareStreamScripts()]).then( + () => + withUniwindConfig(config, { + cssEntryFile: "./global.css", + extraThemes, + polyfills: { rem: 14 }, + }), ); diff --git a/apps/mobile/scripts/generate-device-stream.mts b/apps/mobile/scripts/generate-device-stream.mts index 249bbab6ca58..eede4dbf9bd6 100644 --- a/apps/mobile/scripts/generate-device-stream.mts +++ b/apps/mobile/scripts/generate-device-stream.mts @@ -5,8 +5,8 @@ import { build } from "vite-plus"; const mobileRoot = NodePath.resolve(NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), ".."); -/** Metro embeds the shared browser transport as a small script in the native WebView. */ -export async function generateDeviceStreamScript() { +/** Metro embeds each shared browser transport as a small script in a native WebView. */ +async function generateWebViewScript(entry: string, name: string, output: string) { const result = await build({ configFile: false, logLevel: "silent", @@ -15,8 +15,8 @@ export async function generateDeviceStreamScript() { target: "es2022", minify: true, lib: { - entry: NodePath.join(mobileRoot, "src/features/devices/device-stream.browser.ts"), - name: "T3DeviceStream", + entry: NodePath.join(mobileRoot, entry), + name, formats: ["iife"], }, }, @@ -25,15 +25,29 @@ export async function generateDeviceStreamScript() { const chunk = bundles .flatMap((bundle) => ("output" in bundle ? bundle.output : [])) .find((output) => output.type === "chunk"); - if (!chunk) throw new Error("Device stream build did not emit a script."); - const root = NodePath.join(mobileRoot, ".generated/device-stream"); + if (!chunk) throw new Error(`${name} build did not emit a script.`); + const root = NodePath.join(mobileRoot, output); await NodeFSP.mkdir(root, { recursive: true }); - for (const [name, contents] of [ + for (const [file, contents] of [ ["index.js", `module.exports = ${JSON.stringify(chunk.code)};\n`], ["package.json", '{"main":"index.js"}\n'], ] as const) { - const destination = NodePath.join(root, name); + const destination = NodePath.join(root, file); const previous = await NodeFSP.readFile(destination, "utf8").catch(() => null); if (previous !== contents) await NodeFSP.writeFile(destination, contents); } } + +export const generateDeviceStreamScript = () => + generateWebViewScript( + "src/features/devices/device-stream.browser.ts", + "T3DeviceStream", + ".generated/device-stream", + ); + +export const generatePreviewStreamScript = () => + generateWebViewScript( + "src/features/browser/preview-stream.browser.ts", + "T3PreviewStream", + ".generated/preview-stream", + ); diff --git a/apps/mobile/src/Stack.tsx b/apps/mobile/src/Stack.tsx index 452d1a4dbed5..8dd0bea2f550 100644 --- a/apps/mobile/src/Stack.tsx +++ b/apps/mobile/src/Stack.tsx @@ -43,6 +43,7 @@ import { ReviewCommentComposerSheet } from "./features/review/ReviewCommentCompo import { ReviewSheet } from "./features/review/ReviewSheet"; import { ThreadTerminalRouteScreen } from "./features/terminal/ThreadTerminalRouteScreen"; import { DevicePreviewRouteScreen } from "./features/devices/DevicePreviewRouteScreen"; +import { BrowserPreviewRouteScreen } from "./features/browser/BrowserPreviewRouteScreen"; import { GitBranchesSheet } from "./features/threads/git/GitBranchesSheet"; import { GitCommitSheet } from "./features/threads/git/GitCommitSheet"; import { GitConfirmSheet } from "./features/threads/git/GitConfirmSheet"; @@ -548,6 +549,7 @@ const WORKSPACE_OVERLAY_ROUTES = new Set([ "ThreadQueue", "ThreadReviewComment", "ThreadDevicePreview", + "ThreadBrowserPreview", "ThreadSettingsSheet", ]); @@ -705,6 +707,15 @@ const RootStackConfig = createNativeStackNavigator({ navigationBarHidden: true, }, }), + ThreadBrowserPreview: createNativeStackScreen({ + screen: BrowserPreviewRouteScreen, + linking: `${THREAD_LINKING_PREFIX}/browser`, + options: { + presentation: "fullScreenModal", + headerShown: false, + gestureEnabled: false, + }, + }), ThreadReview: createNativeStackScreen({ screen: ReviewSheet, linking: `${THREAD_LINKING_PREFIX}/review`, diff --git a/apps/mobile/src/features/browser/BrowserPreviewRouteScreen.tsx b/apps/mobile/src/features/browser/BrowserPreviewRouteScreen.tsx new file mode 100644 index 000000000000..27fffbafc67e --- /dev/null +++ b/apps/mobile/src/features/browser/BrowserPreviewRouteScreen.tsx @@ -0,0 +1,311 @@ +import { useIsFocused, useNavigation, type StaticScreenProps } from "@react-navigation/native"; +import { createNativeStackNavigator } from "@react-navigation/native-stack"; +import { EnvironmentId, ThreadId, type PreviewSessionSnapshot } from "@t3tools/contracts"; +import { normalizePreviewUrl } from "@t3tools/shared/preview"; +import { type RefObject, useCallback, useEffect, useRef, useState } from "react"; +import { + ActivityIndicator, + Alert, + AppState, + Platform, + Pressable, + TextInput, + View, +} from "react-native"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; + +import { AppText } from "../../components/AppText"; +import { ControlPill } from "../../components/ControlPill"; +import { ScreenHeader } from "../../components/ScreenHeader"; +import { NativeHeaderToolbar } from "../../native/StackHeader"; +import { + refreshPreviewStreamAccess, + usePreviewStreamAccess, + useThreadServerBrowserTabs, +} from "../../state/preview"; +import { useAppearancePreferences } from "../settings/appearance/AppearancePreferencesProvider"; +import { browserTabTitle, browserTabUrl, latestBrowserTab } from "./browserTabs"; +import { + PreviewStreamWebView, + type PreviewPictureInPictureState, + type PreviewStreamRef, +} from "./PreviewStreamWebView"; + +const BrowserPreviewStack = createNativeStackNavigator<{ BrowserPreview: undefined }>(); + +const NO_PICTURE_IN_PICTURE: PreviewPictureInPictureState = { supported: false, active: false }; + +type BrowserPreviewRouteScreenProps = StaticScreenProps<{ + readonly environmentId: string; + readonly threadId: string; + readonly tabId?: string; +}>; + +/** The nested native stack supplies the navigation bar inside the modal. */ +export function BrowserPreviewRouteScreen({ route }: BrowserPreviewRouteScreenProps) { + const navigation = useNavigation(); + const onClose = useCallback(() => navigation.goBack(), [navigation]); + return ( + + + + {() => ( + + )} + + + + ); +} + +function BrowserPreviewScreen({ + environmentId, + threadId, + initialTabId, + onClose, +}: { + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId; + readonly initialTabId: string | null; + readonly onClose: () => void; +}) { + const insets = useSafeAreaInsets(); + const { themeVariables } = useAppearancePreferences(); + const focused = useIsFocused(); + const [foreground, setForeground] = useState(AppState.currentState !== "background"); + const [selectedTabId, setSelectedTabId] = useState(initialTabId); + const [pictureInPicture, setPictureInPicture] = useState(NO_PICTURE_IN_PICTURE); + const streamRef = useRef(null); + const { tabs, loaded } = useThreadServerBrowserTabs({ environmentId, threadId, enabled: true }); + const tab = tabs.find((entry) => entry.tabId === selectedTabId) ?? latestBrowserTab(tabs); + const tabId = tab?.tabId ?? null; + // Pin the fallback so another tab's activity does not switch the view. + if (tabId !== null && tabId !== selectedTabId) setSelectedTabId(tabId); + useEffect(() => { + const subscription = AppState.addEventListener("change", (state) => + setForeground(state !== "background"), + ); + return () => subscription.remove(); + }, []); + useEffect(() => { + if (focused && loaded && tabs.length === 0) onClose(); + }, [focused, loaded, tabs.length, onClose]); + const onPictureInPicture = useCallback((state: PreviewPictureInPictureState, detail?: string) => { + setPictureInPicture({ supported: state.supported, active: state.active }); + if (detail) Alert.alert("Picture in picture is unavailable", detail); + }, []); + const selectTab = (next: string) => { + setPictureInPicture(NO_PICTURE_IN_PICTURE); + setSelectedTabId(next); + }; + // System picture in picture keeps showing this stream after the app leaves the foreground. + const live = (focused && foreground) || pictureInPicture.active; + + return ( + + streamRef.current?.togglePictureInPicture(), + }, + ] + : [] + } + menus={ + tabs.length > 1 + ? [ + { + title: "Tabs", + icon: "square.on.square", + items: tabs.map((entry) => ({ + id: entry.tabId, + title: browserTabTitle(entry), + subtitle: browserTabUrl(entry), + selected: entry.tabId === tabId, + onPress: () => selectTab(entry.tabId), + })), + }, + ] + : [] + } + /> + {Platform.OS === "ios" ? ( + + + + ) : null} + {tab ? ( + <> + streamRef.current?.command(input)} + /> + {live ? ( + + ) : ( + + )} + + ) : ( + + {focused && foreground ? ( + + ) : null} + + )} + + ); +} + +function BrowserAddressBar({ + tab, + onCommand, +}: { + readonly tab: PreviewSessionSnapshot; + readonly onCommand: PreviewStreamRef["command"]; +}) { + const url = browserTabUrl(tab); + // Null while not editing, so agent navigation keeps the field current. + const [draft, setDraft] = useState(null); + const submit = () => { + const value = draft ?? url; + setDraft(null); + try { + onCommand({ type: "navigate", url: normalizePreviewUrl(value) }); + } catch { + Alert.alert("Could not open this address", "Enter an http or https URL."); + } + }; + return ( + + onCommand({ type: "history", delta: -1 })} + /> + onCommand({ type: "history", delta: 1 })} + /> + setDraft(null)} + onSubmitEditing={submit} + placeholder="Enter a URL" + placeholderTextColorClassName="accent-placeholder" + autoCapitalize="none" + autoCorrect={false} + keyboardType="url" + returnKeyType="go" + selectTextOnFocus + className="h-10 min-w-0 flex-1 rounded-full border border-input-border bg-input px-4 font-sans text-sm text-foreground" + /> + onCommand({ type: "reload" })} + /> + + ); +} + +function OpenBrowserPreview({ + environmentId, + threadId, + tabId, + streamRef, + onPictureInPicture, +}: { + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId; + readonly tabId: string; + readonly streamRef: RefObject; + readonly onPictureInPicture: (state: PreviewPictureInPictureState, detail?: string) => void; +}) { + const { themeVariables } = useAppearancePreferences(); + const { access, error, refresh } = usePreviewStreamAccess(environmentId); + const onUnauthorized = useCallback( + () => refreshPreviewStreamAccess(environmentId), + [environmentId], + ); + return access ? ( + + ) : ( + + {error ? ( + <> + + {error} + + + Retry + + + ) : ( + <> + + Connecting to browser... + + )} + + ); +} diff --git a/apps/mobile/src/features/browser/PreviewStreamWebView.tsx b/apps/mobile/src/features/browser/PreviewStreamWebView.tsx new file mode 100644 index 000000000000..772305a9e1d1 --- /dev/null +++ b/apps/mobile/src/features/browser/PreviewStreamWebView.tsx @@ -0,0 +1,265 @@ +import previewStreamScript from "@t3tools/mobile-preview-stream"; +import type { PreviewStreamInput } from "@t3tools/client-runtime/preview/server-browser-stream"; +import { + useEffect, + useEffectEvent, + useImperativeHandle, + useLayoutEffect, + useMemo, + useRef, + useState, + type Ref, +} from "react"; +import { ActivityIndicator, Platform, Pressable, View } from "react-native"; +import { WebView } from "react-native-webview"; + +import { AppText } from "../../components/AppText"; + +import { + previewStreamDocument, + previewStreamMessage, + type PreviewStreamConfiguration, +} from "./preview-stream-document"; + +export interface PreviewStreamRef { + /** `navigate`, `history`, and `reload` wait for the socket if it is not open yet. */ + readonly command: (input: PreviewStreamInput) => void; + readonly togglePictureInPicture: () => void; +} + +export interface PreviewPictureInPictureState { + readonly supported: boolean; + readonly active: boolean; +} + +type NativeStreamBridge = { + readonly ref?: Ref; + /** Refresh stream access; new access remounts the document with a fresh ticket. */ + readonly onUnauthorized: () => void; + /** The tab was closed on the server. */ + readonly onGone?: () => void; + readonly onViewport?: (viewport: { readonly width: number; readonly height: number }) => void; + readonly onPictureInPicture?: (state: PreviewPictureInPictureState, detail?: string) => void; + /** The floating player shows a spinner without text or a reconnect button. */ + readonly compact?: boolean; +}; + +const UNAUTHORIZED_RETRY_MAX_MS = 30_000; + +/** A server-hosted preview tab streamed into a WebView that runs the shared transport. */ +export function PreviewStreamWebView({ + ref, + ...props +}: PreviewStreamConfiguration & NativeStreamBridge) { + const [attempt, setAttempt] = useState(0); + const processRetried = useRef(false); + const unauthorized = useRef(0); + const refreshTimer = useRef | null>(null); + useEffect( + () => () => { + if (refreshTimer.current !== null) clearTimeout(refreshTimer.current); + }, + [], + ); + const configuration = JSON.stringify({ + access: props.access, + threadId: props.threadId, + tabId: props.tabId, + interactive: props.interactive, + background: props.background, + } satisfies PreviewStreamConfiguration); + return ( + { + // The client has stopped. Restart it, and back off when fresh tickets keep + // getting refused, e.g. a session without operate scope. + const retry = unauthorized.current++; + if (refreshTimer.current !== null) clearTimeout(refreshTimer.current); + refreshTimer.current = setTimeout( + () => { + refreshTimer.current = null; + setAttempt((current) => current + 1); + props.onUnauthorized(); + }, + retry === 0 ? 0 : Math.min(1_000 * 2 ** retry, UNAUTHORIZED_RETRY_MAX_MS), + ); + }} + onRetry={() => { + processRetried.current = false; + setAttempt((current) => current + 1); + props.onUnauthorized(); + }} + onStreaming={() => { + processRetried.current = false; + unauthorized.current = 0; + }} + onRecoverProcess={() => { + if (processRetried.current) return false; + processRetried.current = true; + setAttempt((current) => current + 1); + return true; + }} + /> + ); +} + +function PreviewStreamDocumentView({ + ref, + configuration, + background, + compact, + onUnauthorized, + onGone, + onViewport, + onPictureInPicture, + onRetry, + onStreaming, + onRecoverProcess, +}: Omit & { + readonly configuration: string; + readonly background: string; + readonly compact: boolean; + readonly onRetry: () => void; + readonly onStreaming: () => void; + readonly onRecoverProcess: () => boolean; +}) { + const webView = useRef>(null); + const active = useRef(true); + const failed = useRef(false); + const [status, setStatus] = useState<"connecting" | "streaming" | "error">("connecting"); + const [error, setError] = useState(null); + const [gone, setGone] = useState(false); + const [started, setStarted] = useState(false); + const fail = (message: string) => { + if (!active.current || failed.current) return; + failed.current = true; + webView.current?.injectJavaScript("window.T3PreviewStream?.stop(); true;"); + setError(message); + setStatus("error"); + }; + // The shared transport owns reconnects once the document acknowledges startup. + const bootstrapTimedOut = useEffectEvent(() => + fail("Browser viewer could not start. Reconnect to try again."), + ); + useEffect(() => { + if (started) return; + const timer = setTimeout(bootstrapTimedOut, 15_000); + return () => clearTimeout(timer); + }, [started]); + const source = useMemo( + () => ({ + html: previewStreamDocument(configuration, previewStreamScript), + baseUrl: Platform.OS === "android" ? "https://localhost/" : "file:///", + }), + [configuration], + ); + useImperativeHandle(ref, () => ({ + command: (input) => + webView.current?.injectJavaScript( + `window.T3PreviewStream?.command(${JSON.stringify(input)}); true;`, + ), + togglePictureInPicture: () => + webView.current?.injectJavaScript("window.T3PreviewStream?.pictureInPicture(); true;"), + })); + useLayoutEffect(() => { + active.current = true; + const view = webView.current; + return () => { + active.current = false; + view?.injectJavaScript("window.T3PreviewStream?.stop(); true;"); + }; + }, []); + const processTerminated = () => { + if (!active.current || failed.current) return; + if (!onRecoverProcess()) fail("Browser viewer stopped. Reconnect to try again."); + }; + return ( + + + ref={webView} + source={source} + originWhitelist={["*"]} + scrollEnabled={false} + bounces={false} + mixedContentMode="always" + allowUniversalAccessFromFileURLs + contentInsetAdjustmentBehavior="never" + setSupportMultipleWindows={false} + // Picture in picture plays the canvas as an inline muted video, started from native chrome. + allowsInlineMediaPlayback + allowsPictureInPictureMediaPlayback + mediaPlaybackRequiresUserAction={false} + style={{ flex: 1, backgroundColor: background }} + onError={() => fail("Browser viewer could not load. Reconnect to try again.")} + onHttpError={() => fail("Browser viewer could not load. Reconnect to try again.")} + onContentProcessDidTerminate={processTerminated} + onRenderProcessGone={processTerminated} + onShouldStartLoadWithRequest={(request) => + request.url === "about:blank" || request.url === source.baseUrl + } + onMessage={(event) => { + if (!active.current || failed.current) return; + const message = previewStreamMessage(event.nativeEvent.data); + if (message === null) return; + switch (message.type) { + case "unauthorized": + onUnauthorized(); + return; + case "gone": + setGone(true); + fail("This tab was closed."); + onGone?.(); + return; + case "viewport": + onViewport?.(message); + return; + case "pictureInPicture": + onPictureInPicture?.(message, message.detail); + return; + case "status": + setStarted(true); + if (message.status === "error") { + fail(message.detail ?? "Browser stream failed."); + return; + } + setStatus(message.status); + if (message.status === "streaming") onStreaming(); + } + }} + /> + {status !== "streaming" ? ( + + {status === "connecting" ? : null} + {compact ? null : ( + + {status === "error" ? error : "Connecting to browser..."} + + )} + {status === "error" && !gone && !compact ? ( + + Reconnect + + ) : null} + + ) : null} + + ); +} diff --git a/apps/mobile/src/features/browser/ThreadBrowserFloat.tsx b/apps/mobile/src/features/browser/ThreadBrowserFloat.tsx new file mode 100644 index 000000000000..68624c2c7dc7 --- /dev/null +++ b/apps/mobile/src/features/browser/ThreadBrowserFloat.tsx @@ -0,0 +1,138 @@ +import { useIsFocused } from "@react-navigation/native"; +import type { EnvironmentId, PreviewSessionSnapshot, ThreadId } from "@t3tools/contracts"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { ActivityIndicator, AppState, Pressable, View } from "react-native"; +import Animated, { FadeIn, FadeOut, ReduceMotion } from "react-native-reanimated"; + +import { SymbolView } from "../../components/AppSymbol"; +import { refreshPreviewStreamAccess, usePreviewStreamAccess } from "../../state/preview"; +import { useAppearancePreferences } from "../settings/appearance/AppearancePreferencesProvider"; +import { browserTabTitle } from "./browserTabs"; +import { PreviewStreamWebView } from "./PreviewStreamWebView"; + +const PLAYER_LONG_SIDE = 184; +const PLAYER_ENTERING = FadeIn.duration(180).reduceMotion(ReduceMotion.System); +const PLAYER_EXITING = FadeOut.duration(120).reduceMotion(ReduceMotion.System); + +/** + * Floats a server browser tab over chat when the agent opens one, like web's + * "Float preview over chat". View-only: a tap opens the browser screen. + */ +export function ThreadBrowserFloat(props: { + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId; + readonly tabs: ReadonlyArray; + readonly loaded: boolean; + readonly top: number; + readonly onOpen: (tabId: string) => void; +}) { + const [tabId, setTabId] = useState(null); + const known = useRef | null>(null); + useEffect(() => { + if (!props.loaded) return; + const ids = props.tabs.map((tab) => tab.tabId); + const previous = known.current; + known.current = new Set(ids); + // The first list is a baseline, so reopening a thread does not resurface old tabs. + if (previous === null) return; + const opened = ids.findLast((id) => !previous.has(id)); + if (opened !== undefined) setTabId(opened); + }, [props.loaded, props.tabs]); + const focused = useIsFocused(); + const [foreground, setForeground] = useState(AppState.currentState !== "background"); + useEffect(() => { + const subscription = AppState.addEventListener("change", (state) => + setForeground(state !== "background"), + ); + return () => subscription.remove(); + }, []); + const tab = props.tabs.find((entry) => entry.tabId === tabId) ?? null; + const onOpen = props.onOpen; + const open = useCallback(() => { + if (tabId === null) return; + setTabId(null); + onOpen(tabId); + }, [onOpen, tabId]); + const close = useCallback(() => setTabId(null), []); + if (!tab || !focused) return null; + return ( + + ); +} + +function FloatingBrowserPlayer(props: { + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId; + readonly tab: PreviewSessionSnapshot; + readonly live: boolean; + readonly top: number; + readonly onOpen: () => void; + readonly onClose: () => void; +}) { + const { environmentId } = props; + const { themeVariables } = useAppearancePreferences(); + const { access } = usePreviewStreamAccess(environmentId); + const onUnauthorized = useCallback( + () => refreshPreviewStreamAccess(environmentId), + [environmentId], + ); + const [viewport, setViewport] = useState<{ width: number; height: number } | null>(null); + // The page keeps its own size; the player scales it into a box of the same shape. + const aspect = Math.min(Math.max(viewport ? viewport.width / viewport.height : 16 / 10, 0.5), 2); + const width = aspect >= 1 ? PLAYER_LONG_SIDE : Math.round(PLAYER_LONG_SIDE * aspect); + const height = Math.round(width / aspect); + const background = themeVariables["--color-sheet-solid"]; + return ( + + + + {access && props.live ? ( + + ) : ( + + + + )} + + + + + + + ); +} diff --git a/apps/mobile/src/features/browser/browser-preview-button.tsx b/apps/mobile/src/features/browser/browser-preview-button.tsx new file mode 100644 index 000000000000..78bd6d8eebed --- /dev/null +++ b/apps/mobile/src/features/browser/browser-preview-button.tsx @@ -0,0 +1,37 @@ +import { Pressable, View } from "react-native"; + +import { AppText } from "../../components/AppText"; +import { SymbolView } from "../../components/AppSymbol"; + +/** Opens the thread's server-hosted browser tabs. Sits beside the device preview button. */ +export function BrowserPreviewButton(props: { + readonly count: number; + readonly onPress: () => void; + readonly compact?: boolean; +}) { + const compact = props.compact ?? true; + return ( + + + {!compact ? ( + + {props.count === 1 ? "One tab open" : `${props.count} tabs open`} + + ) : props.count > 1 ? ( + + {props.count} + + ) : null} + + ); +} diff --git a/apps/mobile/src/features/browser/browserTabs.ts b/apps/mobile/src/features/browser/browserTabs.ts new file mode 100644 index 000000000000..509ddad5dcf7 --- /dev/null +++ b/apps/mobile/src/features/browser/browserTabs.ts @@ -0,0 +1,23 @@ +import type { PreviewSessionSnapshot } from "@t3tools/contracts"; + +/** The tab the agent touched last, the default when nothing is selected. */ +export function latestBrowserTab(tabs: ReadonlyArray) { + return tabs.reduce( + (latest, tab) => (latest === null || tab.updatedAt > latest.updatedAt ? tab : latest), + null, + ); +} + +export function browserTabUrl(tab: PreviewSessionSnapshot) { + return tab.navStatus._tag === "Idle" ? "" : tab.navStatus.url; +} + +export function browserTabTitle(tab: PreviewSessionSnapshot) { + if (tab.navStatus._tag === "Idle") return "New tab"; + if (tab.navStatus.title.trim()) return tab.navStatus.title; + try { + return new URL(tab.navStatus.url).host || tab.navStatus.url; + } catch { + return tab.navStatus.url; + } +} diff --git a/apps/mobile/src/features/browser/preview-stream-document.ts b/apps/mobile/src/features/browser/preview-stream-document.ts new file mode 100644 index 000000000000..bd60486751ba --- /dev/null +++ b/apps/mobile/src/features/browser/preview-stream-document.ts @@ -0,0 +1,76 @@ +import type { DeviceHubAccess } from "@t3tools/client-runtime/device/hub-access"; + +export interface PreviewStreamConfiguration { + readonly access: DeviceHubAccess; + readonly threadId: string; + readonly tabId: string; + /** Taps, scrolls, keys, and `resize` to the view size. The floating player only watches. */ + readonly interactive: boolean; + readonly background: string; +} + +/** Messages the WebView document posts to the native view. */ +export type PreviewStreamMessage = + | { + readonly type: "status"; + readonly status: "connecting" | "streaming" | "error"; + readonly detail?: string; + } + | { readonly type: "unauthorized" } + | { readonly type: "gone" } + | { readonly type: "viewport"; readonly width: number; readonly height: number } + | { + readonly type: "pictureInPicture"; + readonly supported: boolean; + readonly active: boolean; + readonly detail?: string; + }; + +export function previewStreamDocument(configuration: string, script: string) { + // Tickets and URLs are data, including any HTML delimiter characters. + const safeConfiguration = configuration.replace(/`; +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null; + +export function previewStreamMessage(data: string): PreviewStreamMessage | null { + let message: unknown; + try { + message = JSON.parse(data); + } catch { + // Ignore messages that are not part of the stream bridge. + return null; + } + if (!isRecord(message)) return null; + const detail = typeof message.detail === "string" ? message.detail : undefined; + switch (message.type) { + case "unauthorized": + case "gone": + return { type: message.type }; + case "status": + return message.status === "connecting" || + message.status === "streaming" || + message.status === "error" + ? { type: "status", status: message.status, ...(detail ? { detail } : {}) } + : null; + case "viewport": + return typeof message.width === "number" && typeof message.height === "number" + ? { type: "viewport", width: message.width, height: message.height } + : null; + case "pictureInPicture": + return typeof message.supported === "boolean" && typeof message.active === "boolean" + ? { + type: "pictureInPicture", + supported: message.supported, + active: message.active, + ...(detail ? { detail } : {}), + } + : null; + default: + return null; + } +} diff --git a/apps/mobile/src/features/browser/preview-stream.browser.ts b/apps/mobile/src/features/browser/preview-stream.browser.ts new file mode 100644 index 000000000000..728d6d31e47e --- /dev/null +++ b/apps/mobile/src/features/browser/preview-stream.browser.ts @@ -0,0 +1,523 @@ +import { + createPreviewFramePainter, + createPreviewStreamClient, + previewStreamModifiers, + type PreviewStreamClient, + type PreviewStreamInput, + type PreviewStreamViewport, +} from "@t3tools/client-runtime/preview/server-browser-stream"; + +import type { PreviewStreamConfiguration, PreviewStreamMessage } from "./preview-stream-document"; + +declare global { + interface Window { + ReactNativeWebView: { postMessage: (message: string) => void }; + } +} + +/** WebKit's presentation API, the only picture in picture entry on older iOS. */ +interface PresentationVideo extends HTMLVideoElement { + webkitSupportsPresentationMode?: (mode: string) => boolean; + webkitSetPresentationMode?: (mode: "inline" | "picture-in-picture") => void; + webkitPresentationMode?: string; +} + +type WheelInput = Extract; + +const RESIZE_DEBOUNCE_MS = 150; +const TAP_SLOP_PX = 8; +const MULTI_CLICK_MS = 500; +const MULTI_CLICK_SLOP_PX = 4; +const WHEEL_LINE_PX = 16; +// JPEG frames past 2x cost bandwidth without a visible gain on a phone. +const MAX_PIXEL_RATIO = 2; + +interface Viewer { + readonly stop: () => void; + readonly command: (input: PreviewStreamInput) => void; + readonly togglePictureInPicture: () => Promise; +} + +let activeViewer: Viewer | null = null; + +export function stop() { + activeViewer?.stop(); + activeViewer = null; +} + +/** Navigation and history from the native chrome. Waits for the socket if it is not open yet. */ +export function command(input: PreviewStreamInput) { + activeViewer?.command(input); +} + +export function pictureInPicture() { + void activeViewer?.togglePictureInPicture(); +} + +/** + * Bundled into a native WebView without React or Expo's web runtime. Draws a + * server tab's JPEG frames into a letterboxed canvas. Interactive viewers turn + * taps into clicks, drags into wheel scrolls, and soft keyboard input into key + * and text messages, and resize fill-mode tabs to the view. + */ +export function start(configuration: PreviewStreamConfiguration) { + stop(); + const post = (message: PreviewStreamMessage) => { + // oxlint-disable-next-line unicorn/require-post-message-target-origin -- The native WebView bridge takes one string. + window.ReactNativeWebView.postMessage(JSON.stringify(message)); + }; + const { interactive } = configuration; + Object.assign(document.documentElement.style, { height: "100%", overflow: "hidden" }); + Object.assign(document.body.style, { + margin: "0", + height: "100%", + overflow: "hidden", + background: configuration.background, + }); + const container = document.createElement("div"); + Object.assign(container.style, { + position: "fixed", + left: "0", + top: "0", + width: "100%", + height: "100%", + overflow: "hidden", + }); + const canvas = document.createElement("canvas"); + canvas.setAttribute("role", "img"); + canvas.setAttribute("aria-label", "Browser page"); + Object.assign(canvas.style, { + position: "absolute", + inset: "0", + width: "100%", + height: "100%", + objectFit: "contain", + touchAction: "none", + userSelect: "none", + webkitUserSelect: "none", + webkitTouchCallout: "none", + }); + container.append(canvas); + // Focus target for page keyboard input, visually hidden like `sr-only`. Pinned + // top-left so focusing it never scrolls; 16px keeps iOS from zooming on focus. + const input = document.createElement("textarea"); + input.setAttribute("aria-label", "Browser page input"); + input.autocapitalize = "off"; + input.autocomplete = "off"; + input.spellcheck = false; + input.setAttribute("autocorrect", "off"); + Object.assign(input.style, { + position: "fixed", + left: "0", + top: "0", + width: "1px", + height: "1px", + padding: "0", + margin: "-1px", + border: "0", + overflow: "hidden", + clip: "rect(0, 0, 0, 0)", + whiteSpace: "nowrap", + fontSize: "16px", + }); + document.body.replaceChildren(container, ...(interactive ? [input] : [])); + + let stopped = false; + let streaming = false; + let viewport: PreviewStreamViewport | null = null; + let size: { width: number; height: number } | null = null; + // Frame cap in device px, fixed per socket. It only grows, so only outgrowing it reconnects. + let cap: { width: number; height: number } | null = null; + let client: PreviewStreamClient | null = null; + let pendingCommand: PreviewStreamInput | null = null; + let resizeTimer: ReturnType | null = null; + let wheelFrame: number | null = null; + let pendingWheel: WheelInput | null = null; + // Whether the page point under a touch takes text; null until the server answers. + let probe: { x: number; y: number; editable: boolean | null; tapped: boolean } | null = null; + let touch: { + pointerId: number; + startX: number; + startY: number; + lastX: number; + lastY: number; + panning: boolean; + } | null = null; + let lastTap: { time: number; x: number; y: number; count: number } | null = null; + + const reportStatus = (status: "connecting" | "streaming") => { + streaming = status === "streaming"; + post({ type: "status", status }); + }; + const painter = createPreviewFramePainter(canvas, () => { + if (!streaming && !stopped) reportStatus("streaming"); + }); + const send = (message: PreviewStreamInput) => client?.send(message) ?? false; + + const connect = () => { + client?.stop(); + if (!cap || stopped) return; + const next = createPreviewStreamClient( + { + access: configuration.access, + threadId: configuration.threadId, + tabId: configuration.tabId, + maxWidth: cap.width, + maxHeight: cap.height, + }, + { + onFrame: (jpeg) => painter.paint(jpeg), + onViewport: (page) => { + if (viewport?.width === page.width && viewport.height === page.height) return; + viewport = page; + post({ type: "viewport", width: page.width, height: page.height }); + }, + onProbe: (result) => { + const current = probe; + if (!current || current.x !== result.x || current.y !== result.y) return; + current.editable = result.editable; + if (!current.tapped) return; + // Late answer: Android still raises the keyboard; iOS waits for the next tap. + probe = null; + if (result.editable) input.focus({ preventScroll: true }); + else input.blur(); + }, + onConnectedChange: (connected) => { + if (!connected) { + if (streaming) reportStatus("connecting"); + return; + } + if (interactive && size) next.send({ type: "resize", ...size }); + const queued = pendingCommand; + pendingCommand = null; + if (queued) next.send(queued); + }, + onUnauthorized: () => post({ type: "unauthorized" }), + onGone: () => post({ type: "gone" }), + }, + ); + client = next; + }; + + const measure = () => { + resizeTimer = null; + const rect = container.getBoundingClientRect(); + if (rect.width < 1 || rect.height < 1) return; + const next = { width: Math.round(rect.width), height: Math.round(rect.height) }; + if (size?.width === next.width && size.height === next.height) return; + size = next; + if (interactive) send({ type: "resize", ...next }); + const ratio = Math.min(window.devicePixelRatio || 1, MAX_PIXEL_RATIO); + // The floating player scales a page of any shape into its box, so its cap is square. + const side = Math.max(next.width, next.height); + const width = Math.round((interactive ? next.width : side) * ratio); + const height = Math.round((interactive ? next.height : side) * ratio); + if (cap && cap.width >= width && cap.height >= height) return; + cap = { width: Math.max(width, cap?.width ?? 0), height: Math.max(height, cap?.height ?? 0) }; + connect(); + }; + const observer = new ResizeObserver(() => { + if (resizeTimer !== null) clearTimeout(resizeTimer); + // The first size connects right away; later ones settle before resizing the page. + resizeTimer = setTimeout(measure, size === null ? 0 : RESIZE_DEBOUNCE_MS); + }); + observer.observe(container); + + // iOS keeps the layout viewport under the soft keyboard; follow the visible area + // so the page resizes above it. + const visualViewport = window.visualViewport; + const followVisualViewport = () => { + if (!visualViewport) return; + container.style.top = `${visualViewport.offsetTop}px`; + container.style.height = `${visualViewport.height}px`; + }; + if (interactive) { + visualViewport?.addEventListener("resize", followVisualViewport); + visualViewport?.addEventListener("scroll", followVisualViewport); + } + + const pagePoint = (clientX: number, clientY: number, clamp: boolean) => { + if (!viewport || canvas.width === 0 || canvas.height === 0) return null; + const rect = canvas.getBoundingClientRect(); + // `object-fit: contain` letterboxes the frame inside the canvas box. + const fit = Math.min(rect.width / canvas.width, rect.height / canvas.height); + const width = canvas.width * fit; + const height = canvas.height * fit; + if (!(width > 0 && height > 0)) return null; + const scale = viewport.width / width; + const x = (clientX - rect.left - (rect.width - width) / 2) * scale; + const y = (clientY - rect.top - (rect.height - height) / 2) * (viewport.height / height); + const inside = x >= 0 && y >= 0 && x <= viewport.width && y <= viewport.height; + if (!inside && !clamp) return null; + return { + x: Math.min(Math.max(x, 0), viewport.width), + y: Math.min(Math.max(y, 0), viewport.height), + scale, + }; + }; + + // Wheel deltas coalesce to one message per animation frame. + const flushWheel = () => { + if (wheelFrame !== null) cancelAnimationFrame(wheelFrame); + wheelFrame = null; + const wheel = pendingWheel; + pendingWheel = null; + if (wheel) send(wheel); + }; + const queueWheel = (point: { x: number; y: number }, deltaX: number, deltaY: number) => { + pendingWheel = { + type: "wheel", + x: point.x, + y: point.y, + deltaX: (pendingWheel?.deltaX ?? 0) + deltaX, + deltaY: (pendingWheel?.deltaY ?? 0) + deltaY, + modifiers: 0, + }; + wheelFrame ??= requestAnimationFrame(flushWheel); + }; + + const onPointerDown = (event: PointerEvent) => { + if (!event.isPrimary) return; + event.preventDefault(); + touch = { + pointerId: event.pointerId, + startX: event.clientX, + startY: event.clientY, + lastX: event.clientX, + lastY: event.clientY, + panning: false, + }; + const point = pagePoint(event.clientX, event.clientY, false); + probe = point ? { x: point.x, y: point.y, editable: null, tapped: false } : null; + if (point) send({ type: "probe", x: point.x, y: point.y }); + }; + const onPointerMove = (event: PointerEvent) => { + if (!touch || touch.pointerId !== event.pointerId) return; + if ( + !touch.panning && + Math.hypot(event.clientX - touch.startX, event.clientY - touch.startY) < TAP_SLOP_PX + ) { + return; + } + touch.panning = true; + const point = pagePoint(event.clientX, event.clientY, true); + // Dragging the page up scrolls it down, following the finger. + if (point) { + queueWheel( + point, + (touch.lastX - event.clientX) * point.scale, + (touch.lastY - event.clientY) * point.scale, + ); + } + touch.lastX = event.clientX; + touch.lastY = event.clientY; + }; + const onPointerUp = (event: PointerEvent) => { + const ended = touch; + if (!ended || ended.pointerId !== event.pointerId) return; + touch = null; + const answered = probe; + if (ended.panning) { + probe = null; + return; + } + const point = pagePoint(event.clientX, event.clientY, false); + if (!point) return; + // Focusing inside the tap's user activation is what lets iOS raise the keyboard. + if (answered?.editable === true) input.focus({ preventScroll: true }); + else if (answered?.editable === false) input.blur(); + if (answered?.editable === null) answered.tapped = true; + else probe = null; + flushWheel(); + const last = lastTap; + const clickCount = + last && + event.timeStamp - last.time < MULTI_CLICK_MS && + Math.hypot(event.clientX - last.x, event.clientY - last.y) < MULTI_CLICK_SLOP_PX + ? last.count + 1 + : 1; + lastTap = { time: event.timeStamp, x: event.clientX, y: event.clientY, count: clickCount }; + const at = { x: point.x, y: point.y, modifiers: 0 }; + send({ type: "mouse", action: "move", ...at, button: "none", buttons: 0, clickCount: 0 }); + send({ type: "mouse", action: "down", ...at, button: "left", buttons: 1, clickCount }); + send({ type: "mouse", action: "up", ...at, button: "left", buttons: 0, clickCount }); + }; + const onPointerCancel = (event: PointerEvent) => { + if (touch?.pointerId === event.pointerId) touch = null; + }; + // Trackpads and mice on tablets scroll with wheel events. + const onWheel = (event: WheelEvent) => { + const point = pagePoint(event.clientX, event.clientY, false); + if (!point) return; + event.preventDefault(); + const unit = + event.deltaMode === WheelEvent.DOM_DELTA_LINE + ? WHEEL_LINE_PX + : event.deltaMode === WheelEvent.DOM_DELTA_PAGE + ? (viewport?.height ?? 0) + : 1; + queueWheel(point, event.deltaX * unit, event.deltaY * unit); + }; + // Keeps focus in the page input and stops native selection and callouts. + const preventDefault = (event: Event) => event.preventDefault(); + + const onKey = (action: "down" | "up", event: KeyboardEvent) => { + // IME and soft keyboards deliver text through composition and input events. + if ( + event.isComposing || + event.keyCode === 229 || + event.key === "Process" || + event.key === "Unidentified" + ) { + return; + } + const shortcut = event.ctrlKey || event.metaKey; + // Enter carries "\r" like Puppeteer's key table, so forms submit and textareas break lines. + const text = shortcut + ? undefined + : [...event.key].length === 1 + ? event.key + : event.key === "Enter" + ? "\r" + : undefined; + send({ + type: "key", + action, + key: event.key, + code: event.code, + keyCode: event.keyCode, + ...(action === "down" && text !== undefined ? { text } : {}), + modifiers: previewStreamModifiers(event), + }); + if (!shortcut) event.preventDefault(); + }; + const onKeyDown = (event: KeyboardEvent) => onKey("down", event); + const onKeyUp = (event: KeyboardEvent) => onKey("up", event); + const onInput = (event: Event) => { + if (event instanceof InputEvent && event.isComposing) return; + if (input.value) send({ type: "text", text: input.value }); + input.value = ""; + }; + const onCompositionEnd = (event: CompositionEvent) => { + if (event.data) send({ type: "text", text: event.data }); + input.value = ""; + }; + + if (interactive) { + canvas.addEventListener("pointerdown", onPointerDown); + canvas.addEventListener("pointermove", onPointerMove); + canvas.addEventListener("pointerup", onPointerUp); + canvas.addEventListener("pointercancel", onPointerCancel); + canvas.addEventListener("wheel", onWheel, { passive: false }); + canvas.addEventListener("mousedown", preventDefault); + canvas.addEventListener("contextmenu", preventDefault); + input.addEventListener("keydown", onKeyDown); + input.addEventListener("keyup", onKeyUp); + input.addEventListener("input", onInput); + input.addEventListener("compositionend", onCompositionEnd); + } + + // Picture in picture plays the canvas as a muted video. It is created on first + // use so a viewer that never pops out pays nothing for it. + const supportsPresentationMode = (element: PresentationVideo) => + element.webkitSupportsPresentationMode?.("picture-in-picture") === true; + const pictureInPictureSupported = + interactive && + typeof canvas.captureStream === "function" && + (document.pictureInPictureEnabled === true || + supportsPresentationMode(document.createElement("video"))); + let video: PresentationVideo | null = null; + const pictureInPictureActive = () => + video !== null && + (document.pictureInPictureElement === video || + video.webkitPresentationMode === "picture-in-picture"); + const reportPictureInPicture = (detail?: string) => + post({ + type: "pictureInPicture", + supported: pictureInPictureSupported, + active: pictureInPictureActive(), + ...(detail ? { detail } : {}), + }); + const ensureVideo = () => { + if (video) return video; + const element: PresentationVideo = document.createElement("video"); + element.muted = true; + element.playsInline = true; + element.autoplay = true; + element.setAttribute("playsinline", ""); + // Under the canvas at full size: WebKit pauses muted video it considers off screen. + Object.assign(element.style, { + position: "absolute", + inset: "0", + width: "100%", + height: "100%", + objectFit: "contain", + pointerEvents: "none", + }); + element.srcObject = canvas.captureStream(); + // A static page sends no new frames; repaint once so the stream has one. + if (canvas.width > 0 && canvas.height > 0) canvas.getContext("2d")?.drawImage(canvas, 0, 0); + for (const name of [ + "enterpictureinpicture", + "leavepictureinpicture", + "webkitpresentationmodechanged", + ]) { + element.addEventListener(name, () => { + // The hidden inline copy stops decoding once the window closes. + if (!pictureInPictureActive()) element.pause(); + reportPictureInPicture(); + }); + } + container.prepend(element); + video = element; + return element; + }; + const togglePictureInPicture = async () => { + if (!pictureInPictureSupported) return; + try { + if (pictureInPictureActive()) { + if (document.pictureInPictureElement) await document.exitPictureInPicture(); + else video?.webkitSetPresentationMode?.("inline"); + return; + } + const element = ensureVideo(); + await element.play(); + if (document.pictureInPictureEnabled) await element.requestPictureInPicture(); + else element.webkitSetPresentationMode?.("picture-in-picture"); + } catch (error) { + reportPictureInPicture( + error instanceof Error ? error.message : "Picture in picture is unavailable.", + ); + } + }; + + const viewer: Viewer = { + stop: () => { + stopped = true; + observer.disconnect(); + visualViewport?.removeEventListener("resize", followVisualViewport); + visualViewport?.removeEventListener("scroll", followVisualViewport); + if (resizeTimer !== null) clearTimeout(resizeTimer); + if (wheelFrame !== null) cancelAnimationFrame(wheelFrame); + painter.stop(); + client?.stop(); + client = null; + if (video) { + for (const track of video.srcObject instanceof MediaStream + ? video.srcObject.getTracks() + : []) { + track.stop(); + } + video.srcObject = null; + } + }, + command: (message) => { + if (!send(message)) pendingCommand = message; + }, + togglePictureInPicture, + }; + activeViewer = viewer; + window.addEventListener("pagehide", stop, { once: true }); + reportStatus("connecting"); + if (interactive) reportPictureInPicture(); +} diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index 129c24150b75..1d3beebe9247 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -95,6 +95,8 @@ import { editPendingThreadMessage } from "../../state/edit-pending-thread-messag import { deviceEnvironment } from "../../state/device"; import { useEnvironmentQuery } from "../../state/query"; import { threadDevicePreviews } from "../devices/threadDevicePreviews"; +import { ThreadBrowserFloat } from "../browser/ThreadBrowserFloat"; +import { useThreadServerBrowserTabs } from "../../state/preview"; import type { QueuedThreadMessage } from "../../state/thread-outbox-model"; import { scopedThreadKey } from "../../lib/scopedEntities"; import { @@ -333,6 +335,22 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread threadId: props.selectedThread.id, }); }, [navigation, props.environmentId, props.selectedThread.id]); + const browserTabs = useThreadServerBrowserTabs({ + environmentId: props.environmentId, + threadId: props.selectedThread.id, + enabled: props.serverConfig?.environment.capabilities.serverBrowser === true, + }); + const openBrowserPreview = useCallback( + (tabId?: string) => { + Keyboard.dismiss(); + navigation.navigate("ThreadBrowserPreview", { + environmentId: props.environmentId, + threadId: props.selectedThread.id, + ...(tabId === undefined ? {} : { tabId }), + }); + }, + [navigation, props.environmentId, props.selectedThread.id], + ); const insets = useSafeAreaInsets(); const isKeyboardVisible = useKeyboardState((state) => state.isVisible); const liveKeyboardHeight = useKeyboardState((state) => state.height); @@ -512,6 +530,7 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread queuedCount > 0 || agentsSegment !== null || devicePreviews.length > 0 || + browserTabs.tabs.length > 0 || props.connectionStateLabel !== "connected" || props.queuedMessages.length > 0 || props.selectedThreadFeed.some( @@ -1129,6 +1148,18 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread )} + {showContent ? ( + + ) : null} + {/* Floating composer — sticks to keyboard via KeyboardStickyView */} {showContent ? ( 0 + ? { count: browserTabs.tabs.length, onPress: () => openBrowserPreview() } + : null + } showScrollToEnd={showScrollToEndButton} onScrollToEnd={handleScrollToEnd} agents={agentsSegment} diff --git a/apps/mobile/src/features/threads/floating-working-control.tsx b/apps/mobile/src/features/threads/floating-working-control.tsx index 27d6757f3158..2e032eb8ba5e 100644 --- a/apps/mobile/src/features/threads/floating-working-control.tsx +++ b/apps/mobile/src/features/threads/floating-working-control.tsx @@ -26,6 +26,7 @@ import { AppText as Text } from "../../components/AppText"; import { SymbolView } from "../../components/AppSymbol"; import { ControlPill } from "../../components/ControlPill"; import { NATIVE_LIQUID_GLASS_SUPPORTED } from "../../native/native-glass"; +import { BrowserPreviewButton } from "../browser/browser-preview-button"; import { DevicePreviewButton } from "../devices/device-preview-button"; import type { FloatingWorkingStatus } from "./floating-working-status"; import { ShimmeringWorkContent } from "./thread-work-log"; @@ -67,6 +68,7 @@ export function FloatingWorkingControl(props: { readonly colorScheme: "light" | "dark"; readonly status: FloatingWorkingStatus | null; readonly devicePreview: { readonly count: number; readonly onPress: () => void } | null; + readonly browserPreview: { readonly count: number; readonly onPress: () => void } | null; readonly showScrollToEnd: boolean; readonly onScrollToEnd: () => void; readonly agents: SubagentPillSegment | null; @@ -81,8 +83,8 @@ export function FloatingWorkingControl(props: { const [queueWidth, setQueueWidth] = useState(0); const [agentsWidth, setAgentsWidth] = useState(0); const hasQueue = props.queuedCount > 0; - const hasDevicePreview = props.devicePreview !== null; - const [deviceWidth, setDeviceWidth] = useState(0); + const hasPreview = props.devicePreview !== null || props.browserPreview !== null; + const [previewWidth, setPreviewWidth] = useState(0); const agents = props.agents; const hasAgents = agents !== null; // Segments keep their measured width; only the status label absorbs the @@ -94,7 +96,7 @@ export function FloatingWorkingControl(props: { 32 - (hasQueue ? queueWidth : 0) - (hasAgents ? agentsWidth : 0) - - (hasDevicePreview ? deviceWidth : 0), + (hasPreview ? previewWidth : 0), ); const separationProgress = useSharedValue(props.showScrollToEnd ? 1 : 0); @@ -131,7 +133,7 @@ export function FloatingWorkingControl(props: { // Forget the width while no label is shown so the next one appears at its // own size instead of animating from the previous label's. const hasStatus = props.status !== null; - const hasCapsule = hasStatus || hasQueue || hasAgents || hasDevicePreview; + const hasCapsule = hasStatus || hasQueue || hasAgents || hasPreview; useEffect(() => { if (!hasStatus) { measuredWidthRef.current = null; @@ -151,9 +153,15 @@ export function FloatingWorkingControl(props: { return null; } + // Both preview buttons fit beside each other only as icons. + const previewCompact = + hasStatus || + hasAgents || + hasQueue || + (props.devicePreview !== null && props.browserPreview !== null); // The queue, agents, and reconnect labels have separate tap targets. const statusInteractive = props.status?.kind === "connection"; - const capsuleInteractive = statusInteractive || hasQueue || hasAgents || hasDevicePreview; + const capsuleInteractive = statusInteractive || hasQueue || hasAgents || hasPreview; // The host stays centered on the capsule, but its measurement constraint // comes from the overlay, independent of the capsule's current width. const statusContent = @@ -184,16 +192,18 @@ export function FloatingWorkingControl(props: { const capsuleContent = ( {statusContent} - {props.devicePreview !== null ? ( + {hasPreview ? ( setDeviceWidth(event.nativeEvent.layout.width)} + onLayout={(event) => setPreviewWidth(event.nativeEvent.layout.width)} > {hasStatus ? : null} - + {props.devicePreview !== null ? ( + + ) : null} + {props.browserPreview !== null ? ( + + ) : null} ) : null} {agents !== null ? ( @@ -205,7 +215,7 @@ export function FloatingWorkingControl(props: { onLayout={(event) => setAgentsWidth(event.nativeEvent.layout.width)} className="h-11 flex-row items-center gap-1.5 px-3 active:opacity-70" > - {hasStatus || hasDevicePreview ? : null} + {hasStatus || hasPreview ? : null} {agents.label} @@ -222,7 +232,7 @@ export function FloatingWorkingControl(props: { style={{ maxWidth: Math.min(overlayWidth, windowWidth) * 0.45 }} className="h-11 flex-row items-center gap-2 px-3 active:opacity-70" > - {hasStatus || hasDevicePreview || hasAgents ? ( + {hasStatus || hasPreview || hasAgents ? ( ) : null} diff --git a/apps/mobile/src/state/preview.ts b/apps/mobile/src/state/preview.ts new file mode 100644 index 000000000000..9640a224e43e --- /dev/null +++ b/apps/mobile/src/state/preview.ts @@ -0,0 +1,187 @@ +import { useAtomValue } from "@effect/atom-react"; +import { parseScopedThreadKey, scopedThreadKey } from "@t3tools/client-runtime/environment"; +import { PREVIEW_STREAM_BASE_PATH } from "@t3tools/client-runtime/preview/server-browser-stream"; +import { createPreviewEnvironmentAtoms } from "@t3tools/client-runtime/state/preview"; +import { resolveDeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; +import type { + EnvironmentId, + PreviewEvent, + PreviewListResult, + PreviewSessionSnapshot, + ThreadId, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import { AsyncResult, Atom } from "effect/unstable/reactivity"; +import { useMemo } from "react"; + +import { connectionAtomRuntime } from "../connection/runtime"; +import { appAtomRegistry } from "./atom-registry"; +import { useEnvironmentQuery } from "./query"; +import { environmentSession, usePreparedConnection } from "./session"; + +export const previewEnvironment = createPreviewEnvironmentAtoms(connectionAtomRuntime); + +interface ThreadPreviewTabs { + /** Null until the first list result or event arrives. */ + readonly serverEpoch: string | null; + readonly revision: number; + readonly sessions: ReadonlyArray; + /** A list result has arrived, so `sessions` covers tabs opened before this view. */ + readonly listed: boolean; +} + +const EMPTY_TABS: ThreadPreviewTabs = { + serverEpoch: null, + revision: 0, + sessions: [], + listed: false, +}; +const emptyTabsAtom = Atom.make(EMPTY_TABS).pipe(Atom.withLabel("mobile-preview-tabs:empty")); + +function reconcileList(current: ThreadPreviewTabs, result: PreviewListResult): ThreadPreviewTabs { + if ( + current.listed && + current.serverEpoch === result.serverEpoch && + result.revision <= current.revision + ) { + return current; + } + return { + serverEpoch: result.serverEpoch, + revision: result.revision, + sessions: result.sessions, + listed: true, + }; +} + +function applyEvent(current: ThreadPreviewTabs, event: PreviewEvent): ThreadPreviewTabs { + if (event.revision <= current.revision) return current; + const others = current.sessions.filter((session) => session.tabId !== event.tabId); + const existing = current.sessions.find((session) => session.tabId === event.tabId); + const sessions = (() => { + switch (event.type) { + case "opened": + case "navigated": + case "resized": + // Keep a tab's place so the picker order is stable. + return existing + ? current.sessions.map((session) => + session.tabId === event.tabId ? event.snapshot : session, + ) + : [...others, event.snapshot]; + case "failed": + return existing + ? current.sessions.map((session) => + session.tabId === event.tabId + ? { + ...session, + navStatus: { + _tag: "LoadFailed" as const, + url: event.url, + title: event.title, + code: event.code, + description: event.description, + }, + updatedAt: event.createdAt, + } + : session, + ) + : current.sessions; + case "closed": + return others; + } + })(); + return { ...current, serverEpoch: event.serverEpoch, revision: event.revision, sessions }; +} + +/** + * One thread's preview tabs: the `preview.list` result kept current by + * environment preview events, the same pairing web's preview session sync uses. + */ +const threadPreviewTabsAtom = Atom.family((threadKey: string) => { + const ref = parseScopedThreadKey(threadKey); + if (ref === null) return emptyTabsAtom; + const listAtom = previewEnvironment.list({ + environmentId: ref.environmentId, + input: { threadId: ref.threadId }, + }); + const eventsAtom = previewEnvironment.events({ environmentId: ref.environmentId, input: {} }); + return Atom.make((get) => { + let disposed = false; + let state = EMPTY_TABS; + const publish = (next: ThreadPreviewTabs) => { + if (next === state) return; + state = next; + get.setSelf(next); + }; + get.addFinalizer(() => { + disposed = true; + }); + get.subscribe(listAtom, (result) => { + if (AsyncResult.isSuccess(result)) publish(reconcileList(state, result.value)); + }); + get.subscribe(eventsAtom, (result) => { + if (!AsyncResult.isSuccess(result) || result.value.threadId !== ref.threadId) return; + // A restarted server resets revisions; only a fresh list is authoritative. + if (state.serverEpoch !== null && result.value.serverEpoch !== state.serverEpoch) { + get.refresh(listAtom); + return; + } + publish(applyEvent(state, result.value)); + }); + const cached = get.once(listAtom); + if (AsyncResult.isSuccess(cached)) state = reconcileList(state, cached.value); + // The cached list can predate an agent-opened tab. + queueMicrotask(() => { + if (!disposed) get.refresh(listAtom); + }); + return state; + }).pipe(Atom.setIdleTTL(1_000), Atom.withLabel(`mobile-preview-tabs:${threadKey}`)); +}); + +/** The thread's server-hosted browser tabs. `loaded` turns true once the tab list has arrived. */ +export function useThreadServerBrowserTabs(input: { + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId; + readonly enabled: boolean; +}) { + const tabs = useAtomValue( + input.enabled + ? threadPreviewTabsAtom( + scopedThreadKey({ environmentId: input.environmentId, threadId: input.threadId }), + ) + : emptyTabsAtom, + ); + const sessions = useMemo( + () => tabs.sessions.filter((session) => session.runtime === "server"), + [tabs.sessions], + ); + return { tabs: sessions, loaded: tabs.listed }; +} + +const previewStreamAccessAtom = Atom.family((environmentId: EnvironmentId) => + connectionAtomRuntime + .atom((get) => { + const prepared = Option.getOrNull( + get(environmentSession.preparedConnectionValueAtom(environmentId)), + ); + return prepared === null + ? Effect.never + : resolveDeviceHubAccess({ prepared, hubBasePath: PREVIEW_STREAM_BASE_PATH }); + }) + .pipe(Atom.setIdleTTL(60_000), Atom.withLabel(`mobile-preview-stream-access:${environmentId}`)), +); + +/** Fetches a fresh stream ticket; a refused socket calls this before reconnecting. */ +export function refreshPreviewStreamAccess(environmentId: EnvironmentId) { + appAtomRegistry.refresh(previewStreamAccessAtom(environmentId)); +} + +/** Stream credentials for server tabs, with the same ticket flow as the device hub. */ +export function usePreviewStreamAccess(environmentId: EnvironmentId) { + const prepared = usePreparedConnection(environmentId); + const query = useEnvironmentQuery(previewStreamAccessAtom(environmentId)); + const access = query.data && query.error === null && Option.isSome(prepared) ? query.data : null; + return { access, error: query.error, refresh: query.refresh }; +} diff --git a/apps/mobile/src/types/mobile-preview-stream.d.ts b/apps/mobile/src/types/mobile-preview-stream.d.ts new file mode 100644 index 000000000000..34a76a995512 --- /dev/null +++ b/apps/mobile/src/types/mobile-preview-stream.d.ts @@ -0,0 +1,4 @@ +declare module "@t3tools/mobile-preview-stream" { + const script: string; + export default script; +} diff --git a/apps/server/package.json b/apps/server/package.json index f62e0d487315..1c86a4b402da 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -42,6 +42,7 @@ "effect": "catalog:", "jose": "catalog:", "node-pty": "^1.2.0-beta.15", + "playwright-core": "1.60.0", "proper-lockfile": "4.1.2", "stream-chain": "^4.2.5", "stream-json": "3.6.0", diff --git a/apps/server/src/auth/http.ts b/apps/server/src/auth/http.ts index 29febe036ae6..b684a2109054 100644 --- a/apps/server/src/auth/http.ts +++ b/apps/server/src/auth/http.ts @@ -157,6 +157,35 @@ export function failEnvironmentNotFound(reason: EnvironmentResourceNotFoundReaso ); } +/** + * `` and WebSocket cannot set headers, so media routes (device hub, + * preview stream) authenticate the way the `/ws` upgrade does: a cookie for browser + * sessions, or a short-lived `wsTicket` minted over authenticated HTTP for + * bearer and DPoP clients. The upgrade authenticator already implements that + * fallback order, so it is used for plain requests as well. + */ +export const authenticateMediaRequest = (requiredScope: AuthEnvironmentScope) => + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; + const session = yield* serverAuth.authenticateWebSocketUpgrade(request).pipe( + Effect.catch((error) => + Effect.gen(function* () { + if (EnvironmentAuth.isServerAuthCredentialError(error)) { + return yield* failEnvironmentAuthInvalid( + EnvironmentAuth.serverAuthCredentialReason(error), + EnvironmentAuth.serverAuthDpopFailureReason(error), + ); + } + return yield* failEnvironmentInternal("internal_error", error); + }), + ), + ); + if (!session.scopes.includes(requiredScope)) { + return yield* failEnvironmentScopeRequired(requiredScope); + } + }); + export function failEnvironmentInternal(reason: EnvironmentInternalErrorReason, error?: unknown) { return Effect.gen(function* () { const traceId = yield* currentEnvironmentTraceId; diff --git a/apps/server/src/device/DeviceHubProxy.ts b/apps/server/src/device/DeviceHubProxy.ts index 5975864d925e..b7821e063ddd 100644 --- a/apps/server/src/device/DeviceHubProxy.ts +++ b/apps/server/src/device/DeviceHubProxy.ts @@ -11,11 +11,7 @@ * Only the routes the Device panel needs are forwarded. Anything under the * hub's dashboard, exec, or WebRTC surface is rejected here. */ -import { - AuthOrchestrationReadScope, - AuthOrchestrationOperateScope, - type AuthEnvironmentScope, -} from "@t3tools/contracts"; +import { AuthOrchestrationReadScope, AuthOrchestrationOperateScope } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; import { @@ -28,12 +24,7 @@ import { import * as Socket from "effect/socket/Socket"; import * as NodeSocket from "@effect/platform-node/NodeSocket"; -import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; -import { - failEnvironmentAuthInvalid, - failEnvironmentInternal, - failEnvironmentScopeRequired, -} from "../auth/http.ts"; +import { authenticateMediaRequest } from "../auth/http.ts"; import * as DeviceService from "./DeviceService.ts"; const ALLOWED_PATHS: ReadonlyArray = [ @@ -80,35 +71,6 @@ const DROPPED_REQUEST_HEADERS = new Set([ const isWebSocketUpgrade = (request: HttpServerRequest.HttpServerRequest) => request.headers.upgrade?.toLowerCase() === "websocket"; -/** - * `` and WebSocket cannot set headers, so every proxied request - * authenticates the way the `/ws` upgrade does: a cookie for browser - * sessions, or a short-lived `wsTicket` minted over authenticated HTTP for - * bearer and DPoP clients. The upgrade authenticator already implements that - * fallback order, so it is used for plain requests as well. - */ -const authenticate = (requiredScope: AuthEnvironmentScope) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; - const session = yield* serverAuth.authenticateWebSocketUpgrade(request).pipe( - Effect.catch((error) => - Effect.gen(function* () { - if (EnvironmentAuth.isServerAuthCredentialError(error)) { - return yield* failEnvironmentAuthInvalid( - EnvironmentAuth.serverAuthCredentialReason(error), - EnvironmentAuth.serverAuthDpopFailureReason(error), - ); - } - return yield* failEnvironmentInternal("internal_error", error); - }), - ), - ); - if (!session.scopes.includes(requiredScope)) { - return yield* failEnvironmentScopeRequired(requiredScope); - } - }); - const forwardHeaders = (request: HttpServerRequest.HttpServerRequest, origin: string) => { const headers: Record = {}; for (const [name, value] of Object.entries(request.headers)) { @@ -206,7 +168,9 @@ const handler = Effect.gen(function* () { const controlsDevice = (upgrade && hubPath !== "/api/devices/ws") || (!readOnly && /\/api\/(stream-(mode|settings)|fold)$/.test(hubPath)); - yield* authenticate(controlsDevice ? AuthOrchestrationOperateScope : AuthOrchestrationReadScope); + yield* authenticateMediaRequest( + controlsDevice ? AuthOrchestrationOperateScope : AuthOrchestrationReadScope, + ); const devices = yield* DeviceService.DeviceService; const ready = yield* devices.currentReadiness(url.value.searchParams.get("hostId") ?? undefined); if (!ready) { diff --git a/apps/server/src/environment/ServerEnvironment.ts b/apps/server/src/environment/ServerEnvironment.ts index 2ccc07b87b79..85d7c09c202d 100644 --- a/apps/server/src/environment/ServerEnvironment.ts +++ b/apps/server/src/environment/ServerEnvironment.ts @@ -20,6 +20,7 @@ import { readAgentActivityPublishingActive } from "../cloud/config.ts"; import { resolveServerSelfUpdateCapability } from "../cloud/selfUpdate.ts"; import { resolveServiceLauncherMode } from "../cloud/serviceLauncherClient.ts"; import * as ServerConfig from "../config.ts"; +import { isServerBrowserEnabled } from "../preview/serverBrowserEnabled.ts"; import * as ProcessRunner from "../processRunner.ts"; import { resolveServerEnvironmentLabel } from "./ServerEnvironmentLabel.ts"; import { detectServerEnvironmentMachineKind } from "./ServerEnvironmentMachine.ts"; @@ -258,6 +259,7 @@ export const make = Effect.gen(function* () { ? { serverSelfUpdateProgress: true } : {}), ...(desktopAppUpdate ? { desktopAppUpdate: true } : {}), + ...(isServerBrowserEnabled(serverConfig.mode) ? { serverBrowser: true } : {}), }, }; diff --git a/apps/server/src/mcp/PreviewAutomationBroker.ts b/apps/server/src/mcp/PreviewAutomationBroker.ts index 2e0534fec25a..bc1ee0722ebc 100644 --- a/apps/server/src/mcp/PreviewAutomationBroker.ts +++ b/apps/server/src/mcp/PreviewAutomationBroker.ts @@ -52,11 +52,21 @@ export interface PreviewAutomationInvokeInput { readonly onTargetTab?: (tabId: PreviewTabId | undefined) => void; } +export interface PreviewAutomationConnectOptions { + /** + * New agent work goes to a preferred host before any desktop. The server's + * own headless browser registers this way so a standalone environment keeps + * browsing when every desktop disconnects. + */ + readonly preferred?: boolean; +} + export class PreviewAutomationBroker extends Context.Service< PreviewAutomationBroker, { readonly connect: ( host: PreviewAutomationHost, + options?: PreviewAutomationConnectOptions, ) => Effect.Effect>; readonly focusHost: (host: PreviewAutomationHostFocus) => Effect.Effect; readonly respond: ( @@ -74,6 +84,7 @@ interface ClientConnection { readonly environmentId: PreviewAutomationHost["environmentId"]; readonly supportedOperations: ReadonlySet; readonly focused: boolean; + readonly preferred: boolean; readonly liveTabs: NonNullable; readonly focusOrder: number; readonly queue: Queue.Queue; @@ -367,6 +378,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { const acquireConnection = Effect.fn("PreviewAutomationBroker.acquireConnection")(function* ( host: PreviewAutomationHost, + options: PreviewAutomationConnectOptions | undefined, ) { const clientId = host.clientId; const queue = yield* Queue.unbounded(); @@ -378,6 +390,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { environmentId: host.environmentId, supportedOperations: new Set(host.supportedOperations ?? PREVIEW_AUTOMATION_V1_OPERATIONS), focused: false, + preferred: options?.preferred ?? false, liveTabs: [], focusOrder: 0, queue, @@ -408,10 +421,10 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { const connect: PreviewAutomationBroker["Service"]["connect"] = Effect.fn( "PreviewAutomationBroker.connect", - )((host) => + )((host, options) => Effect.succeed( Stream.unwrap( - Effect.acquireRelease(acquireConnection(host), (connection) => + Effect.acquireRelease(acquireConnection(host, options), (connection) => disconnect(connection.clientId, connection.queue), ).pipe(Effect.map((connection) => Stream.fromQueue(connection.queue))), ), @@ -518,6 +531,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { (left, right) => Number(ownsTargetTab(right, true)) - Number(ownsTargetTab(left, true)) || Number(ownsTargetTab(right)) - Number(ownsTargetTab(left)) || + Number(right.preferred) - Number(left.preferred) || Number(right.focused) - Number(left.focused) || right.focusOrder - left.focusOrder, )[0]; diff --git a/apps/server/src/preview/Manager.ts b/apps/server/src/preview/Manager.ts index 3a1f870e3904..e0a6aae47a10 100644 --- a/apps/server/src/preview/Manager.ts +++ b/apps/server/src/preview/Manager.ts @@ -21,6 +21,7 @@ import { type PreviewRefreshInput, type PreviewReportStatusInput, type PreviewResizeInput, + type PreviewRuntime, FILL_PREVIEW_VIEWPORT, PreviewSessionLookupError, type PreviewSessionSnapshot, @@ -124,6 +125,7 @@ const buildLoadingSnapshot = (input: { readonly title: string; readonly viewport: PreviewViewportSetting; readonly profileId?: string | undefined; + readonly runtime?: PreviewRuntime | undefined; readonly updatedAt: string; }): PreviewSessionSnapshot => ({ threadId: input.threadId, @@ -133,6 +135,7 @@ const buildLoadingSnapshot = (input: { canGoForward: false, viewport: input.viewport, ...(input.profileId === undefined ? {} : { profileId: input.profileId }), + ...(input.runtime === undefined ? {} : { runtime: input.runtime }), updatedAt: input.updatedAt, }); @@ -141,6 +144,7 @@ const buildIdleSnapshot = (input: { readonly tabId: string; readonly viewport: PreviewViewportSetting; readonly profileId?: string | undefined; + readonly runtime?: PreviewRuntime | undefined; readonly updatedAt: string; }): PreviewSessionSnapshot => ({ threadId: input.threadId, @@ -150,6 +154,7 @@ const buildIdleSnapshot = (input: { canGoForward: false, viewport: input.viewport, ...(input.profileId === undefined ? {} : { profileId: input.profileId }), + ...(input.runtime === undefined ? {} : { runtime: input.runtime }), updatedAt: input.updatedAt, }); @@ -236,6 +241,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { title: "", viewport, profileId: input.profileId, + runtime: input.runtime, updatedAt, }) : buildIdleSnapshot({ @@ -243,6 +249,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { tabId, viewport, profileId: input.profileId, + runtime: input.runtime, updatedAt, }); yield* SynchronizedRef.modifyEffect(stateRef, (state) => @@ -291,6 +298,9 @@ export const make = Effect.gen(function* PreviewManagerMake() { ...(session.snapshot.profileId === undefined ? {} : { profileId: session.snapshot.profileId }), + ...(session.snapshot.runtime === undefined + ? {} + : { runtime: session.snapshot.runtime }), updatedAt, }; return { @@ -327,6 +337,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { ...(session.snapshot.profileId === undefined ? {} : { profileId: session.snapshot.profileId }), + ...(session.snapshot.runtime === undefined ? {} : { runtime: session.snapshot.runtime }), updatedAt, }; const emit: PreviewEventDraft = diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts new file mode 100644 index 000000000000..fa1a25a35582 --- /dev/null +++ b/apps/server/src/preview/ServerBrowser.ts @@ -0,0 +1,1224 @@ +// @effect-diagnostics nodeBuiltinImport:off globalDate:off globalTimers:off - Playwright callbacks run outside the Effect runtime. +/** + * Headless Chromium preview tabs owned by the environment server. + * + * Tabs opened with `runtime: "server"` live here instead of in a desktop + * ``, so agents keep browsing with no client attached and any client + * (web, phone, a desktop on another machine) watches through + * `/api/preview-stream`. PreviewManager stays the tab list: this service + * mirrors it (`opened` creates a page, `resized` applies the viewport, `closed` + * drops it) and reports page state back through `reportStatus`, the same path + * desktops use. Agents reach these tabs through PreviewAutomationBroker, where + * this service registers as a preferred in-process automation host. + * + * Rendering is tuned for servers without a GPU. `--disable-gpu` selects + * software compositing, which measured about six times cheaper than + * SwiftShader GL compositing for a busy page; WebGL still works through + * SwiftShader. Screencast frames ignore emulated device scale, so the process + * renders at a real 2x for crisp screenshots, recordings, and retina viewers; + * small viewers get downscaled frames. + */ +import { + FILL_PREVIEW_VIEWPORT, + PREVIEW_AUTOMATION_OPERATIONS, + PROVIDER_SEND_TURN_MAX_FILE_BYTES, + type PreviewAutomationActionEvent, + type PreviewAutomationClickInput, + type PreviewAutomationConsoleEntry, + type PreviewAutomationEvaluateInput, + type PreviewAutomationNavigateInput, + type PreviewAutomationNetworkEntry, + type PreviewAutomationOpenInput, + type PreviewAutomationPressInput, + type PreviewAutomationRequest, + type PreviewAutomationResizeInput, + type PreviewAutomationScrollInput, + type PreviewAutomationSetColorSchemeInput, + type PreviewAutomationStatus, + type PreviewAutomationTypeInput, + type PreviewAutomationWaitForInput, + type PreviewEvent, + type PreviewNavStatus, + type PreviewSessionSnapshot, + type PreviewViewportSetting, + ThreadId, +} from "@t3tools/contracts"; +import { normalizePreviewUrl } from "@t3tools/shared/preview"; +import { resolvePreviewViewport } from "@t3tools/shared/previewViewport"; +import * as NodeCrypto from "node:crypto"; +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Queue from "effect/Queue"; +import * as Schema from "effect/Schema"; +import type * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; +import type { BrowserContext, CDPSession, Page } from "playwright-core"; + +import { PENDING_ATTACHMENT_THREAD_SEGMENT } from "../attachmentStore.ts"; +import * as ServerConfig from "../config.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as PreviewAutomationBroker from "../mcp/PreviewAutomationBroker.ts"; +import * as PreviewManager from "./Manager.ts"; +import * as ServerBrowserPage from "./ServerBrowserPage.ts"; +import * as ServerBrowserToolchain from "./ServerBrowserToolchain.ts"; +import { isServerBrowserEnabled } from "./serverBrowserEnabled.ts"; + +const SERVER_HOST_CLIENT_ID = "server-browser"; +const RENDER_SCALE = 2; +/** + * Chromium keeps three screencast frames in flight and drops frames past + * that, so holding each ack this long caps a viewer near 30 fps. + */ +const SCREENCAST_ACK_PACE_MS = 100; +/** Quiet time after which a viewer that may have missed frames gets a still. */ +const SCREENCAST_SETTLE_MS = 200; +/** + * This many frames inside the window, while the viewer is scrolling, is + * motion. Motion streams at half size and lower quality, and the sharp frame + * follows once the page settles. Clicks and the page's own animations stay + * sharp. + */ +const SCREENCAST_MOTION_FRAMES = 4; +const SCREENCAST_MOTION_WINDOW_MS = 300; +const SCREENCAST_MOTION_QUALITY = 50; +const RECORDING_SCREENCAST = { format: "jpeg", quality: 90, everyNthFrame: 1 } as const; + +/** Resolves at `deadline`, in epoch ms. */ +const sleepUntil = (deadline: number) => + new Promise((resolve) => setTimeout(resolve, Math.max(0, deadline - Date.now()))); + +/** + * Whether a tap at a page point lands on something that takes text, so a touch + * viewer raises its soft keyboard only then. Frames count as editable because + * their contents are out of reach. + */ +const EDITABLE_AT_POINT_SCRIPT = `(x, y) => { + let element = document.elementFromPoint(x, y); + while (element && element.shadowRoot) { + const inner = element.shadowRoot.elementFromPoint(x, y); + if (!inner || inner === element) break; + element = inner; + } + if (element && element.tagName === "LABEL" && element.control) element = element.control; + if (!element) return false; + if (element.tagName === "IFRAME" || element.tagName === "FRAME") return true; + if (element.isContentEditable) return true; + if (element.tagName === "TEXTAREA") return !element.disabled && !element.readOnly; + if (element.tagName !== "INPUT") return false; + const nonText = ["button", "checkbox", "color", "file", "hidden", "image", "radio", "range", "reset", "submit"]; + return !nonText.includes(element.type) && !element.disabled && !element.readOnly; +}`; +/** Fill-mode size before any viewer reports its panel. */ +const UNATTACHED_FILL_VIEWPORT = { width: 1280, height: 800 } as const; +const NAVIGATION_TIMEOUT_MS = 15_000; +const ACTION_TIMELINE_LIMIT = 50; + +export class ServerBrowserTabNotFoundError extends Schema.TaggedError()( + "ServerBrowserTabNotFoundError", + { threadId: Schema.String, tabId: Schema.String }, +) { + override get message(): string { + return "The server preview tab does not exist."; + } +} + +export class ServerBrowserLaunchError extends Schema.TaggedError()( + "ServerBrowserLaunchError", + { cause: Schema.Defect() }, +) { + override get message(): string { + return "The server preview browser could not start."; + } +} + +export type ServerBrowserViewerOutput = + | { + readonly _tag: "frame"; + readonly data: Uint8Array; + /** Releases the next frame; call after the bytes reach the socket. */ + readonly ack: Effect.Effect; + } + | { readonly _tag: "viewport"; readonly width: number; readonly height: number } + | { + readonly _tag: "probe"; + readonly x: number; + readonly y: number; + readonly editable: boolean; + } + /** The tab closed; the socket ends. */ + | { readonly _tag: "gone" }; + +export interface ServerBrowserViewer { + readonly output: Queue.Dequeue; + readonly input: (message: unknown) => Effect.Effect; +} + +export class ServerBrowser extends Context.Service< + ServerBrowser, + { + readonly enabled: boolean; + /** Streams one tab to a viewer until the scope closes. */ + readonly attachViewer: (input: { + readonly threadId: string; + readonly tabId: string; + readonly maxWidth: number; + readonly maxHeight: number; + readonly quality: number; + }) => Effect.Effect< + ServerBrowserViewer, + ServerBrowserTabNotFoundError | ServerBrowserLaunchError, + Scope.Scope + >; + } +>()("t3/preview/ServerBrowser") {} + +interface ViewerState { + readonly push: (output: ServerBrowserViewerOutput) => void; + /** Stops and restarts this viewer's screencast around a scaled capture. */ + readonly pause: () => Promise; + readonly resume: () => Promise; + /** Last wheel input from this viewer, for motion mode. */ + scrolledAt: number; +} + +interface Recording { + readonly encoder: Page; + readonly session: CDPSession; + readonly startedAt: string; +} + +interface ServerTab { + readonly threadId: ThreadId; + readonly tabId: string; + readonly page: Page; + readonly cdp: CDPSession; + readonly createdAt: number; + readonly viewers: Set; + readonly consoleEntries: Array; + readonly networkEntries: Array; + readonly actionTimeline: Array; + setting: PreviewViewportSetting; + loading: boolean; + closing: boolean; + recording: Recording | null; +} + +const tabKey = (threadId: string, tabId: string) => `${threadId}\u0000${tabId}`; + +const pushBounded = ( + buffer: Array, + entry: A, + limit = ServerBrowserPage.DIAGNOSTIC_BUFFER_LIMIT, +) => { + buffer.push(entry); + if (buffer.length > limit) buffer.splice(0, buffer.length - limit); +}; + +const fixedViewportSize = (setting: PreviewViewportSetting | undefined) => + setting === undefined || setting._tag === "fill" + ? null + : { width: setting.width, height: setting.height }; + +const asRecord = (value: unknown): Record | null => + typeof value === "object" && value !== null ? (value as Record) : null; + +const num = (value: unknown, fallback = 0) => + typeof value === "number" && Number.isFinite(value) ? value : fallback; + +const modifiersOf = (message: Record) => { + const value = num(message.modifiers); + return Number.isInteger(value) && value >= 0 && value < 16 ? value : 0; +}; + +const make = Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const enabled = isServerBrowserEnabled(config.mode); + const manager = yield* PreviewManager.PreviewManager; + const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker; + const environment = yield* ServerEnvironment.ServerEnvironment; + const toolchain = yield* ServerBrowserToolchain.ServerBrowserToolchain; + const runFork = Effect.runForkWith(yield* Effect.context()); + + const tabs = new Map(); + const pendingTabs = new Map>(); + const contexts = new Map>(); + let hostConnectionId: string | null = null; + + const profilesDir = NodePath.join(config.stateDir, "server-browser", "profiles"); + + const launchContext = async (profileId: string): Promise => { + const resolved = await Effect.runPromise(toolchain.resolve); + const userDataDir = NodePath.join(profilesDir, encodeURIComponent(profileId)); + await NodeFSP.mkdir(userDataDir, { recursive: true }); + // Loaded on first use so servers that never open a server tab skip it. + const { chromium } = await import("playwright-core"); + const launch = (chromiumSandbox: boolean) => + chromium.launchPersistentContext(userDataDir, { + executablePath: resolved.executablePath, + env: { ...process.env, ...resolved.env }, + args: [...resolved.args, "--disable-gpu", `--force-device-scale-factor=${RENDER_SCALE}`], + headless: true, + chromiumSandbox, + viewport: UNATTACHED_FILL_VIEWPORT, + deviceScaleFactor: RENDER_SCALE, + }); + // The namespace sandbox needs unprivileged user namespaces, which some + // containers disable. Running unsandboxed there matches what the agent + // already has: a shell as the same user. + const context = await launch(true).catch(() => launch(false)); + for (const page of context.pages()) await page.close().catch(() => undefined); + context.on("close", () => { + contexts.delete(profileId); + for (const tab of tabs.values()) { + if (tab.page.context() === context) dropTab(tab, true); + } + }); + return context; + }; + + const contextFor = (profileId: string) => { + let context = contexts.get(profileId); + if (!context) { + context = launchContext(profileId); + contexts.set(profileId, context); + context.catch(() => contexts.delete(profileId)); + } + return context; + }; + + const report = (tab: ServerTab, navStatus: PreviewNavStatus) => { + void tab.cdp + .send("Page.getNavigationHistory") + .catch(() => null) + .then((history) => { + const index = history?.currentIndex ?? 0; + const count = history?.entries.length ?? 0; + runFork( + manager + .reportStatus({ + threadId: tab.threadId, + tabId: tab.tabId, + navStatus, + canGoBack: index > 0, + canGoForward: index < count - 1, + }) + .pipe(Effect.ignore), + ); + }); + }; + + const reportLoaded = async (tab: ServerTab) => { + const url = tab.page.url(); + if (url === "about:blank") return; + const title = (await tab.page.title().catch(() => "")).slice(0, 512); + report(tab, { _tag: "Success", url: url.slice(0, 2048), title }); + }; + + const reportLiveTabs = () => { + const connectionId = hostConnectionId; + if (connectionId === null) return; + runFork( + environment.getEnvironmentId.pipe( + Effect.flatMap((environmentId) => + broker.focusHost({ + clientId: SERVER_HOST_CLIENT_ID, + environmentId, + connectionId, + focused: true, + liveTabs: [...tabs.values()].map((tab) => ({ + threadId: tab.threadId, + tabId: tab.tabId, + visible: tab.viewers.size > 0, + })), + }), + ), + ), + ); + }; + + const broadcastViewport = (tab: ServerTab) => { + const size = tab.page.viewportSize(); + if (!size) return; + for (const viewer of tab.viewers) viewer.push({ _tag: "viewport", ...size }); + }; + + const applySetting = async (tab: ServerTab, setting: PreviewViewportSetting) => { + tab.setting = setting; + const size = fixedViewportSize(setting); + if (size) await tab.page.setViewportSize(size); + broadcastViewport(tab); + }; + + const dropTab = (tab: ServerTab, closeSession: boolean) => { + const key = tabKey(tab.threadId, tab.tabId); + if (tabs.get(key) !== tab) return; + tabs.delete(key); + tab.closing = true; + for (const viewer of tab.viewers) viewer.push({ _tag: "gone" }); + void tab.page.close().catch(() => undefined); + void tab.recording?.encoder.close().catch(() => undefined); + reportLiveTabs(); + if (closeSession) { + runFork(manager.close({ threadId: tab.threadId, tabId: tab.tabId }).pipe(Effect.ignore)); + } + }; + + const createTab = async (snapshot: PreviewSessionSnapshot): Promise => { + const context = await contextFor(snapshot.profileId ?? "default"); + const page = await context.newPage(); + const cdp = await context.newCDPSession(page); + const tab: ServerTab = { + threadId: ThreadId.make(snapshot.threadId), + tabId: snapshot.tabId, + page, + cdp, + createdAt: Date.now(), + viewers: new Set(), + consoleEntries: [], + networkEntries: [], + actionTimeline: [], + setting: snapshot.viewport ?? FILL_PREVIEW_VIEWPORT, + loading: false, + closing: false, + recording: null, + }; + await page.setViewportSize(fixedViewportSize(tab.setting) ?? UNATTACHED_FILL_VIEWPORT); + const isMainNavigation = (request: { isNavigationRequest(): boolean; frame(): unknown }) => + request.isNavigationRequest() && request.frame() === page.mainFrame(); + page.on("request", (request) => { + if (!isMainNavigation(request)) return; + tab.loading = true; + report(tab, { _tag: "Loading", url: request.url().slice(0, 2048), title: "" }); + }); + page.on("load", () => { + tab.loading = false; + void reportLoaded(tab); + }); + page.on("framenavigated", (frame) => { + // Same-document navigations (SPA routes) fire no load event. + if (frame === page.mainFrame() && !tab.loading) void reportLoaded(tab); + }); + page.on("requestfailed", (request) => { + const errorText = request.failure()?.errorText ?? ""; + pushBounded(tab.networkEntries, { + url: request.url(), + method: request.method(), + status: null, + failed: true, + errorText, + timestamp: new Date().toISOString(), + }); + if (!isMainNavigation(request) || errorText.includes("ERR_ABORTED")) return; + tab.loading = false; + const { code, description } = ServerBrowserPage.parseNetError(errorText); + report(tab, { + _tag: "LoadFailed", + url: request.url().slice(0, 2048), + title: "", + code, + description, + }); + }); + page.on("response", (response) => { + pushBounded(tab.networkEntries, { + url: response.url(), + method: response.request().method(), + status: response.status(), + failed: false, + timestamp: new Date().toISOString(), + }); + }); + page.on("console", (message) => { + pushBounded(tab.consoleEntries, { + level: message.type(), + text: message.text().slice(0, 2_000), + timestamp: new Date().toISOString(), + }); + }); + // A blocking dialog would freeze the page for every viewer and agent. + page.on("dialog", (dialog) => void dialog.dismiss().catch(() => undefined)); + // One page per tab: a popup becomes a navigation of its opener. + page.on("popup", (popup) => { + void popup + .waitForURL((url) => url.href !== "about:blank", { + timeout: NAVIGATION_TIMEOUT_MS, + waitUntil: "commit", + }) + .catch(() => undefined) + .then(async () => { + const url = popup.url(); + await popup.close().catch(() => undefined); + if (url !== "about:blank") await page.goto(url).catch(() => undefined); + }); + }); + page.on("close", () => { + if (!tab.closing) dropTab(tab, true); + }); + tabs.set(tabKey(tab.threadId, tab.tabId), tab); + reportLiveTabs(); + if (snapshot.navStatus._tag === "Loading") { + void page.goto(snapshot.navStatus.url).catch(() => undefined); + } + return tab; + }; + + const ensureTab = (snapshot: PreviewSessionSnapshot): Promise => { + const key = tabKey(snapshot.threadId, snapshot.tabId); + const existing = tabs.get(key); + if (existing) return Promise.resolve(existing); + let pending = pendingTabs.get(key); + if (!pending) { + pending = createTab(snapshot).finally(() => pendingTabs.delete(key)); + pendingTabs.set(key, pending); + } + return pending; + }; + + /** Server tabs that exist in PreviewManager but not yet here (e.g. after a crash). */ + const findTab = (threadId: string, tabId: string) => + Effect.gen(function* () { + const existing = tabs.get(tabKey(threadId, tabId)); + if (existing) return existing; + const { sessions } = yield* manager.list({ threadId: ThreadId.make(threadId) }); + const snapshot = sessions.find( + (session) => session.tabId === tabId && session.runtime === "server", + ); + if (!snapshot) return yield* new ServerBrowserTabNotFoundError({ threadId, tabId }); + return yield* Effect.tryPromise({ + try: () => ensureTab(snapshot), + catch: (cause) => new ServerBrowserLaunchError({ cause }), + }); + }); + + const latestThreadTab = (threadId: string) => + [...tabs.values()] + .filter((tab) => tab.threadId === threadId) + .sort((left, right) => right.createdAt - left.createdAt)[0]; + + const statusOf = (tab: ServerTab | undefined): PreviewAutomationStatus => { + if (!tab) { + return { + available: false, + visible: false, + tabId: null, + url: null, + title: null, + loading: false, + }; + } + const url = tab.page.url(); + const viewport = tab.page.viewportSize(); + return { + available: true, + visible: tab.viewers.size > 0, + tabId: tab.tabId, + url: url === "about:blank" ? null : url, + title: null, + loading: tab.loading, + viewportSetting: tab.setting, + ...(viewport ? { viewport } : {}), + }; + }; + + const statusWithTitle = async (tab: ServerTab | undefined) => { + const status = statusOf(tab); + if (!tab || status.url === null) return status; + return { ...status, title: (await tab.page.title().catch(() => "")) || null }; + }; + + const resolveNavigationUrl = (input: PreviewAutomationNavigateInput) => { + if (input.url !== undefined) return normalizePreviewUrl(input.url); + const target = input.target!; + if (target.kind === "url") return normalizePreviewUrl(target.url); + // The browser runs inside the environment, so its ports are loopback. + const path = target.path ?? ""; + return `${target.protocol ?? "http"}://localhost:${target.port}${path.startsWith("/") || path === "" ? path : `/${path}`}`; + }; + + const navigate = async ( + tab: ServerTab, + url: string, + readiness: "load" | "domContentLoaded" | "none", + timeout: number, + ) => { + const navigation = tab.page.goto(url, { + timeout, + waitUntil: + readiness === "domContentLoaded" + ? "domcontentloaded" + : readiness === "none" + ? "commit" + : "load", + }); + if (readiness === "none") { + void navigation.catch(() => undefined); + return; + } + await navigation.catch((cause: unknown) => { + const message = cause instanceof Error ? cause.message : String(cause); + if (/ERR_[A-Z_]+/.test(message)) { + throw new ServerBrowserPage.ServerBrowserOperationError( + "PreviewAutomationExecutionError", + `Navigation to ${url} failed: ${ServerBrowserPage.parseNetError(message).description}`, + ); + } + throw cause; + }); + }; + + const moveAgentCursor = ( + tab: ServerTab, + point: { readonly x: number; readonly y: number }, + click: boolean, + ) => { + void tab.recording?.encoder + .evaluate( + ([x, y, isClick]) => { + const recorder = ( + globalThis as unknown as { + __t3Recorder?: { cursor(x: number, y: number, click: boolean): void }; + } + ).__t3Recorder; + recorder?.cursor(x, y, isClick); + }, + [point.x, point.y, click] as const, + ) + .catch(() => undefined); + }; + + const startRecording = async (tab: ServerTab) => { + if (tab.recording) return tab.recording; + const encoder = await tab.page.context().newPage(); + await encoder.evaluate(ServerBrowserPage.RECORDING_ENCODER_SCRIPT); + const session = await tab.page.context().newCDPSession(tab.page); + session.on("Page.screencastFrame", (frame) => { + const cssWidth = tab.page.viewportSize()?.width ?? frame.metadata.deviceWidth; + void encoder + .evaluate( + ([data, width]) => { + const recorder = ( + globalThis as unknown as { + __t3Recorder: { frame(data: string, width: number): Promise }; + } + ).__t3Recorder; + return recorder.frame(data, width); + }, + [frame.data, cssWidth] as const, + ) + .catch(() => undefined) + .finally(() => { + void session + .send("Page.screencastFrameAck", { sessionId: frame.sessionId }) + .catch(() => undefined); + }); + }); + await session.send("Page.startScreencast", RECORDING_SCREENCAST); + const recording: Recording = { encoder, session, startedAt: new Date().toISOString() }; + tab.recording = recording; + return recording; + }; + + /** + * Runs a scaled capture with the tab's screencasts stopped, so viewers and + * recordings never receive the frame it renders at the capture scale. + */ + const withScreencastsPaused = async (tab: ServerTab, capture: () => Promise) => { + const viewers = [...tab.viewers]; + const recording = tab.recording; + await Promise.all([ + ...viewers.map((viewer) => viewer.pause()), + recording?.session.send("Page.stopScreencast").catch(() => undefined), + ]); + try { + return await capture(); + } finally { + await Promise.all([ + ...viewers.map((viewer) => viewer.resume()), + recording && tab.recording === recording + ? recording.session + .send("Page.startScreencast", RECORDING_SCREENCAST) + .catch(() => undefined) + : undefined, + ]); + } + }; + + const stopRecording = async (tab: ServerTab) => { + const recording = tab.recording; + if (!recording) { + throw new ServerBrowserPage.ServerBrowserOperationError( + "PreviewAutomationRecordingNotActiveError", + "No recording is active for this tab.", + ); + } + tab.recording = null; + await recording.session.send("Page.stopScreencast").catch(() => undefined); + await recording.session.detach().catch(() => undefined); + type EncoderWindow = { + __t3Recorder: { + stop(): Promise<{ mimeType: string | null; count: number }>; + chunk(index: number): Promise; + }; + }; + const { mimeType, count } = await recording.encoder.evaluate(() => + (globalThis as unknown as EncoderWindow).__t3Recorder.stop(), + ); + const chunks: Array = []; + for (let index = 0; index < count; index += 1) { + const chunk = await recording.encoder.evaluate( + (chunkIndex) => (globalThis as unknown as EncoderWindow).__t3Recorder.chunk(chunkIndex), + index, + ); + chunks.push(Buffer.from(chunk, "base64")); + } + await recording.encoder.close().catch(() => undefined); + const data = Buffer.concat(chunks); + if (!mimeType || data.byteLength === 0) { + throw new ServerBrowserPage.ServerBrowserOperationError( + "PreviewAutomationExecutionError", + "The recording captured no frames.", + ); + } + if (data.byteLength > PROVIDER_SEND_TURN_MAX_FILE_BYTES) { + throw new ServerBrowserPage.ServerBrowserOperationError( + "PreviewAutomationRecordingTooLargeError", + "The recording is larger than the attachment limit.", + ); + } + // Written where a desktop upload would land, so the MCP handler claims + // both the same way. + const extension = mimeType.startsWith("video/mp4") ? "mp4" : "webm"; + const pendingId = `${PENDING_ATTACHMENT_THREAD_SEGMENT}-${NodeCrypto.randomUUID()}-${extension}`; + const path = NodePath.join(config.attachmentsDir, `${pendingId}.${extension}`); + await NodeFSP.mkdir(config.attachmentsDir, { recursive: true }); + await NodeFSP.writeFile(path, data); + return { + id: pendingId, + tabId: tab.tabId, + path, + mimeType: mimeType.split(";")[0]!, + sizeBytes: data.byteLength, + createdAt: new Date().toISOString(), + uploadedAttachmentId: pendingId, + }; + }; + + const recordAction = (tab: ServerTab, action: string, run: () => Promise): Promise => { + const event: { + -readonly [K in keyof PreviewAutomationActionEvent]: PreviewAutomationActionEvent[K]; + } = { + id: NodeCrypto.randomUUID(), + action, + status: "running", + startedAt: new Date().toISOString(), + }; + pushBounded(tab.actionTimeline, event, ACTION_TIMELINE_LIMIT); + return run().then( + (result) => { + event.status = "succeeded"; + event.completedAt = new Date().toISOString(); + return result; + }, + (cause: unknown) => { + event.status = "failed"; + event.completedAt = new Date().toISOString(); + event.error = cause instanceof Error ? cause.message.split("\n")[0] : String(cause); + throw cause; + }, + ); + }; + + const requireTab = async (request: PreviewAutomationRequest) => { + const tab = + request.tabId === undefined + ? latestThreadTab(request.threadId) + : (tabs.get(tabKey(request.threadId, request.tabId)) ?? + (await Effect.runPromise( + findTab(request.threadId, request.tabId).pipe(Effect.orElseSucceed(() => undefined)), + ))); + if (!tab) { + throw new ServerBrowserPage.ServerBrowserOperationError( + "PreviewAutomationTabNotFoundError", + "No server preview tab is open for this thread. Call preview_open first.", + ); + } + return tab; + }; + + const runOperation = async (request: PreviewAutomationRequest): Promise => { + const input = request.input; + switch (request.operation) { + case "status": + return statusWithTitle( + request.tabId === undefined + ? latestThreadTab(request.threadId) + : tabs.get(tabKey(request.threadId, request.tabId)), + ); + case "open": { + const open = input as PreviewAutomationOpenInput; + const url = open.url === undefined ? undefined : normalizePreviewUrl(open.url); + const reuse = open.reuseExistingTab ?? true; + const existing = + reuse && request.tabId !== undefined + ? tabs.get(tabKey(request.threadId, request.tabId)) + : undefined; + if (existing) { + if (url) await navigate(existing, url, "load", request.timeoutMs); + return statusWithTitle(existing); + } + const snapshot = await Effect.runPromise( + manager.open({ + threadId: request.threadId, + ...(url ? { url } : {}), + runtime: "server", + }), + ); + const tab = await ensureTab(snapshot); + if (url) { + await tab.page + .waitForLoadState("load", { timeout: request.timeoutMs }) + .catch(() => undefined); + } + return statusWithTitle(tab); + } + case "navigate": { + const tab = await requireTab(request); + const navigateInput = input as PreviewAutomationNavigateInput; + await recordAction(tab, "navigate", () => + navigate( + tab, + resolveNavigationUrl(navigateInput), + navigateInput.readiness ?? "load", + navigateInput.timeoutMs ?? request.timeoutMs, + ), + ); + return statusWithTitle(tab); + } + case "resize": { + const tab = await requireTab(request); + const setting = resolvePreviewViewport(input as PreviewAutomationResizeInput); + await Effect.runPromise( + manager.resize({ threadId: tab.threadId, tabId: tab.tabId, viewport: setting }), + ); + await applySetting(tab, setting); + return { + tabId: tab.tabId, + setting, + viewport: tab.page.viewportSize() ?? UNATTACHED_FILL_VIEWPORT, + }; + } + case "setColorScheme": { + const tab = await requireTab(request); + const { colorScheme } = input as PreviewAutomationSetColorSchemeInput; + await tab.page.emulateMedia({ colorScheme: colorScheme === "system" ? null : colorScheme }); + return { tabId: tab.tabId, colorScheme }; + } + case "snapshot": { + const tab = await requireTab(request); + return withScreencastsPaused(tab, () => + ServerBrowserPage.snapshot({ + page: tab.page, + cdp: tab.cdp, + renderScale: RENDER_SCALE, + consoleEntries: tab.consoleEntries, + networkEntries: tab.networkEntries, + actionTimeline: tab.actionTimeline, + }), + ); + } + case "click": { + const tab = await requireTab(request); + const clickInput = input as PreviewAutomationClickInput; + const point = await recordAction(tab, "click", () => + ServerBrowserPage.click(tab.page, clickInput), + ); + moveAgentCursor(tab, point, true); + return undefined; + } + case "type": { + const tab = await requireTab(request); + await recordAction(tab, "type", () => + ServerBrowserPage.type(tab.page, input as PreviewAutomationTypeInput), + ); + return undefined; + } + case "press": { + const tab = await requireTab(request); + await recordAction(tab, "press", () => + ServerBrowserPage.press(tab.page, input as PreviewAutomationPressInput), + ); + return undefined; + } + case "scroll": { + const tab = await requireTab(request); + await recordAction(tab, "scroll", () => + ServerBrowserPage.scroll(tab.page, input as PreviewAutomationScrollInput), + ); + return undefined; + } + case "evaluate": { + const tab = await requireTab(request); + return ServerBrowserPage.evaluate(tab.cdp, input as PreviewAutomationEvaluateInput); + } + case "waitFor": { + const tab = await requireTab(request); + await ServerBrowserPage.waitFor(tab.page, input as PreviewAutomationWaitForInput); + return undefined; + } + case "recordingStart": { + const tab = await requireTab(request); + const recording = await startRecording(tab); + return { tabId: tab.tabId, recording: true, startedAt: recording.startedAt }; + } + case "recordingStop": { + const tab = + request.tabId === undefined + ? [...tabs.values()].find( + (candidate) => candidate.threadId === request.threadId && candidate.recording, + ) + : await requireTab(request); + if (!tab) { + throw new ServerBrowserPage.ServerBrowserOperationError( + "PreviewAutomationRecordingNotActiveError", + "No recording is active for this thread.", + ); + } + return stopRecording(tab); + } + } + }; + + const handleRequest = (connectionId: string, request: PreviewAutomationRequest) => + Effect.tryPromise({ + try: () => runOperation(request), + catch: ServerBrowserPage.toOperationError, + }).pipe( + Effect.match({ + onSuccess: (result) => ({ ok: true as const, result }), + onFailure: (error) => ({ + ok: false as const, + error: { + _tag: error.tag, + message: error.message, + ...(error.detail === undefined ? {} : { detail: error.detail }), + }, + }), + }), + Effect.flatMap((outcome) => + broker.respond({ + clientId: SERVER_HOST_CLIENT_ID, + connectionId, + requestId: request.requestId, + ...outcome, + }), + ), + Effect.ignore, + ); + + const mirrorManagerEvent = (event: PreviewEvent) => + Effect.promise(async () => { + if (event.type === "opened" && event.snapshot.runtime === "server") { + await ensureTab(event.snapshot).catch(() => undefined); + return; + } + const tab = tabs.get(tabKey(event.threadId, event.tabId)); + if (!tab) return; + if (event.type === "resized" && event.snapshot.viewport) { + await applySetting(tab, event.snapshot.viewport).catch(() => undefined); + } else if (event.type === "closed") { + dropTab(tab, false); + } + }); + + const dispatchViewerInput = async ( + tab: ServerTab, + session: CDPSession, + viewer: ViewerState, + raw: unknown, + ) => { + const message = asRecord(raw); + if (!message) return; + const modifiers = modifiersOf(message); + switch (message.type) { + case "mouse": { + const action = message.action; + const type = + action === "down" ? "mousePressed" : action === "up" ? "mouseReleased" : "mouseMoved"; + const button = ["none", "left", "middle", "right"].includes(String(message.button)) + ? (message.button as "none" | "left" | "middle" | "right") + : "none"; + await session.send("Input.dispatchMouseEvent", { + type, + x: num(message.x), + y: num(message.y), + button, + buttons: num(message.buttons), + clickCount: num(message.clickCount, type === "mouseMoved" ? 0 : 1), + modifiers, + }); + return; + } + case "wheel": + viewer.scrolledAt = Date.now(); + await session.send("Input.dispatchMouseEvent", { + type: "mouseWheel", + x: num(message.x), + y: num(message.y), + deltaX: num(message.deltaX), + deltaY: num(message.deltaY), + modifiers, + }); + return; + case "key": { + const key = typeof message.key === "string" ? message.key : ""; + const code = typeof message.code === "string" ? message.code : ""; + const text = typeof message.text === "string" ? message.text : undefined; + if (message.action === "up") { + await session.send("Input.dispatchKeyEvent", { type: "keyUp", key, code, modifiers }); + return; + } + await session.send("Input.dispatchKeyEvent", { + type: text ? "keyDown" : "rawKeyDown", + key, + code, + modifiers, + ...(text ? { text, unmodifiedText: text } : {}), + windowsVirtualKeyCode: num( + message.keyCode, + key.length === 1 ? key.toUpperCase().charCodeAt(0) : 0, + ), + }); + return; + } + case "text": + if (typeof message.text === "string" && message.text.length > 0) { + await session.send("Input.insertText", { text: message.text.slice(0, 10_000) }); + } + return; + case "resize": { + const width = Math.round(num(message.width)); + const height = Math.round(num(message.height)); + if (tab.setting._tag !== "fill" || width < 100 || height < 100) return; + const current = tab.page.viewportSize(); + if (current?.width === width && current.height === height) return; + await tab.page.setViewportSize({ + width: Math.min(width, 3840), + height: Math.min(height, 2160), + }); + broadcastViewport(tab); + return; + } + case "navigate": + if (typeof message.url === "string") { + const url = normalizePreviewUrl(message.url); + void tab.page.goto(url).catch(() => undefined); + } + return; + case "history": + void (num(message.delta) < 0 ? tab.page.goBack() : tab.page.goForward()).catch( + () => undefined, + ); + return; + case "reload": + void tab.page.reload().catch(() => undefined); + return; + case "probe": { + const x = num(message.x); + const y = num(message.y); + const result = await session.send("Runtime.evaluate", { + expression: `(${EDITABLE_AT_POINT_SCRIPT})(${x}, ${y})`, + returnByValue: true, + }); + viewer.push({ _tag: "probe", x, y, editable: result.result.value === true }); + return; + } + } + }; + + const attachViewer: ServerBrowser["Service"]["attachViewer"] = (input) => + Effect.gen(function* () { + const tab = yield* findTab(input.threadId, input.tabId); + const output = yield* Queue.unbounded(); + const session = yield* Effect.acquireRelease( + Effect.tryPromise({ + try: () => tab.page.context().newCDPSession(tab.page), + catch: (cause) => new ServerBrowserLaunchError({ cause }), + }), + (session) => + Effect.promise(() => + session + .send("Page.stopScreencast") + .catch(() => undefined) + .then(() => session.detach()) + .catch(() => undefined), + ), + ); + const viewer: ViewerState = { + push: (next) => void Queue.offerUnsafe(output, next), + pause: () => { + screencastParams = screencastParams.then(() => + session.send("Page.stopScreencast").then( + () => undefined, + () => undefined, + ), + ); + return screencastParams; + }, + resume: () => startScreencast(motion ? 0.5 : 1), + scrolledAt: 0, + }; + yield* Effect.acquireRelease( + Effect.sync(() => { + tab.viewers.add(viewer); + reportLiveTabs(); + }), + () => + Effect.sync(() => { + tab.viewers.delete(viewer); + reportLiveTabs(); + }), + ); + const quality = Math.min(100, Math.max(1, Math.round(input.quality))); + // Full scale: a scaled capture would flash in every other viewer. + const pushStill = async () => { + const data = await ServerBrowserPage.captureViewport(tab.page, session, { + format: "jpeg", + quality, + scale: 1, + }).catch(() => null); + if (data) + viewer.push({ _tag: "frame", data: Buffer.from(data, "base64"), ack: Effect.void }); + }; + // Frames that arrive while two are in flight may have pushed later ones + // out, and a page that then goes still never repaints them. A still + // after the burst shows the final state. + let framesInFlight = 0; + let mayHaveDropped = false; + let motion = false; + const recentFrames: Array = []; + let screencastParams = Promise.resolve(); + const startScreencast = (scale: number) => { + screencastParams = screencastParams.then(async () => { + await session.send("Page.stopScreencast").catch(() => undefined); + await session + .send("Page.startScreencast", { + format: "jpeg", + quality: scale < 1 ? Math.min(quality, SCREENCAST_MOTION_QUALITY) : quality, + maxWidth: Math.max(1, Math.round(input.maxWidth * scale)), + maxHeight: Math.max(1, Math.round(input.maxHeight * scale)), + }) + .catch(() => undefined); + }); + return screencastParams; + }; + let settleTimer: ReturnType | null = null; + yield* Effect.addFinalizer(() => + Effect.sync(() => { + if (settleTimer !== null) clearTimeout(settleTimer); + }), + ); + let screencastStarted = false; + session.on("Page.screencastFrame", (frame) => { + screencastStarted = true; + if (framesInFlight > 0) mayHaveDropped = true; + framesInFlight += 1; + const arrivedAt = Date.now(); + recentFrames.push(arrivedAt); + while (recentFrames[0]! < arrivedAt - SCREENCAST_MOTION_WINDOW_MS) recentFrames.shift(); + if ( + !motion && + recentFrames.length >= SCREENCAST_MOTION_FRAMES && + arrivedAt - viewer.scrolledAt < SCREENCAST_MOTION_WINDOW_MS + ) { + motion = true; + void startScreencast(0.5); + } else if (motion && arrivedAt - viewer.scrolledAt > SCREENCAST_SETTLE_MS) { + // The page keeps animating after the scroll; its own frames are sharp again. + motion = false; + recentFrames.length = 0; + void startScreencast(1); + } + if (settleTimer !== null) clearTimeout(settleTimer); + settleTimer = setTimeout(() => { + settleTimer = null; + if (!motion && !mayHaveDropped) return; + mayHaveDropped = false; + if (motion) { + motion = false; + recentFrames.length = 0; + void startScreencast(1); + } + void pushStill(); + }, SCREENCAST_SETTLE_MS); + viewer.push({ + _tag: "frame", + data: Buffer.from(frame.data, "base64"), + ack: Effect.promise(() => + sleepUntil(arrivedAt + SCREENCAST_ACK_PACE_MS).then(() => { + framesInFlight -= 1; + return session + .send("Page.screencastFrameAck", { sessionId: frame.sessionId }) + .catch(() => undefined); + }), + ), + }); + }); + broadcastViewport(tab); + yield* Effect.promise(() => startScreencast(1)); + // An idle page does not repaint for a new screencast, so the viewer + // starts from a still unless a live frame beat it. + if (!screencastStarted) yield* Effect.promise(pushStill); + return { + output, + input: (message: unknown) => + Effect.promise(() => + dispatchViewerInput(tab, session, viewer, message).catch(() => undefined), + ), + } satisfies ServerBrowserViewer; + }); + + if (enabled) { + yield* manager.events.pipe(Stream.runForEach(mirrorManagerEvent), Effect.forkScoped); + const environmentId = yield* environment.getEnvironmentId; + yield* broker + .connect( + { + clientId: SERVER_HOST_CLIENT_ID, + environmentId, + supportedOperations: [...PREVIEW_AUTOMATION_OPERATIONS], + }, + { preferred: true }, + ) + .pipe( + Effect.flatMap((events) => + events.pipe( + Stream.runForEach((event) => { + if (event.type === "connected") { + hostConnectionId = event.connectionId; + return Effect.sync(reportLiveTabs); + } + return handleRequest(event.connectionId, event.request).pipe( + Effect.forkScoped, + Effect.asVoid, + ); + }), + ), + ), + Effect.forkScoped, + ); + yield* Effect.addFinalizer(() => + Effect.promise(async () => { + for (const context of contexts.values()) { + await context.then((resolved) => resolved.close()).catch(() => undefined); + } + }), + ); + } + + return ServerBrowser.of({ enabled, attachViewer }); +}); + +export const layer = Layer.effect(ServerBrowser, make); diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts new file mode 100644 index 000000000000..90ce25adda26 --- /dev/null +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -0,0 +1,422 @@ +// @effect-diagnostics globalDate:off - Playwright callbacks run outside the Effect runtime. +/** + * Page-level automation for server-hosted preview tabs. + * + * Plain async helpers over one Playwright page, called by `ServerBrowser`. + * Results mirror what desktop hosts return for the same operations, so the + * MCP preview tools cannot tell the two runtimes apart. Failures carry the + * broker's error tags (see `PreviewAutomationBroker.classifyResponseError`). + */ +import type { + PreviewAutomationClickInput, + PreviewAutomationConsoleEntry, + PreviewAutomationEvaluateInput, + PreviewAutomationNetworkEntry, + PreviewAutomationPressInput, + PreviewAutomationScrollInput, + PreviewAutomationSnapshot, + PreviewAutomationTypeInput, + PreviewAutomationWaitForInput, +} from "@t3tools/contracts"; +import type { CDPSession, Locator, Page } from "playwright-core"; + +const MAX_EVALUATION_BYTES = 64_000; +const MAX_VISIBLE_TEXT_LENGTH = 20_000; +const MAX_INTERACTIVE_ELEMENTS = 200; +const MAX_INTERACTIVE_ELEMENT_NAME_LENGTH = 200; +const MAX_SCREENSHOT_WIDTH = 1280; +export const DIAGNOSTIC_BUFFER_LIMIT = 200; + +export class ServerBrowserOperationError extends Error { + readonly tag: string; + readonly detail: unknown; + + constructor(tag: string, message: string, detail?: unknown) { + super(message); + this.tag = tag; + this.detail = detail; + } +} + +/** Maps a Playwright failure onto the tag the broker classifies. */ +export const toOperationError = (cause: unknown): ServerBrowserOperationError => { + if (cause instanceof ServerBrowserOperationError) return cause; + const message = cause instanceof Error ? cause.message : String(cause); + const firstLine = message.split("\n")[0] ?? message; + if (cause instanceof Error && cause.name === "TimeoutError") { + return new ServerBrowserOperationError("PreviewAutomationTimeoutError", firstLine); + } + if (/while parsing selector|Unknown engine|Unexpected token/i.test(message)) { + return new ServerBrowserOperationError("PreviewAutomationInvalidSelectorError", firstLine); + } + if (/not an |not editable|not an editable/i.test(message)) { + return new ServerBrowserOperationError("PreviewAutomationTargetNotEditableError", firstLine); + } + return new ServerBrowserOperationError("PreviewAutomationExecutionError", firstLine); +}; + +const DEFAULT_TIMEOUT_MS = 15_000; + +const targetLocator = ( + page: Page, + input: { readonly locator?: string | undefined; readonly selector?: string | undefined }, +): Locator | null => { + const selector = input.locator ?? input.selector; + return selector === undefined ? null : page.locator(selector).first(); +}; + +const SNAPSHOT_SCRIPT = `(() => { + const selectorFor = (element) => { + if (element.id) return "#" + CSS.escape(element.id); + for (const attribute of ["data-testid", "name"]) { + const value = element.getAttribute(attribute); + if (value) return element.tagName.toLowerCase() + "[" + attribute + "=" + JSON.stringify(value) + "]"; + } + const parts = []; + for (let current = element; current && current.nodeType === Node.ELEMENT_NODE && parts.length < 8; current = current.parentElement) { + const parent = current.parentElement; + const siblings = parent ? Array.from(parent.children).filter((child) => child.tagName === current.tagName) : []; + const base = current.tagName.toLowerCase(); + parts.unshift(siblings.length > 1 ? base + ":nth-of-type(" + (siblings.indexOf(current) + 1) + ")" : base); + } + return parts.join(" > "); + }; + const visible = (element) => { + const style = getComputedStyle(element); + const rect = element.getBoundingClientRect(); + return style.visibility !== "hidden" && style.display !== "none" && rect.width > 0 && rect.height > 0; + }; + const elements = Array.from(document.querySelectorAll("a[href],button,input,textarea,select,[role],[tabindex]")) + .filter(visible) + .slice(0, ${MAX_INTERACTIVE_ELEMENTS}) + .map((element) => { + const rect = element.getBoundingClientRect(); + return { + tag: element.tagName.toLowerCase(), + role: element.getAttribute("role"), + name: (element.getAttribute("aria-label") || element.innerText || element.getAttribute("name") || "").slice(0, ${MAX_INTERACTIVE_ELEMENT_NAME_LENGTH}), + selector: selectorFor(element), + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + }; + }); + return { + url: location.href, + title: document.title, + loading: document.readyState !== "complete", + visibleText: (document.body?.innerText || "").slice(0, ${MAX_VISIBLE_TEXT_LENGTH}), + interactiveElements: elements, + }; +})()`; + +/** + * The visible viewport as an image, `scale` relative to the rendered pixels. + * + * A scaled capture briefly re-renders the page at that scale, and every live + * screencast on the page streams that frame; callers pause them around it. + * At full scale there is no clip and nothing re-renders. The clip is + * document-relative, so it starts at the scroll offset. + */ +export const captureViewport = async ( + page: Page, + cdp: CDPSession, + options: { readonly format: "png" | "jpeg"; readonly quality?: number; readonly scale: number }, +) => { + const quality = options.quality === undefined ? {} : { quality: options.quality }; + if (options.scale >= 1) { + const { data } = await cdp.send("Page.captureScreenshot", { + format: options.format, + ...quality, + }); + return data; + } + const viewport = page.viewportSize() ?? { width: 1280, height: 800 }; + const { cssVisualViewport } = await cdp.send("Page.getLayoutMetrics"); + const { data } = await cdp.send("Page.captureScreenshot", { + format: options.format, + ...quality, + clip: { + x: cssVisualViewport.pageX, + y: cssVisualViewport.pageY, + width: viewport.width, + height: viewport.height, + scale: options.scale, + }, + }); + return data; +}; + +/** + * Screenshot of the current viewport at device resolution, scaled down to at + * most `MAX_SCREENSHOT_WIDTH` like desktop snapshots. `clip.scale` multiplies + * the render scale, so this needs no image library on the server. + */ + +export const captureViewportPng = async (page: Page, cdp: CDPSession, renderScale: number) => { + const viewport = page.viewportSize() ?? { width: 1280, height: 800 }; + const scale = Math.min(1, MAX_SCREENSHOT_WIDTH / (viewport.width * renderScale)); + const data = await captureViewport(page, cdp, { format: "png", scale }); + return { + mimeType: "image/png" as const, + data, + width: Math.round(viewport.width * renderScale * scale), + height: Math.round(viewport.height * renderScale * scale), + }; +}; + +export const snapshot = async (input: { + readonly page: Page; + readonly cdp: CDPSession; + readonly renderScale: number; + readonly consoleEntries: ReadonlyArray; + readonly networkEntries: ReadonlyArray; + readonly actionTimeline: PreviewAutomationSnapshot["actionTimeline"]; +}): Promise => { + const [page, screenshot] = await Promise.all([ + input.page.evaluate(SNAPSHOT_SCRIPT) as Promise< + Pick< + PreviewAutomationSnapshot, + "url" | "title" | "loading" | "visibleText" | "interactiveElements" + > + >, + captureViewportPng(input.page, input.cdp, input.renderScale), + ]); + return { + ...page, + // The MCP layer drops the AX tree before it reaches the agent. + accessibilityTree: null, + consoleEntries: [...input.consoleEntries], + networkEntries: [...input.networkEntries], + actionTimeline: [...input.actionTimeline], + screenshot, + }; +}; + +/** Resolves the viewport point an action targets, for the agent cursor. */ +export const click = async ( + page: Page, + input: PreviewAutomationClickInput, +): Promise<{ readonly x: number; readonly y: number }> => { + const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const locator = targetLocator(page, input); + if (locator === null) { + await page.mouse.click(input.x!, input.y!); + return { x: input.x!, y: input.y! }; + } + await locator.scrollIntoViewIfNeeded({ timeout }); + const box = await locator.boundingBox({ timeout }); + await locator.click({ timeout }); + return box ? { x: box.x + box.width / 2, y: box.y + box.height / 2 } : { x: 0, y: 0 }; +}; + +export const type = async (page: Page, input: PreviewAutomationTypeInput) => { + const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const locator = targetLocator(page, input); + if (locator !== null) { + if (input.clear) { + await locator.fill(input.text, { timeout }); + return; + } + await locator.focus({ timeout }); + } else if (input.clear) { + await page.keyboard.press("ControlOrMeta+A"); + await page.keyboard.press("Delete"); + } + await page.keyboard.insertText(input.text); +}; + +export const press = async (page: Page, input: PreviewAutomationPressInput) => { + await page.keyboard.press([...(input.modifiers ?? []), input.key].join("+")); +}; + +export const scroll = async (page: Page, input: PreviewAutomationScrollInput) => { + const delta = [input.deltaX ?? 0, input.deltaY ?? 0] as const; + const locator = targetLocator(page, input); + if (locator === null) { + // Page-side code is passed as source: the server compiles without DOM types. + await page.evaluate(`scrollBy(${delta[0]}, ${delta[1]})`); + return; + } + await locator.evaluate((element, [x, y]) => element.scrollBy(x, y), delta); +}; + +export const evaluate = async (cdp: CDPSession, input: PreviewAutomationEvaluateInput) => { + const result = await cdp.send("Runtime.evaluate", { + expression: input.expression, + awaitPromise: input.awaitPromise ?? true, + returnByValue: input.returnByValue ?? true, + }); + if (result.exceptionDetails) { + throw new ServerBrowserOperationError( + "PreviewAutomationExecutionError", + result.exceptionDetails.exception?.description ?? result.exceptionDetails.text, + ); + } + const value = + "value" in result.result ? result.result.value : (result.result.description ?? null); + const actualBytes = Buffer.byteLength(JSON.stringify(value ?? null), "utf8"); + if (actualBytes > MAX_EVALUATION_BYTES) { + throw new ServerBrowserOperationError( + "PreviewAutomationResultTooLargeError", + `Evaluation result is ${actualBytes} bytes; the limit is ${MAX_EVALUATION_BYTES}.`, + { maximumBytes: MAX_EVALUATION_BYTES }, + ); + } + return value; +}; + +export const waitFor = async (page: Page, input: PreviewAutomationWaitForInput) => { + const deadline = Date.now() + (input.timeoutMs ?? DEFAULT_TIMEOUT_MS); + const remaining = () => Math.max(1, deadline - Date.now()); + const locator = targetLocator(page, input); + if (locator !== null) await locator.waitFor({ state: "visible", timeout: remaining() }); + if (input.text !== undefined) { + await page.waitForFunction( + `(document.body?.innerText ?? "").includes(${JSON.stringify(input.text)})`, + undefined, + { timeout: remaining() }, + ); + } + if (input.urlIncludes !== undefined) { + await page.waitForFunction( + `location.href.includes(${JSON.stringify(input.urlIncludes)})`, + undefined, + { + timeout: remaining(), + }, + ); + } +}; + +/** Chromium net error names reported for failed main-frame loads. */ +const NET_ERROR_CODES: Readonly> = { + ERR_FAILED: -2, + ERR_TIMED_OUT: -7, + ERR_CONNECTION_CLOSED: -100, + ERR_CONNECTION_RESET: -101, + ERR_CONNECTION_REFUSED: -102, + ERR_NAME_NOT_RESOLVED: -105, + ERR_INTERNET_DISCONNECTED: -106, + ERR_ADDRESS_UNREACHABLE: -109, + ERR_CERT_AUTHORITY_INVALID: -202, + ERR_EMPTY_RESPONSE: -324, +}; + +export const parseNetError = (errorText: string) => { + const description = /ERR_[A-Z_]+/.exec(errorText)?.[0] ?? errorText; + return { description, code: NET_ERROR_CODES[description] ?? -2 }; +}; + +/** + * In-page video encoder for recordings. Screencast frames are drawn onto a + * canvas and encoded by the browser's own MediaRecorder, so recording needs + * no ffmpeg on the server and costs about the same CPU as piping to x264. + * The agent cursor is drawn on the canvas, not injected into the page. The + * video keeps its first frame's size; a page resized mid-recording is + * letterboxed into it. + */ +export const RECORDING_ENCODER_SCRIPT = `(() => { + const canvas = document.createElement("canvas"); + const context = canvas.getContext("2d"); + const chunks = []; + let recorder = null; + let frame = null; + let cursor = null; + let ring = null; + let ringTimer = null; + let scale = 1; + let offsetX = 0; + let offsetY = 0; + const paint = () => { + if (!frame) return; + const fit = Math.min(canvas.width / frame.width, canvas.height / frame.height); + const width = frame.width * fit; + const height = frame.height * fit; + offsetX = (canvas.width - width) / 2; + offsetY = (canvas.height - height) / 2; + if (width < canvas.width || height < canvas.height) { + context.fillStyle = "#000"; + context.fillRect(0, 0, canvas.width, canvas.height); + } + context.drawImage(frame, offsetX, offsetY, width, height); + scale = (width / frame.width) * frame.cssScale; + if (ring) { + const age = (performance.now() - ring.at) / 400; + if (age < 1) { + context.beginPath(); + context.arc(offsetX + ring.x * scale, offsetY + ring.y * scale, (8 + 18 * age) * scale, 0, Math.PI * 2); + context.strokeStyle = "rgba(59,130,246," + (0.9 * (1 - age)) + ")"; + context.lineWidth = 3 * scale; + context.stroke(); + } + } + if (cursor) { + const x = offsetX + cursor.x * scale; + const y = offsetY + cursor.y * scale; + const s = scale; + context.beginPath(); + context.moveTo(x, y); + context.lineTo(x, y + 18 * s); + context.lineTo(x + 4.5 * s, y + 14 * s); + context.lineTo(x + 8 * s, y + 21 * s); + context.lineTo(x + 11 * s, y + 19.5 * s); + context.lineTo(x + 7.5 * s, y + 12.5 * s); + context.lineTo(x + 13 * s, y + 12.5 * s); + context.closePath(); + context.fillStyle = "#111"; + context.strokeStyle = "#fff"; + context.lineWidth = 1.5 * s; + context.fill(); + context.stroke(); + } + }; + window.__t3Recorder = { + async frame(base64, cssWidth) { + const bytes = Uint8Array.from(atob(base64), (char) => char.charCodeAt(0)); + const bitmap = await createImageBitmap(new Blob([bytes], { type: "image/jpeg" })); + if (!recorder) { + canvas.width = bitmap.width; + canvas.height = bitmap.height; + const mimeType = ["video/mp4;codecs=avc1.640033", "video/webm;codecs=vp9", "video/webm"].find((type) => MediaRecorder.isTypeSupported(type)); + const bitsPerSecond = Math.min(50e6, Math.max(2.5e6, bitmap.width * bitmap.height * 30 * 0.05)); + recorder = new MediaRecorder(canvas.captureStream(30), { mimeType, videoBitsPerSecond: bitsPerSecond }); + recorder.ondataavailable = (event) => { if (event.data.size > 0) chunks.push(event.data); }; + recorder.start(1000); + } + // Frame px per page CSS px, before fitting into the canvas. + bitmap.cssScale = bitmap.width / cssWidth; + frame?.close(); + frame = bitmap; + paint(); + }, + cursor(x, y, click) { + cursor = { x, y }; + if (click) { + ring = { x, y, at: performance.now() }; + clearInterval(ringTimer); + ringTimer = setInterval(() => { + paint(); + if (performance.now() - ring.at > 400) { ring = null; clearInterval(ringTimer); paint(); } + }, 33); + } + paint(); + }, + stop() { + return new Promise((resolve) => { + if (!recorder) return resolve({ mimeType: null, count: 0 }); + recorder.onstop = () => resolve({ mimeType: recorder.mimeType, count: chunks.length }); + recorder.stop(); + }); + }, + async chunk(index) { + const bytes = new Uint8Array(await chunks[index].arrayBuffer()); + let binary = ""; + for (let offset = 0; offset < bytes.length; offset += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000)); + } + return btoa(binary); + }, + }; +})()`; diff --git a/apps/server/src/preview/ServerBrowserStream.ts b/apps/server/src/preview/ServerBrowserStream.ts new file mode 100644 index 000000000000..a1c45f5da89d --- /dev/null +++ b/apps/server/src/preview/ServerBrowserStream.ts @@ -0,0 +1,131 @@ +/** + * `/api/preview-stream/ws`: one server preview tab over a WebSocket. + * + * Frames go out as binary JPEG messages, viewport changes as JSON text, and + * viewer input comes back as JSON text. Each frame is acknowledged to Chromium + * only after the socket write drains, so a slow link (phone over T3 Connect) + * gets fewer frames instead of a growing buffer. Authentication matches the + * device hub proxy; the socket drives the page, so it needs operate scope. + */ +import { AuthOrchestrationOperateScope } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; +import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; +import * as Socket from "effect/unstable/socket/Socket"; + +import { authenticateMediaRequest } from "../auth/http.ts"; +import * as ServerBrowser from "./ServerBrowser.ts"; + +export const PREVIEW_STREAM_ROUTE_PREFIX = "/api/preview-stream"; +/** Matches `PREVIEW_STREAM_TAB_GONE_CODE` in the client. */ +const TAB_GONE_CODE = 4404; + +const DEFAULT_QUALITY = 70; +const textDecoder = new TextDecoder(); + +const intParam = (params: URLSearchParams, name: string, fallback: number, max: number) => { + const value = Number(params.get(name)); + return Number.isFinite(value) && value > 0 ? Math.min(Math.round(value), max) : fallback; +}; + +const parseMessage = (chunk: Uint8Array | string): unknown => { + try { + return JSON.parse(typeof chunk === "string" ? chunk : textDecoder.decode(chunk)); + } catch { + return null; + } +}; + +const makeHandler = (browser: ServerBrowser.ServerBrowser["Service"]) => + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const url = HttpServerRequest.toURL(request); + if (Option.isNone(url)) return HttpServerResponse.text("Bad Request", { status: 400 }); + if ( + url.value.pathname !== `${PREVIEW_STREAM_ROUTE_PREFIX}/ws` || + request.headers.upgrade?.toLowerCase() !== "websocket" + ) { + return HttpServerResponse.text("Not Found", { status: 404 }); + } + yield* authenticateMediaRequest(AuthOrchestrationOperateScope); + const params = url.value.searchParams; + const threadId = params.get("threadId") ?? ""; + const tabId = params.get("tabId") ?? ""; + if (!browser.enabled || threadId.length === 0 || tabId.length === 0) { + return HttpServerResponse.text("Not Found", { status: 404 }); + } + return yield* Effect.scoped( + Effect.gen(function* () { + const attached = yield* browser + .attachViewer({ + threadId, + tabId, + maxWidth: intParam(params, "maxWidth", 1280, 7680), + maxHeight: intParam(params, "maxHeight", 800, 4320), + quality: intParam(params, "quality", DEFAULT_QUALITY, 100), + }) + .pipe( + Effect.map(Option.some), + Effect.catchTag("ServerBrowserTabNotFoundError", () => Effect.succeedNone), + ); + const socket = yield* request.upgrade; + const writer = yield* socket.writer; + // A refused upgrade reads as an auth failure to ticket clients, so a + // missing tab is a close code they stop on. + const gone = writer.write(new Socket.CloseEvent(TAB_GONE_CODE, "tab closed")); + if (Option.isNone(attached)) { + yield* gone; + return HttpServerResponse.empty(); + } + const viewer = attached.value; + const reader = yield* socket.reader; + const sendOutput = Queue.take(viewer.output).pipe( + Effect.flatMap((output) => { + switch (output._tag) { + case "frame": + return writer.write(output.data).pipe(Effect.ensuring(output.ack)); + case "viewport": + return writer.write( + JSON.stringify({ type: "viewport", width: output.width, height: output.height }), + ); + case "probe": + return writer.write( + JSON.stringify({ + type: "probe", + x: output.x, + y: output.y, + editable: output.editable, + }), + ); + case "gone": + return gone.pipe(Effect.andThen(Effect.interrupt)); + } + }), + ); + const receiveInput = reader.pull.pipe( + Effect.flatMap((chunks) => + Effect.forEach(chunks, (chunk) => viewer.input(parseMessage(chunk)), { discard: true }), + ), + ); + // Whichever side ends first closes the other through scope teardown. + return yield* Effect.raceFirst(Effect.forever(sendOutput), Effect.forever(receiveInput)); + }), + ).pipe( + Effect.catchTag("ServerBrowserLaunchError", (error) => + Effect.logWarning("server preview browser failed to start", { cause: error.cause }).pipe( + Effect.as(HttpServerResponse.text("Service Unavailable", { status: 503 })), + ), + ), + // A dropped socket is a normal end of viewing. + Effect.catch(() => Effect.succeed(HttpServerResponse.empty())), + ); + }); + +// Route handlers only see request-scoped services, so the browser is captured +// when the route is registered. +export const routeLayer = HttpRouter.use((router) => + Effect.flatMap(ServerBrowser.ServerBrowser, (browser) => + router.add("GET", `${PREVIEW_STREAM_ROUTE_PREFIX}/*`, makeHandler(browser)), + ), +); diff --git a/apps/server/src/preview/ServerBrowserToolchain.ts b/apps/server/src/preview/ServerBrowserToolchain.ts new file mode 100644 index 000000000000..6dc645660ef7 --- /dev/null +++ b/apps/server/src/preview/ServerBrowserToolchain.ts @@ -0,0 +1,600 @@ +/** + * The Chromium the environment server runs for server-side preview tabs. + * + * Servers are often GPU-less containers without root or Chrome's system + * libraries, so on glibc Linux we install a pinned headless build from the + * `@sparticuz/chromium` GitHub release (its npm package ships x64 only): the + * executable with SwiftShader beside it, and NSS/NSPR/expat in `lib/` for + * `LD_LIBRARY_PATH`. A private fontconfig file layers optional T3-owned emoji + * and CJK fonts over whatever the system has. Everywhere else nothing is + * downloaded: we use an installed Chrome, Chromium, or Edge, or a Playwright + * cache build. + * + * Installs follow the pinned-runtime recipe: stage into a temp sibling, write + * the sentinel only after the tree is complete (and, for the browser, after + * the executable runs), then rename into place. + */ +import * as NodeStream from "@effect/platform-node/NodeStream"; +import { + HostProcessArchitecture, + HostProcessEnvironment, + HostProcessPlatform, +} from "@t3tools/shared/hostProcess"; +import { resolveCommandPath } from "@t3tools/shared/shell"; +import * as Context from "effect/Context"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; +import * as NodeCrypto from "node:crypto"; +import * as NodeUtil from "node:util"; +import * as NodeZlib from "node:zlib"; + +import * as ServerConfig from "../config.ts"; +import * as ProcessRunner from "../processRunner.ts"; + +/** `@sparticuz/chromium` release; ships Chromium 153.0.8010.0. */ +export const SERVER_BROWSER_VERSION = "153.0.0"; +/** Bump when `FONTS` changes. */ +const FONTS_VERSION = "1"; + +interface PinnedAsset { + readonly url: string; + readonly sha256: string; +} + +const RELEASE_URL = `https://github.com/Sparticuz/chromium/releases/download/v${SERVER_BROWSER_VERSION}`; +const BROWSER_PACKS: Partial> = { + x64: { + url: `${RELEASE_URL}/chromium-v${SERVER_BROWSER_VERSION}-pack.x64.tar`, + sha256: "91b9f56d35a2cbb14279a1cbdaf1c86c0faa5fd82315bdd7334ff93bf35f224d", + }, + arm64: { + url: `${RELEASE_URL}/chromium-v${SERVER_BROWSER_VERSION}-pack.arm64.tar`, + sha256: "b6874fc0d7af15e16a15ef1b73177731f698c296c73d26aa4a90840c0777a338", + }, +}; + +// Emoji must be the CBDT build; this Chromium does not render the COLRv1 one. +const FONTS: ReadonlyArray = [ + { + file: "NotoColorEmoji.ttf", + url: "https://raw.githubusercontent.com/googlefonts/noto-emoji/v2.051/fonts/NotoColorEmoji.ttf", + sha256: "72a635cb3d2f3524c51620cdde406b217204e8a6a06c6a096ff8ed4b5fd6e27b", + }, + { + file: "NotoSansCJK-Regular.ttc", + url: "https://raw.githubusercontent.com/notofonts/noto-cjk/Sans2.004/Sans/OTC/NotoSansCJK-Regular.ttc", + sha256: "b76b0433203017ca80401b2ee0dd69350349871c4b19d504c34dbdd80541690a", + }, + { + file: "NotoSansCJK-Bold.ttc", + url: "https://raw.githubusercontent.com/notofonts/noto-cjk/Sans2.004/Sans/OTC/NotoSansCJK-Bold.ttc", + sha256: "faa5f3656a78b2e2d450d27fe8382c778bc2b6bb5ea29c986664a6a435056ceb", + }, +]; + +const BROWSER_TOOL = "server-browser"; +const FONTS_TOOL = "server-browser-fonts"; +const SENTINEL = ".install-complete"; + +const xmlText = (value: string) => + value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">"); + +/** + * Our cache dir comes first so fontconfig writes there (a relative cachedir + * resolves against the cwd, so it must be absolute). is scanned + * recursively and dot entries are skipped, so the fonts tool's staging dirs + * stay invisible and whichever fonts version is installed gets picked up. + */ +const fontconfigFile = ( + browserDir: string, + fontsToolDir: string, + path: Path.Path, +) => ` + + + ${xmlText(path.join(browserDir, "fontconfig-cache"))} + /etc/fonts/fonts.conf + ${xmlText(path.join(browserDir, "fonts"))} + ${xmlText(fontsToolDir)} + +`; + +const BROWSER_INSTALL_TIMEOUT = Duration.minutes(10); +const FONTS_INSTALL_TIMEOUT = Duration.minutes(3); +const browserInstallLock = Semaphore.makeUnsafe(1); +const fontsInstallLock = Semaphore.makeUnsafe(1); + +export type ServerBrowserSource = "override" | "bundled" | "system" | "playwright-cache"; + +export interface ServerBrowserLaunch { + readonly executablePath: string; + /** Merge over the server's environment when spawning. */ + readonly env: Readonly>; + /** Flags this build needs beyond the launcher's own. */ + readonly args: ReadonlyArray; + readonly source: ServerBrowserSource; +} + +export class ServerBrowserInstallError extends Schema.TaggedError()( + "ServerBrowserInstallError", + { + tool: Schema.String, + step: Schema.String, + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + return `Installing ${this.tool} failed while ${this.step}.`; + } +} + +export class ServerBrowserNotFoundError extends Schema.TaggedError()( + "ServerBrowserNotFoundError", + { + platform: Schema.String, + architecture: Schema.String, + override: Schema.optional(Schema.String), + }, +) { + override get message(): string { + return this.override === undefined + ? `No Chrome, Chromium, or Edge was found on ${this.platform}/${this.architecture}. Install one or set T3CODE_PREVIEW_BROWSER_PATH.` + : `T3CODE_PREVIEW_BROWSER_PATH points at '${this.override}', which is not an executable.`; + } +} + +export type ServerBrowserToolchainError = ServerBrowserInstallError | ServerBrowserNotFoundError; + +export class ServerBrowserToolchain extends Context.Service< + ServerBrowserToolchain, + { + /** Installs the bundled build on first use; later calls only read sentinels. */ + readonly resolve: Effect.Effect; + } +>()("t3/preview/ServerBrowserToolchain") {} + +interface TarEntry { + readonly name: string; + readonly type: string; + readonly mode: number; + readonly data: Uint8Array; +} + +/** Reads the plain GNU/ustar tarballs in the pinned pack: files, directories, long names. */ +const readTar = (archive: Uint8Array): ReadonlyArray => { + const decoder = new TextDecoder(); + const text = (bytes: Uint8Array) => decoder.decode(bytes).replace(/\0[\s\S]*$/, ""); + const entries: Array = []; + let longName: string | undefined; + let offset = 0; + while (offset + 512 <= archive.length) { + const header = archive.subarray(offset, offset + 512); + if (header.every((byte) => byte === 0)) break; + const field = (start: number, length: number) => text(header.subarray(start, start + length)); + const size = Number.parseInt(field(124, 12).trim() || "0", 8); + const data = archive.subarray(offset + 512, offset + 512 + size); + offset += 512 + Math.ceil(size / 512) * 512; + const type = field(156, 1) || "0"; + if (type === "L") { + longName = text(data); + continue; + } + // Only POSIX ustar has a prefix field; old GNU headers keep other data there. + const prefix = field(257, 6) === "ustar" ? field(345, 155) : ""; + const name = longName ?? (prefix ? `${prefix}/${field(0, 100)}` : field(0, 100)); + longName = undefined; + entries.push({ name, type, mode: Number.parseInt(field(100, 8).trim() || "644", 8), data }); + } + return entries; +}; + +const brotliDecompress = NodeUtil.promisify(NodeZlib.brotliDecompress); + +const hostUsesGlibc = (): boolean => { + try { + const report = process.report?.getReport() as + | { readonly header?: { readonly glibcVersionRuntime?: unknown } } + | undefined; + return typeof report?.header?.glibcVersionRuntime === "string"; + } catch { + return false; + } +}; + +const systemBrowserCandidates = ( + platform: NodeJS.Platform, + env: NodeJS.ProcessEnv, + path: Path.Path, +): ReadonlyArray => { + if (platform === "darwin") { + const apps = [ + "Google Chrome.app/Contents/MacOS/Google Chrome", + "Chromium.app/Contents/MacOS/Chromium", + "Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + ]; + const roots = ["/Applications", ...(env.HOME ? [path.join(env.HOME, "Applications")] : [])]; + return apps.flatMap((app) => roots.map((root) => path.join(root, app))); + } + if (platform === "win32") { + const apps = [ + "Google\\Chrome\\Application\\chrome.exe", + "Microsoft\\Edge\\Application\\msedge.exe", + "Chromium\\Application\\chrome.exe", + ]; + const roots = [env.ProgramFiles, env["ProgramFiles(x86)"], env.LOCALAPPDATA].filter( + (root): root is string => Boolean(root), + ); + return apps.flatMap((app) => roots.map((root) => path.join(root, app))); + } + return [ + "google-chrome-stable", + "google-chrome", + "chromium", + "chromium-browser", + "microsoft-edge-stable", + "microsoft-edge", + ]; +}; + +/** `chromium_headless_shell-/`, from playwright-core's registry. */ +const PLAYWRIGHT_HEADLESS_SHELL: Partial>> = { + "linux-x64": ["chrome-headless-shell-linux64", "chrome-headless-shell"], + "linux-arm64": ["chrome-linux", "headless_shell"], + "darwin-x64": ["chrome-headless-shell-mac-x64", "chrome-headless-shell"], + "darwin-arm64": ["chrome-headless-shell-mac-arm64", "chrome-headless-shell"], + "win32-x64": ["chrome-headless-shell-win64", "chrome-headless-shell.exe"], +}; + +const playwrightCacheDir = ( + platform: NodeJS.Platform, + env: NodeJS.ProcessEnv, + path: Path.Path, +): string | undefined => { + if (env.PLAYWRIGHT_BROWSERS_PATH && env.PLAYWRIGHT_BROWSERS_PATH !== "0") { + return env.PLAYWRIGHT_BROWSERS_PATH; + } + if (platform === "win32") { + return env.LOCALAPPDATA ? path.join(env.LOCALAPPDATA, "ms-playwright") : undefined; + } + if (!env.HOME) return undefined; + return platform === "darwin" + ? path.join(env.HOME, "Library", "Caches", "ms-playwright") + : path.join(env.XDG_CACHE_HOME || path.join(env.HOME, ".cache"), "ms-playwright"); +}; + +const make = Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const httpClient = yield* HttpClient.HttpClient; + const runner = yield* ProcessRunner.ProcessRunner; + const platform = yield* HostProcessPlatform; + const architecture = yield* HostProcessArchitecture; + const hostEnv = yield* HostProcessEnvironment; + + const toolsDir = path.join(config.baseDir, "tools"); + const browserDir = path.join(toolsDir, BROWSER_TOOL, SERVER_BROWSER_VERSION); + const browserPack = + platform === "linux" && hostUsesGlibc() ? BROWSER_PACKS[architecture] : undefined; + + const isInstalled = (installDir: string, version: string) => + fs.readFileString(path.join(installDir, SENTINEL)).pipe( + Effect.map((sentinel) => sentinel.trim() === version), + Effect.orElseSucceed(() => false), + ); + + /** Stages, publishes, and prunes `/tools//` unless it is already complete. */ + const installOnce = Effect.fn("ServerBrowserToolchain.installOnce")(function* ( + tool: string, + version: string, + populate: (stagingDir: string) => Effect.Effect, + ) { + const fail = (step: string) => (cause: unknown) => + new ServerBrowserInstallError({ tool, step, cause }); + const parentDir = path.join(toolsDir, tool); + const installDir = path.join(parentDir, version); + if (yield* isInstalled(installDir, version)) return; + + yield* fs + .remove(installDir, { recursive: true, force: true }) + .pipe(Effect.mapError(fail("removing an incomplete install"))); + yield* fs + .makeDirectory(parentDir, { recursive: true }) + .pipe(Effect.mapError(fail("preparing the install directory"))); + const stagingDir = yield* fs + .makeTempDirectory({ directory: parentDir, prefix: ".staging-" }) + .pipe(Effect.mapError(fail("preparing the install directory"))); + + yield* Effect.gen(function* () { + yield* populate(stagingDir); + yield* fs + .writeFileString(path.join(stagingDir, SENTINEL), `${version}\n`) + .pipe(Effect.mapError(fail("recording the completed install"))); + yield* fs.rename(stagingDir, installDir).pipe( + Effect.catch((cause) => + // A concurrent server may have published the same version first. + isInstalled(installDir, version).pipe( + Effect.flatMap((published) => + published ? Effect.void : Effect.fail(fail("publishing the install")(cause)), + ), + ), + ), + ); + // Only runs right after an upgrade; dot entries are other installs' staging dirs. + const names = yield* fs.readDirectory(parentDir).pipe(Effect.orElseSucceed(() => [])); + yield* Effect.forEach( + names.filter((name) => !name.startsWith(".") && name !== version), + (name) => fs.remove(path.join(parentDir, name), { recursive: true, force: true }), + ).pipe(Effect.ignore); + }).pipe( + Effect.ensuring(fs.remove(stagingDir, { recursive: true, force: true }).pipe(Effect.ignore)), + ); + }); + + const download = Effect.fn("ServerBrowserToolchain.download")(function* ( + tool: string, + asset: PinnedAsset, + ) { + const bytes = yield* httpClient.execute(HttpClientRequest.get(asset.url)).pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap((response) => response.arrayBuffer), + Effect.map((buffer) => new Uint8Array(buffer)), + Effect.mapError( + (cause) => new ServerBrowserInstallError({ tool, step: `downloading ${asset.url}`, cause }), + ), + ); + if (NodeCrypto.createHash("sha256").update(bytes).digest("hex") !== asset.sha256) { + return yield* new ServerBrowserInstallError({ tool, step: `verifying ${asset.url}` }); + } + return bytes; + }); + + const launchEnv = (installDir: string): Record => ({ + LD_LIBRARY_PATH: [path.join(installDir, "lib"), hostEnv.LD_LIBRARY_PATH] + .filter(Boolean) + .join(":"), + FONTCONFIG_FILE: path.join(installDir, "fonts.conf"), + }); + + const populateBrowser = (pack: PinnedAsset) => + Effect.fn("ServerBrowserToolchain.populateBrowser")(function* (stagingDir: string) { + const fail = (step: string) => (cause: unknown) => + new ServerBrowserInstallError({ tool: BROWSER_TOOL, step, cause }); + const members = new Map( + readTar(yield* download(BROWSER_TOOL, pack)).map((entry) => [entry.name, entry.data]), + ); + const member = (name: string) => { + const data = members.get(name); + return data === undefined + ? Effect.fail( + new ServerBrowserInstallError({ tool: BROWSER_TOOL, step: `finding ${name}` }), + ) + : Effect.succeed(Buffer.from(data.buffer, data.byteOffset, data.byteLength)); + }; + + // Stream the ~200 MB executable to disk instead of inflating it in memory. + const executablePath = path.join(stagingDir, "chromium"); + const chromium = yield* member("chromium.br"); + yield* Stream.succeed(chromium).pipe( + NodeStream.pipeThroughSimple(() => NodeZlib.createBrotliDecompress()), + Stream.run(fs.sink(executablePath, { mode: 0o755 })), + Effect.mapError(fail("unpacking chromium.br")), + ); + + // SwiftShader sits beside the executable, NSS/NSPR/expat land in lib/, + // and fonts.tar adds Open Sans for hosts with no fonts at all. + for (const name of ["swiftshader.tar.br", "al2023.tar.br", "fonts.tar.br"]) { + const compressed = yield* member(name); + const archive = yield* Effect.tryPromise({ + try: () => brotliDecompress(compressed), + catch: fail(`unpacking ${name}`), + }); + for (const entry of readTar(archive)) { + const target = path.resolve(stagingDir, entry.name); + if ( + !target.startsWith(stagingDir + path.sep) || + (entry.type !== "0" && entry.type !== "5") + ) { + return yield* fail(`unpacking ${name}`)( + `Unsupported entry ${entry.type} ${entry.name}`, + ); + } + yield* ( + entry.type === "5" + ? fs.makeDirectory(target, { recursive: true }) + : fs + .makeDirectory(path.dirname(target), { recursive: true }) + .pipe(Effect.andThen(fs.writeFile(target, entry.data, { mode: entry.mode }))) + ).pipe(Effect.mapError(fail(`unpacking ${name}`))); + } + } + // Replaces the Lambda-specific fonts.conf from fonts.tar. + yield* fs + .writeFileString( + path.join(stagingDir, "fonts.conf"), + fontconfigFile(browserDir, path.join(toolsDir, FONTS_TOOL), path), + ) + .pipe(Effect.mapError(fail("writing fonts.conf"))); + + // Proves the loader finds every library before the install counts. + const result = yield* runner + .run({ + command: executablePath, + args: ["--version"], + env: launchEnv(stagingDir), + timeout: Duration.seconds(30), + }) + .pipe(Effect.mapError(fail("running chromium --version"))); + if (result.code !== 0 || !result.stdout.includes("Chromium")) { + return yield* fail("running chromium --version")(result); + } + }); + + const populateFonts = Effect.fn("ServerBrowserToolchain.populateFonts")(function* ( + stagingDir: string, + ) { + yield* Effect.forEach( + FONTS, + (font) => + download(FONTS_TOOL, font).pipe( + Effect.flatMap((bytes) => + fs.writeFile(path.join(stagingDir, font.file), bytes).pipe( + Effect.mapError( + (cause) => + new ServerBrowserInstallError({ + tool: FONTS_TOOL, + step: `saving ${font.file}`, + cause, + }), + ), + ), + ), + ), + { concurrency: "unbounded", discard: true }, + ); + }); + + // Fonts are optional: one attempt per server process that never fails the + // browser. It runs detached so a cancelled or failed resolve cannot + // interrupt it, which would cache the interruption for every later call. + const fontsFiber = yield* Effect.cached( + Effect.forkDetach( + fontsInstallLock.withPermit(installOnce(FONTS_TOOL, FONTS_VERSION, populateFonts)).pipe( + Effect.timeoutOrElse({ + duration: FONTS_INSTALL_TIMEOUT, + orElse: () => + Effect.fail( + new ServerBrowserInstallError({ tool: FONTS_TOOL, step: "downloading (timed out)" }), + ), + }), + Effect.catch((cause) => + Effect.logWarning("Preview browser fonts unavailable; emoji and CJK may not render", { + cause, + }), + ), + ), + ), + ); + const ensureFonts = Effect.flatMap(fontsFiber, Fiber.join); + + const bundledLaunch: ServerBrowserLaunch = { + executablePath: path.join(browserDir, "chromium"), + env: launchEnv(browserDir), + args: [], + source: "bundled", + }; + const bundled = (pack: PinnedAsset) => + Effect.all( + [ + browserInstallLock + .withPermit(installOnce(BROWSER_TOOL, SERVER_BROWSER_VERSION, populateBrowser(pack))) + .pipe( + Effect.timeoutOrElse({ + duration: BROWSER_INSTALL_TIMEOUT, + orElse: () => + Effect.fail( + new ServerBrowserInstallError({ + tool: BROWSER_TOOL, + step: "installing (timed out)", + }), + ), + }), + ), + ensureFonts, + ], + { concurrency: "unbounded", discard: true }, + ).pipe(Effect.as(bundledLaunch)); + + /** Resolves bare names through PATH and checks absolute paths are executable. */ + const executable = (candidate: string) => + resolveCommandPath(candidate).pipe( + Effect.option, + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ); + const firstExecutable = Effect.fn("ServerBrowserToolchain.firstExecutable")(function* ( + candidates: ReadonlyArray, + ) { + for (const candidate of candidates) { + const resolved = yield* executable(candidate); + if (Option.isSome(resolved)) return resolved.value; + } + return undefined; + }); + + const discover = Effect.gen(function* () { + const system = yield* firstExecutable(systemBrowserCandidates(platform, hostEnv, path)); + if (system !== undefined) { + return { + executablePath: system, + env: {}, + args: [], + source: "system", + } satisfies ServerBrowserLaunch; + } + const cacheDir = playwrightCacheDir(platform, hostEnv, path); + const relative = PLAYWRIGHT_HEADLESS_SHELL[`${platform}-${architecture}`]; + if (cacheDir !== undefined && relative !== undefined) { + const revisions = (yield* fs.readDirectory(cacheDir).pipe(Effect.orElseSucceed(() => []))) + .flatMap((name) => { + const match = /^chromium_headless_shell-(\d+)$/.exec(name); + return match ? [{ name, revision: Number(match[1]) }] : []; + }) + .sort((left, right) => right.revision - left.revision); + const cached = yield* firstExecutable( + revisions.map(({ name }) => path.join(cacheDir, name, ...relative)), + ); + if (cached !== undefined) { + return { + executablePath: cached, + env: {}, + args: [], + source: "playwright-cache", + } satisfies ServerBrowserLaunch; + } + } + return yield* new ServerBrowserNotFoundError({ platform, architecture }); + }); + + const resolve: ServerBrowserToolchain["Service"]["resolve"] = Effect.gen(function* () { + const override = hostEnv.T3CODE_PREVIEW_BROWSER_PATH; + if (override) { + const resolved = yield* executable(override); + if (Option.isNone(resolved)) { + return yield* new ServerBrowserNotFoundError({ platform, architecture, override }); + } + const libraryPath = hostEnv.T3CODE_PREVIEW_BROWSER_LD_LIBRARY_PATH; + return { + executablePath: resolved.value, + env: libraryPath ? { LD_LIBRARY_PATH: libraryPath } : {}, + args: [], + source: "override", + } satisfies ServerBrowserLaunch; + } + if (browserPack === undefined) return yield* discover; + // Offline with a system browser still works; otherwise report the install failure. + return yield* bundled(browserPack).pipe( + Effect.catch((installError) => + Effect.logWarning("Bundled preview browser unavailable; looking for a system browser", { + cause: installError, + }).pipe( + Effect.andThen(discover), + Effect.catchTag("ServerBrowserNotFoundError", () => Effect.fail(installError)), + ), + ), + ); + }); + + return ServerBrowserToolchain.of({ resolve }); +}); + +export const layer = Layer.effect(ServerBrowserToolchain, make); diff --git a/apps/server/src/preview/serverBrowserEnabled.ts b/apps/server/src/preview/serverBrowserEnabled.ts new file mode 100644 index 000000000000..63bca9a8e985 --- /dev/null +++ b/apps/server/src/preview/serverBrowserEnabled.ts @@ -0,0 +1,13 @@ +import type { RuntimeMode } from "../config.ts"; + +/** + * Standalone servers host `runtime: "server"` preview tabs. A desktop-managed + * server leaves the browser to its Electron app. `T3CODE_SERVER_BROWSER=0|1` + * overrides either. + */ +export const isServerBrowserEnabled = (mode: RuntimeMode): boolean => { + const override = process.env.T3CODE_SERVER_BROWSER; + if (override === "0") return false; + if (override === "1") return true; + return mode === "web"; +}; diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 3f528cd85f15..49acd5190de8 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -63,6 +63,9 @@ import * as DeviceService from "./device/DeviceService.ts"; import * as DeviceHubProxy from "./device/DeviceHubProxy.ts"; import * as PreviewManager from "./preview/Manager.ts"; import * as PortScanner from "./preview/PortScanner.ts"; +import * as ServerBrowser from "./preview/ServerBrowser.ts"; +import * as ServerBrowserStream from "./preview/ServerBrowserStream.ts"; +import * as ServerBrowserToolchain from "./preview/ServerBrowserToolchain.ts"; import * as ProcessRunner from "./processRunner.ts"; import * as GitManager from "./git/GitManager.ts"; import * as EnvironmentTheme from "./environmentTheme.ts"; @@ -661,6 +664,13 @@ const layerMakeRoutes = Layer.mergeAll( ServerHttp.layerAssetRoute, ServerHttp.layerAttachmentUploadRoute, DeviceHubProxy.layer, + ServerBrowserStream.routeLayer.pipe( + Layer.provide( + ServerBrowser.layer.pipe( + Layer.provide(ServerBrowserToolchain.layer.pipe(Layer.provide(ProcessRunner.layer))), + ), + ), + ), ServerHttp.layerStaticAndDevRoute, Ws.layer, ), diff --git a/apps/web/src/browser/ElectronBrowserHost.tsx b/apps/web/src/browser/ElectronBrowserHost.tsx index de7e23603298..e2325cf5fbd2 100644 --- a/apps/web/src/browser/ElectronBrowserHost.tsx +++ b/apps/web/src/browser/ElectronBrowserHost.tsx @@ -20,19 +20,22 @@ export function ElectronBrowserHost() { () => Object.entries(previewByThreadKey).flatMap(([threadKey, previewState]) => { const threadRef = parseScopedThreadKey(threadKey); + // Server tabs stream from the environment and never get a local webview. return threadRef - ? Object.values(previewState.sessions).map((snapshot) => ({ - threadRef, - snapshot, - runtimeTabId: previewRuntimeTabId( + ? Object.values(previewState.sessions) + .filter((snapshot) => snapshot.runtime !== "server") + .map((snapshot) => ({ threadRef, - previewState.serverEpoch, - snapshot.tabId, - ), - pictureInPicture: - previewState.desktopByTabId[snapshot.tabId]?.pictureInPicture ?? false, - zoomFactor: previewState.desktopByTabId[snapshot.tabId]?.zoomFactor ?? 1, - })) + snapshot, + runtimeTabId: previewRuntimeTabId( + threadRef, + previewState.serverEpoch, + snapshot.tabId, + ), + pictureInPicture: + previewState.desktopByTabId[snapshot.tabId]?.pictureInPicture ?? false, + zoomFactor: previewState.desktopByTabId[snapshot.tabId]?.zoomFactor ?? 1, + })) : []; }), [previewByThreadKey], diff --git a/apps/web/src/browser/ServerBrowserSurface.tsx b/apps/web/src/browser/ServerBrowserSurface.tsx new file mode 100644 index 000000000000..5d8315a9d701 --- /dev/null +++ b/apps/web/src/browser/ServerBrowserSurface.tsx @@ -0,0 +1,576 @@ +"use client"; + +import { + createPreviewFramePainter, + createPreviewStreamClient, + previewStreamModifiers, + type PreviewStreamClient, + type PreviewStreamInput, + type PreviewStreamMouseButton, + type PreviewStreamViewport, +} from "@t3tools/client-runtime/preview/server-browser-stream"; +import type { EnvironmentId } from "@t3tools/contracts"; +import { + type ClipboardEvent, + type CompositionEvent, + type FormEvent, + type KeyboardEvent, + type PointerEvent, + type Ref, + useCallback, + useEffect, + useEffectEvent, + useImperativeHandle, + useRef, + useState, +} from "react"; + +import { cn } from "~/lib/utils"; +import { refreshPreviewStreamAccess, usePreviewStreamAccess } from "~/state/previewStream"; + +/** Chrome-row controls for a server tab. A command issued before the socket opens waits for it. */ +export interface ServerBrowserHandle { + readonly navigate: (url: string) => void; + readonly history: (delta: -1 | 1) => void; + readonly reload: () => void; + /** The canvas showing the latest frame. */ + readonly canvas: () => HTMLCanvasElement | null; +} + +const RESIZE_DEBOUNCE_MS = 150; +const ACCESS_RETRY_MS = 10_000; +const UNAUTHORIZED_RETRY_MAX_MS = 30_000; +const TAP_SLOP_PX = 8; +const MULTI_CLICK_MS = 500; +const MULTI_CLICK_SLOP_PX = 4; +const WHEEL_LINE_PX = 16; + +type MouseInput = Extract; +type WheelInput = Extract; + +interface PagePoint { + readonly x: number; + readonly y: number; + /** Page CSS px per client px. */ + readonly scale: number; +} + +interface TouchGesture { + readonly pointerId: number; + readonly startX: number; + readonly startY: number; + lastX: number; + lastY: number; + panning: boolean; +} + +/** Whether the page point under a touch takes text; null until the server answers. */ +interface TouchProbe { + readonly x: number; + readonly y: number; + editable: boolean | null; + /** The tap ended before the answer arrived. */ + tapped: boolean; +} + +const buttonOf = (button: number): PreviewStreamMouseButton => + button === 0 ? "left" : button === 1 ? "middle" : button === 2 ? "right" : "none"; + +const pressedButtonOf = (buttons: number): PreviewStreamMouseButton => + buttons & 1 ? "left" : buttons & 2 ? "right" : buttons & 4 ? "middle" : "none"; + +/** + * A server-hosted preview tab: JPEG frames from the environment's headless + * Chromium drawn into a canvas, with pointer, wheel, and keyboard input sent + * back in page coordinates. Touch taps click and touch drags scroll; the soft + * keyboard opens only for taps the server reports as landing on a text field. + * `visible=false` closes the stream so a hidden panel decodes nothing. + */ +export function ServerBrowserSurface(props: { + readonly environmentId: EnvironmentId; + readonly threadId: string; + readonly tabId: string; + readonly visible: boolean; + /** Sends the surface size so fill-mode tabs follow it. The floating player scales the page instead. */ + readonly followSize?: boolean; + readonly onFirstFrame?: () => void; + readonly onViewport?: (viewport: PreviewStreamViewport) => void; + readonly className?: string; + readonly ref?: Ref; +}) { + const { + environmentId, + threadId, + tabId, + visible, + followSize = true, + onFirstFrame, + onViewport, + className, + ref, + } = props; + const access = usePreviewStreamAccess(environmentId); + const containerRef = useRef(null); + const canvasRef = useRef(null); + const inputRef = useRef(null); + const clientRef = useRef(null); + const viewportRef = useRef(null); + const sizeRef = useRef<{ width: number; height: number } | null>(null); + const hasFrameRef = useRef(false); + const pendingCommandRef = useRef(null); + const unauthorizedRef = useRef(0); + const pendingMoveRef = useRef(null); + const pendingWheelRef = useRef(null); + const inputFrameRef = useRef(null); + const mousePressedRef = useRef(false); + const lastClickRef = useRef<{ + button: PreviewStreamMouseButton; + time: number; + x: number; + y: number; + count: number; + } | null>(null); + const touchRef = useRef(null); + const probeRef = useRef(null); + const firstFrame = useEffectEvent(() => onFirstFrame?.()); + const viewportChanged = useEffectEvent((viewport: PreviewStreamViewport) => + onViewport?.(viewport), + ); + // Frame cap in device px, fixed per socket. It grows with the surface and + // never shrinks, so only outgrowing it reconnects. + const [cap, setCap] = useState<{ width: number; height: number } | null>(null); + + const send = useCallback((input: PreviewStreamInput) => { + clientRef.current?.send(input); + }, []); + + const flushInput = useCallback(() => { + if (inputFrameRef.current !== null) cancelAnimationFrame(inputFrameRef.current); + inputFrameRef.current = null; + const move = pendingMoveRef.current; + const wheel = pendingWheelRef.current; + pendingMoveRef.current = null; + pendingWheelRef.current = null; + if (move) send(move); + if (wheel) send(wheel); + }, [send]); + + // Moves and wheel deltas coalesce to one message each per animation frame. + // `flushInput` only closes over refs and `send`, so it never changes. + const scheduleFlush = useCallback(() => { + inputFrameRef.current ??= requestAnimationFrame(flushInput); + }, []); + + const queueWheel = useCallback( + (point: PagePoint, deltaX: number, deltaY: number, modifiers: number) => { + const pending = pendingWheelRef.current; + pendingWheelRef.current = { + type: "wheel", + x: point.x, + y: point.y, + deltaX: (pending?.deltaX ?? 0) + deltaX, + deltaY: (pending?.deltaY ?? 0) + deltaY, + modifiers, + }; + scheduleFlush(); + }, + [scheduleFlush], + ); + + const pagePoint = useCallback( + (clientX: number, clientY: number, clamp: boolean): PagePoint | null => { + const canvas = canvasRef.current; + const viewport = viewportRef.current; + if (!canvas || !viewport || !hasFrameRef.current) return null; + const rect = canvas.getBoundingClientRect(); + // `object-contain` letterboxes the frame inside the canvas box. + const fit = Math.min(rect.width / canvas.width, rect.height / canvas.height); + const width = canvas.width * fit; + const height = canvas.height * fit; + if (!(width > 0 && height > 0)) return null; + const scale = viewport.width / width; + const x = (clientX - rect.left - (rect.width - width) / 2) * scale; + const y = (clientY - rect.top - (rect.height - height) / 2) * (viewport.height / height); + const inside = x >= 0 && y >= 0 && x <= viewport.width && y <= viewport.height; + if (!inside && !clamp) return null; + return { + x: Math.min(Math.max(x, 0), viewport.width), + y: Math.min(Math.max(y, 0), viewport.height), + scale, + }; + }, + [], + ); + + const countClick = (button: PreviewStreamMouseButton, x: number, y: number, time: number) => { + const last = lastClickRef.current; + const count = + last && + last.button === button && + time - last.time < MULTI_CLICK_MS && + Math.hypot(x - last.x, y - last.y) < MULTI_CLICK_SLOP_PX + ? last.count + 1 + : 1; + lastClickRef.current = { button, time, x, y, count }; + return count; + }; + + const focusInput = () => inputRef.current?.focus({ preventScroll: true }); + + useImperativeHandle( + ref, + () => { + const command = (input: PreviewStreamInput) => { + if (clientRef.current?.send(input)) return; + pendingCommandRef.current = input; + }; + return { + navigate: (url) => command({ type: "navigate", url }), + history: (delta) => command({ type: "history", delta }), + reload: () => command({ type: "reload" }), + canvas: () => (hasFrameRef.current ? canvasRef.current : null), + }; + }, + [], + ); + + useEffect(() => { + const element = containerRef.current; + if (!element) return; + let timer: ReturnType | null = null; + const measure = () => { + timer = null; + const rect = element.getBoundingClientRect(); + if (rect.width < 1 || rect.height < 1) return; + const size = { width: Math.round(rect.width), height: Math.round(rect.height) }; + const previous = sizeRef.current; + if (previous?.width === size.width && previous.height === size.height) return; + sizeRef.current = size; + if (followSize) clientRef.current?.send({ type: "resize", ...size }); + const ratio = window.devicePixelRatio || 1; + const width = Math.round(size.width * ratio); + const height = Math.round(size.height * ratio); + setCap((current) => + current !== null && current.width >= width && current.height >= height + ? current + : { + width: Math.max(width, current?.width ?? 0), + height: Math.max(height, current?.height ?? 0), + }, + ); + }; + const observer = new ResizeObserver(() => { + if (timer !== null) clearTimeout(timer); + // The first size connects right away; later ones settle before resizing the page. + timer = setTimeout(measure, sizeRef.current === null ? 0 : RESIZE_DEBOUNCE_MS); + }); + observer.observe(element); + return () => { + observer.disconnect(); + if (timer !== null) clearTimeout(timer); + }; + }, [followSize]); + + useEffect(() => { + const canvas = canvasRef.current; + if (!visible || !access || !cap || !canvas) return; + let refreshTimer: ReturnType | null = null; + const painter = createPreviewFramePainter(canvas, () => { + if (hasFrameRef.current) return; + hasFrameRef.current = true; + firstFrame(); + }); + const client = createPreviewStreamClient( + { access, threadId, tabId, maxWidth: cap.width, maxHeight: cap.height }, + { + onFrame: (jpeg) => { + unauthorizedRef.current = 0; + painter.paint(jpeg); + }, + onProbe: (result) => { + const probe = probeRef.current; + if (!probe || probe.x !== result.x || probe.y !== result.y) return; + probe.editable = result.editable; + if (!probe.tapped) return; + // Late answer: Android still raises the keyboard; iOS waits for the next tap. + probeRef.current = null; + if (result.editable) inputRef.current?.focus({ preventScroll: true }); + else inputRef.current?.blur(); + }, + onViewport: (viewport) => { + viewportRef.current = viewport; + viewportChanged(viewport); + }, + onConnectedChange: (connected) => { + if (!connected) return; + const size = sizeRef.current; + if (followSize && size) client.send({ type: "resize", ...size }); + const command = pendingCommandRef.current; + pendingCommandRef.current = null; + if (command) client.send(command); + }, + onUnauthorized: () => { + // A fresh ticket re-runs this effect through `access`. Back off when + // new tickets keep getting refused, e.g. a session without operate scope. + const attempt = unauthorizedRef.current++; + refreshTimer = setTimeout( + () => refreshPreviewStreamAccess(environmentId), + attempt === 0 ? 0 : Math.min(1_000 * 2 ** attempt, UNAUTHORIZED_RETRY_MAX_MS), + ); + }, + }, + ); + clientRef.current = client; + return () => { + painter.stop(); + if (refreshTimer !== null) clearTimeout(refreshTimer); + client.stop(); + if (clientRef.current === client) clientRef.current = null; + }; + }, [access, cap, environmentId, followSize, tabId, threadId, visible]); + + useEffect(() => { + if (!visible || access !== null) return; + // A failed ticket mint, e.g. while the server restarts, never retries on its own. + const timer = setTimeout(() => refreshPreviewStreamAccess(environmentId), ACCESS_RETRY_MS); + return () => clearTimeout(timer); + }, [access, environmentId, visible]); + + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas) return; + // Registered natively: React's wheel listener is passive and cannot stop the panel scrolling. + const onWheel = (event: WheelEvent) => { + const point = pagePoint(event.clientX, event.clientY, false); + if (!point) return; + event.preventDefault(); + const unit = + event.deltaMode === WheelEvent.DOM_DELTA_LINE + ? WHEEL_LINE_PX + : event.deltaMode === WheelEvent.DOM_DELTA_PAGE + ? (viewportRef.current?.height ?? 0) + : 1; + queueWheel(point, event.deltaX * unit, event.deltaY * unit, previewStreamModifiers(event)); + }; + canvas.addEventListener("wheel", onWheel, { passive: false }); + return () => canvas.removeEventListener("wheel", onWheel); + }, [pagePoint, queueWheel]); + + useEffect( + () => () => { + if (inputFrameRef.current !== null) cancelAnimationFrame(inputFrameRef.current); + }, + [], + ); + + const handlePointerDown = (event: PointerEvent) => { + if (event.pointerType === "touch") { + if (!event.isPrimary) return; + touchRef.current = { + pointerId: event.pointerId, + startX: event.clientX, + startY: event.clientY, + lastX: event.clientX, + lastY: event.clientY, + panning: false, + }; + const point = pagePoint(event.clientX, event.clientY, false); + probeRef.current = point ? { x: point.x, y: point.y, editable: null, tapped: false } : null; + if (point) send({ type: "probe", x: point.x, y: point.y }); + return; + } + focusInput(); + const point = pagePoint(event.clientX, event.clientY, false); + if (!point) return; + event.currentTarget.setPointerCapture(event.pointerId); + flushInput(); + mousePressedRef.current = true; + const button = buttonOf(event.button); + send({ + type: "mouse", + action: "down", + x: point.x, + y: point.y, + button, + buttons: event.buttons, + clickCount: countClick(button, event.clientX, event.clientY, event.timeStamp), + modifiers: previewStreamModifiers(event), + }); + }; + + const handlePointerMove = (event: PointerEvent) => { + if (event.pointerType === "touch") { + const touch = touchRef.current; + if (!touch || touch.pointerId !== event.pointerId) return; + if ( + !touch.panning && + Math.hypot(event.clientX - touch.startX, event.clientY - touch.startY) < TAP_SLOP_PX + ) { + return; + } + touch.panning = true; + const point = pagePoint(event.clientX, event.clientY, true); + // Dragging the page up scrolls it down, following the finger. + if (point) { + queueWheel( + point, + (touch.lastX - event.clientX) * point.scale, + (touch.lastY - event.clientY) * point.scale, + 0, + ); + } + touch.lastX = event.clientX; + touch.lastY = event.clientY; + return; + } + const point = pagePoint(event.clientX, event.clientY, event.buttons !== 0); + if (!point) return; + pendingMoveRef.current = { + type: "mouse", + action: "move", + x: point.x, + y: point.y, + button: pressedButtonOf(event.buttons), + buttons: event.buttons, + clickCount: 0, + modifiers: previewStreamModifiers(event), + }; + scheduleFlush(); + }; + + const handlePointerUp = (event: PointerEvent) => { + if (event.pointerType === "touch") { + const touch = touchRef.current; + if (!touch || touch.pointerId !== event.pointerId) return; + touchRef.current = null; + const probe = probeRef.current; + if (touch.panning) { + probeRef.current = null; + return; + } + const point = pagePoint(event.clientX, event.clientY, false); + if (!point) return; + // Focusing inside the tap's user activation is what lets iOS raise the keyboard. + if (probe?.editable === true) focusInput(); + else if (probe?.editable === false) inputRef.current?.blur(); + if (probe?.editable === null) probe.tapped = true; + else probeRef.current = null; + flushInput(); + const clickCount = countClick("left", event.clientX, event.clientY, event.timeStamp); + const at = { x: point.x, y: point.y, modifiers: 0 }; + send({ type: "mouse", action: "move", ...at, button: "none", buttons: 0, clickCount: 0 }); + send({ type: "mouse", action: "down", ...at, button: "left", buttons: 1, clickCount }); + send({ type: "mouse", action: "up", ...at, button: "left", buttons: 0, clickCount }); + return; + } + if (!mousePressedRef.current) return; + mousePressedRef.current = false; + const point = pagePoint(event.clientX, event.clientY, true); + if (!point) return; + flushInput(); + send({ + type: "mouse", + action: "up", + x: point.x, + y: point.y, + button: buttonOf(event.button), + buttons: event.buttons, + clickCount: lastClickRef.current?.count ?? 1, + modifiers: previewStreamModifiers(event), + }); + }; + + const handlePointerCancel = (event: PointerEvent) => { + if (touchRef.current?.pointerId === event.pointerId) touchRef.current = null; + }; + + const handleKey = (action: "down" | "up", event: KeyboardEvent) => { + // IME and soft keyboards deliver text through composition and input events. + if ( + event.nativeEvent.isComposing || + event.keyCode === 229 || + event.key === "Process" || + event.key === "Unidentified" + ) { + return; + } + const shortcut = event.ctrlKey || event.metaKey; + // Paste arrives as a paste event carrying this device's clipboard. + if (shortcut && event.key.toLowerCase() === "v") return; + // Enter carries "\r" like Puppeteer's key table, so forms submit and textareas break lines. + const text = shortcut + ? undefined + : [...event.key].length === 1 + ? event.key + : event.key === "Enter" + ? "\r" + : undefined; + send({ + type: "key", + action, + key: event.key, + code: event.code, + keyCode: event.keyCode, + ...(action === "down" && text !== undefined ? { text } : {}), + modifiers: previewStreamModifiers(event), + }); + // Shortcuts stay with the app (copy, paste, keybindings); other keys belong to the page. + if (shortcut) return; + event.preventDefault(); + event.stopPropagation(); + }; + + const handleInput = (event: FormEvent) => { + if (event.nativeEvent instanceof InputEvent && event.nativeEvent.isComposing) return; + const textarea = event.currentTarget; + if (textarea.value) send({ type: "text", text: textarea.value }); + textarea.value = ""; + }; + + const handleCompositionEnd = (event: CompositionEvent) => { + if (event.data) send({ type: "text", text: event.data }); + event.currentTarget.value = ""; + }; + + const handlePaste = (event: ClipboardEvent) => { + event.preventDefault(); + const text = event.clipboardData.getData("text/plain"); + if (text) send({ type: "text", text }); + }; + + return ( +
+ event.preventDefault()} + onContextMenu={(event) => event.preventDefault()} + /> + {/* Focus target for page keyboard input. Pinned top-left so focusing it never + scrolls the surface; 16px keeps iOS from zooming the app on focus. */} +