From 6b45a659b144d495c8c274bee3df95b116b56c17 Mon Sep 17 00:00:00 2001 From: Nicholas Tindle Date: Fri, 9 Oct 2026 06:59:38 -0500 Subject: [PATCH 1/3] feat(mobile): paste, copy and password fill on the Browser page The Browser page draws a page that runs on the environment, so this device's paste menu and AutoFill never reach its fields: a login needs the password typed by hand. - While a page field has the keyboard, a clipboard button sits above it. Its menu pastes this device's clipboard into the field, copies the page selection back (Cmd+C, which the server already runs as the copy command), or fills a saved password. - Fill password opens a card whose username and password fields take iOS Password AutoFill (or Android's autofill service). Fill sends one fillLogin message: the server types the password straight into a focused password field; otherwise it types the username, presses Tab, and types the password only if Tab reached a password field, so a password never lands in a field that would show it. A server without fillLogin ignores the message. - The viewer script reports when its input gains or loses focus, so the button only shows while typing into the page. A page copy that reaches this device now gives haptic feedback, like the app's other copies. Co-Authored-By: Claude Opus 5.5 --- apps/mobile/src/components/AppSymbol.tsx | 4 + .../features/browser/BrowserClipboardMenu.tsx | 40 +++++++++ .../features/browser/BrowserPasswordFill.tsx | 89 +++++++++++++++++++ .../browser/BrowserPreviewRouteScreen.tsx | 45 ++++++++++ .../features/browser/PreviewStreamWebView.tsx | 11 ++- .../browser/preview-stream-document.ts | 2 + .../browser/preview-stream.browser.ts | 3 + apps/server/src/preview/ServerBrowser.test.ts | 57 ++++++++++++ apps/server/src/preview/ServerBrowser.ts | 42 +++++++++ docs/user/remote-access.md | 5 ++ .../src/preview/serverBrowserStream.ts | 6 ++ 11 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 apps/mobile/src/features/browser/BrowserClipboardMenu.tsx create mode 100644 apps/mobile/src/features/browser/BrowserPasswordFill.tsx diff --git a/apps/mobile/src/components/AppSymbol.tsx b/apps/mobile/src/components/AppSymbol.tsx index 122ad1658e27..087a5a1b1161 100644 --- a/apps/mobile/src/components/AppSymbol.tsx +++ b/apps/mobile/src/components/AppSymbol.tsx @@ -30,6 +30,7 @@ import IconBrain from "@tabler/icons-react-native/IconBrain"; import IconCamera from "@tabler/icons-react-native/IconCamera"; import IconChartBar from "@tabler/icons-react-native/IconChartBar"; import IconCheck from "@tabler/icons-react-native/IconCheck"; +import IconClipboard from "@tabler/icons-react-native/IconClipboard"; import IconCloud from "@tabler/icons-react-native/IconCloud"; import IconChevronDown from "@tabler/icons-react-native/IconChevronDown"; import IconChevronLeft from "@tabler/icons-react-native/IconChevronLeft"; @@ -65,6 +66,7 @@ import IconGitPullRequest from "@tabler/icons-react-native/IconGitPullRequest"; import IconHammer from "@tabler/icons-react-native/IconHammer"; import IconHome from "@tabler/icons-react-native/IconHome"; import IconInfoCircle from "@tabler/icons-react-native/IconInfoCircle"; +import IconKey from "@tabler/icons-react-native/IconKey"; import IconKeyboard from "@tabler/icons-react-native/IconKeyboard"; import IconKeyboardHide from "@tabler/icons-react-native/IconKeyboardHide"; import IconLock from "@tabler/icons-react-native/IconLock"; @@ -164,6 +166,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = { "chevron.up": IconChevronUp, desktopcomputer: IconDeviceDesktop, doc: IconFileText, + "doc.on.clipboard": IconClipboard, "doc.on.doc": IconCopy, "doc.text": IconFileText, ellipsis: IconDots, @@ -180,6 +183,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = { hammer: IconHammer, house: IconHome, "info.circle": IconInfoCircle, + key: IconKey, internaldrive: IconDatabase, keyboard: IconKeyboard, laptopcomputer: IconDeviceLaptop, diff --git a/apps/mobile/src/features/browser/BrowserClipboardMenu.tsx b/apps/mobile/src/features/browser/BrowserClipboardMenu.tsx new file mode 100644 index 000000000000..caeeab7b3bd6 --- /dev/null +++ b/apps/mobile/src/features/browser/BrowserClipboardMenu.tsx @@ -0,0 +1,40 @@ +import type { MenuAction } from "@react-native-menu/menu"; + +import { ControlPill, ControlPillMenu } from "../../components/ControlPill"; + +const ACTIONS: MenuAction[] = [ + { id: "paste", title: "Paste from clipboard", image: "doc.on.clipboard" }, + { id: "copy", title: "Copy selection", image: "doc.on.doc" }, + { id: "password", title: "Fill password", image: "key" }, +]; + +/** + * Clipboard and password tools for the page field that has the keyboard. The + * page runs on the environment, out of reach of this device's paste menu and + * AutoFill. + */ +export function BrowserClipboardMenu(props: { + readonly onPaste: () => void; + readonly onCopy: () => void; + readonly onFillPassword: () => void; +}) { + return ( + { + if (nativeEvent.event === "paste") props.onPaste(); + else if (nativeEvent.event === "copy") props.onCopy(); + else if (nativeEvent.event === "password") props.onFillPassword(); + }} + > + + + ); +} diff --git a/apps/mobile/src/features/browser/BrowserPasswordFill.tsx b/apps/mobile/src/features/browser/BrowserPasswordFill.tsx new file mode 100644 index 000000000000..91af2d8a2c80 --- /dev/null +++ b/apps/mobile/src/features/browser/BrowserPasswordFill.tsx @@ -0,0 +1,89 @@ +import { useRef, useState } from "react"; +import { Pressable, View, type TextInputInstance } from "react-native"; + +import { AppText, AppTextInput } from "../../components/AppText"; + +export interface BrowserLogin { + readonly username: string; + readonly password: string; +} + +/** + * Takes a saved login from this device's AutoFill (the Passwords key above the + * iOS keyboard, or Android's autofill service) and hands it to the page. + */ +export function BrowserPasswordFill(props: { + readonly onFill: (login: BrowserLogin) => void; + readonly onCancel: () => void; +}) { + const [username, setUsername] = useState(""); + const [password, setPassword] = useState(""); + const passwordInput = useRef(null); + // Empty the fields, then close on the next frame: iOS offers to save a password + // field's value when the field leaves the screen, and this is not a login here. + const close = (fill: boolean) => { + const login = { username, password }; + setUsername(""); + setPassword(""); + requestAnimationFrame(() => (fill ? props.onFill(login) : props.onCancel())); + }; + const canFill = username.length > 0 || password.length > 0; + return ( + // Below the address bar and control row, where the page's dialog cards sit. + + + Fill password + + Choose a saved login above the keyboard. The page gets the username, then the password, + which only goes into a password field. + + + passwordInput.current?.focus()} + /> + canFill && close(true)} + /> + + close(false)}> + Cancel + + close(true)} + > + Fill + + + + ); +} diff --git a/apps/mobile/src/features/browser/BrowserPreviewRouteScreen.tsx b/apps/mobile/src/features/browser/BrowserPreviewRouteScreen.tsx index 1d4f3fb3ea28..16e795f25029 100644 --- a/apps/mobile/src/features/browser/BrowserPreviewRouteScreen.tsx +++ b/apps/mobile/src/features/browser/BrowserPreviewRouteScreen.tsx @@ -6,9 +6,12 @@ import { ThreadId, type PreviewSessionSnapshot, } from "@t3tools/contracts"; +import type { PreviewStreamInput } from "@t3tools/client-runtime/preview/server-browser-stream"; import { normalizePreviewUrl } from "@t3tools/shared/preview"; +import * as Clipboard from "expo-clipboard"; import { useCallback, useEffect, useRef, useState } from "react"; import { ActivityIndicator, Alert, AppState, Platform, TextInput, View } from "react-native"; +import { KeyboardStickyView, useKeyboardState } from "react-native-keyboard-controller"; import { useSafeAreaInsets } from "react-native-safe-area-context"; import { ControlPill } from "../../components/ControlPill"; @@ -16,6 +19,8 @@ import { ScreenHeader } from "../../components/ScreenHeader"; import { NativeHeaderToolbar } from "../../native/StackHeader"; import { useThreadServerBrowserTabs } from "../../state/preview"; import { useAppearancePreferences } from "../settings/appearance/AppearancePreferencesProvider"; +import { BrowserClipboardMenu } from "./BrowserClipboardMenu"; +import { BrowserPasswordFill, type BrowserLogin } from "./BrowserPasswordFill"; import { BrowserTabMenu } from "./BrowserTabMenu"; import { browserTabTitle, browserTabUrl, latestBrowserTab } from "./browserTabs"; import { @@ -27,6 +32,8 @@ import { const BrowserPreviewStack = createNativeStackNavigator<{ BrowserPreview: undefined }>(); const NO_PICTURE_IN_PICTURE: PreviewPictureInPictureState = { supported: false, active: false }; +// Cmd+C runs as the page's copy command, and the page sends its selection back. +const COPY_KEY = { key: "c", code: "KeyC", keyCode: 67, modifiers: 4 } as const; type BrowserPreviewRouteScreenProps = StaticScreenProps<{ readonly environmentId: string; @@ -84,6 +91,10 @@ function BrowserPreviewScreen({ // Address bar commands only reach a page that is streaming. const [streaming, setStreaming] = useState(false); const [canControl, setCanControl] = useState(false); + // A page field has this device's keyboard. + const [typing, setTyping] = useState(false); + const [fillingPassword, setFillingPassword] = useState(false); + const keyboardVisible = useKeyboardState((state) => state.isVisible); const streamRef = useRef(null); const { tabs, loaded } = useThreadServerBrowserTabs({ environmentId, threadId, enabled: true }); const tab = tabs.find((entry) => entry.tabId === selectedTabId) ?? latestBrowserTab(tabs); @@ -91,6 +102,7 @@ function BrowserPreviewScreen({ // Pin the fallback so another tab's activity does not switch the view. if (tabId !== null && tabId !== selectedTabId) { setPictureInPicture(NO_PICTURE_IN_PICTURE); + setFillingPassword(false); setSelectedTabId(tabId); } useEffect(() => { @@ -108,10 +120,24 @@ function BrowserPreviewScreen({ }, []); const selectTab = (next: string) => { setPictureInPicture(NO_PICTURE_IN_PICTURE); + setFillingPassword(false); setSelectedTabId(next); }; // System picture in picture keeps showing this stream after the app leaves the foreground. const live = (focused && foreground) || pictureInPicture.active; + const send = (input: PreviewStreamInput) => streamRef.current?.command(input); + const pasteFromClipboard = async () => { + const text = await Clipboard.getStringAsync().catch(() => ""); + if (text) send({ type: "text", text }); + }; + const copySelection = () => { + send({ type: "key", action: "down", ...COPY_KEY }); + send({ type: "key", action: "up", ...COPY_KEY }); + }; + const fillLogin = (login: BrowserLogin) => { + setFillingPassword(false); + send({ type: "fillLogin", ...login }); + }; return ( @@ -187,11 +213,30 @@ function BrowserPreviewScreen({ background={themeVariables["--color-sheet-solid"]} onPictureInPicture={onPictureInPicture} onStreamingChange={setStreaming} + onPageInput={setTyping} onControl={(control) => setCanControl(control?.controller === "you")} /> ) : ( )} + {live && canControl && fillingPassword ? ( + setFillingPassword(false)} /> + ) : null} + {live && canControl && typing && keyboardVisible && !fillingPassword ? ( + + + void pasteFromClipboard()} + onCopy={copySelection} + onFillPassword={() => setFillingPassword(true)} + /> + + + ) : null} ) : ( diff --git a/apps/mobile/src/features/browser/PreviewStreamWebView.tsx b/apps/mobile/src/features/browser/PreviewStreamWebView.tsx index 27063f24f1b3..6734bf981912 100644 --- a/apps/mobile/src/features/browser/PreviewStreamWebView.tsx +++ b/apps/mobile/src/features/browser/PreviewStreamWebView.tsx @@ -24,6 +24,7 @@ import * as Clipboard from "expo-clipboard"; import { AppText } from "../../components/AppText"; import { downloadAndShareAttachment } from "../../lib/attachmentDownload"; +import { copyTextWithHaptic } from "../../lib/copyTextWithHaptic"; import { beginForegroundHandoff } from "../../lib/foreground-handoff"; import { usePreviewStreamAccess } from "../../state/preview"; @@ -55,6 +56,8 @@ type NativeStreamBridge = { readonly onPictureInPicture?: (state: PreviewPictureInPictureState, detail?: string) => void; /** True while frames show, so commands reach the page. Pass a stable function. */ readonly onStreamingChange?: (streaming: boolean) => void; + /** A page field took or lost this device's keyboard. Pass a stable function. */ + readonly onPageInput?: (focused: boolean) => void; /** The floating player shows a spinner without text or a reconnect button. */ readonly compact?: boolean; }; @@ -235,6 +238,7 @@ function PreviewStreamDocumentView({ onControl, onPictureInPicture, onStreamingChange, + onPageInput, onRetry, onStreaming, onRecoverProcess, @@ -279,6 +283,7 @@ function PreviewStreamDocumentView({ failed.current = true; webView.current?.injectJavaScript("window.T3PreviewStream?.stop(); true;"); onStreamingChange?.(false); + onPageInput?.(false); setControl(null); setFileChooser(null); onControl?.(null); @@ -315,6 +320,7 @@ function PreviewStreamDocumentView({ }; }, []); useEffect(() => () => onStreamingChange?.(false), [onStreamingChange]); + useEffect(() => () => onPageInput?.(false), [onPageInput]); useEffect(() => () => controlChanged(null), []); const processTerminated = () => { if (!active.current || failed.current) return; @@ -394,7 +400,10 @@ function PreviewStreamDocumentView({ onViewport?.(message); return; case "clipboard": - void Clipboard.setStringAsync(message.text).catch(() => undefined); + copyTextWithHaptic(message.text, { target: "browser page selection" }); + return; + case "input": + onPageInput?.(message.focused); return; case "download": offerDownload(message); diff --git a/apps/mobile/src/features/browser/preview-stream-document.ts b/apps/mobile/src/features/browser/preview-stream-document.ts index 8b404825707b..c6af70045738 100644 --- a/apps/mobile/src/features/browser/preview-stream-document.ts +++ b/apps/mobile/src/features/browser/preview-stream-document.ts @@ -28,6 +28,8 @@ export type PreviewStreamMessage = | ({ readonly type: "hostSetup" } & PreviewStreamHostSetup) | { readonly type: "viewport"; readonly width: number; readonly height: number } | { readonly type: "clipboard"; readonly text: string } + /** The input that types into page fields gained or lost focus. */ + | { readonly type: "input"; readonly focused: boolean } | ({ readonly type: "download" } & PreviewStreamDownload) | { readonly type: "fileChooser"; readonly chooser: PreviewStreamFileChooser | null } | { diff --git a/apps/mobile/src/features/browser/preview-stream.browser.ts b/apps/mobile/src/features/browser/preview-stream.browser.ts index 91fa5d7a7085..b8f5532e7d80 100644 --- a/apps/mobile/src/features/browser/preview-stream.browser.ts +++ b/apps/mobile/src/features/browser/preview-stream.browser.ts @@ -610,6 +610,9 @@ export function start(configuration: PreviewStreamConfiguration) { input.addEventListener("input", onInput); input.addEventListener("compositionend", onCompositionEnd); input.addEventListener("focus", resetInput); + // Native shows its clipboard and password tools while a page field has the keyboard. + input.addEventListener("focus", () => post({ type: "input", focused: true })); + input.addEventListener("blur", () => post({ type: "input", focused: false })); // Copying the input would put its sentinel on this device's clipboard. input.addEventListener("copy", preventDefault); input.addEventListener("cut", preventDefault); diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index f66e255c51b4..2f9c30c871ec 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -1035,6 +1035,63 @@ it.live("page copies reach only the controlling viewer right after its input", ( ).pipe(Effect.provide(layer)), ); +it.live.each([ + { + name: "a username field gets the username, then Tab, then the password field the password", + focused: "text", + afterTab: "password", + typed: ["nick", "hunter2"], + }, + { + name: "the password is never typed where Tab lands outside a password field", + focused: "text", + afterTab: "text", + typed: ["nick"], + }, + { + name: "a focused password field gets only the password", + focused: "password", + afterTab: "password", + typed: ["hunter2"], + }, +] as const)("$name", ({ focused, afterTab, typed }) => + Effect.scoped( + Effect.gen(function* () { + const { browser, tabId } = yield* ready; + const viewer = yield* browser.attachViewer(viewerInput(tabId, true)); + yield* viewer.input({ type: "takeControl" }); + const cdp = contexts[0]!.sessions.at(-1)!; + const send = cdp.send.getMockImplementation()!; + // Stands in for the page: Tab moves focus to the next field. + let field: "text" | "password" = focused; + cdp.send.mockImplementation(async (method, input) => { + const params = (input ?? {}) as { expression?: string; type?: string; key?: string }; + if (method === "Runtime.evaluate" && params.expression?.includes("activeElement")) { + return { result: { value: field === "password" } }; + } + if ( + method === "Input.dispatchKeyEvent" && + params.key === "Tab" && + params.type === "keyUp" + ) { + field = afterTab; + } + return send(method, input); + }); + yield* viewer.input({ type: "fillLogin", username: "nick", password: "hunter2" }); + const inserted = cdp.send.mock.calls + .filter(([method]) => method === "Input.insertText") + .map(([, input]) => (input as { text: string }).text); + expect(inserted).toEqual(typed); + const tabs = cdp.send.mock.calls.filter( + ([method, input]) => + method === "Input.dispatchKeyEvent" && (input as { key?: string }).key === "Tab", + ); + expect(tabs).toHaveLength(focused === "password" ? 0 : 2); + }), + ).pipe(Effect.provide(layer)), +); + it.live("a page download is saved, offered to the controller, and listed for the agent", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 072a022f90c0..461225f9e565 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -121,6 +121,18 @@ const EDITABLE_AT_POINT_SCRIPT = `(x, y) => { const nonText = ["button", "checkbox", "color", "file", "hidden", "image", "radio", "range", "reset", "submit"]; return !nonText.includes(element.type) && !element.disabled && !element.readOnly; }`; +// A viewer's saved password goes only into a focused password field, found through +// shadow roots and same-origin frames; a cross-origin frame stays opaque. +const FOCUSED_PASSWORD_SCRIPT = `() => { + let element = document.activeElement; + while (element) { + const frame = element.tagName === "IFRAME" || element.tagName === "FRAME"; + const inner = element.shadowRoot?.activeElement ?? (frame ? element.contentDocument?.activeElement : null); + if (!inner || inner === element) break; + element = inner; + } + return element?.tagName === "INPUT" && element.type === "password" && !element.disabled && !element.readOnly; +}`; const UNATTACHED_FILL_VIEWPORT = { width: 1280, height: 800 } as const; const NAVIGATION_TIMEOUT_MS = 15_000; const VIEWER_NAVIGATION_OPTIONS = { waitUntil: "commit", timeout: NAVIGATION_TIMEOUT_MS } as const; @@ -2100,6 +2112,36 @@ const make = Effect.gen(function* () { await session.send("Input.insertText", { text: message.text.slice(0, 10_000) }); } return; + case "fillLogin": { + const username = typeof message.username === "string" ? message.username : ""; + const password = typeof message.password === "string" ? message.password : ""; + const passwordFocused = async () => { + const result = await session.send("Runtime.evaluate", { + expression: `(${FOCUSED_PASSWORD_SCRIPT})()`, + returnByValue: true, + }); + return result.result.value === true; + }; + if (await passwordFocused()) { + if (password) await session.send("Input.insertText", { text: password.slice(0, 10_000) }); + return; + } + if (username) await session.send("Input.insertText", { text: username.slice(0, 10_000) }); + if (!password) return; + for (const type of ["rawKeyDown", "keyUp"] as const) { + await session.send("Input.dispatchKeyEvent", { + type, + key: "Tab", + code: "Tab", + windowsVirtualKeyCode: 9, + }); + } + // A password never lands in a field that would show it. + if (await passwordFocused()) { + await session.send("Input.insertText", { text: password.slice(0, 10_000) }); + } + return; + } case "resize": { const width = Math.min(Math.round(num(message.width)), 3840); const height = Math.min(Math.round(num(message.height)), 2160); diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index 231823f13dcc..0254ea2b6155 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -205,6 +205,11 @@ picker, and a finished download is offered for you to save. Popups such as sign-in windows open as their own tabs. Downloads stay on the host until the tab closes. Audio does not play on your device. +On a phone, while you type into the page, the clipboard button above the +keyboard pastes your clipboard, copies the page's selection, or fills a login +saved in your password manager. A filled password only goes into a password +field. + On a phone, tap the floating preview's corner dot to show its controls, then **Pop into separate window** to keep watching in picture-in-picture over other apps. diff --git a/packages/client-runtime/src/preview/serverBrowserStream.ts b/packages/client-runtime/src/preview/serverBrowserStream.ts index 34a3786976d0..5f7f72f984a1 100644 --- a/packages/client-runtime/src/preview/serverBrowserStream.ts +++ b/packages/client-runtime/src/preview/serverBrowserStream.ts @@ -87,6 +87,12 @@ export type PreviewStreamInput = readonly modifiers: number; } | { readonly type: "text"; readonly text: string } + /** + * A saved login from the viewer's AutoFill. A focused password field gets the + * password; another field gets the username, then Tab, then the password only + * if Tab reached a password field. + */ + | { readonly type: "fillLogin"; readonly username: string; readonly password: string } | { readonly type: "resize"; readonly width: number; readonly height: number } | { readonly type: "navigate"; readonly url: string } | { readonly type: "history"; readonly delta: -1 | 1 } From 637bbab920404f8af7fdefda37da716fe2e9fd74 Mon Sep 17 00:00:00 2001 From: Nicholas Tindle Date: Fri, 9 Oct 2026 08:22:36 -0500 Subject: [PATCH 2/3] fix(mobile): fill a login only on the site its card shows - The fill card shows the page's site, and fillLogin carries that origin. The server fills only while the page's own URL is on that origin, and never into a field inside a cross-origin frame. - A paste whose clipboard read finishes after the tab changed is dropped instead of reaching the new tab. Co-Authored-By: Claude Opus 5.5 --- .../features/browser/BrowserClipboardMenu.tsx | 8 ++-- .../features/browser/BrowserPasswordFill.tsx | 9 +++- .../browser/BrowserPreviewRouteScreen.tsx | 31 ++++++++----- .../src/features/browser/browserTabs.ts | 10 ++++ apps/server/src/preview/ServerBrowser.test.ts | 46 +++++++++++++++---- apps/server/src/preview/ServerBrowser.ts | 42 ++++++++++++----- docs/user/remote-access.md | 5 +- .../src/preview/serverBrowserStream.ts | 12 +++-- 8 files changed, 122 insertions(+), 41 deletions(-) diff --git a/apps/mobile/src/features/browser/BrowserClipboardMenu.tsx b/apps/mobile/src/features/browser/BrowserClipboardMenu.tsx index caeeab7b3bd6..7aebdc37c843 100644 --- a/apps/mobile/src/features/browser/BrowserClipboardMenu.tsx +++ b/apps/mobile/src/features/browser/BrowserClipboardMenu.tsx @@ -16,18 +16,20 @@ const ACTIONS: MenuAction[] = [ export function BrowserClipboardMenu(props: { readonly onPaste: () => void; readonly onCopy: () => void; - readonly onFillPassword: () => void; + /** Null on a page without an http(s) origin, which no login belongs to. */ + readonly onFillPassword: (() => void) | null; }) { + const { onFillPassword } = props; return ( action.id !== "password")} onPressAction={({ nativeEvent }) => { if (nativeEvent.event === "paste") props.onPaste(); else if (nativeEvent.event === "copy") props.onCopy(); - else if (nativeEvent.event === "password") props.onFillPassword(); + else if (nativeEvent.event === "password") onFillPassword?.(); }} > void; readonly onCancel: () => void; }) { @@ -28,14 +30,17 @@ export function BrowserPasswordFill(props: { requestAnimationFrame(() => (fill ? props.onFill(login) : props.onCancel())); }; const canFill = username.length > 0 || password.length > 0; + // An https site shows as its host; http keeps its scheme, so it reads as unencrypted. + const site = props.origin.replace(/^https:\/\//, ""); return ( // Below the address bar and control row, where the page's dialog cards sit. Fill password + {site} - Choose a saved login above the keyboard. The page gets the username, then the password, - which only goes into a password field. + Choose a saved login above the keyboard. Only this site gets it: the username, then the + password, which only goes into a password field. (null); const keyboardVisible = useKeyboardState((state) => state.isVisible); const streamRef = useRef(null); const { tabs, loaded } = useThreadServerBrowserTabs({ environmentId, threadId, enabled: true }); @@ -102,7 +103,7 @@ function BrowserPreviewScreen({ // Pin the fallback so another tab's activity does not switch the view. if (tabId !== null && tabId !== selectedTabId) { setPictureInPicture(NO_PICTURE_IN_PICTURE); - setFillingPassword(false); + setFillOrigin(null); setSelectedTabId(tabId); } useEffect(() => { @@ -120,23 +121,27 @@ function BrowserPreviewScreen({ }, []); const selectTab = (next: string) => { setPictureInPicture(NO_PICTURE_IN_PICTURE); - setFillingPassword(false); + setFillOrigin(null); setSelectedTabId(next); }; // System picture in picture keeps showing this stream after the app leaves the foreground. const live = (focused && foreground) || pictureInPicture.active; const send = (input: PreviewStreamInput) => streamRef.current?.command(input); const pasteFromClipboard = async () => { + // Reading can wait on the paste prompt; if the tab changes meanwhile, nothing is pasted. + const stream = streamRef.current; const text = await Clipboard.getStringAsync().catch(() => ""); - if (text) send({ type: "text", text }); + if (text && stream && streamRef.current === stream) stream.command({ type: "text", text }); }; const copySelection = () => { send({ type: "key", action: "down", ...COPY_KEY }); send({ type: "key", action: "up", ...COPY_KEY }); }; + const pageOrigin = tab ? browserTabOrigin(tab) : null; const fillLogin = (login: BrowserLogin) => { - setFillingPassword(false); - send({ type: "fillLogin", ...login }); + if (fillOrigin === null) return; + setFillOrigin(null); + send({ type: "fillLogin", origin: fillOrigin, ...login }); }; return ( @@ -219,10 +224,14 @@ function BrowserPreviewScreen({ ) : ( )} - {live && canControl && fillingPassword ? ( - setFillingPassword(false)} /> + {live && canControl && fillOrigin !== null ? ( + setFillOrigin(null)} + /> ) : null} - {live && canControl && typing && keyboardVisible && !fillingPassword ? ( + {live && canControl && typing && keyboardVisible && fillOrigin === null ? ( void pasteFromClipboard()} onCopy={copySelection} - onFillPassword={() => setFillingPassword(true)} + onFillPassword={pageOrigin === null ? null : () => setFillOrigin(pageOrigin)} /> diff --git a/apps/mobile/src/features/browser/browserTabs.ts b/apps/mobile/src/features/browser/browserTabs.ts index 509ddad5dcf7..f2eaf0bc7f64 100644 --- a/apps/mobile/src/features/browser/browserTabs.ts +++ b/apps/mobile/src/features/browser/browserTabs.ts @@ -12,6 +12,16 @@ export function browserTabUrl(tab: PreviewSessionSnapshot) { return tab.navStatus._tag === "Idle" ? "" : tab.navStatus.url; } +/** The page's http(s) origin, the only site a saved login can be filled into. */ +export function browserTabOrigin(tab: PreviewSessionSnapshot) { + try { + const url = new URL(browserTabUrl(tab)); + return url.protocol === "http:" || url.protocol === "https:" ? url.origin : null; + } catch { + return null; + } +} + export function browserTabTitle(tab: PreviewSessionSnapshot) { if (tab.navStatus._tag === "Idle") return "New tab"; if (tab.navStatus.title.trim()) return tab.navStatus.title; diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 2f9c30c871ec..3b68d905fbd2 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -1038,36 +1038,59 @@ it.live("page copies reach only the controlling viewer right after its input", ( it.live.each([ { name: "a username field gets the username, then Tab, then the password field the password", - focused: "text", + url: "https://example.com/login", + focused: "field", afterTab: "password", typed: ["nick", "hunter2"], + tabs: 2, }, { name: "the password is never typed where Tab lands outside a password field", - focused: "text", - afterTab: "text", + url: "https://example.com/login", + focused: "field", + afterTab: "field", typed: ["nick"], + tabs: 2, }, { name: "a focused password field gets only the password", + url: "https://example.com/login", focused: "password", afterTab: "password", typed: ["hunter2"], + tabs: 0, }, -] as const)("$name", ({ focused, afterTab, typed }) => + { + name: "a page that left the confirmed site gets nothing", + url: "https://example.net/login", + focused: "password", + afterTab: "password", + typed: [], + tabs: 0, + }, + { + name: "a field in a cross-origin frame gets nothing", + url: "https://example.com/login", + focused: "frame", + afterTab: "password", + typed: [], + tabs: 0, + }, +] as const)("$name", ({ url, focused, afterTab, typed, tabs }) => Effect.scoped( Effect.gen(function* () { const { browser, tabId } = yield* ready; + yield* Effect.promise(() => contexts[0]!.page.goto(url)); const viewer = yield* browser.attachViewer(viewerInput(tabId, true)); yield* viewer.input({ type: "takeControl" }); const cdp = contexts[0]!.sessions.at(-1)!; const send = cdp.send.getMockImplementation()!; // Stands in for the page: Tab moves focus to the next field. - let field: "text" | "password" = focused; + let field: "field" | "password" | "frame" = focused; cdp.send.mockImplementation(async (method, input) => { const params = (input ?? {}) as { expression?: string; type?: string; key?: string }; if (method === "Runtime.evaluate" && params.expression?.includes("activeElement")) { - return { result: { value: field === "password" } }; + return { result: { value: field } }; } if ( method === "Input.dispatchKeyEvent" && @@ -1078,16 +1101,21 @@ it.live.each([ } return send(method, input); }); - yield* viewer.input({ type: "fillLogin", username: "nick", password: "hunter2" }); + yield* viewer.input({ + type: "fillLogin", + origin: "https://example.com", + username: "nick", + password: "hunter2", + }); const inserted = cdp.send.mock.calls .filter(([method]) => method === "Input.insertText") .map(([, input]) => (input as { text: string }).text); expect(inserted).toEqual(typed); - const tabs = cdp.send.mock.calls.filter( + const tabPresses = cdp.send.mock.calls.filter( ([method, input]) => method === "Input.dispatchKeyEvent" && (input as { key?: string }).key === "Tab", ); - expect(tabs).toHaveLength(focused === "password" ? 0 : 2); + expect(tabPresses).toHaveLength(tabs); }), ).pipe(Effect.provide(layer)), ); diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 461225f9e565..c26b7d65ac96 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -121,18 +121,32 @@ const EDITABLE_AT_POINT_SCRIPT = `(x, y) => { const nonText = ["button", "checkbox", "color", "file", "hidden", "image", "radio", "range", "reset", "submit"]; return !nonText.includes(element.type) && !element.disabled && !element.readOnly; }`; -// A viewer's saved password goes only into a focused password field, found through -// shadow roots and same-origin frames; a cross-origin frame stays opaque. -const FOCUSED_PASSWORD_SCRIPT = `() => { +// Where a viewer's saved login would land, found through shadow roots and same-origin +// frames: "password" for a password field, "field" for anything else, "frame" when focus is +// in a cross-origin frame, which stays opaque and gets nothing. +const FOCUSED_FIELD_SCRIPT = `() => { let element = document.activeElement; while (element) { - const frame = element.tagName === "IFRAME" || element.tagName === "FRAME"; - const inner = element.shadowRoot?.activeElement ?? (frame ? element.contentDocument?.activeElement : null); + if (element.tagName === "IFRAME" || element.tagName === "FRAME") { + if (!element.contentDocument) return "frame"; + element = element.contentDocument.activeElement; + continue; + } + const inner = element.shadowRoot?.activeElement; if (!inner || inner === element) break; element = inner; } - return element?.tagName === "INPUT" && element.type === "password" && !element.disabled && !element.readOnly; + const password = element?.tagName === "INPUT" && element.type === "password" && !element.disabled && !element.readOnly; + return password ? "password" : "field"; }`; +const httpOrigin = (url: string) => { + try { + const parsed = new URL(url); + return parsed.protocol === "http:" || parsed.protocol === "https:" ? parsed.origin : null; + } catch { + return null; + } +}; const UNATTACHED_FILL_VIEWPORT = { width: 1280, height: 800 } as const; const NAVIGATION_TIMEOUT_MS = 15_000; const VIEWER_NAVIGATION_OPTIONS = { waitUntil: "commit", timeout: NAVIGATION_TIMEOUT_MS } as const; @@ -2115,14 +2129,20 @@ const make = Effect.gen(function* () { case "fillLogin": { const username = typeof message.username === "string" ? message.username : ""; const password = typeof message.password === "string" ? message.password : ""; - const passwordFocused = async () => { + // The site the viewer confirmed. The page's own URL decides, which its scripts cannot fake. + const origin = typeof message.origin === "string" ? message.origin : ""; + const target = async () => { + if (!origin || httpOrigin(tab.page.url()) !== origin) return null; const result = await session.send("Runtime.evaluate", { - expression: `(${FOCUSED_PASSWORD_SCRIPT})()`, + expression: `(${FOCUSED_FIELD_SCRIPT})()`, returnByValue: true, }); - return result.result.value === true; + const field: unknown = result.result.value; + return field === "password" || field === "field" ? field : null; }; - if (await passwordFocused()) { + const first = await target(); + if (first === null) return; + if (first === "password") { if (password) await session.send("Input.insertText", { text: password.slice(0, 10_000) }); return; } @@ -2137,7 +2157,7 @@ const make = Effect.gen(function* () { }); } // A password never lands in a field that would show it. - if (await passwordFocused()) { + if ((await target()) === "password") { await session.send("Input.insertText", { text: password.slice(0, 10_000) }); } return; diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index 0254ea2b6155..17eedb545981 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -207,8 +207,9 @@ tab closes. Audio does not play on your device. On a phone, while you type into the page, the clipboard button above the keyboard pastes your clipboard, copies the page's selection, or fills a login -saved in your password manager. A filled password only goes into a password -field. +saved in your password manager. The fill card shows the site the login goes to, +and nothing is filled if the page has left that site by then. The password only +goes into a password field. On a phone, tap the floating preview's corner dot to show its controls, then **Pop into separate window** to keep watching in picture-in-picture over other diff --git a/packages/client-runtime/src/preview/serverBrowserStream.ts b/packages/client-runtime/src/preview/serverBrowserStream.ts index 5f7f72f984a1..583935ff775d 100644 --- a/packages/client-runtime/src/preview/serverBrowserStream.ts +++ b/packages/client-runtime/src/preview/serverBrowserStream.ts @@ -88,11 +88,17 @@ export type PreviewStreamInput = } | { readonly type: "text"; readonly text: string } /** - * A saved login from the viewer's AutoFill. A focused password field gets the + * A saved login from the viewer's AutoFill, for the page only while it is on + * `origin`, the site the viewer confirmed. A focused password field gets the * password; another field gets the username, then Tab, then the password only - * if Tab reached a password field. + * if Tab reached a password field. A field in a cross-origin frame gets nothing. */ - | { readonly type: "fillLogin"; readonly username: string; readonly password: string } + | { + readonly type: "fillLogin"; + readonly origin: string; + readonly username: string; + readonly password: string; + } | { readonly type: "resize"; readonly width: number; readonly height: number } | { readonly type: "navigate"; readonly url: string } | { readonly type: "history"; readonly delta: -1 | 1 } From 9306f3f78205221dddd5929c810ee1e22fa72eb0 Mon Sep 17 00:00:00 2001 From: Nicholas Tindle Date: Sat, 10 Oct 2026 02:41:25 -0500 Subject: [PATCH 3/3] feat(mobile): passkeys for Browser page sites from the phone's passkey sheet The server's headless browser has no authenticator, so a page's passkey sign-in or sign-up has nowhere to go. - Server browser pages get a WebAuthn bridge, modeled on the desktop's preview bridge: create() and get() go to the server, which checks the page's real origin (top-level, https or loopback), the RP ID (no public suffixes), a recent touch from the controlling viewer, and one request at a time, then forwards the request to that viewer. The device's answer must carry this page's origin, challenge and RP ID hash before the page sees it. Conditional requests stay native. - With no passkey-capable viewer in control, pages keep Chromium's own WebAuthn, and isUserVerifyingPlatformAuthenticatorAvailable() stays native unless such a viewer is watching the tab. - iOS: a T3Passkeys module answers with AuthenticationServices' browser requests (client data with the page's origin, iOS 17.4+), offering device passkeys and security keys. - iOS allows that for any site only with Apple's managed default-browser entitlement, so it stays off unless the build sets T3CODE_IOS_BROWSER_PASSKEYS=1 (README covers the Apple request). - Viewers offer passkeys with a passkeys=true stream parameter; only viewers that may operate the page keep it. Co-Authored-By: Claude Opus 5.5 --- apps/mobile/.swiftlint.yml | 1 + apps/mobile/README.md | 25 + apps/mobile/app.config.ts | 7 + .../t3-passkeys/expo-module.config.json | 4 + .../t3-passkeys/ios/T3Passkeys.podspec | 20 + .../t3-passkeys/ios/T3PasskeysModule.swift | 294 +++++++++++ .../features/browser/PreviewStreamWebView.tsx | 20 +- .../src/features/browser/browserPasskeys.ts | 50 ++ .../browser/preview-stream-document.ts | 5 + .../browser/preview-stream.browser.ts | 3 + apps/server/package.json | 1 + apps/server/src/preview/ServerBrowser.test.ts | 154 +++++- apps/server/src/preview/ServerBrowser.ts | 130 ++++- .../src/preview/ServerBrowserPasskeys.test.ts | 271 ++++++++++ .../src/preview/ServerBrowserPasskeys.ts | 486 ++++++++++++++++++ .../src/preview/ServerBrowserStream.test.ts | 4 +- .../server/src/preview/ServerBrowserStream.ts | 3 + docs/user/remote-access.md | 5 + .../src/preview/serverBrowserStream.test.ts | 37 ++ .../src/preview/serverBrowserStream.ts | 65 +++ pnpm-lock.yaml | 3 + 21 files changed, 1580 insertions(+), 8 deletions(-) create mode 100644 apps/mobile/modules/t3-passkeys/expo-module.config.json create mode 100644 apps/mobile/modules/t3-passkeys/ios/T3Passkeys.podspec create mode 100644 apps/mobile/modules/t3-passkeys/ios/T3PasskeysModule.swift create mode 100644 apps/mobile/src/features/browser/browserPasskeys.ts create mode 100644 apps/server/src/preview/ServerBrowserPasskeys.test.ts create mode 100644 apps/server/src/preview/ServerBrowserPasskeys.ts diff --git a/apps/mobile/.swiftlint.yml b/apps/mobile/.swiftlint.yml index c56a56251f73..1822d6e0b310 100644 --- a/apps/mobile/.swiftlint.yml +++ b/apps/mobile/.swiftlint.yml @@ -2,6 +2,7 @@ included: - ios/T3Code - modules/t3-composer-editor/ios - modules/t3-native-controls/ios + - modules/t3-passkeys/ios - modules/t3-terminal/ios - modules/t3-review-diff/ios diff --git a/apps/mobile/README.md b/apps/mobile/README.md index 20f98c6c9e34..9533d4402c2b 100644 --- a/apps/mobile/README.md +++ b/apps/mobile/README.md @@ -129,6 +129,31 @@ For preview or production EAS environments, set `T3CODE_CLERK_PUBLISHABLE_KEY`, `T3CODE_CLERK_JWT_TEMPLATE`, and `T3CODE_RELAY_URL` as EAS environment variables. Expo config maps the canonical values into the mobile build. +### Passkeys on the Browser page + +The Browser page shows a page that runs in the server's headless browser, which has no +authenticator. iOS builds can answer that page's passkey requests with the phone's own passkey +sheet (iCloud Keychain, password managers, security keys), but iOS lets an app use passkeys for any +site only with Apple's managed default-browser entitlement, `com.apple.developer.web-browser`. +The Mac counterpart that the desktop app uses, `com.apple.developer.web-browser.public-key-credential`, +does not exist on iOS. + +Builds leave this off. `T3CODE_IOS_BROWSER_PASSKEYS=1` adds the entitlement, and the app then +offers passkeys to the server. Set it only for a bundle identifier whose provisioning profile grants +the entitlement: a profile without it cannot sign the build. To get it: + +- The Account Holder of the T3 Tools Apple Developer organization requests it for the app's App ID + through Apple's + [default browser entitlement request form](https://developer.apple.com/contact/request/default-browser-entitlement/). +- Apple's [criteria](https://developer.apple.com/documentation/xcode/preparing-your-app-to-be-the-default-browser) + make this a product decision to settle first. The app must declare the `http` and `https` URL + schemes, open such links straight to the page, and offer a URL field on launch, so T3 Code + becomes a default browser choice in iOS Settings. iOS also ignores an entitled app's own + Universal Links, which the app's `applinks:` domain relies on today. +- After approval, regenerate the profiles and build with `T3CODE_IOS_BROWSER_PASSKEYS=1`. + +Simulator builds are not signed against a profile, so they accept the flag for local testing. + Create a PR preview dev-client build manually: ```bash diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index 57717d2d8458..63902d8c0db9 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -10,6 +10,10 @@ Object.assign(process.env, repoEnv); const APP_VARIANT = resolveAppVariant(repoEnv.APP_VARIANT); const isIosPersonalTeamBuild = repoEnv.T3CODE_IOS_PERSONAL_TEAM === "1"; +// Passkeys for Browser page sites need Apple's managed default-browser entitlement. +// A profile without it cannot sign the build, so it stays off until Apple grants it +// (README.md, "Passkeys on the Browser page"). +const iosBrowserPasskeys = repoEnv.T3CODE_IOS_BROWSER_PASSKEYS === "1"; const runtimeVersionPolicy = process.env.MOBILE_VERSION_POLICY ?? (APP_VARIANT === "development" ? "appVersion" : "fingerprint"); @@ -262,8 +266,11 @@ const config: ExpoConfig = { ], entitlements: { "keychain-access-groups": [`$(AppIdentifierPrefix)${variant.iosBundleIdentifier}`], + ...(iosBrowserPasskeys ? { "com.apple.developer.web-browser": true } : {}), }, infoPlist: { + // Read by modules/t3-passkeys, so the app offers passkeys only when signed for them. + ...(iosBrowserPasskeys ? { T3BrowserPasskeys: true } : {}), UISupportedInterfaceOrientations: [ "UIInterfaceOrientationPortrait", "UIInterfaceOrientationLandscapeLeft", diff --git a/apps/mobile/modules/t3-passkeys/expo-module.config.json b/apps/mobile/modules/t3-passkeys/expo-module.config.json new file mode 100644 index 000000000000..912fa67ba83e --- /dev/null +++ b/apps/mobile/modules/t3-passkeys/expo-module.config.json @@ -0,0 +1,4 @@ +{ + "platforms": ["apple"], + "apple": { "modules": ["T3PasskeysModule"] } +} diff --git a/apps/mobile/modules/t3-passkeys/ios/T3Passkeys.podspec b/apps/mobile/modules/t3-passkeys/ios/T3Passkeys.podspec new file mode 100644 index 000000000000..4b823d76b308 --- /dev/null +++ b/apps/mobile/modules/t3-passkeys/ios/T3Passkeys.podspec @@ -0,0 +1,20 @@ +Pod::Spec.new do |s| + s.name = 'T3Passkeys' + s.version = '1.0.0' + s.summary = 'Passkeys for T3 Code server browser pages.' + s.description = 'Answers a server browser page\'s WebAuthn requests with the system passkey sheet.' + s.author = 'T3 Tools' + s.homepage = 'https://t3tools.com' + s.platforms = { + :ios => '18.0', + } + s.source = { :path => '.' } + s.static_framework = true + + s.dependency 'ExpoModulesCore' + s.frameworks = 'AuthenticationServices' + s.pod_target_xcconfig = { + 'DEFINES_MODULE' => 'YES', + } + s.source_files = '**/*.{h,m,mm,swift,hpp,cpp}' +end diff --git a/apps/mobile/modules/t3-passkeys/ios/T3PasskeysModule.swift b/apps/mobile/modules/t3-passkeys/ios/T3PasskeysModule.swift new file mode 100644 index 000000000000..41691f23ab1b --- /dev/null +++ b/apps/mobile/modules/t3-passkeys/ios/T3PasskeysModule.swift @@ -0,0 +1,294 @@ +import AuthenticationServices +import ExpoModulesCore +import UIKit + +/// Answers a server browser page's WebAuthn request with the system passkey +/// sheet, for the page's own origin. AuthenticationServices serves any site only +/// to apps that hold Apple's browser entitlement, so builds without it report +/// unavailable and are never asked. +public final class T3PasskeysModule: Module { + private var ceremony: T3PasskeyCeremony? + + public func definition() -> ModuleDefinition { + Name("T3Passkeys") + + Constants { + // app.config.ts sets this only in builds signed with the entitlement. + ["available": Bundle.main.object(forInfoDictionaryKey: "T3BrowserPasskeys") as? Bool == true] + } + + // Both run on the main queue, where AuthenticationServices presents its sheet. + AsyncFunction("perform") { (id: String, kind: String, origin: String, options: String, promise: Promise) in + MainActor.assumeIsolated { + // One sheet at a time: a newer request replaces one still unanswered. + self.ceremony?.cancel() + // The ceremony drops this completion once it runs, which ends the cycle back to the module. + let ceremony = T3PasskeyCeremony(id: id, kind: kind, origin: origin, options: options) { result in + if self.ceremony?.id == id { self.ceremony = nil } + promise.resolve(result) + } + self.ceremony = ceremony + ceremony.start() + } + }.runOnQueue(.main) + + AsyncFunction("cancel") { (id: String) in + MainActor.assumeIsolated { + if self.ceremony?.id == id { self.ceremony?.cancel() } + } + }.runOnQueue(.main) + + OnDestroy { + let ceremony = self.ceremony + DispatchQueue.main.async { ceremony?.cancel() } + } + } +} + +/// One WebAuthn ceremony. The server has checked the origin and RP ID; the +/// result goes back as a `PreviewStreamPasskeyResult` in JSON. +final class T3PasskeyCeremony: NSObject, ASAuthorizationControllerDelegate, + ASAuthorizationControllerPresentationContextProviding +{ + let id: String + private let kind: String + private let origin: String + private let options: String + private var completion: ((String) -> Void)? + private var controller: ASAuthorizationController? + + private static let notAllowed: [String: Any] = ["success": false, "error": "NotAllowedError"] + private static let transports: [String: ASAuthorizationSecurityKeyPublicKeyCredentialDescriptor.Transport] = [ + "usb": .usb, "nfc": .nfc, "ble": .bluetooth, + ] + + init(id: String, kind: String, origin: String, options: String, completion: @escaping (String) -> Void) { + self.id = id + self.kind = kind + self.origin = origin + self.options = options + self.completion = completion + } + + func start() { + guard let requests = makeRequests(), !requests.isEmpty else { + finish(["success": false, "error": "TypeError"]) + return + } + let controller = ASAuthorizationController(authorizationRequests: requests) + controller.delegate = self + controller.presentationContextProvider = self + self.controller = controller + controller.performRequests() + } + + func cancel() { + controller?.cancel() + finish(Self.notAllowed) + } + + // Passkeys on the device and security keys are offered together, as Safari does, + // unless the page asked for one kind of authenticator. + private func makeRequests() -> [ASAuthorizationRequest]? { + let json = Data(options.utf8) + if kind == "create" { + guard + let options = try? JSONDecoder().decode(CreationOptions.self, from: json), + let challenge = Data(base64URL: options.challenge), + let userID = Data(base64URL: options.user.id) + else { return nil } + let clientData = ASPublicKeyCredentialClientData(challenge: challenge, origin: origin) + let selection = options.authenticatorSelection + let verification = ASAuthorizationPublicKeyCredentialUserVerificationPreference( + rawValue: selection?.userVerification ?? "preferred") + let attestation = ASAuthorizationPublicKeyCredentialAttestationKind(rawValue: options.attestation ?? "none") + let excluded = (options.excludeCredentials ?? []).compactMap { descriptor in + Data(base64URL: descriptor.id).map { (descriptor, $0) } + } + var requests: [ASAuthorizationRequest] = [] + if selection?.authenticatorAttachment != "cross-platform" { + let request = ASAuthorizationPlatformPublicKeyCredentialProvider(relyingPartyIdentifier: options.rp.id) + .createCredentialRegistrationRequest(clientData: clientData, name: options.user.name, userID: userID) + request.displayName = options.user.displayName + request.userVerificationPreference = verification + request.attestationPreference = attestation + request.excludedCredentials = excluded.map { + ASAuthorizationPlatformPublicKeyCredentialDescriptor(credentialID: $0.1) + } + requests.append(request) + } + if selection?.authenticatorAttachment != "platform" { + let request = ASAuthorizationSecurityKeyPublicKeyCredentialProvider(relyingPartyIdentifier: options.rp.id) + .createCredentialRegistrationRequest( + clientData: clientData, + displayName: options.user.displayName ?? options.user.name, + name: options.user.name, + userID: userID + ) + request.credentialParameters = [ASAuthorizationPublicKeyCredentialParameters(algorithm: .ES256)] + request.excludedCredentials = excluded.map { securityKey($0.0, id: $0.1) } + request.residentKeyPreference = ASAuthorizationPublicKeyCredentialResidentKeyPreference( + rawValue: selection?.residentKey ?? (selection?.requireResidentKey == true ? "required" : "discouraged")) + request.userVerificationPreference = verification + request.attestationPreference = attestation + requests.append(request) + } + return requests + } + guard + let options = try? JSONDecoder().decode(RequestOptions.self, from: json), + let challenge = Data(base64URL: options.challenge) + else { return nil } + let clientData = ASPublicKeyCredentialClientData(challenge: challenge, origin: origin) + let verification = ASAuthorizationPublicKeyCredentialUserVerificationPreference( + rawValue: options.userVerification ?? "preferred") + let allowed = (options.allowCredentials ?? []).compactMap { descriptor in + Data(base64URL: descriptor.id).map { (descriptor, $0) } + } + let platform = ASAuthorizationPlatformPublicKeyCredentialProvider(relyingPartyIdentifier: options.rpId) + .createCredentialAssertionRequest(clientData: clientData) + platform.allowedCredentials = allowed.map { ASAuthorizationPlatformPublicKeyCredentialDescriptor(credentialID: $0.1) } + platform.userVerificationPreference = verification + let key = ASAuthorizationSecurityKeyPublicKeyCredentialProvider(relyingPartyIdentifier: options.rpId) + .createCredentialAssertionRequest(clientData: clientData) + key.allowedCredentials = allowed.map { securityKey($0.0, id: $0.1) } + key.userVerificationPreference = verification + return [platform, key] + } + + private func securityKey(_ descriptor: Descriptor, id: Data) -> ASAuthorizationSecurityKeyPublicKeyCredentialDescriptor { + let transports = (descriptor.transports ?? []).compactMap { Self.transports[$0] } + return ASAuthorizationSecurityKeyPublicKeyCredentialDescriptor( + credentialID: id, + transports: transports.isEmpty ? ASAuthorizationSecurityKeyPublicKeyCredentialDescriptor.Transport.allSupported : transports + ) + } + + func authorizationController(controller: ASAuthorizationController, didCompleteWithAuthorization authorization: ASAuthorization) { + switch authorization.credential { + case let registration as ASAuthorizationPlatformPublicKeyCredentialRegistration: + guard let attestation = registration.rawAttestationObject else { return finish(Self.notAllowed) } + finish(succeeded([ + "id": registration.credentialID.base64URL, + "clientDataJSON": registration.rawClientDataJSON.base64URL, + "attestationObject": attestation.base64URL, + "authenticatorAttachment": Self.attachment(registration.attachment), + "transports": ["hybrid", "internal"], + ])) + case let registration as ASAuthorizationSecurityKeyPublicKeyCredentialRegistration: + guard let attestation = registration.rawAttestationObject else { return finish(Self.notAllowed) } + finish(succeeded([ + "id": registration.credentialID.base64URL, + "clientDataJSON": registration.rawClientDataJSON.base64URL, + "attestationObject": attestation.base64URL, + "authenticatorAttachment": "cross-platform", + "transports": registration.transports.map(\.rawValue), + ])) + case let assertion as ASAuthorizationPlatformPublicKeyCredentialAssertion: + finish(succeeded([ + "id": assertion.credentialID.base64URL, + "clientDataJSON": assertion.rawClientDataJSON.base64URL, + "authenticatorData": assertion.rawAuthenticatorData.base64URL, + "signature": assertion.signature.base64URL, + "userHandle": assertion.userID.base64URL, + "authenticatorAttachment": Self.attachment(assertion.attachment), + ])) + case let assertion as ASAuthorizationSecurityKeyPublicKeyCredentialAssertion: + finish(succeeded([ + "id": assertion.credentialID.base64URL, + "clientDataJSON": assertion.rawClientDataJSON.base64URL, + "authenticatorData": assertion.rawAuthenticatorData.base64URL, + "signature": assertion.signature.base64URL, + "userHandle": assertion.userID.base64URL, + "authenticatorAttachment": "cross-platform", + ])) + default: + finish(Self.notAllowed) + } + } + + func authorizationController(controller: ASAuthorizationController, didCompleteWithError error: Error) { + // WebAuthn reports a passkey this site already has as InvalidStateError; anything else stays opaque. + let excluded = (error as? ASAuthorizationError)?.code == .matchedExcludedCredential + finish(excluded ? ["success": false, "error": "InvalidStateError"] : Self.notAllowed) + } + + func presentationAnchor(for controller: ASAuthorizationController) -> ASPresentationAnchor { + UIApplication.shared.connectedScenes + .compactMap { $0 as? UIWindowScene } + .flatMap(\.windows) + .first(where: \.isKeyWindow) ?? ASPresentationAnchor() + } + + private func succeeded(_ credential: [String: Any]) -> [String: Any] { + ["success": true, "credential": credential] + } + + private static func attachment(_ attachment: ASAuthorizationPublicKeyCredentialAttachment) -> String { + attachment == .crossPlatform ? "cross-platform" : "platform" + } + + private func finish(_ result: [String: Any]) { + guard let completion else { return } + self.completion = nil + controller = nil + let data = (try? JSONSerialization.data(withJSONObject: result)) + ?? Data(#"{"success":false,"error":"NotAllowedError"}"#.utf8) + completion(String(decoding: data, as: UTF8.self)) + } +} + +private struct Descriptor: Decodable { + let id: String + let transports: [String]? +} + +/// The members of `PublicKeyCredentialCreationOptionsJSON` the system sheet uses. +private struct CreationOptions: Decodable { + struct RelyingParty: Decodable { + let id: String + } + + struct User: Decodable { + let id: String + let name: String + let displayName: String? + } + + struct Selection: Decodable { + let authenticatorAttachment: String? + let residentKey: String? + let requireResidentKey: Bool? + let userVerification: String? + } + + let rp: RelyingParty + let user: User + let challenge: String + let excludeCredentials: [Descriptor]? + let authenticatorSelection: Selection? + let attestation: String? +} + +/// The members of `PublicKeyCredentialRequestOptionsJSON` the system sheet uses. +private struct RequestOptions: Decodable { + let rpId: String + let challenge: String + let allowCredentials: [Descriptor]? + let userVerification: String? +} + +private extension Data { + init?(base64URL: String) { + var base64 = base64URL.replacingOccurrences(of: "-", with: "+").replacingOccurrences(of: "_", with: "/") + base64 += String(repeating: "=", count: (4 - base64.count % 4) % 4) + self.init(base64Encoded: base64) + } + + var base64URL: String { + base64EncodedString() + .replacingOccurrences(of: "+", with: "-") + .replacingOccurrences(of: "/", with: "_") + .replacingOccurrences(of: "=", with: "") + } +} diff --git a/apps/mobile/src/features/browser/PreviewStreamWebView.tsx b/apps/mobile/src/features/browser/PreviewStreamWebView.tsx index 6734bf981912..7e8c0ccc333a 100644 --- a/apps/mobile/src/features/browser/PreviewStreamWebView.tsx +++ b/apps/mobile/src/features/browser/PreviewStreamWebView.tsx @@ -28,6 +28,11 @@ import { copyTextWithHaptic } from "../../lib/copyTextWithHaptic"; import { beginForegroundHandoff } from "../../lib/foreground-handoff"; import { usePreviewStreamAccess } from "../../state/preview"; +import { + browserPasskeysAvailable, + cancelBrowserPasskey, + performBrowserPasskey, +} from "./browserPasskeys"; import { previewStreamDocument, previewStreamMessage, @@ -117,7 +122,7 @@ function offerDownload(download: PreviewStreamDownload) { const MAX_REFUSALS = 3; export function PreviewStreamWebView( - props: Omit & + props: Omit & Omit & { readonly environmentId: EnvironmentId; readonly paused?: boolean; @@ -164,7 +169,7 @@ export function PreviewStreamWebView( function AuthorizedPreviewStream({ ref, ...props -}: PreviewStreamConfiguration & NativeStreamBridge) { +}: Omit & NativeStreamBridge) { const [attempt, setAttempt] = useState(0); const [previousAccess, setPreviousAccess] = useState(props.access); // Wait for refreshed access, including cookie credentials with unchanged JSON. @@ -187,6 +192,7 @@ function AuthorizedPreviewStream({ tabId: props.tabId, interactive: props.interactive, background: props.background, + passkeys: props.interactive && browserPasskeysAvailable, } satisfies PreviewStreamConfiguration); return ( + command({ type: "passkeyResult", id: request.id, result }), + ); + return; + } + case "passkeyCancel": + cancelBrowserPasskey(message.id); + return; case "pictureInPicture": onPictureInPicture?.(message, message.detail); return; diff --git a/apps/mobile/src/features/browser/browserPasskeys.ts b/apps/mobile/src/features/browser/browserPasskeys.ts new file mode 100644 index 000000000000..897a8600e27a --- /dev/null +++ b/apps/mobile/src/features/browser/browserPasskeys.ts @@ -0,0 +1,50 @@ +import type { + PreviewStreamPasskeyRequest, + PreviewStreamPasskeyResult, +} from "@t3tools/client-runtime/preview/server-browser-stream"; +import { requireOptionalNativeModule } from "expo"; +import { Platform } from "react-native"; + +interface T3Passkeys { + /** Set only in builds signed with Apple's browser entitlement; see app.config.ts. */ + readonly available: boolean; + /** Answers with a `PreviewStreamPasskeyResult` in JSON. */ + readonly perform: ( + id: string, + kind: "create" | "get", + origin: string, + options: string, + ) => Promise; + readonly cancel: (id: string) => Promise; +} + +const native = Platform.OS === "ios" ? requireOptionalNativeModule("T3Passkeys") : null; + +/** This device answers server browser pages' passkey requests with its own passkeys. */ +export const browserPasskeysAvailable = native?.available === true; + +const NOT_ALLOWED: PreviewStreamPasskeyResult = { success: false, error: "NotAllowedError" }; + +/** Runs a page's passkey request through the system passkey sheet. */ +export async function performBrowserPasskey( + request: PreviewStreamPasskeyRequest, +): Promise { + if (!native?.available) return NOT_ALLOWED; + try { + const result = await native.perform( + request.id, + request.kind, + request.origin, + JSON.stringify(request.publicKey), + ); + // The server checks the credential before the page sees it. + return JSON.parse(result) as PreviewStreamPasskeyResult; + } catch { + return NOT_ALLOWED; + } +} + +/** Closes the sheet for a request the page gave up on. */ +export function cancelBrowserPasskey(id: string) { + void native?.cancel(id).catch(() => undefined); +} diff --git a/apps/mobile/src/features/browser/preview-stream-document.ts b/apps/mobile/src/features/browser/preview-stream-document.ts index c6af70045738..1e7a91322422 100644 --- a/apps/mobile/src/features/browser/preview-stream-document.ts +++ b/apps/mobile/src/features/browser/preview-stream-document.ts @@ -4,6 +4,7 @@ import type { PreviewStreamControl, PreviewStreamDownload, PreviewStreamFileChooser, + PreviewStreamPasskeyRequest, } from "@t3tools/client-runtime/preview/server-browser-stream"; export interface PreviewStreamConfiguration { @@ -13,6 +14,8 @@ export interface PreviewStreamConfiguration { /** Taps, scrolls, keys, and `resize` to the view size. The floating player only watches. */ readonly interactive: boolean; readonly background: string; + /** This device answers the page's passkey requests; offered to the server with control. */ + readonly passkeys: boolean; } /** Messages the WebView document posts to the native view. */ @@ -32,6 +35,8 @@ export type PreviewStreamMessage = | { readonly type: "input"; readonly focused: boolean } | ({ readonly type: "download" } & PreviewStreamDownload) | { readonly type: "fileChooser"; readonly chooser: PreviewStreamFileChooser | null } + | { readonly type: "passkey"; readonly request: PreviewStreamPasskeyRequest } + | { readonly type: "passkeyCancel"; readonly id: string } | { readonly type: "pictureInPicture"; readonly supported: boolean; diff --git a/apps/mobile/src/features/browser/preview-stream.browser.ts b/apps/mobile/src/features/browser/preview-stream.browser.ts index b8f5532e7d80..aff2fbef3230 100644 --- a/apps/mobile/src/features/browser/preview-stream.browser.ts +++ b/apps/mobile/src/features/browser/preview-stream.browser.ts @@ -179,6 +179,7 @@ export function start(configuration: PreviewStreamConfiguration) { maxWidth: cap.width, maxHeight: cap.height, interactive, + passkeys: configuration.passkeys, }, { onFrame: (jpeg) => painter.paint(jpeg), @@ -186,6 +187,8 @@ export function start(configuration: PreviewStreamConfiguration) { onClipboard: (text) => post({ type: "clipboard", text }), onDownload: (download) => post({ type: "download", ...download }), onFileChooser: (chooser) => post({ type: "fileChooser", chooser }), + onPasskey: (request) => post({ type: "passkey", request }), + onPasskeyCancel: (id) => post({ type: "passkeyCancel", id }), onViewport: (page) => { if (viewport?.width === page.width && viewport.height === page.height) return; viewport = page; diff --git a/apps/server/package.json b/apps/server/package.json index 3dd02fcdcb5e..e57a4e086a8d 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -58,6 +58,7 @@ "proper-lockfile": "4.1.2", "stream-chain": "^4.2.5", "stream-json": "3.6.0", + "tldts": "catalog:", "yaml": "catalog:", "yauzl": "^3.4.0" }, diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 3b68d905fbd2..99f120717183 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -20,7 +20,7 @@ import * as PubSub from "effect/PubSub"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; import type { BrowserContext, Page } from "playwright-core"; -import { beforeEach, expect, vi } from "vite-plus/test"; +import { assert, beforeEach, expect, vi } from "vite-plus/test"; import * as ServerConfig from "../config.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; @@ -118,6 +118,7 @@ function makeContext(onClose?: (context: BrowserContext) => void) { throw new Error("Unexpected locator action"); }), isClosed: () => closed, + isDetached: () => closed, close: vi.fn(async () => { if (!closed) { closed = true; @@ -130,8 +131,9 @@ function makeContext(onClose?: (context: BrowserContext) => void) { sessions, newPage: async () => page as unknown as Page, grantPermissions: vi.fn(async () => {}), - exposeBinding: vi.fn(async (_name: string, binding: ClipboardBinding) => { - clipboardBinding = binding; + exposeBinding: vi.fn(async (name: string, binding: unknown) => { + if (name === "__t3PreviewClipboard") clipboardBinding = binding as ClipboardBinding; + if (name === "__t3PreviewPasskey") passkeyBinding = binding as PasskeyBinding; }), addInitScript: vi.fn(async () => {}), newCDPSession: async () => { @@ -155,6 +157,13 @@ const contextRequests: Array<{ profileId: string; isolated: boolean }> = []; let contextGate: PromiseWithResolvers | null = null; type ClipboardBinding = (source: { page: unknown }, text: unknown) => void; let clipboardBinding: ClipboardBinding | null = null; +type PasskeyBinding = ( + source: { page: unknown; frame: unknown }, + kind: unknown, + ticket?: unknown, + options?: unknown, +) => Promise; +let passkeyBinding: PasskeyBinding | null = null; let contextFailure: Error | null = null; /** Server tabs the fake desktop renders, and the endpoints the server connected to. */ let desktopRendersNext = false; @@ -258,10 +267,11 @@ const stallViewer = ( const output = viewer.output as unknown as Queue.Queue; while (Queue.offerUnsafe(output, item)); }; -const viewerInput = (tabId: string, canOperate: boolean) => ({ +const viewerInput = (tabId: string, canOperate: boolean, passkeys = false) => ({ threadId: scope.thread.threadId, tabId, canOperate, + passkeys, maxWidth: 1280, maxHeight: 800, quality: 70, @@ -1120,6 +1130,142 @@ it.live.each([ ).pipe(Effect.provide(layer)), ); +/** SHA-256 of `example.com`, which authenticator data starts with. */ +const EXAMPLE_RP_ID_HASH = "a379a6f6eeafb9a55e378c118034e2751e682fab9f2d30ab13d2125586ce1947"; +/** What the phone's authenticator would sign for a sign-in on example.com. */ +const exampleAssertion = (challenge: string, origin = "https://example.com") => ({ + success: true, + credential: { + id: "Y3JlZA", + clientDataJSON: Buffer.from( + JSON.stringify({ type: "webauthn.get", challenge, origin, crossOrigin: false }), + ).toString("base64url"), + authenticatorData: Buffer.from(`${EXAMPLE_RP_ID_HASH}0500000001`, "hex").toString("base64url"), + signature: "c2ln", + userHandle: "dXNlcg", + authenticatorAttachment: "platform", + }, +}); + +/** Opens example.com with a viewer in control, then touches the page so it may ask. */ +const passkeyViewer = (passkeys: boolean) => + Effect.gen(function* () { + const { browser, tabId } = yield* ready; + const page = contexts[0]!.page; + yield* Effect.promise(() => page.goto("https://example.com/login")); + const viewer = yield* browser.attachViewer(viewerInput(tabId, true, passkeys)); + yield* viewer.input({ type: "takeControl" }); + yield* viewer.input({ type: "mouse", action: "move", x: 1, y: 1, button: "none", buttons: 0 }); + return { page, viewer }; + }); + +const nextPasskeyOutput = (viewer: ServerBrowser.ServerBrowserViewer) => + Effect.gen(function* () { + let output = yield* Queue.take(viewer.output); + while (output._tag !== "passkey" && output._tag !== "passkeyCancel") { + output = yield* Queue.take(viewer.output); + } + return output; + }); + +it.live("a page's passkey request goes to the controlling viewer's device and back", () => + Effect.scoped( + Effect.gen(function* () { + const { page, viewer } = yield* passkeyViewer(true); + expect(yield* Effect.promise(() => passkeyBinding!({ page, frame: page }, "available"))).toBe( + true, + ); + const answer = passkeyBinding!({ page, frame: page }, "get", 1, { + challenge: "Y2hhbGxlbmdl", + userVerification: "required", + }); + const request = yield* nextPasskeyOutput(viewer); + assert(request._tag === "passkey"); + expect(request).toMatchObject({ + kind: "get", + origin: "https://example.com", + publicKey: { rpId: "example.com", challenge: "Y2hhbGxlbmdl", userVerification: "required" }, + }); + yield* viewer.input({ + type: "passkeyResult", + id: request.id, + result: exampleAssertion("Y2hhbGxlbmdl"), + }); + expect(yield* Effect.promise(() => answer)).toMatchObject({ + credential: { + id: "Y3JlZA", + rawId: "Y3JlZA", + type: "public-key", + authenticatorAttachment: "platform", + response: { signature: "c2ln", userHandle: "dXNlcg" }, + }, + }); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("without a passkey device in control the page keeps Chromium's own WebAuthn", () => + Effect.scoped( + Effect.gen(function* () { + const { page } = yield* passkeyViewer(false); + expect(yield* Effect.promise(() => passkeyBinding!({ page, frame: page }, "available"))).toBe( + false, + ); + expect( + yield* Effect.promise(() => + passkeyBinding!({ page, frame: page }, "get", 1, { challenge: "Y2hhbGxlbmdl" }), + ), + ).toEqual({ native: true }); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("only a secure top-level page asks, and the device's answer must be for it", () => + Effect.scoped( + Effect.gen(function* () { + const { page, viewer } = yield* passkeyViewer(true); + const ask = (frame: unknown, options: unknown) => + Effect.promise(() => passkeyBinding!({ page, frame }, "get", 1, options)); + const notAllowed = { error: "NotAllowedError" }; + // A frame inside the page cannot open the sheet. + expect(yield* ask({}, { challenge: "Y2hhbGxlbmdl" })).toEqual(notAllowed); + // Another site's RP ID is refused before the device is asked. + expect(yield* ask(page, { challenge: "Y2hhbGxlbmdl", rpId: "example.net" })).toEqual({ + error: "SecurityError", + }); + // An answer signed for another origin never reaches the page. + const fiber = yield* Effect.forkScoped(ask(page, { challenge: "Y2hhbGxlbmdl" })); + const request = yield* nextPasskeyOutput(viewer); + assert(request._tag === "passkey"); + yield* viewer.input({ + type: "passkeyResult", + id: request.id, + result: exampleAssertion("Y2hhbGxlbmdl", "https://example.net"), + }); + expect(yield* Fiber.join(fiber)).toEqual(notAllowed); + // Plain http is not a secure context. + yield* Effect.promise(() => page.goto("http://example.com/login")); + expect(yield* ask(page, { challenge: "Y2hhbGxlbmdl" })).toEqual(notAllowed); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("releasing control cancels the request on the device and refuses the page", () => + Effect.scoped( + Effect.gen(function* () { + const { page, viewer } = yield* passkeyViewer(true); + const answer = passkeyBinding!({ page, frame: page }, "get", 1, { + challenge: "Y2hhbGxlbmdl", + }); + const request = yield* nextPasskeyOutput(viewer); + assert(request._tag === "passkey"); + yield* viewer.input({ type: "releaseControl" }); + expect(yield* nextPasskeyOutput(viewer)).toEqual({ _tag: "passkeyCancel", id: request.id }); + expect(yield* Effect.promise(() => answer)).toEqual({ error: "NotAllowedError" }); + }), + ).pipe(Effect.provide(layer)), +); + it.live("a page download is saved, offered to the controller, and listed for the agent", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index c26b7d65ac96..8e290f83490a 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -66,6 +66,7 @@ import type { Dialog, Download, FileChooser, + Frame, Page, } from "playwright-core"; @@ -77,6 +78,7 @@ import * as PreviewAutomationBroker from "../mcp/PreviewAutomationBroker.ts"; import * as DesktopBrowserChannel from "./DesktopBrowserChannel.ts"; import * as PreviewManager from "./Manager.ts"; import * as ServerBrowserPage from "./ServerBrowserPage.ts"; +import * as ServerBrowserPasskeys from "./ServerBrowserPasskeys.ts"; import * as PreviewBrowser from "./PreviewBrowser.ts"; import * as PreviewBrowserHost from "./PreviewBrowserHost.ts"; import { presentAsChrome, ServerBrowserContexts } from "./ServerBrowserContexts.ts"; @@ -217,6 +219,15 @@ export type ServerBrowserViewerOutput = readonly accept: string; } | { readonly _tag: "fileChooserClosed"; readonly id: string } + /** The page asked for a passkey; only a controlling viewer that offered passkeys gets it. */ + | { + readonly _tag: "passkey"; + readonly id: string; + readonly kind: "create" | "get"; + readonly origin: string; + readonly publicKey: Readonly>; + } + | { readonly _tag: "passkeyCancel"; readonly id: string } /** The page this viewer controls opened a new tab; the viewer shows it, as a browser would. */ | { readonly _tag: "popup"; readonly tabId: string } | { @@ -244,6 +255,8 @@ export class ServerBrowser extends Context.Service< readonly maxHeight: number; readonly quality: number; readonly canOperate: boolean; + /** The viewer's device answers the page's passkey requests while it has control. */ + readonly passkeys?: boolean; }) => Effect.Effect< ServerBrowserViewer, ServerBrowserTabNotFoundError | ServerBrowserLaunchError, @@ -290,6 +303,8 @@ interface ViewerState { scrolledAt: number; /** Last input from this viewer; page copies reach its clipboard only right after. */ inputAt: number; + /** This viewer's device answers the page's passkey requests while it has control. */ + readonly passkeys: boolean; /** Panel bounds, retained in fixed mode; passive viewers never request a size. */ requestedSize: { width: number; height: number; order: number } | null; } @@ -343,6 +358,15 @@ interface ServerTab { readonly accept: string; } | null; dialog: Dialog | null; + /** A page's passkey request, waiting on the controlling viewer's device. */ + passkey: { + readonly id: string; + readonly ticket: unknown; + readonly frame: Frame; + readonly viewerId: string; + readonly request: ServerBrowserPasskeys.PasskeyRequest; + readonly settle: (answer: ServerBrowserPasskeys.PasskeyAnswer) => void; + } | null; setting: PreviewViewportSetting; colorScheme: PreviewAppearancePreference; zoomFactor: number; @@ -550,7 +574,19 @@ const make = Effect.gen(function* () { } : null; + /** Ends the tab's passkey request, closing its viewer's sheet. The page gets NotAllowedError. */ + const cancelPasskey = (tab: ServerTab) => { + const pending = tab.passkey; + if (!pending) return; + tab.passkey = null; + for (const viewer of tab.viewers) + if (viewer.id === pending.viewerId) viewer.push({ _tag: "passkeyCancel", id: pending.id }); + pending.settle({ error: "NotAllowedError" }); + }; + const broadcastControl = (tab: ServerTab) => { + // A request belongs to the viewer that had control when the page asked. + if (tab.passkey && tab.passkey.viewerId !== tab.control.controller) cancelPasskey(tab); for (const viewer of tab.viewers) viewer.push({ _tag: "control", @@ -711,6 +747,7 @@ const make = Effect.gen(function* () { tabs.delete(key); tab.closing = true; clearAbortedNavigation(tab); + cancelPasskey(tab); // A desktop page outlives the connection unless its session closed with it. const end = tab.desktop && !closeSession && sessionOpen(tab) ? "reconnect" : "gone"; for (const viewer of tab.viewers) viewer.push({ _tag: end }); @@ -816,6 +853,7 @@ const make = Effect.gen(function* () { downloads: [], fileChooser: null, dialog: null, + passkey: null, setting: snapshot.viewport ?? FILL_PREVIEW_VIEWPORT, colorScheme: "system", zoomFactor: 1, @@ -1126,6 +1164,71 @@ const make = Effect.gen(function* () { : Option.some({ path: download.path, fileName: download.fileName }); }); + /** + * Serves a page's passkey request with the controlling viewer's device. With + * no such viewer, the page keeps Chromium's own WebAuthn. + */ + const answerPasskey = async ( + page: Page, + frame: Frame, + kind: unknown, + ticket: unknown, + options: unknown, + ): Promise => { + const tab = [...tabs.values()].find((candidate) => candidate.page === page); + if (kind === "abort") { + const pending = tab?.passkey; + if (tab && pending && pending.ticket === ticket && pending.frame === frame) { + cancelPasskey(tab); + } + return null; + } + // Pages often check once, on load, so a device watching the tab counts before it takes control. + if (kind === "available") { + return tab !== undefined && [...tab.viewers].some((viewer) => viewer.passkeys); + } + const controller = tab + ? [...tab.viewers].find((viewer) => viewer.id === tab.control.controller && viewer.passkeys) + : undefined; + if (!tab || !controller) return { native: true }; + if (kind !== "create" && kind !== "get") return { error: "TypeError" }; + // Chromium names the origin, never the page. Like Chromium's own sheet, only a + // top-level page asks, one request at a time, right after the person touched it. + const origin = + frame === page.mainFrame() ? ServerBrowserPasskeys.passkeyOrigin(frame.url()) : null; + if ( + origin === null || + tab.passkey !== null || + Date.now() - controller.inputAt > ServerBrowserPasskeys.PASSKEY_GESTURE_MS + ) { + return { error: "NotAllowedError" }; + } + const request = ServerBrowserPasskeys.passkeyRequest(kind, origin, options); + if ("error" in request) return request; + const id = NodeCrypto.randomUUID(); + const answer = await new Promise((resolve) => { + const timer = setTimeout(() => cancelPasskey(tab), request.timeoutMs); + tab.passkey = { + id, + ticket, + frame, + viewerId: controller.id, + request, + settle: (next) => { + clearTimeout(timer); + if (tab.passkey?.id === id) tab.passkey = null; + resolve(next); + }, + }; + controller.push({ _tag: "passkey", id, kind, origin, publicKey: request.publicKey }); + }); + // A credential for a page that navigated away belongs to nobody. + return "credential" in answer && + (frame.isDetached() || ServerBrowserPasskeys.passkeyOrigin(frame.url()) !== origin) + ? { error: "NotAllowedError" } + : answer; + }; + const preparedContexts = new WeakSet(); const prepareContext = async (context: BrowserContext) => { if (preparedContexts.has(context)) return; @@ -1141,6 +1244,12 @@ const make = Effect.gen(function* () { controller.push({ _tag: "clipboard", text: text.slice(0, CLIPBOARD_TEXT_LIMIT) }); }); await context.addInitScript(CLIPBOARD_SCRIPT); + await context.exposeBinding( + ServerBrowserPasskeys.PASSKEY_BINDING, + ({ page, frame }, kind: unknown, ticket: unknown, options: unknown) => + answerPasskey(page, frame, kind, ticket, options), + ); + await context.addInitScript(ServerBrowserPasskeys.PASSKEY_SCRIPT); }; const adoptPopup = async (opener: ServerTab, popup: Page) => { @@ -2271,13 +2380,24 @@ const make = Effect.gen(function* () { if (next._tag === "frame") runFork(next.ack); // State a stalled viewer cannot miss replaces its backlog. It runs // synchronously so an older replacement can never land after a newer one. - else if (next._tag === "gone" || next._tag === "control" || next._tag === "fileChooser") { + else if ( + next._tag === "gone" || + next._tag === "control" || + next._tag === "fileChooser" || + next._tag === "passkey" || + next._tag === "passkeyCancel" + ) { const dropped = Effect.runSyncExit(Queue.clear(output)); if (dropped._tag === "Failure") return; Queue.offerUnsafe(output, next); // The controller's open picker may have been in the dropped backlog. const chooser = next._tag === "control" ? fileChooserMessage(tab) : null; if (chooser && tab.control.controller === viewer.id) Queue.offerUnsafe(output, chooser); + // So may a passkey request the page still waits on; sent twice, it would open twice. + const request = dropped.value.find((item) => item._tag === "passkey"); + if (request && next._tag !== "passkeyCancel" && tab.passkey?.id === request.id) { + Queue.offerUnsafe(output, request); + } for (const item of dropped.value) if (item._tag === "frame") runFork(item.ack); } }, @@ -2290,6 +2410,7 @@ const make = Effect.gen(function* () { resume: () => startScreencast(screencastScale), scrolledAt: 0, inputAt: 0, + passkeys: input.canOperate && input.passkeys === true, requestedSize: null, }; yield* Effect.acquireRelease( @@ -2409,6 +2530,13 @@ const make = Effect.gen(function* () { ? { promptText: message.promptText } : {}), }); + } else if (message.type === "passkeyResult") { + const pending = tab.passkey; + if (pending && pending.id === message.id && pending.viewerId === viewer.id) { + pending.settle( + ServerBrowserPasskeys.passkeyAnswer(pending.request, message.result), + ); + } } else { await tab.control.human(viewer.id, () => dispatchViewerInput(tab, session, viewer, message), diff --git a/apps/server/src/preview/ServerBrowserPasskeys.test.ts b/apps/server/src/preview/ServerBrowserPasskeys.test.ts new file mode 100644 index 000000000000..778a917a1663 --- /dev/null +++ b/apps/server/src/preview/ServerBrowserPasskeys.test.ts @@ -0,0 +1,271 @@ +// @effect-diagnostics nodeBuiltinImport:off - Builds real P-256 keys and hashes, as an authenticator would. +import * as NodeCrypto from "node:crypto"; + +import { describe, expect, it } from "vite-plus/test"; + +import { + PASSKEY_SCRIPT, + passkeyAnswer, + passkeyOrigin, + passkeyRequest, + relyingPartyIdError, + type PasskeyRequest, +} from "./ServerBrowserPasskeys.ts"; + +const base64Url = (bytes: Uint8Array | string) => Buffer.from(bytes).toString("base64url"); + +/** Encodes the CBOR subset attestation objects use. */ +const cbor = (value: unknown): Buffer => { + const head = (major: number, length: number) => + length < 24 + ? Buffer.from([(major << 5) | length]) + : length < 256 + ? Buffer.from([(major << 5) | 24, length]) + : Buffer.from([(major << 5) | 25, length >> 8, length & 255]); + if (typeof value === "number") return value >= 0 ? head(0, value) : head(1, -1 - value); + if (typeof value === "string") { + const bytes = Buffer.from(value); + return Buffer.concat([head(3, bytes.length), bytes]); + } + if (value instanceof Uint8Array) return Buffer.concat([head(2, value.length), value]); + if (value instanceof Map) { + return Buffer.concat([ + head(5, value.size), + ...[...value].flatMap(([key, member]) => [cbor(key), cbor(member)]), + ]); + } + throw new Error("Unsupported CBOR value"); +}; + +const rpIdHash = (rpId: string) => NodeCrypto.createHash("sha256").update(rpId).digest(); +const clientData = (type: string, challenge: string, origin = "https://example.com") => + base64Url(JSON.stringify({ type, challenge, origin, crossOrigin: false })); + +const request = (kind: "create" | "get", options: Record) => { + const result = passkeyRequest(kind, "https://example.com", options); + if ("error" in result) throw new Error(result.error); + return result; +}; + +const creation = { + challenge: "Y2hhbGxlbmdl", + rp: { name: "Example" }, + user: { id: base64Url("user-1"), name: "alice", displayName: "Alice" }, + pubKeyCredParams: [ + { type: "public-key", alg: -257 }, + { type: "public-key", alg: -7 }, + ], +}; + +describe("passkeyOrigin", () => { + it("allows https and loopback http only", () => { + expect(passkeyOrigin("https://example.com/login?next=1")).toBe("https://example.com"); + expect(passkeyOrigin("http://localhost:5173/")).toBe("http://localhost:5173"); + expect(passkeyOrigin("http://127.0.0.1:3000/a")).toBe("http://127.0.0.1:3000"); + expect(passkeyOrigin("http://example.com/")).toBeNull(); + expect(passkeyOrigin("about:blank")).toBeNull(); + expect(passkeyOrigin("not a url")).toBeNull(); + }); +}); + +describe("relyingPartyIdError", () => { + it("accepts the host and its registrable parents", () => { + expect(relyingPartyIdError("https://example.com", "example.com")).toBeNull(); + expect(relyingPartyIdError("https://login.example.com", "example.com")).toBeNull(); + expect(relyingPartyIdError("http://localhost:5173", "localhost")).toBeNull(); + }); + + it("refuses public suffixes, other sites, and lookalikes", () => { + expect(relyingPartyIdError("https://alice.github.io", "github.io")).toBe("SecurityError"); + expect(relyingPartyIdError("https://example.com", "com")).toBe("SecurityError"); + expect(relyingPartyIdError("https://example.com", "example.net")).toBe("SecurityError"); + expect(relyingPartyIdError("https://notexample.com", "example.com")).toBe("SecurityError"); + expect(relyingPartyIdError("https://example.com", "login.example.com")).toBe("SecurityError"); + }); +}); + +describe("passkeyRequest", () => { + it("fills in a sign-in's RP ID and clamps its timeout", () => { + const signIn = request("get", { + challenge: "Y2hhbGxlbmdl", + timeout: 1, + allowCredentials: [ + { type: "public-key", id: "Y3JlZA" }, + { type: "other", id: "eA" }, + ], + }); + expect(signIn).toMatchObject({ + rpId: "example.com", + publicKey: { rpId: "example.com" }, + timeoutMs: 10_000, + allowCredentials: ["Y3JlZA"], + }); + }); + + it("refuses options a browser would", () => { + expect(passkeyRequest("get", "https://example.com", { challenge: 5 })).toEqual({ + error: "TypeError", + }); + expect( + passkeyRequest("get", "https://example.com", { challenge: "eA", rpId: "example.net" }), + ).toEqual({ error: "SecurityError" }); + expect( + passkeyRequest("create", "https://example.com", { + ...creation, + user: { ...creation.user, id: "" }, + }), + ).toEqual({ error: "TypeError" }); + expect( + passkeyRequest("create", "https://example.com", { + ...creation, + pubKeyCredParams: [{ type: "public-key", alg: -257 }], + }), + ).toEqual({ error: "NotSupportedError" }); + }); + + it("asks the device for a P-256 key for the page's site", () => { + expect(request("create", creation)).toMatchObject({ + rpId: "example.com", + publicKey: { + rp: { id: "example.com", name: "Example" }, + pubKeyCredParams: [{ type: "public-key", alg: -7 }], + }, + }); + }); +}); + +describe("passkeyAnswer", () => { + const signIn = request("get", { challenge: "Y2hhbGxlbmdl", allowCredentials: [] }); + const assertion = (overrides: Record = {}) => ({ + success: true, + credential: { + id: "Y3JlZA", + clientDataJSON: clientData("webauthn.get", "Y2hhbGxlbmdl"), + authenticatorData: base64Url( + Buffer.concat([rpIdHash("example.com"), Buffer.from([0x05, 0, 0, 0, 1])]), + ), + signature: "c2ln", + userHandle: "dXNlcg", + authenticatorAttachment: "platform", + ...overrides, + }, + }); + + it("hands the page a sign-in signed for this ceremony", () => { + expect(passkeyAnswer(signIn, assertion())).toEqual({ + credential: { + id: "Y3JlZA", + rawId: "Y3JlZA", + type: "public-key", + authenticatorAttachment: "platform", + response: { + clientDataJSON: clientData("webauthn.get", "Y2hhbGxlbmdl"), + authenticatorData: assertion().credential.authenticatorData, + signature: "c2ln", + userHandle: "dXNlcg", + }, + clientExtensionResults: {}, + }, + }); + }); + + it("drops a credential signed for another page, ceremony, or site", () => { + const notAllowed = { error: "NotAllowedError" }; + const wrong = [ + { clientDataJSON: clientData("webauthn.get", "Y2hhbGxlbmdl", "https://example.net") }, + { clientDataJSON: clientData("webauthn.get", "b3RoZXI") }, + { clientDataJSON: clientData("webauthn.create", "Y2hhbGxlbmdl") }, + { authenticatorData: base64Url(Buffer.concat([rpIdHash("example.net"), Buffer.alloc(5)])) }, + { signature: undefined }, + ]; + for (const overrides of wrong) { + expect(passkeyAnswer(signIn, assertion(overrides))).toEqual(notAllowed); + } + const listed = request("get", { + challenge: "Y2hhbGxlbmdl", + allowCredentials: [{ type: "public-key", id: "b3RoZXI" }], + }); + expect(passkeyAnswer(listed, assertion())).toEqual(notAllowed); + }); + + it("passes the device's WebAuthn errors through and hides anything else", () => { + expect(passkeyAnswer(signIn, { success: false, error: "InvalidStateError" })).toEqual({ + error: "InvalidStateError", + }); + expect(passkeyAnswer(signIn, { success: false, error: "ASAuthorizationError 1004" })).toEqual({ + error: "NotAllowedError", + }); + expect(passkeyAnswer(signIn, "garbage")).toEqual({ error: "NotAllowedError" }); + }); + + it("reads a new passkey's public key out of its attestation", () => { + const signUp: PasskeyRequest = request("create", { + ...creation, + extensions: { credProps: true }, + }); + const keys = NodeCrypto.generateKeyPairSync("ec", { namedCurve: "P-256" }); + const jwk = keys.publicKey.export({ format: "jwk" }); + const credentialId = Buffer.from("credential-1"); + const coseKey = new Map([ + [1, 2], + [3, -7], + [-1, 1], + [-2, Buffer.from(jwk.x!, "base64url")], + [-3, Buffer.from(jwk.y!, "base64url")], + ]); + const authData = Buffer.concat([ + rpIdHash("example.com"), + Buffer.from([0x45, 0, 0, 0, 0]), + Buffer.alloc(16), + Buffer.from([0, credentialId.length]), + credentialId, + cbor(coseKey), + ]); + const attestationObject = base64Url( + cbor( + new Map([ + ["fmt", "none"], + ["attStmt", new Map()], + ["authData", authData], + ]), + ), + ); + const registration = (id: string) => ({ + success: true, + credential: { + id, + clientDataJSON: clientData("webauthn.create", "Y2hhbGxlbmdl"), + attestationObject, + authenticatorAttachment: "platform", + transports: ["hybrid", "internal", 5], + }, + }); + expect(passkeyAnswer(signUp, registration(base64Url(credentialId)))).toEqual({ + credential: { + id: base64Url(credentialId), + rawId: base64Url(credentialId), + type: "public-key", + authenticatorAttachment: "platform", + response: { + clientDataJSON: clientData("webauthn.create", "Y2hhbGxlbmdl"), + attestationObject, + authenticatorData: base64Url(authData), + transports: ["hybrid", "internal"], + publicKeyAlgorithm: -7, + publicKey: base64Url(keys.publicKey.export({ format: "der", type: "spki" })), + }, + clientExtensionResults: { credProps: { rk: true } }, + }, + }); + // The credential ID the device reports must be the one it attested. + expect(passkeyAnswer(signUp, registration(base64Url("another")))).toEqual({ + error: "NotAllowedError", + }); + }); +}); + +describe("PASSKEY_SCRIPT", () => { + it("parses, and leaves a page without WebAuthn alone", () => { + expect(() => new Function(PASSKEY_SCRIPT)()).not.toThrow(); + }); +}); diff --git a/apps/server/src/preview/ServerBrowserPasskeys.ts b/apps/server/src/preview/ServerBrowserPasskeys.ts new file mode 100644 index 000000000000..2e8edc26a68a --- /dev/null +++ b/apps/server/src/preview/ServerBrowserPasskeys.ts @@ -0,0 +1,486 @@ +// @effect-diagnostics nodeBuiltinImport:off - Runs inside Playwright binding callbacks, outside the Effect runtime. +import * as NodeCrypto from "node:crypto"; +import { getPublicSuffix, parse } from "tldts"; + +/** + * Passkeys for server browser pages, served by the controlling viewer's device. + * Headless Chromium has no authenticator, so a page's WebAuthn request travels + * to the viewer, whose system passkey sheet answers it. This module holds the + * page script and the checks a browser owes the relying party: the origin comes + * from Chromium, never from the page, the RP ID must belong to it, and a + * credential must be the one this ceremony asked for. + */ + +export const PASSKEY_BINDING = "__t3PreviewPasskey"; +/** A page's request reaches a viewer only this soon after it last touched the page. */ +export const PASSKEY_GESTURE_MS = 10_000; +// Chromium clamps WebAuthn timeouts to the same range. +const MIN_TIMEOUT_MS = 10_000; +const DEFAULT_TIMEOUT_MS = 5 * 60_000; +const MAX_TIMEOUT_MS = 10 * 60_000; +// Passkeys on Apple devices are P-256 keys. +const ES256 = -7; +const ERROR_NAMES = [ + "AbortError", + "InvalidStateError", + "NotAllowedError", + "NotSupportedError", + "SecurityError", + "TypeError", +]; + +/** + * Routes the page's WebAuthn ceremonies through the binding. The server answers + * `native` when no viewer holds passkeys, and the page keeps Chromium's own + * WebAuthn. Conditional requests (autofill, automatic upgrades) always stay + * native: the device's sheet must not open on its own. + */ +export const PASSKEY_SCRIPT = `(() => { + if (typeof CredentialsContainer === "undefined" || typeof PublicKeyCredential === "undefined") return; + const ask = (...args) => new Promise((resolve) => resolve(globalThis.${PASSKEY_BINDING}(...args))); + const bytesOf = (value) => + value instanceof ArrayBuffer ? new Uint8Array(value) : new Uint8Array(value.buffer, value.byteOffset, value.byteLength); + const toBase64Url = (value) => { + let binary = ""; + for (const byte of bytesOf(value)) binary += String.fromCharCode(byte); + return btoa(binary).replaceAll("+", "-").replaceAll("/", "_").replace(/=+$/, ""); + }; + const fromBase64Url = (text) => { + const binary = atob(text.replaceAll("-", "+").replaceAll("_", "/") + "===".slice((text.length + 3) % 4)); + return Uint8Array.from(binary, (character) => character.charCodeAt(0)).buffer; + }; + // Options cross in WebAuthn's JSON form: binary members as base64url. + const toJson = (value) => { + if (value instanceof ArrayBuffer || ArrayBuffer.isView(value)) return toBase64Url(value); + if (Array.isArray(value)) return value.map(toJson); + if (typeof value !== "object" || value === null) return value; + const json = {}; + for (const [key, member] of Object.entries(value)) + if (typeof member !== "function" && member !== undefined) json[key] = toJson(member); + return json; + }; + const failure = (name) => + name === "TypeError" + ? new TypeError("The passkey request options are invalid.") + : new DOMException( + "The operation either timed out or was not allowed. See: https://www.w3.org/TR/webauthn-2/#sctn-privacy-considerations-client.", + ${JSON.stringify(ERROR_NAMES)}.includes(name) ? name : "NotAllowedError", + ); + // Own properties shadow the native getters, which only work on objects Chromium made. + const withValues = (target, values) => { + for (const [key, value] of Object.entries(values)) + Object.defineProperty(target, key, { value, enumerable: true, configurable: true }); + return target; + }; + const credentialFrom = (json) => { + const response = json.response; + return withValues(Object.create(PublicKeyCredential.prototype), { + id: json.id, + rawId: fromBase64Url(json.rawId), + type: "public-key", + authenticatorAttachment: json.authenticatorAttachment, + response: + "attestationObject" in response + ? withValues(Object.create(AuthenticatorAttestationResponse.prototype), { + clientDataJSON: fromBase64Url(response.clientDataJSON), + attestationObject: fromBase64Url(response.attestationObject), + getTransports: () => [...response.transports], + getAuthenticatorData: () => fromBase64Url(response.authenticatorData), + getPublicKey: () => (response.publicKey === undefined ? null : fromBase64Url(response.publicKey)), + getPublicKeyAlgorithm: () => response.publicKeyAlgorithm, + }) + : withValues(Object.create(AuthenticatorAssertionResponse.prototype), { + clientDataJSON: fromBase64Url(response.clientDataJSON), + authenticatorData: fromBase64Url(response.authenticatorData), + signature: fromBase64Url(response.signature), + userHandle: response.userHandle === undefined ? null : fromBase64Url(response.userHandle), + }), + getClientExtensionResults: () => structuredClone(json.clientExtensionResults), + toJSON: () => structuredClone(json), + }); + }; + let tickets = 0; + const run = async (native, self, kind, options) => { + const signal = options.signal; + const abortReason = () => signal.reason ?? new DOMException("The operation was aborted.", "AbortError"); + if (signal?.aborted) throw abortReason(); + let request; + try { + request = toJson(options.publicKey); + } catch { + throw failure("TypeError"); + } + const ticket = ++tickets; + const answered = ask(kind, ticket, request).catch(() => ({ error: "NotAllowedError" })); + let onAbort = null; + const aborted = signal + ? new Promise((resolve) => { + onAbort = () => resolve({ aborted: true }); + signal.addEventListener("abort", onAbort, { once: true }); + }) + : null; + const answer = await (aborted ? Promise.race([answered, aborted]) : answered); + if (onAbort) signal.removeEventListener("abort", onAbort); + if (answer?.aborted) { + void ask("abort", ticket).catch(() => {}); + throw abortReason(); + } + if (answer?.native) return Reflect.apply(native, self, [options]); + if (answer?.credential) return credentialFrom(answer.credential); + throw failure(answer?.error); + }; + const container = CredentialsContainer.prototype; + for (const [kind, native] of [["create", container.create], ["get", container.get]]) { + Object.defineProperty(container, kind, { + configurable: true, + writable: true, + value: function (options) { + return options?.publicKey && options.mediation !== "conditional" + ? run(native, this, kind, options) + : Reflect.apply(native, this, [options]); + }, + }); + } + const available = () => ask("available").then((value) => value === true, () => false); + const nativeAvailable = PublicKeyCredential.isUserVerifyingPlatformAuthenticatorAvailable; + if (typeof nativeAvailable === "function") { + Object.defineProperty(PublicKeyCredential, "isUserVerifyingPlatformAuthenticatorAvailable", { + configurable: true, + writable: true, + value: async () => (await available()) || Reflect.apply(nativeAvailable, PublicKeyCredential, []), + }); + } + const nativeCapabilities = PublicKeyCredential.getClientCapabilities; + if (typeof nativeCapabilities === "function") { + Object.defineProperty(PublicKeyCredential, "getClientCapabilities", { + configurable: true, + writable: true, + value: async () => { + const capabilities = await Reflect.apply(nativeCapabilities, PublicKeyCredential, []); + return (await available()) + ? { ...capabilities, passkeyPlatformAuthenticator: true, userVerifyingPlatformAuthenticator: true } + : capabilities; + }, + }); + } +})();`; + +type Json = Readonly>; + +/** A page's request that passed the server's checks, ready for the viewer. */ +export interface PasskeyRequest { + readonly kind: "create" | "get"; + readonly origin: string; + readonly rpId: string; + readonly challenge: string; + /** The page's options in WebAuthn's JSON form, with the RP ID filled in. */ + readonly publicKey: Json; + readonly timeoutMs: number; + /** Credential IDs a sign-in accepts; empty means any of this site's. */ + readonly allowCredentials: ReadonlyArray; + readonly credProps: boolean; +} + +/** What the page script receives: a credential in `PublicKeyCredential.toJSON()` form, or an error name. */ +export type PasskeyAnswer = { readonly error: string } | { readonly credential: Json }; + +const NOT_ALLOWED = { error: "NotAllowedError" } as const; +const TYPE_ERROR = { error: "TypeError" } as const; + +const record = (value: unknown): Json | undefined => + typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Json) + : undefined; + +const isBase64Url = (value: unknown): value is string => + typeof value === "string" && /^[A-Za-z0-9_-]*$/u.test(value); + +const fromBase64Url = (value: string) => Buffer.from(value, "base64url"); + +const isPublicSuffix = (domain: string) => + getPublicSuffix(domain, { allowPrivateDomains: true }) === domain; + +/** The origin of a page that may use passkeys: https, or http on this machine. Null otherwise. */ +export const passkeyOrigin = (url: string) => { + if (!URL.canParse(url)) return null; + const { protocol, hostname, origin } = new URL(url); + const loopback = hostname === "localhost" || hostname === "127.0.0.1" || hostname === "[::1]"; + return protocol === "https:" || (protocol === "http:" && loopback) ? origin : null; +}; + +/** + * WebAuthn's rule: the RP ID is the page's host or a registrable parent of it, + * and never a public suffix, so one github.io site cannot sign in for another. + */ +export const relyingPartyIdError = (origin: string, rpId: string) => { + const host = new URL(origin).hostname; + if (rpId === host) return host === "localhost" || !isPublicSuffix(host) ? null : "SecurityError"; + return !parse(host).isIp && host.endsWith(`.${rpId}`) && !isPublicSuffix(rpId) + ? null + : "SecurityError"; +}; + +const credentialIds = (list: unknown) => + Array.isArray(list) + ? list.flatMap((item) => { + const descriptor = record(item); + return descriptor?.type === "public-key" && isBase64Url(descriptor.id) + ? [descriptor.id] + : []; + }) + : []; + +/** Checks a page's options and settles what the viewer's device is asked for. */ +export const passkeyRequest = ( + kind: "create" | "get", + origin: string, + options: unknown, +): PasskeyRequest | { readonly error: string } => { + const publicKey = record(options); + if (!publicKey || !isBase64Url(publicKey.challenge) || publicKey.challenge.length === 0) { + return TYPE_ERROR; + } + const host = new URL(origin).hostname; + const timeoutMs = + typeof publicKey.timeout === "number" && Number.isFinite(publicKey.timeout) + ? Math.min(Math.max(publicKey.timeout, MIN_TIMEOUT_MS), MAX_TIMEOUT_MS) + : DEFAULT_TIMEOUT_MS; + if (kind === "get") { + const rpId = publicKey.rpId ?? host; + if (typeof rpId !== "string") return TYPE_ERROR; + const rpError = relyingPartyIdError(origin, rpId); + if (rpError) return { error: rpError }; + return { + kind, + origin, + rpId, + challenge: publicKey.challenge, + publicKey: { ...publicKey, rpId }, + timeoutMs, + allowCredentials: credentialIds(publicKey.allowCredentials), + credProps: false, + }; + } + const rp = record(publicKey.rp); + const user = record(publicKey.user); + if (!rp || !user || typeof rp.name !== "string" || typeof user.name !== "string") { + return TYPE_ERROR; + } + const userIdLength = isBase64Url(user.id) ? fromBase64Url(user.id).length : 0; + if (userIdLength < 1 || userIdLength > 64) return TYPE_ERROR; + const rpId = rp.id ?? host; + if (typeof rpId !== "string") return TYPE_ERROR; + const rpError = relyingPartyIdError(origin, rpId); + if (rpError) return { error: rpError }; + const params = publicKey.pubKeyCredParams; + if (params !== undefined && !Array.isArray(params)) return TYPE_ERROR; + const allowsEs256 = + !Array.isArray(params) || + params.length === 0 || + params.some((param) => record(param)?.alg === ES256); + if (!allowsEs256) return { error: "NotSupportedError" }; + return { + kind, + origin, + rpId, + challenge: publicKey.challenge, + publicKey: { + ...publicKey, + rp: { ...rp, id: rpId }, + pubKeyCredParams: [{ type: "public-key", alg: ES256 }], + }, + timeoutMs, + allowCredentials: [], + credProps: record(publicKey.extensions)?.credProps === true, + }; +}; + +/** Decodes the first CBOR item, in the definite-length subset authenticators emit. */ +const decodeCbor = (bytes: Uint8Array): unknown => { + let offset = 0; + const malformed = () => new Error("Malformed CBOR"); + const item = (depth: number): unknown => { + const initial = bytes[offset++]; + if (initial === undefined || depth > 16) throw malformed(); + const major = initial >> 5; + const info = initial & 31; + let length = info; + if (info >= 24) { + if (info > 27) throw malformed(); + const size = 1 << (info - 24); + if (offset + size > bytes.length) throw malformed(); + length = 0; + for (let index = 0; index < size; index++) length = length * 256 + (bytes[offset++] ?? 0); + } + switch (major) { + case 0: + return length; + case 1: + return -1 - length; + case 2: + case 3: { + if (offset + length > bytes.length) throw malformed(); + const value = bytes.subarray(offset, offset + length); + offset += length; + return major === 2 ? value : new TextDecoder().decode(value); + } + case 4: + case 5: { + // Every member takes at least a byte, which also bounds hostile lengths. + if (length > bytes.length - offset) throw malformed(); + if (major === 4) return Array.from({ length }, () => item(depth + 1)); + const map = new Map(); + for (let index = 0; index < length; index++) map.set(item(depth + 1), item(depth + 1)); + return map; + } + case 6: + return item(depth + 1); + default: + return info === 21 ? true : info === 20 ? false : null; + } + }; + try { + return item(0); + } catch { + return undefined; + } +}; + +// SubjectPublicKeyInfo header for an uncompressed P-256 point, the form Chromium's getPublicKey() returns. +const P256_SPKI_PREFIX = Buffer.from( + "3059301306072a8648ce3d020106082a8648ce3d03010703420004", + "hex", +); + +/** Reads the attested credential out of registration authenticator data. */ +const attestedCredential = (authData: Uint8Array) => { + // rpIdHash 32, flags 1, sign count 4, AAGUID 16, then a length-prefixed credential ID and its COSE key. + if (authData.length < 55 || ((authData[32] ?? 0) & 0x40) === 0) return undefined; + const idLength = ((authData[53] ?? 0) << 8) | (authData[54] ?? 0); + const credentialId = authData.subarray(55, 55 + idLength); + if (credentialId.length !== idLength) return undefined; + const key = decodeCbor(authData.subarray(55 + idLength)); + const algorithm = key instanceof Map ? key.get(3) : undefined; + if (!(key instanceof Map) || typeof algorithm !== "number") return undefined; + const x = key.get(-2); + const y = key.get(-3); + const p256 = + key.get(1) === 2 && + key.get(-1) === 1 && + x instanceof Uint8Array && + y instanceof Uint8Array && + x.length === 32 && + y.length === 32; + return { + credentialId: Buffer.from(credentialId).toString("base64url"), + algorithm, + publicKey: p256 ? Buffer.concat([P256_SPKI_PREFIX, x, y]).toString("base64url") : undefined, + }; +}; + +const parseJson = (text: string) => { + try { + return record(JSON.parse(text)); + } catch { + return undefined; + } +}; + +/** + * Turns the device's result into what the page receives. The device must have + * signed this ceremony for this page: its client data names the page's origin + * and challenge, and its authenticator data the RP ID's hash. + */ +export const passkeyAnswer = (request: PasskeyRequest, result: unknown): PasskeyAnswer => { + const answer = record(result); + if (answer?.success !== true) { + const error = answer?.error; + return typeof error === "string" && ERROR_NAMES.includes(error) ? { error } : NOT_ALLOWED; + } + const credential = record(answer.credential); + if ( + !credential || + !isBase64Url(credential.id) || + credential.id.length === 0 || + !isBase64Url(credential.clientDataJSON) + ) { + return NOT_ALLOWED; + } + const clientData = parseJson(fromBase64Url(credential.clientDataJSON).toString("utf8")); + if ( + clientData?.type !== (request.kind === "create" ? "webauthn.create" : "webauthn.get") || + clientData.origin !== request.origin || + clientData.challenge !== request.challenge + ) { + return NOT_ALLOWED; + } + const rpIdHash = NodeCrypto.createHash("sha256").update(request.rpId).digest(); + const authenticatorAttachment = + credential.authenticatorAttachment === "platform" || + credential.authenticatorAttachment === "cross-platform" + ? credential.authenticatorAttachment + : null; + + if (request.kind === "create") { + if (!isBase64Url(credential.attestationObject)) return NOT_ALLOWED; + const attestation = decodeCbor(fromBase64Url(credential.attestationObject)); + const authData = attestation instanceof Map ? attestation.get("authData") : undefined; + if (!(authData instanceof Uint8Array) || !rpIdHash.equals(authData.subarray(0, 32))) { + return NOT_ALLOWED; + } + const attested = attestedCredential(authData); + if (attested?.credentialId !== credential.id) return NOT_ALLOWED; + return { + credential: { + id: credential.id, + rawId: credential.id, + type: "public-key", + authenticatorAttachment, + response: { + clientDataJSON: credential.clientDataJSON, + attestationObject: credential.attestationObject, + authenticatorData: Buffer.from(authData).toString("base64url"), + transports: Array.isArray(credential.transports) + ? credential.transports.filter((transport) => typeof transport === "string") + : [], + publicKeyAlgorithm: attested.algorithm, + ...(attested.publicKey === undefined ? {} : { publicKey: attested.publicKey }), + }, + // Passkeys on Apple devices are always discoverable. + clientExtensionResults: + request.credProps && authenticatorAttachment === "platform" + ? { credProps: { rk: true } } + : {}, + }, + }; + } + + if ( + !isBase64Url(credential.authenticatorData) || + !isBase64Url(credential.signature) || + (credential.userHandle !== undefined && !isBase64Url(credential.userHandle)) + ) { + return NOT_ALLOWED; + } + const authData = fromBase64Url(credential.authenticatorData); + if (authData.length < 37 || !rpIdHash.equals(authData.subarray(0, 32))) return NOT_ALLOWED; + // A security key can answer with a credential the page did not list. + if (request.allowCredentials.length > 0 && !request.allowCredentials.includes(credential.id)) { + return NOT_ALLOWED; + } + return { + credential: { + id: credential.id, + rawId: credential.id, + type: "public-key", + authenticatorAttachment, + response: { + clientDataJSON: credential.clientDataJSON, + authenticatorData: credential.authenticatorData, + signature: credential.signature, + ...(credential.userHandle ? { userHandle: credential.userHandle } : {}), + }, + clientExtensionResults: {}, + }, + }; +}; diff --git a/apps/server/src/preview/ServerBrowserStream.test.ts b/apps/server/src/preview/ServerBrowserStream.test.ts index 0654ea62a2b8..118903fff092 100644 --- a/apps/server/src/preview/ServerBrowserStream.test.ts +++ b/apps/server/src/preview/ServerBrowserStream.test.ts @@ -110,7 +110,8 @@ it.effect.each([ ); const server = Context.get(services, HttpServer.HttpServer); const origin = HttpServer.formatAddress(server.address).replace(/^http/, "ws"); - const resource = `/api/preview-stream/ws?threadId=thread&tabId=tab&wsTicket=one-use-ticket${interactive ? "" : "&interactive=false"}`; + // Every viewer offers passkeys here; only one that may operate the page keeps them. + const resource = `/api/preview-stream/ws?threadId=thread&tabId=tab&passkeys=true&wsTicket=one-use-ticket${interactive ? "" : "&interactive=false"}`; const received = Promise.withResolvers(); const socket = yield* Effect.acquireRelease( Effect.sync(() => new WebSocket(`${origin}${resource}`)), @@ -146,6 +147,7 @@ it.effect.each([ maxHeight: 800, quality: 70, canOperate, + passkeys: canOperate, }, ]); // In particular, a one-use ticket must never be authenticated a second diff --git a/apps/server/src/preview/ServerBrowserStream.ts b/apps/server/src/preview/ServerBrowserStream.ts index 964f143f5ef6..2d27233130b4 100644 --- a/apps/server/src/preview/ServerBrowserStream.ts +++ b/apps/server/src/preview/ServerBrowserStream.ts @@ -86,6 +86,7 @@ const makeHandler = (browser: ServerBrowser.ServerBrowser["Service"]) => threadId, tabId, canOperate, + passkeys: canOperate && params.get("passkeys") === "true", maxWidth: intParam(params, "maxWidth", 1280, 7680), maxHeight: intParam(params, "maxHeight", 800, 4320), quality: intParam(params, "quality", DEFAULT_QUALITY, 100), @@ -156,6 +157,8 @@ const makeHandler = (browser: ServerBrowser.ServerBrowser["Service"]) => case "download": case "fileChooser": case "fileChooserClosed": + case "passkey": + case "passkeyCancel": case "popup": case "pointer": case "probe": { diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index 17eedb545981..bcbfcb37500f 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -211,6 +211,11 @@ saved in your password manager. The fill card shows the site the login goes to, and nothing is filled if the page has left that site by then. The password only goes into a password field. +The host's browser has no passkeys of its own. On an iPhone build that carries +Apple's browser entitlement, a page's passkey sign-in or sign-up opens your +phone's passkey sheet while you have control, for that page's site only. Other +builds, and Android, leave passkeys to the host's browser. + On a phone, tap the floating preview's corner dot to show its controls, then **Pop into separate window** to keep watching in picture-in-picture over other apps. diff --git a/packages/client-runtime/src/preview/serverBrowserStream.test.ts b/packages/client-runtime/src/preview/serverBrowserStream.test.ts index 8a644b22d649..a983fce7e916 100644 --- a/packages/client-runtime/src/preview/serverBrowserStream.test.ts +++ b/packages/client-runtime/src/preview/serverBrowserStream.test.ts @@ -172,6 +172,43 @@ describe("preview stream downloads", () => { }); }); +describe("preview stream passkeys", () => { + beforeEach(() => vi.stubGlobal("WebSocket", FakeSocket)); + afterEach(() => vi.unstubAllGlobals()); + + it("offers passkeys only when asked, and hands over well-formed requests", () => { + const events = { + onFrame: vi.fn(), + onViewport: vi.fn(), + onConnectedChange: vi.fn(), + onUnauthorized: vi.fn(), + }; + createPreviewStreamClient(target, events).stop(); + expect(new URL(FakeSocket.current.url).searchParams.has("passkeys")).toBe(false); + const onPasskey = vi.fn(); + const onPasskeyCancel = vi.fn(); + const client = createPreviewStreamClient( + { ...target, passkeys: true }, + { ...events, onPasskey, onPasskeyCancel }, + ); + const socket = FakeSocket.current; + expect(new URL(socket.url).searchParams.get("passkeys")).toBe("true"); + const request = { + id: "p1", + kind: "get", + origin: "https://example.com", + publicKey: { challenge: "Y2hhbGxlbmdl", rpId: "example.com" }, + }; + socket.message(JSON.stringify({ type: "passkey", ...request, kind: "register" })); + socket.message(JSON.stringify({ type: "passkey", ...request, publicKey: null })); + socket.message(JSON.stringify({ type: "passkey", ...request })); + socket.message(JSON.stringify({ type: "passkeyCancel", id: "p1" })); + expect(onPasskey).toHaveBeenCalledExactlyOnceWith(request); + expect(onPasskeyCancel).toHaveBeenCalledExactlyOnceWith("p1"); + client.stop(); + }); +}); + describe("preview stream agent pointer", () => { beforeEach(() => vi.stubGlobal("WebSocket", FakeSocket)); afterEach(() => vi.unstubAllGlobals()); diff --git a/packages/client-runtime/src/preview/serverBrowserStream.ts b/packages/client-runtime/src/preview/serverBrowserStream.ts index 583935ff775d..e9250d0e3862 100644 --- a/packages/client-runtime/src/preview/serverBrowserStream.ts +++ b/packages/client-runtime/src/preview/serverBrowserStream.ts @@ -99,6 +99,12 @@ export type PreviewStreamInput = readonly username: string; readonly password: string; } + /** Answer to a `passkey` request (see `PreviewStreamTarget.passkeys`). */ + | { + readonly type: "passkeyResult"; + readonly id: string; + readonly result: PreviewStreamPasskeyResult; + } | { readonly type: "resize"; readonly width: number; readonly height: number } | { readonly type: "navigate"; readonly url: string } | { readonly type: "history"; readonly delta: -1 | 1 } @@ -165,6 +171,39 @@ export const previewStreamDownloadUrl = ( target.access, ); +/** + * A page's WebAuthn request, for the controlling viewer's own passkeys. The + * server has checked the origin and RP ID. Binary members are base64url, as in + * WebAuthn's JSON forms of the options. + */ +export interface PreviewStreamPasskeyRequest { + readonly id: string; + readonly kind: "create" | "get"; + /** The page's origin, as the server's browser reports it, never as the page claims it. */ + readonly origin: string; + /** `PublicKeyCredentialCreationOptionsJSON` or `PublicKeyCredentialRequestOptionsJSON`. */ + readonly publicKey: Readonly>; +} + +/** What the device's authenticator returned, base64url throughout. */ +export interface PreviewStreamPasskeyCredential { + readonly id: string; + readonly clientDataJSON: string; + /** Registrations only. */ + readonly attestationObject?: string; + /** Assertions only. */ + readonly authenticatorData?: string; + readonly signature?: string; + readonly userHandle?: string; + readonly authenticatorAttachment?: "platform" | "cross-platform"; + readonly transports?: ReadonlyArray; +} + +export type PreviewStreamPasskeyResult = + | { readonly success: true; readonly credential: PreviewStreamPasskeyCredential } + /** A WebAuthn DOMException name, such as `NotAllowedError`. */ + | { readonly success: false; readonly error: string }; + /** Answer to a `probe`, echoing its point. */ export interface PreviewStreamProbe { readonly x: number; @@ -201,6 +240,8 @@ export interface PreviewStreamTarget { readonly maxHeight: number; /** Passive viewers reduce their own access, including automatic control grants. */ readonly interactive?: boolean; + /** This device answers the page's passkey requests while it has control. */ + readonly passkeys?: boolean; } export interface PreviewStreamEvents { @@ -219,6 +260,10 @@ export interface PreviewStreamEvents { readonly onPopup?: (tabId: string) => void; /** The page opened a file picker (`null` once answered or replaced). */ readonly onFileChooser?: (chooser: PreviewStreamFileChooser | null) => void; + /** The page asked for a passkey; answer with a `passkeyResult` input. */ + readonly onPasskey?: (request: PreviewStreamPasskeyRequest) => void; + /** The page gave up on a passkey request: it aborted, timed out, navigated, or control moved. */ + readonly onPasskeyCancel?: (id: string) => void; /** Input sent while disconnected is dropped. */ readonly onConnectedChange: (connected: boolean) => void; /** The upgrade was refused; refresh access and start a new client. */ @@ -247,6 +292,7 @@ export function createPreviewStreamClient( maxHeight: String(Math.max(1, Math.round(target.maxHeight))), }); if (target.interactive === false) query.set("interactive", "false"); + if (target.passkeys) query.set("passkeys", "true"); const url = withDeviceHubQuery(`${target.access.wsBase}/ws?${query.toString()}`, target.access); let stopped = false; let socket: WebSocket | null = null; @@ -302,8 +348,27 @@ export function createPreviewStreamClient( phase, sequence, tabId, + kind, + origin, + publicKey, } = message as Record; if ( + type === "passkey" && + typeof id === "string" && + (kind === "create" || kind === "get") && + typeof origin === "string" && + typeof publicKey === "object" && + publicKey !== null + ) { + events.onPasskey?.({ + id, + kind, + origin, + publicKey: publicKey as Readonly>, + }); + } else if (type === "passkeyCancel" && typeof id === "string") { + events.onPasskeyCancel?.(id); + } else if ( type === "fileChooser" && typeof id === "string" && typeof multiple === "boolean" && diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2e44380dde31..c71d040599f8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -687,6 +687,9 @@ importers: stream-json: specifier: 3.6.0 version: 3.6.0 + tldts: + specifier: 'catalog:' + version: 7.4.2 yaml: specifier: ^2.9.0 version: 2.9.0