fix(web): app shortcuts keep working after clicking into an HTML render

apps/web/src/components/files/BrowserDocumentFrame.tsx

@@ -1,18 +1,35 @@
-import type { ScopedThreadRef } from "@t3tools/contracts";
+import { useAtomValue } from "@effect/atom-react";
 import {
+  type KeybindingCommand,
+  type KeybindingShortcut,
+  MODEL_PICKER_JUMP_KEYBINDING_COMMANDS,
+  type ScopedThreadRef,
+  THREAD_JUMP_KEYBINDING_COMMANDS,
+} from "@t3tools/contracts";
+import {
+  type HtmlRenderShortcut,
+  htmlRenderShortcutsMessage,
   htmlRenderThemeFragment,
   htmlRenderThemeMessage,
   htmlRenderResult,
   readHtmlRenderContentHeight,
+  readHtmlRenderKeyDown,
   readHtmlRenderLinkRequest,
 } from "@t3tools/shared/htmlRender";
-import { useEffect, useEffectEvent, useLayoutEffect, useRef, useState } from "react";
+import {
+  isMacPlatform,
+  matchesKeybindingShortcut,
+  normalizeEventKey,
+} from "@t3tools/shared/keybindings";
+import { useEffect, useEffectEvent, useLayoutEffect, useMemo, useRef, useState } from "react";
 
 import { useOpenLink } from "~/browser/useOpenLink";
 import { useHtmlRenderTheme } from "~/hooks/useHtmlRenderTheme";
+import { shortcutConflictKey } from "~/keybindings";
 import { useOpenChangeRequestLink } from "~/lib/openPullRequestLink";
 import { readLocalApi } from "~/localApi";
 import { cn } from "~/lib/utils";
+import { primaryServerKeybindingsAtom, serverEnvironment } from "~/state/server";
 
 /**
  * Chromium's viewer opens with its own toolbar, a thumbnail rail and a small
@@ -22,6 +39,32 @@ import { cn } from "~/lib/utils";
  */
 const PDF_VIEWER_FRAGMENT = "#toolbar=0&view=FitH";
 
+/**
+ * Shortcuts that still work while a focused HTML render holds the keyboard.
+ * The page reports the keys, and a page script can report any key it likes,
+ * so only commands that change what the reader sees are here, never one that
+ * changes a thread.
+ */
+const HTML_RENDER_SHORTCUT_COMMANDS: ReadonlySet<KeybindingCommand> = new Set([
+  "commandPalette.toggle",
+  "filePicker.toggle",
+  "projectSearch.toggle",
+  "chat.find",
+  "sidebar.toggle",
+  "rightPanel.toggle",
+  "rightPanel.toggleMaximized",
+  "threadPanel.toggle",
+  "terminal.toggle",
+  "preview.toggle",
+  "navigation.back",
+  "navigation.forward",
+  "thread.previous",
+  "thread.next",
+  ...THREAD_JUMP_KEYBINDING_COMMANDS,
+  // Share Cmd/Ctrl+digit with thread jumps, and act only while the picker is open.
+  ...MODEL_PICKER_JUMP_KEYBINDING_COMMANDS,
+]);
+
 export const isPdfPreviewFile = (path: string): boolean =>
   /\.pdf$/i.test(path.split(/[?#]/, 1)[0] ?? "");
 
@@ -90,6 +133,63 @@ export function HtmlRenderDocument(props: {
     frameRef.current?.contentWindow?.postMessage(htmlRenderThemeMessage(theme), "*");
   };
   useEffect(postTheme, [theme]);
+  const keybindings = useAtomValue(primaryServerKeybindingsAtom);
+  // Chat also resolves project scripts from the thread's own environment.
+  const environmentKeybindings = useAtomValue(
+    serverEnvironment.configValueAtom(props.threadRef?.environmentId ?? null),
+  )?.keybindings;
+  // The app resolves a replayed key in its live context, so every allowed
+  // binding counts whatever its `when`, and a key that any binding gives to
+  // another command stays with the page. Only Cmd/Ctrl chords: a printable key
+  // would type into the composer instead.
+  const { shortcuts, blocked } = useMemo(() => {
+    const mac = isMacPlatform(navigator.platform);
+    const resolve = (shortcut: KeybindingShortcut): HtmlRenderShortcut => ({
+      key: shortcut.key,
+      metaKey: shortcut.metaKey || (shortcut.modKey && mac),
+      ctrlKey: shortcut.ctrlKey || (shortcut.modKey && !mac),
+      shiftKey: shortcut.shiftKey,
+      altKey: shortcut.altKey,
+    });
+    const blocked = [...keybindings, ...(environmentKeybindings ?? [])]
+      .filter((binding) => !HTML_RENDER_SHORTCUT_COMMANDS.has(binding.command))
+      .map((binding) => resolve(binding.shortcut));
+    const shortcuts = new Map<string, HtmlRenderShortcut>();
+    for (const { command, shortcut } of keybindings) {
+      if (
+        HTML_RENDER_SHORTCUT_COMMANDS.has(command) &&
+        (shortcut.metaKey || shortcut.ctrlKey || shortcut.modKey)
+      ) {
+        shortcuts.set(shortcutConflictKey(shortcut), resolve(shortcut));
+      }
+    }
+    return { shortcuts: [...shortcuts.values()], blocked };
+  }, [environmentKeybindings, keybindings]);
+  const postShortcuts = () => {
+    frameRef.current?.contentWindow?.postMessage(
+      htmlRenderShortcutsMessage(shortcuts, blocked),
+      "*",
+    );
+  };
+  useEffect(postShortcuts, [blocked, shortcuts]);
+  // The page reports the key and code, and a page script can report any pair,
+  // so a key that could match another command's binding is never replayed. A
+  // named key (Backspace, Enter) must be the shortcut's own, since some app
+  // handlers read `event.key` directly. A dead key matches by its code.
+  const isShortcut = useEffectEvent(
+    (key: NonNullable<ReturnType<typeof readHtmlRenderKeyDown>>) => {
+      const matches = (shortcut: HtmlRenderShortcut) =>
+        matchesKeybindingShortcut(key, { ...shortcut, modKey: false }, navigator.platform);
+      const allowed = shortcuts.filter(matches);
+      return (
+        allowed.length > 0 &&
+        !blocked.some(matches) &&
+        (key.key.length === 1 ||
+          key.key === "Dead" ||
+          allowed.some((shortcut) => shortcut.key === normalizeEventKey(key.key)))
+      );
+    },
+  );
   const openChangeRequestLink = useOpenChangeRequestLink(props.threadRef);
   const openLink = useOpenLink(props.threadRef);
   // Same rules as a chat link: a pull request opens in the right panel, any
@@ -140,6 +240,31 @@ export function HtmlRenderDocument(props: {
     window.addEventListener("message", openLink);
     return () => window.removeEventListener("message", openLink);
   }, []);
+  // A focused frame takes every key, so app shortcuts would stop until the
+  // reader clicked out. The page passes on the chords posted to it, which replay
+  // on the frame element and bubble to the app's window listeners. A report is
+  // gated like a link and must be one of those chords.
+  useEffect(() => {
+    const replayKey = (event: MessageEvent) => {
+      const frame = frameRef.current;
+      const key = readHtmlRenderKeyDown(event.data);
+      if (
+        key === undefined ||
+        frame === null ||
+        event.source !== frame.contentWindow ||
+        document.activeElement !== frame ||
+        navigator.userActivation?.isActive === false
+      ) {
+        return;
+      }
+      if (!isShortcut(key)) return;
+      frame.dispatchEvent(
+        new KeyboardEvent("keydown", { ...key, bubbles: true, cancelable: true }),
+      );
+    };
+    window.addEventListener("message", replayKey);
+    return () => window.removeEventListener("message", replayKey);
+  }, []);
   const { onContentHeight } = props;
   // A page posts its height once per change, so listen from the commit that
   // inserts the frame; a passive effect could run after a fast page's first post.
@@ -166,6 +291,7 @@ export function HtmlRenderDocument(props: {
         setLoaded(true);
         // Covers a theme change that landed while the page was loading.
         postTheme();
+        postShortcuts();
       }}
       // A frame whose color scheme differs from its document's paints an opaque
       // canvas, so the blank document a frame starts with would flash white in

packages/shared/src/htmlRender.test.ts

@@ -9,6 +9,7 @@ import {
   injectHtmlRenderBootstrap,
   htmlRenderThemeMessage,
   readHtmlRenderContentHeight,
+  readHtmlRenderKeyDown,
   readHtmlRenderLinkRequest,
   readHtmlRenderReference,
 } from "./htmlRender.ts";
@@ -106,6 +107,29 @@ describe("readHtmlRenderLinkRequest", () => {
   });
 });
 
+describe("readHtmlRenderKeyDown", () => {
+  it("reads the key of a keydown notification and nothing else", () => {
+    const notification = (params: unknown) => ({
+      jsonrpc: "2.0",
+      method: "t3/notifications/keydown",
+      params,
+    });
+    expect(readHtmlRenderKeyDown(notification({ key: "k", code: "KeyK", metaKey: true }))).toEqual({
+      key: "k",
+      code: "KeyK",
+      metaKey: true,
+      ctrlKey: false,
+      shiftKey: false,
+      altKey: false,
+      repeat: false,
+    });
+    expect(readHtmlRenderKeyDown(notification({ key: 1, code: "KeyK" }))).toBeUndefined();
+    expect(
+      readHtmlRenderKeyDown({ jsonrpc: "2.0", method: "ui/open-link", params: { key: "k" } }),
+    ).toBeUndefined();
+  });
+});
+
 describe("readHtmlRenderContentHeight", () => {
   it("reads only the height of an MCP Apps size-changed notification", () => {
     const notification = (params: unknown) => ({

packages/shared/src/htmlRender.ts

@@ -4,6 +4,7 @@ import {
   type ThemeAppearance,
   type ThemeColors,
 } from "./themePalettes.ts";
+import { EVENT_CODE_SHORTCUT_KEYS } from "./keybindings.ts";
 
 /**
  * Agent-authored HTML pages ("HTML renders") are self-contained documents an
@@ -274,6 +275,11 @@ export const HTML_RENDER_LAYOUT_GUIDE = [
 const HOST_CONTEXT_CHANGED_METHOD = "ui/notifications/host-context-changed";
 const OPEN_LINK_METHOD = "ui/open-link";
 const SIZE_CHANGED_METHOD = "ui/notifications/size-changed";
+// Not part of MCP Apps: a focused frame keeps its keys from the app, so the
+// client tells the page which chords are app shortcuts and the page passes
+// those on.
+const SHORTCUTS_METHOD = "t3/notifications/shortcuts";
+const KEY_DOWN_METHOD = "t3/notifications/keydown";
 
 /** The content height in a framed render's `ui/notifications/size-changed` notification. */
 export function readHtmlRenderContentHeight(data: unknown): number | undefined {
@@ -311,6 +317,60 @@ export function readHtmlRenderLinkRequest(data: unknown):
     : undefined;
 }
 
+/** The key a framed render passed on in a `t3/notifications/keydown` notification. */
+export function readHtmlRenderKeyDown(data: unknown):
+  | {
+      readonly key: string;
+      readonly code: string;
+      readonly metaKey: boolean;
+      readonly ctrlKey: boolean;
+      readonly shiftKey: boolean;
+      readonly altKey: boolean;
+      readonly repeat: boolean;
+    }
+  | undefined {
+  if (typeof data !== "object" || data === null) return undefined;
+  const { jsonrpc, method, params } = data as Record<string, unknown>;
+  if (jsonrpc !== "2.0" || method !== KEY_DOWN_METHOD) return undefined;
+  if (typeof params !== "object" || params === null) return undefined;
+  const { key, code, metaKey, ctrlKey, shiftKey, altKey, repeat } = params as Record<
+    string,
+    unknown
+  >;
+  if (typeof key !== "string" || typeof code !== "string") return undefined;
+  return {
+    key,
+    code,
+    metaKey: metaKey === true,
+    ctrlKey: ctrlKey === true,
+    shiftKey: shiftKey === true,
+    altKey: altKey === true,
+    repeat: repeat === true,
+  };
+}
+
+/** A chord as the page sees it: `mod` already resolved to Meta or Control. */
+export interface HtmlRenderShortcut {
+  /** Lowercase key name, as in a keybinding: `k`, `[`, `enter`. */
+  readonly key: string;
+  readonly metaKey: boolean;
+  readonly ctrlKey: boolean;
+  readonly shiftKey: boolean;
+  readonly altKey: boolean;
+}
+
+/**
+ * The `t3/notifications/shortcuts` notification a client posts into a render:
+ * the chords it may pass on, and chords of other commands that the page keeps
+ * even when one also matches an allowed chord.
+ */
+export function htmlRenderShortcutsMessage(
+  shortcuts: ReadonlyArray<HtmlRenderShortcut>,
+  blocked: ReadonlyArray<HtmlRenderShortcut>,
+) {
+  return { jsonrpc: "2.0", method: SHORTCUTS_METHOD, params: { shortcuts, blocked } } as const;
+}
+
 /** The empty result a client sends back for a render's request. */
 export function htmlRenderResult(id: string | number) {
   return { jsonrpc: "2.0", id, result: {} } as const;
@@ -355,8 +415,12 @@ function rootRule(theme: HtmlRenderTheme): string {
 // a framed page asks its client to open it, and a top-level page (mobile)
 // opens it as a new window, which the client sends to the browser. A framed
 // page also reports its content height, measured as the server measures it,
-// so its client can fit the frame to the page.
-const BOOTSTRAP_SCRIPT = `(function(){var s=document.getElementById("t3-theme"),n=0;if(!s)return;var b=${JSON.stringify(BASE_CSS)};function a(t){if(!t||typeof t!=="object"||!t.variables||typeof t.variables!=="object")return;var c=":root{color-scheme:"+(t.appearance==="light"?"light":"dark")+";";for(var k in t.variables){if(/^--[a-z0-9-]+$/.test(k))c+=k+":"+String(t.variables[k]).replace(/[;{}<>]/g,"")+";";}s.textContent=c+"}"+b;}try{var m=/[#&]${THEME_FRAGMENT_KEY}=([^&]*)/.exec(location.hash);if(m){a(JSON.parse(decodeURIComponent(m[1])));history.replaceState(history.state,"",location.pathname+location.search);}}catch(e){}window.addEventListener("message",function(e){var d=e.data,p=d&&d.params;if(d&&d.jsonrpc==="2.0"&&d.method===${JSON.stringify(HOST_CONTEXT_CHANGED_METHOD)}&&p&&p.styles)a({appearance:p.theme,variables:p.styles.variables});});document.addEventListener("click",function(e){var l=e.isTrusted?e.composedPath().find(function(t){return t&&t.matches&&t.matches("a[href]");}):null,u;if(!l)return;try{u=new URL(l.getAttribute("href"),document.baseURI);}catch(x){return;}if(!/^https?:$/.test(u.protocol)||u.href.split("#")[0]===location.href.split("#")[0])return;if(window.parent!==window){e.preventDefault();window.parent.postMessage({jsonrpc:"2.0",id:"t3-link-"+(++n),method:${JSON.stringify(OPEN_LINK_METHOD)},params:{url:u.href,metaKey:e.metaKey,ctrlKey:e.ctrlKey}},"*");}else{l.setAttribute("target","_blank");l.setAttribute("rel","noopener");}},true);if(window.parent!==window){var h,o,z=function(){var r=document.documentElement,v=Math.ceil(r.scrollHeight>r.clientHeight?r.scrollHeight:r.getBoundingClientRect().height);if(v===h)return;h=v;window.parent.postMessage({jsonrpc:"2.0",method:${JSON.stringify(SIZE_CHANGED_METHOD)},params:{height:v}},"*");};if(window.ResizeObserver){o=new ResizeObserver(z);o.observe(document.documentElement);}document.addEventListener("DOMContentLoaded",function(){if(o&&document.body)o.observe(document.body);z();});window.addEventListener("load",z);}})();`;
+// so its client can fit the frame to the page. A chord the client named as an
+// app shortcut is passed on rather than left to the browser, unless a text
+// field has focus or the page handled it: the deciding listener is added while
+// the key dispatches, so it runs after every listener the page has. Keys
+// match as resolveEventKeys reads them, so the client accepts what it claims.
+const BOOTSTRAP_SCRIPT = `(function(){var s=document.getElementById("t3-theme"),n=0;if(!s)return;var b=${JSON.stringify(BASE_CSS)};function a(t){if(!t||typeof t!=="object"||!t.variables||typeof t.variables!=="object")return;var c=":root{color-scheme:"+(t.appearance==="light"?"light":"dark")+";";for(var k in t.variables){if(/^--[a-z0-9-]+$/.test(k))c+=k+":"+String(t.variables[k]).replace(/[;{}<>]/g,"")+";";}s.textContent=c+"}"+b;}try{var m=/[#&]${THEME_FRAGMENT_KEY}=([^&]*)/.exec(location.hash);if(m){a(JSON.parse(decodeURIComponent(m[1])));history.replaceState(history.state,"",location.pathname+location.search);}}catch(e){}window.addEventListener("message",function(e){var d=e.data,p=d&&d.params;if(d&&d.jsonrpc==="2.0"&&d.method===${JSON.stringify(HOST_CONTEXT_CHANGED_METHOD)}&&p&&p.styles)a({appearance:p.theme,variables:p.styles.variables});});document.addEventListener("click",function(e){var l=e.isTrusted?e.composedPath().find(function(t){return t&&t.matches&&t.matches("a[href]");}):null,u;if(!l)return;try{u=new URL(l.getAttribute("href"),document.baseURI);}catch(x){return;}if(!/^https?:$/.test(u.protocol)||u.href.split("#")[0]===location.href.split("#")[0])return;if(window.parent!==window){e.preventDefault();window.parent.postMessage({jsonrpc:"2.0",id:"t3-link-"+(++n),method:${JSON.stringify(OPEN_LINK_METHOD)},params:{url:u.href,metaKey:e.metaKey,ctrlKey:e.ctrlKey}},"*");}else{l.setAttribute("target","_blank");l.setAttribute("rel","noopener");}},true);if(window.parent!==window){var h,o,z=function(){var r=document.documentElement,v=Math.ceil(r.scrollHeight>r.clientHeight?r.scrollHeight:r.getBoundingClientRect().height);if(v===h)return;h=v;window.parent.postMessage({jsonrpc:"2.0",method:${JSON.stringify(SIZE_CHANGED_METHOD)},params:{height:v}},"*");};if(window.ResizeObserver){o=new ResizeObserver(z);o.observe(document.documentElement);}document.addEventListener("DOMContentLoaded",function(){if(o&&document.body)o.observe(document.body);z();});window.addEventListener("load",z);var K=[],B=[],C=${JSON.stringify(EVENT_CODE_SHORTCUT_KEYS)};window.addEventListener("message",function(e){var d=e.data,p=d&&d.params;if(e.source===window.parent&&d&&d.jsonrpc==="2.0"&&d.method===${JSON.stringify(SHORTCUTS_METHOD)}&&p&&Array.isArray(p.shortcuts)&&Array.isArray(p.blocked)){K=p.shortcuts;B=p.blocked;}});window.addEventListener("keydown",function(e){var t=e.composedPath()[0],c=e.code||"",l=e.key.toLowerCase(),x=/^[a-z]$/.test(l)?[l]:[l,/^Key[A-Z]$/.test(c)?c.slice(3).toLowerCase():"",C.hasOwnProperty(c)?C[c]:""],f,m=function(s){return s&&s.metaKey===e.metaKey&&s.ctrlKey===e.ctrlKey&&s.shiftKey===e.shiftKey&&s.altKey===e.altKey&&x.indexOf(s.key)>-1;};if(!e.isTrusted||e.isComposing||e.getModifierState("AltGraph")&&!/^[a-z0-9]$/i.test(e.key)||t&&(t.isContentEditable||/^(?:INPUT|TEXTAREA|SELECT)$/.test(t.tagName))||!K.some(m)||B.some(m))return;f=function(v){window.removeEventListener("keydown",f);if(v!==e||e.defaultPrevented)return;e.preventDefault();window.parent.postMessage({jsonrpc:"2.0",method:${JSON.stringify(KEY_DOWN_METHOD)},params:{key:e.key,code:e.code,metaKey:e.metaKey,ctrlKey:e.ctrlKey,shiftKey:e.shiftKey,altKey:e.altKey,repeat:e.repeat}},"*");};window.addEventListener("keydown",f);setTimeout(function(){window.removeEventListener("keydown",f);});},true);}})();`;
 
 function bootstrapMarkup(markup: string): string {
   const dark = htmlRenderTheme(T3_CODE_DARK_THEME_COLORS, "dark");

packages/shared/src/keybindings.ts

@@ -29,7 +29,7 @@ export interface ShortcutModifierStateLike {
   altKey: boolean;
 }
 
-const EVENT_CODE_SHORTCUT_KEYS: Readonly<Record<string, string>> = {
+export const EVENT_CODE_SHORTCUT_KEYS: Readonly<Record<string, string>> = {
   Backquote: "`",
   Backslash: "\\",
   BracketLeft: "[",