feat(voice): talk to existing threads with GPT-Live

README.md

@@ -79,6 +79,7 @@ Full docs live in [docs/](./docs). There's no docs site yet.
 - [Install and first run](./docs/user/install.md)
 - [Permission modes](./docs/user/permission-modes.md)
 - [Keyboard shortcuts](./docs/user/keybindings.md)
+- [Live voice (experimental)](./docs/user/live-voice.md)
 - [Project settings](./docs/user/project-settings.md)
 - [Remote access from a phone or another machine](./docs/user/remote-access.md)
 - [Keeping app and server in sync](./docs/user/updating.md)

apps/mobile/app.config.ts

@@ -358,21 +358,34 @@ const config: ExpoConfig = {
       "expo-audio",
       {
         microphonePermission: "Allow T3 Code to use your microphone for voice input.",
-        recordAudioAndroid: false,
+        recordAudioAndroid: true,
         enableBackgroundPlayback: false,
         enableBackgroundRecording: false,
       },
     ],
+    [
+      "@config-plugins/react-native-webrtc",
+      {
+        microphonePermission: "Allow T3 Code to use your microphone for voice input.",
+        cameraPermission: "Allow T3 Code to access your camera so you can scan pairing QR codes.",
+      },
+    ],
     [
       "expo-camera",
       {
         cameraPermission: "Allow T3 Code to access your camera so you can scan pairing QR codes.",
-        microphonePermission: false,
+        microphonePermission: "Allow T3 Code to use your microphone for voice input.",
         barcodeScannerEnabled: true,
         recordAudioAndroid: false,
       },
     ],
-    ["expo-image-picker", { photosPermission: false, microphonePermission: false }],
+    [
+      "expo-image-picker",
+      {
+        photosPermission: false,
+        microphonePermission: "Allow T3 Code to use your microphone for voice input.",
+      },
+    ],
     [
       "expo-splash-screen",
       {

apps/mobile/package.json

@@ -46,6 +46,7 @@
   },
   "dependencies": {
     "@clerk/expo": "catalog:",
+    "@config-plugins/react-native-webrtc": "15.0.2",
     "@effect/atom-react": "catalog:",
     "@expo-google-fonts/dm-sans": "^0.4.2",
     "@expo/metro-runtime": "~57.0.14",
@@ -120,6 +121,7 @@
     "react-native-screens": "~4.26.0",
     "react-native-shiki-engine": "^0.3.12",
     "react-native-svg": "15.15.4",
+    "react-native-webrtc": "124.0.8",
     "react-native-webview": "^13.16.1",
     "react-native-worklets": "0.10.1",
     "shiki": "4.2.0",

apps/mobile/src/features/live-voice/LiveVoiceCall.tsx

@@ -0,0 +1,105 @@
+import type { LiveVoiceState } from "@t3tools/client-runtime/live-voice";
+import { useRef } from "react";
+import { Modal, Pressable, ScrollView, View } from "react-native";
+import { useSafeAreaInsets } from "react-native-safe-area-context";
+
+import { AppText as Text } from "../../components/AppText";
+
+export function LiveVoiceStartButton(props: { disabled: boolean; onPress: () => void }) {
+  return (
+    <Pressable
+      accessibilityRole="button"
+      accessibilityLabel="Start voice chat"
+      accessibilityState={{ disabled: props.disabled }}
+      disabled={props.disabled}
+      onPress={props.onPress}
+      className="h-11 justify-center rounded-xl px-2 active:bg-subtle"
+      style={{ opacity: props.disabled ? 0.45 : 1 }}
+    >
+      <Text className="text-sm font-t3-medium text-foreground-muted">Voice</Text>
+    </Pressable>
+  );
+}
+
+export function LiveVoiceCall(props: {
+  visible: boolean;
+  state: LiveVoiceState;
+  threadTitle: string;
+  onClose: () => void;
+  onMute: (muted: boolean) => void;
+}) {
+  const captions = useRef<ScrollView>(null);
+  const insets = useSafeAreaInsets();
+  const active = props.state.status === "connecting" || props.state.status === "connected";
+  return (
+    <Modal visible={props.visible} animationType="slide" onRequestClose={props.onClose}>
+      <View
+        className="flex-1 bg-screen"
+        style={{ paddingTop: insets.top, paddingBottom: insets.bottom }}
+      >
+        <View className="gap-2 border-b border-border p-5">
+          <Text className="text-xl font-t3-medium text-foreground">Voice chat</Text>
+          <Text className="text-sm text-foreground-muted" numberOfLines={2}>
+            {props.threadTitle}
+          </Text>
+          <Text accessibilityLiveRegion="polite" className="text-sm text-foreground-muted">
+            {props.state.status === "connecting"
+              ? "Connecting…"
+              : props.state.status === "connected"
+                ? props.state.muted
+                  ? "Microphone muted"
+                  : "Listening"
+                : props.state.status === "error"
+                  ? "Could not continue voice chat"
+                  : "Call ended"}
+          </Text>
+        </View>
+        <ScrollView
+          ref={captions}
+          className="flex-1"
+          contentContainerStyle={{ padding: 20, gap: 16 }}
+          onContentSizeChange={() => captions.current?.scrollToEnd({ animated: false })}
+        >
+          {props.state.transcript.length === 0 ? (
+            <Text className="text-foreground-muted">
+              Speak naturally about this thread. Captions appear here during your call.
+            </Text>
+          ) : null}
+          <Text className="text-base text-foreground" selectable>
+            {props.state.transcript
+              .map((entry) => `${entry.role === "user" ? "You" : "Assistant"}\n${entry.text}`)
+              .join("\n\n")}
+          </Text>
+          {props.state.error ? (
+            <Text accessibilityRole="alert" className="text-foreground">
+              {props.state.error}
+            </Text>
+          ) : null}
+        </ScrollView>
+        <View className="gap-3 border-t border-border p-5">
+          {active ? (
+            <Pressable
+              accessibilityRole="button"
+              accessibilityLabel={props.state.muted ? "Unmute microphone" : "Mute microphone"}
+              accessibilityState={{ selected: props.state.muted }}
+              onPress={() => props.onMute(!props.state.muted)}
+              className="min-h-12 items-center justify-center rounded-xl bg-subtle px-4"
+            >
+              <Text className="font-t3-medium text-foreground">
+                {props.state.muted ? "Unmute" : "Mute"}
+              </Text>
+            </Pressable>
+          ) : null}
+          <Pressable
+            accessibilityRole="button"
+            accessibilityLabel={active ? "End voice chat" : "Close voice chat"}
+            onPress={props.onClose}
+            className="min-h-12 items-center justify-center rounded-xl bg-subtle px-4"
+          >
+            <Text className="font-t3-medium text-foreground">{active ? "End call" : "Close"}</Text>
+          </Pressable>
+        </View>
+      </View>
+    </Modal>
+  );
+}

apps/mobile/src/features/live-voice/microphoneReservation.ts

@@ -0,0 +1,24 @@
+let reserved = false;
+const listeners = new Set<() => void>();
+
+export const isLiveVoiceMicrophoneReserved = () => reserved;
+export function subscribeLiveVoiceMicrophone(listener: () => void) {
+  listeners.add(listener);
+  return () => {
+    listeners.delete(listener);
+  };
+}
+
+/** Keeps dictation blocked across navigation until a late permission result is released. */
+export function reserveLiveVoiceMicrophone() {
+  if (reserved) throw new Error("The microphone is still finishing the previous voice chat.");
+  reserved = true;
+  listeners.forEach((listener) => listener());
+  let released = false;
+  return () => {
+    if (released) return;
+    released = true;
+    reserved = false;
+    listeners.forEach((listener) => listener());
+  };
+}

apps/mobile/src/features/live-voice/nativeLiveVoiceTransport.test.ts

@@ -0,0 +1,172 @@
+import { beforeEach, describe, expect, it, vi } from "vite-plus/test";
+
+const rtc = vi.hoisted(() => {
+  const peerListeners = new Map<string, () => void>();
+  const channelListeners = new Map<string, (event: { data?: unknown }) => void>();
+  const track = { enabled: true, stop: vi.fn(), addEventListener: vi.fn() };
+  const stream = { getTracks: () => [track], getAudioTracks: () => [track], release: vi.fn() };
+  const channel = {
+    close: vi.fn(),
+    set onopen(handler: (event: { data?: unknown }) => void) {
+      channelListeners.set("open", handler);
+    },
+    set onclose(handler: (event: { data?: unknown }) => void) {
+      channelListeners.set("close", handler);
+    },
+    set onerror(handler: (event: { data?: unknown }) => void) {
+      channelListeners.set("error", handler);
+    },
+    set onmessage(handler: (event: { data?: unknown }) => void) {
+      channelListeners.set("message", handler);
+    },
+    addEventListener: vi.fn((name: string, handler: (event: { data?: unknown }) => void) => {
+      channelListeners.set(name, handler);
+    }),
+  };
+  const peer = {
+    connectionState: "new",
+    iceGatheringState: "complete",
+    localDescription: { sdp: "final-offer-with-candidates" },
+    addTrack: vi.fn(),
+    createDataChannel: vi.fn(() => channel),
+    createOffer: vi.fn(async () => ({ type: "offer", sdp: "initial-offer" })),
+    setLocalDescription: vi.fn(async () => undefined),
+    setRemoteDescription: vi.fn(async () => undefined),
+    addEventListener: vi.fn((name: string, handler: () => void) => {
+      peerListeners.set(name, handler);
+    }),
+    removeEventListener: vi.fn((name: string) => {
+      peerListeners.delete(name);
+    }),
+    close: vi.fn(),
+    set onconnectionstatechange(handler: () => void) {
+      peerListeners.set("connectionstatechange", handler);
+    },
+    set onicegatheringstatechange(handler: (() => void) | null) {
+      if (handler) peerListeners.set("icegatheringstatechange", handler);
+      else peerListeners.delete("icegatheringstatechange");
+    },
+  };
+  const getUserMedia = vi.fn(async () => stream);
+  return { track, stream, channel, peer, getUserMedia, peerListeners, channelListeners };
+});
+
+vi.mock("react-native-webrtc", () => ({
+  mediaDevices: { getUserMedia: rtc.getUserMedia },
+  RTCPeerConnection: function () {
+    return rtc.peer;
+  },
+}));
+
+import { createNativeLiveVoiceTransport } from "./nativeLiveVoiceTransport";
+import { isLiveVoiceMicrophoneReserved } from "./microphoneReservation";
+
+function setup() {
+  const callbacks = { onEvent: vi.fn(), onConnectionState: vi.fn() };
+  return { callbacks, transport: createNativeLiveVoiceTransport(callbacks) };
+}
+
+describe("native live voice transport", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    rtc.peerListeners.clear();
+    rtc.channelListeners.clear();
+    rtc.peer.iceGatheringState = "complete";
+    rtc.track.enabled = true;
+    rtc.getUserMedia.mockResolvedValue(rtc.stream);
+  });
+
+  it("negotiates audio only using the final gathered SDP and releases capture once", async () => {
+    const { transport } = setup();
+    transport.setMuted(true);
+    expect(await transport.createOffer()).toBe("final-offer-with-candidates");
+    expect(rtc.getUserMedia).toHaveBeenCalledWith({ audio: true, video: false });
+    expect(rtc.track.enabled).toBe(false);
+    await transport.acceptAnswer("remote-answer");
+    expect(rtc.peer.setRemoteDescription).toHaveBeenCalledWith({
+      type: "answer",
+      sdp: "remote-answer",
+    });
+    transport.setMuted(false);
+    expect(rtc.track.enabled).toBe(true);
+    transport.close();
+    transport.close();
+    expect(rtc.track.stop).toHaveBeenCalledOnce();
+    expect(rtc.stream.release).toHaveBeenCalledOnce();
+    expect(rtc.peer.close).toHaveBeenCalledOnce();
+  });
+
+  it("waits for ICE gathering and removes its listener after completion", async () => {
+    rtc.peer.iceGatheringState = "gathering";
+    const { transport } = setup();
+    const pending = transport.createOffer();
+    await vi.waitFor(() => expect(rtc.peerListeners.has("icegatheringstatechange")).toBe(true));
+    rtc.peer.iceGatheringState = "complete";
+    rtc.peerListeners.get("icegatheringstatechange")?.();
+    expect(await pending).toBe("final-offer-with-candidates");
+    expect(rtc.peerListeners.has("icegatheringstatechange")).toBe(false);
+    transport.close();
+  });
+
+  it("cancels an outstanding ICE wait immediately", async () => {
+    rtc.peer.iceGatheringState = "gathering";
+    const { transport } = setup();
+    const pending = transport.createOffer();
+    const rejection = expect(pending).rejects.toThrow("Voice chat ended");
+    await vi.waitFor(() => expect(rtc.peerListeners.has("icegatheringstatechange")).toBe(true));
+    transport.close();
+    await rejection;
+    expect(rtc.peerListeners.has("icegatheringstatechange")).toBe(false);
+    expect(rtc.stream.release).toHaveBeenCalledOnce();
+  });
+
+  it("releases a microphone granted after cancellation without creating a peer", async () => {
+    let grant!: (value: typeof rtc.stream) => void;
+    rtc.getUserMedia.mockImplementationOnce(
+      () =>
+        new Promise((resolve) => {
+          grant = resolve;
+        }),
+    );
+    const { transport } = setup();
+    const pending = transport.createOffer();
+    const rejection = expect(pending).rejects.toThrow("Voice chat ended");
+    await vi.waitFor(() => expect(rtc.getUserMedia).toHaveBeenCalledOnce());
+    transport.close();
+    expect(isLiveVoiceMicrophoneReserved()).toBe(true);
+    const nextThread = setup().transport;
+    await expect(nextThread.createOffer()).rejects.toThrow("finishing the previous voice chat");
+    expect(rtc.getUserMedia).toHaveBeenCalledOnce();
+    grant(rtc.stream);
+    await rejection;
+    expect(rtc.stream.release).toHaveBeenCalledOnce();
+    expect(rtc.peer.addTrack).not.toHaveBeenCalled();
+    expect(isLiveVoiceMicrophoneReserved()).toBe(false);
+    await nextThread.createOffer();
+    nextThread.close();
+  });
+
+  it("forwards channel events and ignores them after closing", async () => {
+    const { transport, callbacks } = setup();
+    await transport.createOffer();
+    rtc.channelListeners.get("open")?.({});
+    expect(callbacks.onConnectionState).toHaveBeenCalledWith("connected");
+    rtc.channelListeners.get("message")?.({ data: '{"type":"session.started"}' });
+    expect(callbacks.onEvent).toHaveBeenCalledWith({ type: "session.started" });
+    rtc.channelListeners.get("message")?.({ data: "invalid" });
+    transport.close();
+    rtc.channelListeners.get("message")?.({ data: '{"type":"session.started"}' });
+    rtc.channelListeners.get("close")?.({});
+    expect(callbacks.onEvent).toHaveBeenCalledOnce();
+    expect(callbacks.onConnectionState).toHaveBeenCalledOnce();
+  });
+
+  it("releases native capture if offer generation fails", async () => {
+    rtc.peer.createOffer.mockRejectedValueOnce(new Error("Native negotiation failed"));
+    const { transport } = setup();
+    await expect(transport.createOffer()).rejects.toThrow("Native negotiation failed");
+    expect(rtc.stream.release).toHaveBeenCalledOnce();
+    expect(rtc.peer.close).toHaveBeenCalledOnce();
+    expect(isLiveVoiceMicrophoneReserved()).toBe(false);
+  });
+});