From e2cea12c274b2117dd9db05449473d772b3ed81b Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 10 Oct 2026 12:14:13 -0700 Subject: [PATCH 1/4] feat(mobile): choose the microphone order for voice input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Voice input recorded from whatever input iOS routed to, which is the car's or AirPods' Bluetooth microphone whenever one is connected. Settings → Microphone now lists Wired, Bluetooth, iPhone, and CarPlay in a draggable order; when recording starts, voice input selects the first connected input of the highest-ranked kind. The default ranks CarPlay below the iPhone microphone. Co-Authored-By: Claude Opus 5.5 --- apps/mobile/src/Stack.tsx | 8 + apps/mobile/src/components/AppSymbol.tsx | 7 + .../settings/EnvironmentRoutesSection.tsx | 65 +----- .../SettingsMicrophoneRouteScreen.tsx | 193 ++++++++++++++++++ .../features/settings/SettingsRouteScreen.tsx | 5 +- .../components/SettingsDragHandle.tsx | 64 ++++++ .../components/settings-sheet-targets.ts | 1 + .../voice-input/VoiceInputProvider.tsx | 15 ++ .../voice-input/lazyVoiceRecorder.test.ts | 35 +++- .../features/voice-input/lazyVoiceRecorder.ts | 22 +- .../mobile/src/lib/microphonePriority.test.ts | 43 ++++ apps/mobile/src/lib/microphonePriority.ts | 46 +++++ .../src/persistence/mobile-preferences.ts | 7 + docs/user/composer.md | 4 + 14 files changed, 449 insertions(+), 66 deletions(-) create mode 100644 apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx create mode 100644 apps/mobile/src/features/settings/components/SettingsDragHandle.tsx create mode 100644 apps/mobile/src/lib/microphonePriority.test.ts create mode 100644 apps/mobile/src/lib/microphonePriority.ts diff --git a/apps/mobile/src/Stack.tsx b/apps/mobile/src/Stack.tsx index ec4b2565e174..c089945ccc8f 100644 --- a/apps/mobile/src/Stack.tsx +++ b/apps/mobile/src/Stack.tsx @@ -102,6 +102,7 @@ import { import { ScheduledTaskEditorProvider } from "./features/settings/scheduled-task-editor"; import { SettingsKeyboardRouteScreen } from "./features/settings/SettingsKeyboardRouteScreen"; import { SettingsLegalRouteScreen } from "./features/settings/SettingsLegalRouteScreen"; +import { SettingsMicrophoneRouteScreen } from "./features/settings/SettingsMicrophoneRouteScreen"; import { SettingsOpenSourceLicenseRouteScreen, SettingsOpenSourceLicensesRouteScreen, @@ -310,6 +311,13 @@ const SettingsContentStack = createV5SheetStackNavigator({ title: "Keyboard", }, }), + SettingsMicrophone: createNativeStackScreen({ + screen: SettingsMicrophoneRouteScreen, + linking: "microphone", + options: { + title: "Microphone", + }, + }), SettingsFollowUp: createNativeStackScreen({ screen: SettingsFollowUpRouteScreen, linking: "follow-ups", diff --git a/apps/mobile/src/components/AppSymbol.tsx b/apps/mobile/src/components/AppSymbol.tsx index 122ad1658e27..06c2e9e42761 100644 --- a/apps/mobile/src/components/AppSymbol.tsx +++ b/apps/mobile/src/components/AppSymbol.tsx @@ -28,6 +28,7 @@ import IconBolt from "@tabler/icons-react-native/IconBolt"; import IconBox from "@tabler/icons-react-native/IconBox"; import IconBrain from "@tabler/icons-react-native/IconBrain"; import IconCamera from "@tabler/icons-react-native/IconCamera"; +import IconCar from "@tabler/icons-react-native/IconCar"; import IconChartBar from "@tabler/icons-react-native/IconChartBar"; import IconCheck from "@tabler/icons-react-native/IconCheck"; import IconCloud from "@tabler/icons-react-native/IconCloud"; @@ -58,6 +59,7 @@ import IconFilter from "@tabler/icons-react-native/IconFilter"; import IconFilterFilled from "@tabler/icons-react-native/IconFilterFilled"; import IconFolder from "@tabler/icons-react-native/IconFolder"; import IconFolderOpen from "@tabler/icons-react-native/IconFolderOpen"; +import IconHeadphones from "@tabler/icons-react-native/IconHeadphones"; import IconFolderPlus from "@tabler/icons-react-native/IconFolderPlus"; import IconGitBranch from "@tabler/icons-react-native/IconGitBranch"; import IconGitMerge from "@tabler/icons-react-native/IconGitMerge"; @@ -83,6 +85,7 @@ import IconMoon from "@tabler/icons-react-native/IconMoon"; import IconNetwork from "@tabler/icons-react-native/IconNetwork"; import IconPalette from "@tabler/icons-react-native/IconPalette"; import IconPencil from "@tabler/icons-react-native/IconPencil"; +import IconPlug from "@tabler/icons-react-native/IconPlug"; import IconPhoto from "@tabler/icons-react-native/IconPhoto"; import IconPin from "@tabler/icons-react-native/IconPin"; import IconPinnedOff from "@tabler/icons-react-native/IconPinnedOff"; @@ -140,13 +143,16 @@ const ANDROID_ICON_BY_SF_SYMBOL = { "arrow.up.right.circle": IconArrowUpRightCircle, "arrow.uturn.backward": IconArrowBackUp, "arrow.uturn.forward": IconArrowForwardUp, + airpods: IconHeadphones, archivebox: IconArchive, "archivebox.fill": IconArchive, "bell.badge": IconBellRinging, "bolt.circle": IconBolt, "bolt.horizontal.circle": IconBolt, brain: IconBrain, + "cable.connector": IconPlug, camera: IconCamera, + car: IconCar, "chart.bar.xaxis": IconChartBar, checkmark: IconCheck, "checkmark.circle": IconCircleCheck, @@ -181,6 +187,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = { house: IconHome, "info.circle": IconInfoCircle, internaldrive: IconDatabase, + iphone: IconDeviceMobile, keyboard: IconKeyboard, laptopcomputer: IconDeviceLaptop, link: IconLink, diff --git a/apps/mobile/src/features/settings/EnvironmentRoutesSection.tsx b/apps/mobile/src/features/settings/EnvironmentRoutesSection.tsx index b20699ecb26b..17b6bcadd880 100644 --- a/apps/mobile/src/features/settings/EnvironmentRoutesSection.tsx +++ b/apps/mobile/src/features/settings/EnvironmentRoutesSection.tsx @@ -13,9 +13,8 @@ import { type EnvironmentId, sessionGrantsScope } from "@t3tools/contracts"; import { AUTH_SCOPE_OPTIONS } from "@t3tools/shared/authScopeOptions"; import { AsyncResult } from "effect/reactivity"; import * as Option from "effect/Option"; -import { useEffect, useMemo, useRef, useState } from "react"; +import { useState } from "react"; import { Alert, Platform, Pressable, View } from "react-native"; -import { Gesture, GestureDetector } from "react-native-gesture-handler"; import Reanimated, { ReduceMotion, useAnimatedStyle, withTiming } from "react-native-reanimated"; import { type AppSymbolName, SymbolView } from "../../components/AppSymbol"; @@ -27,6 +26,7 @@ import { environmentSession } from "../../state/session"; import { useAtomCommand } from "../../state/use-atom-command"; import { connectionTone } from "../connection/connectionTone"; import { SettingsActionRow } from "./components/SettingsActionRow"; +import { SettingsDragHandle } from "./components/SettingsDragHandle"; import { SettingsSection } from "./components/SettingsSection"; const ICON_SIZE = Platform.OS === "android" ? 24 : 22; @@ -324,7 +324,7 @@ function RouteRow(props: { {props.editing ? ( - 1} canMoveDown={props.position < props.count} @@ -337,62 +337,3 @@ function RouteRow(props: { ); } - -/** Pan recognition wins over the settings scroll view only inside the handle. */ -function DragHandle(props: { - readonly title: string; - readonly canMoveUp: boolean; - readonly canMoveDown: boolean; - readonly onStart: () => void; - readonly onMove: (translation: number) => void; - readonly onEnd: (translation: number, cancelled: boolean) => void; - readonly onStep: (direction: "up" | "down") => void; -}) { - const latest = useRef(props); - useEffect(() => { - latest.current = props; - }); - const translation = useRef(0); - const gesture = useMemo( - () => - Gesture.Pan() - .minDistance(0) - .shouldCancelWhenOutside(false) - .runOnJS(true) - .onStart(() => { - translation.current = 0; - latest.current.onStart(); - }) - .onUpdate((event) => { - translation.current = event.translationY; - latest.current.onMove(event.translationY); - }) - .onFinalize((_, success) => latest.current.onEnd(translation.current, !success)), - [], - ); - return ( - - { - if (nativeEvent.actionName === "decrement" && props.canMoveUp) props.onStep("up"); - if (nativeEvent.actionName === "increment" && props.canMoveDown) props.onStep("down"); - }} - style={{ width: 48, alignSelf: "stretch", alignItems: "center", justifyContent: "center" }} - > - - - - ); -} diff --git a/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx new file mode 100644 index 000000000000..174531ed9424 --- /dev/null +++ b/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx @@ -0,0 +1,193 @@ +import { useAtomSet, useAtomValue } from "@effect/atom-react"; +import { AsyncResult } from "effect/reactivity"; +import { useState } from "react"; +import { ScrollView, View } from "react-native"; +import Reanimated, { ReduceMotion, useAnimatedStyle, withTiming } from "react-native-reanimated"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; + +import { type AppSymbolName, SymbolView } from "../../components/AppSymbol"; +import { AppText as Text } from "../../components/AppText"; +import { cn } from "../../lib/cn"; +import { type MicrophoneKind, resolveMicrophonePriority } from "../../lib/microphonePriority"; +import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences"; +import { SettingsDragHandle } from "./components/SettingsDragHandle"; +import { SettingsSection } from "./components/SettingsSection"; + +const MICROPHONES: Record< + MicrophoneKind, + { readonly label: string; readonly description: string; readonly icon: AppSymbolName } +> = { + wired: { + label: "Wired", + description: "Headsets and USB microphones", + icon: "cable.connector", + }, + bluetooth: { + label: "Bluetooth", + description: "AirPods, headsets, and cars without CarPlay", + icon: "airpods", + }, + builtIn: { + label: "iPhone", + description: "The microphone on this device", + icon: "iphone", + }, + carPlay: { + label: "CarPlay", + description: "The car’s microphone", + icon: "car", + }, +}; + +/** Voice input records from the first connected microphone in this order. iOS only. */ +export function SettingsMicrophoneRouteScreen() { + const insets = useSafeAreaInsets(); + const preferencesResult = useAtomValue(mobilePreferencesAtom); + const savePreferences = useAtomSet(updateMobilePreferencesAtom); + const ready = AsyncResult.isSuccess(preferencesResult) && !preferencesResult.waiting; + const order = resolveMicrophonePriority( + AsyncResult.isSuccess(preferencesResult) ? preferencesResult.value.microphonePriority : [], + ); + const [drag, setDrag] = useState<{ + readonly kind: MicrophoneKind; + readonly translation: number; + } | null>(null); + const [rowHeight, setRowHeight] = useState(0); + // Each drop remounts the rows so the new order and the cleared offsets land in one frame. + const [drops, setDrops] = useState(0); + + const move = (from: number, to: number) => { + if (to < 0 || to >= order.length || from === to) return; + const next = [...order]; + const [moved] = next.splice(from, 1); + next.splice(to, 0, moved!); + savePreferences({ microphonePriority: next }); + }; + // A lifted row takes a neighbour's slot once it has moved past half of that row. + const dropIndex = (from: number, translation: number) => + rowHeight === 0 + ? from + : Math.min(order.length - 1, Math.max(0, from + Math.round(translation / rowHeight))); + + const dragFrom = drag === null ? -1 : order.indexOf(drag.kind); + const dragTo = drag === null ? -1 : dropIndex(dragFrom, drag.translation); + + return ( + + + + {order.map((kind, index) => { + const shift = + dragFrom === -1 || index === dragFrom + ? 0 + : dragFrom < dragTo && index > dragFrom && index <= dragTo + ? -rowHeight + : dragFrom > dragTo && index < dragFrom && index >= dragTo + ? rowHeight + : 0; + return ( + setDrag({ kind, translation: 0 })} + onDragMove={(translation) => setDrag({ kind, translation })} + onDragEnd={(translation, cancelled) => { + setDrag(null); + setDrops((count) => count + 1); + if (!cancelled) move(index, dropIndex(index, translation)); + }} + onStep={(direction) => move(index, direction === "up" ? index - 1 : index + 1)} + /> + ); + })} + + + Voice input records from the first connected microphone in this list. Drag to reorder. + + + + ); +} + +function MicrophoneRow(props: { + readonly kind: MicrophoneKind; + readonly position: number; + readonly count: number; + readonly disabled: boolean; + readonly offset: number; + readonly lifted: boolean; + readonly onHeight: (height: number) => void; + readonly onDragStart: () => void; + readonly onDragMove: (translation: number) => void; + readonly onDragEnd: (translation: number, cancelled: boolean) => void; + readonly onStep: (direction: "up" | "down") => void; +}) { + const { lifted, offset } = props; + const microphone = MICROPHONES[props.kind]; + const style = useAnimatedStyle(() => ({ + transform: [ + { + translateY: lifted + ? offset + : withTiming(offset, { duration: 160, reduceMotion: ReduceMotion.System }), + }, + ], + zIndex: lifted ? 1 : 0, + })); + return ( + props.onHeight(event.nativeEvent.layout.height)} + className={cn( + "flex-row items-center gap-4 pl-4", + props.position > 1 && !lifted && "border-t border-border-subtle", + lifted && "bg-grouped-card shadow-md", + )} + > + + + + + {microphone.label} + + + {microphone.description} + + + + {props.disabled ? null : ( + 1} + canMoveDown={props.position < props.count} + onStart={props.onDragStart} + onMove={props.onDragMove} + onEnd={props.onDragEnd} + onStep={props.onStep} + /> + )} + + ); +} diff --git a/apps/mobile/src/features/settings/SettingsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsRouteScreen.tsx index 1f474a9d56b8..adb6551cf5fa 100644 --- a/apps/mobile/src/features/settings/SettingsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsRouteScreen.tsx @@ -154,7 +154,10 @@ function SettingsIndexSections() { {Platform.OS === "ios" ? ( - + <> + + + ) : null} diff --git a/apps/mobile/src/features/settings/components/SettingsDragHandle.tsx b/apps/mobile/src/features/settings/components/SettingsDragHandle.tsx new file mode 100644 index 000000000000..84b9260074ed --- /dev/null +++ b/apps/mobile/src/features/settings/components/SettingsDragHandle.tsx @@ -0,0 +1,64 @@ +import { useEffect, useMemo, useRef } from "react"; +import { View } from "react-native"; +import { Gesture, GestureDetector } from "react-native-gesture-handler"; + +import { SymbolView } from "../../../components/AppSymbol"; + +/** Pan recognition wins over the settings scroll view only inside the handle. */ +export function SettingsDragHandle(props: { + readonly title: string; + readonly canMoveUp: boolean; + readonly canMoveDown: boolean; + readonly onStart: () => void; + readonly onMove: (translation: number) => void; + readonly onEnd: (translation: number, cancelled: boolean) => void; + readonly onStep: (direction: "up" | "down") => void; +}) { + const latest = useRef(props); + useEffect(() => { + latest.current = props; + }); + const translation = useRef(0); + const gesture = useMemo( + () => + Gesture.Pan() + .minDistance(0) + .shouldCancelWhenOutside(false) + .runOnJS(true) + .onStart(() => { + translation.current = 0; + latest.current.onStart(); + }) + .onUpdate((event) => { + translation.current = event.translationY; + latest.current.onMove(event.translationY); + }) + .onFinalize((_, success) => latest.current.onEnd(translation.current, !success)), + [], + ); + return ( + + { + if (nativeEvent.actionName === "decrement" && props.canMoveUp) props.onStep("up"); + if (nativeEvent.actionName === "increment" && props.canMoveDown) props.onStep("down"); + }} + style={{ width: 48, alignSelf: "stretch", alignItems: "center", justifyContent: "center" }} + > + + + + ); +} diff --git a/apps/mobile/src/features/settings/components/settings-sheet-targets.ts b/apps/mobile/src/features/settings/components/settings-sheet-targets.ts index ad0197d634d8..611a1b3410c7 100644 --- a/apps/mobile/src/features/settings/components/settings-sheet-targets.ts +++ b/apps/mobile/src/features/settings/components/settings-sheet-targets.ts @@ -13,6 +13,7 @@ export type SettingsSheetTarget = | "SettingsEnvironmentMaintenance" | "SettingsProviderAccounts" | "SettingsKeyboard" + | "SettingsMicrophone" | "SettingsFollowUp" | "SettingsScheduledTasks" | "SettingsProjectGrouping" diff --git a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx index ebd628e4bad4..5b1607612e4b 100644 --- a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx +++ b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx @@ -1,3 +1,4 @@ +import { RegistryContext } from "@effect/atom-react"; import { AudioModule, RecordingPresets, @@ -20,9 +21,12 @@ import { type ReactNode, } from "react"; import { AppState, Platform } from "react-native"; +import { AsyncResult } from "effect/reactivity"; import { useSharedValue } from "react-native-reanimated"; +import { preferredMicrophoneInput, resolveMicrophonePriority } from "../../lib/microphonePriority"; import { getLocalVoiceTranscriber } from "../../native/voiceTranscription"; +import { mobilePreferencesAtom } from "../../state/preferences"; import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene"; import { VOICE_RECORDING_LIMIT_SECONDS, @@ -91,6 +95,7 @@ export function useGlobalVoiceInput() { } function useVoiceInputRuntime() { + const registry = use(RegistryContext); const [{ state, ownerKey, label }, setState] = useState({ state: INITIAL_STATE, ownerKey: null as string | null, @@ -126,6 +131,16 @@ function useVoiceInputRuntime() { url: status.url, }); }, + // Read when recording starts so preference changes do not re-render every composer. + selectInput: (inputs) => { + const preferences = registry.get(mobilePreferencesAtom); + return preferredMicrophoneInput( + inputs, + resolveMicrophonePriority( + AsyncResult.isSuccess(preferences) ? preferences.value.microphonePriority : undefined, + ), + ); + }, }); recorderRef.current = recorder; sessionRef.current = new VoiceInputSession({ diff --git a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts index ffba8faefffd..076d26cd6401 100644 --- a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts +++ b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vite-plus/test"; import { VoiceInputController } from "@t3tools/client-runtime/voice-input"; -import { createLazyVoiceRecorder } from "./lazyVoiceRecorder"; +import { createLazyVoiceRecorder, type VoiceRecordingInput } from "./lazyVoiceRecorder"; type Status = { readonly isFinished: boolean }; type State = { readonly isRecording: boolean }; @@ -30,12 +30,19 @@ function createNativeRecorder(uri: string) { listeners.add(listener); return { remove: () => listeners.delete(listener) }; }), + getAvailableInputs: vi.fn((): ReadonlyArray => [ + { uid: "car", type: "CarAudio" }, + { uid: "built-in", type: "MicrophoneBuiltIn" }, + ]), + setInput: vi.fn((_uid: string) => undefined), release: vi.fn(), emit: (status: Status) => listeners.forEach((listener) => listener(status)), }; } -function createHarness() { +function createHarness( + selectInput?: (inputs: ReadonlyArray) => VoiceRecordingInput | null, +) { const created: Array> = []; const statuses: Status[] = []; const recorder = createLazyVoiceRecorder({ @@ -45,6 +52,7 @@ function createHarness() { return native; }, onStatus: (status) => statuses.push(status), + ...(selectInput ? { selectInput } : {}), }); return { recorder, created, statuses }; } @@ -79,6 +87,29 @@ describe("createLazyVoiceRecorder", () => { expect(created[0]!.prepareToRecordAsync).toHaveBeenCalledTimes(2); }); + it("selects the preferred input after preparing, when the session lists inputs", async () => { + const { recorder, created } = createHarness( + (inputs) => inputs.find((input) => input.type === "MicrophoneBuiltIn") ?? null, + ); + + await recorder.prepareToRecordAsync(); + + expect(created[0]!.setInput).toHaveBeenCalledWith("built-in"); + }); + + it("keeps the system input when none is preferred or the preferred one disconnects", async () => { + const unpreferred = createHarness(() => null); + await unpreferred.recorder.prepareToRecordAsync(); + expect(unpreferred.created[0]!.setInput).not.toHaveBeenCalled(); + + const disconnected = createHarness((inputs) => inputs[1] ?? null); + await disconnected.recorder.prepareToRecordAsync(); + disconnected.created[0]!.setInput.mockImplementationOnce(() => { + throw new Error("Preferred input not found"); + }); + await expect(disconnected.recorder.prepareToRecordAsync()).resolves.toBeUndefined(); + }); + it("forwards status events until the recorder is released", async () => { const { recorder, created, statuses } = createHarness(); await recorder.prepareToRecordAsync(); diff --git a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts index d6deb2163c63..eae72b884ce2 100644 --- a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts +++ b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts @@ -2,6 +2,11 @@ import type { VoiceRecorder } from "@t3tools/client-runtime/voice-input"; type Subscription = { remove(): void }; +export interface VoiceRecordingInput { + readonly uid: string; + readonly type: string; +} + // The subset of expo-audio's native `AudioRecorder` the voice input hook uses. export interface NativeVoiceRecorder { readonly uri: string | null; @@ -13,6 +18,8 @@ export interface NativeVoiceRecorder { eventName: "recordingStatusUpdate", listener: (status: TStatus) => void, ): Subscription; + getAvailableInputs(): ReadonlyArray; + setInput(uid: string): void; release(): void; } @@ -34,6 +41,8 @@ export interface LazyVoiceRecorder extends VoiceRecorder { export function createLazyVoiceRecorder(input: { readonly create: () => NativeVoiceRecorder; readonly onStatus: (status: TStatus) => void; + /** Picks the input to record from once the session lists them, or null to keep the system's. */ + readonly selectInput?: (inputs: ReadonlyArray) => VoiceRecordingInput | null; }): LazyVoiceRecorder { let recorder: NativeVoiceRecorder | null = null; let subscription: Subscription | null = null; @@ -57,7 +66,18 @@ export function createLazyVoiceRecorder(input: { recorder = input.create(); subscription = recorder.addListener("recordingStatusUpdate", input.onStatus); } - return track(recorder.prepareToRecordAsync()); + const prepared = recorder; + return track( + prepared.prepareToRecordAsync().then(() => { + if (!input.selectInput) return; + try { + const selected = input.selectInput(prepared.getAvailableInputs()); + if (selected) prepared.setInput(selected.uid); + } catch { + // The input can disconnect in between. The system's route still records. + } + }), + ); }, record(options) { if (!recorder) throw new Error("Voice recorder is not prepared."); diff --git a/apps/mobile/src/lib/microphonePriority.test.ts b/apps/mobile/src/lib/microphonePriority.test.ts new file mode 100644 index 000000000000..0970a60af533 --- /dev/null +++ b/apps/mobile/src/lib/microphonePriority.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + DEFAULT_MICROPHONE_PRIORITY, + preferredMicrophoneInput, + resolveMicrophonePriority, +} from "./microphonePriority"; + +const builtIn = { uid: "built-in", type: "MicrophoneBuiltIn" }; +const airPods = { uid: "airpods", type: "BluetoothHFP" }; +const carPlay = { uid: "car", type: "CarAudio" }; +const usb = { uid: "usb", type: "USBAudio" }; + +describe("resolveMicrophonePriority", () => { + it("keeps the saved order and appends kinds it does not mention", () => { + expect(resolveMicrophonePriority(["builtIn", "carPlay", "builtIn"])).toEqual([ + "builtIn", + "carPlay", + "wired", + "bluetooth", + ]); + expect(resolveMicrophonePriority(undefined)).toEqual(DEFAULT_MICROPHONE_PRIORITY); + }); +}); + +describe("preferredMicrophoneInput", () => { + it("records from the device instead of CarPlay by default", () => { + const priority = resolveMicrophonePriority(undefined); + expect(preferredMicrophoneInput([carPlay, builtIn], priority)).toBe(builtIn); + expect(preferredMicrophoneInput([builtIn, airPods], priority)).toBe(airPods); + }); + + it("follows the saved order", () => { + const priority = resolveMicrophonePriority(["builtIn"]); + expect(preferredMicrophoneInput([airPods, usb, builtIn], priority)).toBe(builtIn); + expect(preferredMicrophoneInput([airPods, usb], priority)).toBe(usb); + }); + + it("leaves unknown inputs to iOS", () => { + const priority = resolveMicrophonePriority(undefined); + expect(preferredMicrophoneInput([{ uid: "x", type: "Virtual" }], priority)).toBeNull(); + }); +}); diff --git a/apps/mobile/src/lib/microphonePriority.ts b/apps/mobile/src/lib/microphonePriority.ts new file mode 100644 index 000000000000..43f83b7bad2f --- /dev/null +++ b/apps/mobile/src/lib/microphonePriority.ts @@ -0,0 +1,46 @@ +/** + * The kinds of microphone voice input can record from, in the order the user + * prefers them. iOS reports each input's port type; several port types share + * a kind. A car without CarPlay connects as a Bluetooth headset, so only + * CarPlay can be ranked on its own. + */ +export const MICROPHONE_KINDS = ["wired", "bluetooth", "builtIn", "carPlay"] as const; + +export type MicrophoneKind = (typeof MICROPHONE_KINDS)[number]; + +/** CarPlay ranks below the device microphone so dictation in the car avoids the car's microphone. */ +export const DEFAULT_MICROPHONE_PRIORITY: ReadonlyArray = MICROPHONE_KINDS; + +// Raw values of AVAudioSession.Port. +const KIND_BY_PORT_TYPE: Readonly> = { + MicrophoneBuiltIn: "builtIn", + BluetoothHFP: "bluetooth", + BluetoothLE: "bluetooth", + MicrophoneWired: "wired", + USBAudio: "wired", + LineIn: "wired", + CarAudio: "carPlay", +}; + +export function isMicrophoneKind(value: unknown): value is MicrophoneKind { + return (MICROPHONE_KINDS as ReadonlyArray).includes(value); +} + +/** The saved order, with kinds it does not mention appended in their default order. */ +export function resolveMicrophonePriority( + saved: ReadonlyArray | undefined, +): ReadonlyArray { + return [...new Set([...(saved ?? []), ...DEFAULT_MICROPHONE_PRIORITY])]; +} + +/** The available input of the highest-ranked kind, or null to leave the route to iOS. */ +export function preferredMicrophoneInput( + inputs: ReadonlyArray, + priority: ReadonlyArray, +): Input | null { + for (const kind of priority) { + const input = inputs.find((candidate) => KIND_BY_PORT_TYPE[candidate.type] === kind); + if (input) return input; + } + return null; +} diff --git a/apps/mobile/src/persistence/mobile-preferences.ts b/apps/mobile/src/persistence/mobile-preferences.ts index ee96bce2d5ce..c53c806b06cb 100644 --- a/apps/mobile/src/persistence/mobile-preferences.ts +++ b/apps/mobile/src/persistence/mobile-preferences.ts @@ -8,6 +8,7 @@ import * as Semaphore from "effect/Semaphore"; import type { ProviderInstanceId, SidebarProjectGroupingMode } from "@t3tools/contracts"; import type { ComposerEnterBehavior } from "../lib/composerEnterBehavior"; import type { FollowUpBehavior } from "../lib/followUpBehavior"; +import { isMicrophoneKind, type MicrophoneKind } from "../lib/microphonePriority"; import { MOBILE_THEME_IDS, type MobileThemeId, type MobileThemeMode } from "../lib/mobileTheme"; import * as MobileDatabase from "./mobile-database"; import * as MobileSecureStorage from "./mobile-secure-storage"; @@ -36,6 +37,8 @@ export interface Preferences { * message sent during a running turn queues behind it or steers it. */ readonly followUpBehavior?: FollowUpBehavior; + /** The order voice input tries microphone kinds in, most preferred first. iOS only. */ + readonly microphonePriority?: ReadonlyArray; /** @deprecated Kept temporarily so older OTA bundles retain the selected mode. */ readonly projectGroupingEnabled?: boolean; readonly projectGroupingMode?: SidebarProjectGroupingMode; @@ -107,6 +110,7 @@ function sanitizePreferences(parsed: Preferences): Preferences { collapsedProjectGroups?: readonly string[]; composerEnterBehavior?: ComposerEnterBehavior; followUpBehavior?: FollowUpBehavior; + microphonePriority?: ReadonlyArray; projectGroupingEnabled?: boolean; projectGroupingMode?: SidebarProjectGroupingMode; planModeEnabled?: boolean; @@ -172,6 +176,9 @@ function sanitizePreferences(parsed: Preferences): Preferences { if (parsed.followUpBehavior === "queue" || parsed.followUpBehavior === "steer") { preferences.followUpBehavior = parsed.followUpBehavior; } + if (Array.isArray(parsed.microphonePriority)) { + preferences.microphonePriority = parsed.microphonePriority.filter(isMicrophoneKind); + } if (typeof parsed.projectGroupingEnabled === "boolean") { preferences.projectGroupingEnabled = parsed.projectGroupingEnabled; } diff --git a/docs/user/composer.md b/docs/user/composer.md index ba8d26fd81c2..5c51d1ff2bd6 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -164,6 +164,10 @@ minutes long. Canceling, leaving the screen, or an audio interruption discards t recording and preserves your existing draft. While recording, the screen stays awake; it can sleep normally once recording stops. +Voice input records from the first connected microphone in **Settings → Microphone**. +Drag the list to choose, for example, the iPhone microphone over AirPods or your car. +By default, CarPlay comes after the iPhone microphone. + Transcription runs on your device. T3 Code deletes the temporary audio after transcription or cancellation; only the message text is sent when you submit. From 2b14a86e198db187eaf9d4d7c9b9fb91569484dd Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 10 Oct 2026 12:32:06 -0700 Subject: [PATCH 2/4] feat(mobile): rank remembered microphones by device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kinds could not tell AirPods from a car without CarPlay, since both connect as Bluetooth headsets. Voice input now remembers each connected microphone by its iOS port identifier and name the first time it records near it, and Settings → Microphone lists those devices for the user to drag into order or forget. A new device joins after the last one of its kind, so an existing preference carries over; CarPlay still starts below the built-in microphone. Co-Authored-By: Claude Opus 5.5 --- apps/mobile/src/components/AppSymbol.tsx | 2 +- .../SettingsMicrophoneRouteScreen.tsx | 227 +++++++++++------- .../voice-input/VoiceInputProvider.tsx | 26 +- .../voice-input/lazyVoiceRecorder.test.ts | 4 +- .../features/voice-input/lazyVoiceRecorder.ts | 1 + .../mobile/src/lib/microphonePriority.test.ts | 85 ++++--- apps/mobile/src/lib/microphonePriority.ts | 86 +++++-- .../src/persistence/mobile-preferences.ts | 19 +- docs/user/composer.md | 6 +- 9 files changed, 293 insertions(+), 163 deletions(-) diff --git a/apps/mobile/src/components/AppSymbol.tsx b/apps/mobile/src/components/AppSymbol.tsx index 06c2e9e42761..c9457a55c83a 100644 --- a/apps/mobile/src/components/AppSymbol.tsx +++ b/apps/mobile/src/components/AppSymbol.tsx @@ -143,7 +143,6 @@ const ANDROID_ICON_BY_SF_SYMBOL = { "arrow.up.right.circle": IconArrowUpRightCircle, "arrow.uturn.backward": IconArrowBackUp, "arrow.uturn.forward": IconArrowForwardUp, - airpods: IconHeadphones, archivebox: IconArchive, "archivebox.fill": IconArchive, "bell.badge": IconBellRinging, @@ -184,6 +183,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = { gearshape: IconSettings, globe: IconWorld, hammer: IconHammer, + headphones: IconHeadphones, house: IconHome, "info.circle": IconInfoCircle, internaldrive: IconDatabase, diff --git a/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx index 174531ed9424..e8a448e3850e 100644 --- a/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx @@ -1,76 +1,76 @@ import { useAtomSet, useAtomValue } from "@effect/atom-react"; import { AsyncResult } from "effect/reactivity"; import { useState } from "react"; -import { ScrollView, View } from "react-native"; +import { Pressable, ScrollView, View } from "react-native"; import Reanimated, { ReduceMotion, useAnimatedStyle, withTiming } from "react-native-reanimated"; import { useSafeAreaInsets } from "react-native-safe-area-context"; import { type AppSymbolName, SymbolView } from "../../components/AppSymbol"; import { AppText as Text } from "../../components/AppText"; import { cn } from "../../lib/cn"; -import { type MicrophoneKind, resolveMicrophonePriority } from "../../lib/microphonePriority"; +import type { MicrophoneKind, RememberedMicrophone } from "../../lib/microphonePriority"; import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences"; import { SettingsDragHandle } from "./components/SettingsDragHandle"; import { SettingsSection } from "./components/SettingsSection"; -const MICROPHONES: Record< - MicrophoneKind, - { readonly label: string; readonly description: string; readonly icon: AppSymbolName } -> = { - wired: { - label: "Wired", - description: "Headsets and USB microphones", - icon: "cable.connector", - }, - bluetooth: { - label: "Bluetooth", - description: "AirPods, headsets, and cars without CarPlay", - icon: "airpods", - }, - builtIn: { - label: "iPhone", - description: "The microphone on this device", - icon: "iphone", - }, - carPlay: { - label: "CarPlay", - description: "The car’s microphone", - icon: "car", - }, +const REMOVE_SIZE = 20; + +const KINDS: Record = { + wired: { label: "Wired", icon: "cable.connector" }, + bluetooth: { label: "Bluetooth", icon: "headphones" }, + builtIn: { label: "Built-in", icon: "iphone" }, + carPlay: { label: "CarPlay", icon: "car" }, }; -/** Voice input records from the first connected microphone in this order. iOS only. */ +/** + * Microphones voice input has seen, in the order it prefers them. iOS only. + * A microphone joins the list the first time it is connected during dictation. + */ export function SettingsMicrophoneRouteScreen() { const insets = useSafeAreaInsets(); const preferencesResult = useAtomValue(mobilePreferencesAtom); const savePreferences = useAtomSet(updateMobilePreferencesAtom); const ready = AsyncResult.isSuccess(preferencesResult) && !preferencesResult.waiting; - const order = resolveMicrophonePriority( - AsyncResult.isSuccess(preferencesResult) ? preferencesResult.value.microphonePriority : [], + const microphones = AsyncResult.isSuccess(preferencesResult) + ? (preferencesResult.value.microphones ?? []) + : []; + const [editing, setEditing] = useState(false); + const [drag, setDrag] = useState<{ readonly uid: string; readonly translation: number } | null>( + null, ); - const [drag, setDrag] = useState<{ - readonly kind: MicrophoneKind; - readonly translation: number; - } | null>(null); const [rowHeight, setRowHeight] = useState(0); // Each drop remounts the rows so the new order and the cleared offsets land in one frame. const [drops, setDrops] = useState(0); - const move = (from: number, to: number) => { - if (to < 0 || to >= order.length || from === to) return; - const next = [...order]; - const [moved] = next.splice(from, 1); - next.splice(to, 0, moved!); - savePreferences({ microphonePriority: next }); - }; + // Transforms apply to the stored list, so a microphone recorded meanwhile is kept. + const move = (uid: string, steps: number) => + savePreferences({ + transform: (current) => { + const next = [...(current.microphones ?? [])]; + const from = next.findIndex((microphone) => microphone.uid === uid); + const to = Math.min(next.length - 1, Math.max(0, from + steps)); + if (from === -1 || from === to) return {}; + const [moved] = next.splice(from, 1); + next.splice(to, 0, moved!); + return { microphones: next }; + }, + }); + const forget = (uid: string) => + savePreferences({ + transform: (current) => ({ + microphones: (current.microphones ?? []).filter((microphone) => microphone.uid !== uid), + }), + }); // A lifted row takes a neighbour's slot once it has moved past half of that row. - const dropIndex = (from: number, translation: number) => + const dropSteps = (from: number, translation: number) => rowHeight === 0 - ? from - : Math.min(order.length - 1, Math.max(0, from + Math.round(translation / rowHeight))); + ? 0 + : Math.min(microphones.length - 1, Math.max(0, from + Math.round(translation / rowHeight))) - + from; - const dragFrom = drag === null ? -1 : order.indexOf(drag.kind); - const dragTo = drag === null ? -1 : dropIndex(dragFrom, drag.translation); + const dragFrom = + drag === null ? -1 : microphones.findIndex((microphone) => microphone.uid === drag.uid); + const dragTo = drag === null ? -1 : dragFrom + dropSteps(dragFrom, drag.translation); return ( @@ -81,51 +81,78 @@ export function SettingsMicrophoneRouteScreen() { contentContainerClassName="gap-3 px-5 pt-4" contentContainerStyle={{ paddingBottom: Math.max(insets.bottom, 18) + 18 }} > - - {order.map((kind, index) => { - const shift = - dragFrom === -1 || index === dragFrom - ? 0 - : dragFrom < dragTo && index > dragFrom && index <= dragTo - ? -rowHeight - : dragFrom > dragTo && index < dragFrom && index >= dragTo - ? rowHeight - : 0; - return ( - setDrag({ kind, translation: 0 })} - onDragMove={(translation) => setDrag({ kind, translation })} - onDragEnd={(translation, cancelled) => { - setDrag(null); - setDrops((count) => count + 1); - if (!cancelled) move(index, dropIndex(index, translation)); - }} - onStep={(direction) => move(index, direction === "up" ? index - 1 : index + 1)} - /> - ); - })} - - - Voice input records from the first connected microphone in this list. Drag to reorder. - + {microphones.length === 0 ? ( + + {ready + ? "Microphones appear here after you use voice input with them connected." + : null} + + ) : ( + <> + setEditing((value) => !value)} + className="px-2 py-1 active:opacity-70" + > + + {editing ? "Done" : "Edit"} + + + } + > + {microphones.map((microphone, index) => { + const shift = + dragFrom === -1 || index === dragFrom + ? 0 + : dragFrom < dragTo && index > dragFrom && index <= dragTo + ? -rowHeight + : dragFrom > dragTo && index < dragFrom && index >= dragTo + ? rowHeight + : 0; + return ( + 1} + editing={editing} + offset={index === dragFrom ? (drag?.translation ?? 0) : shift} + lifted={index === dragFrom} + onHeight={setRowHeight} + onDragStart={() => setDrag({ uid: microphone.uid, translation: 0 })} + onDragMove={(translation) => setDrag({ uid: microphone.uid, translation })} + onDragEnd={(translation, cancelled) => { + setDrag(null); + setDrops((count) => count + 1); + if (!cancelled) move(microphone.uid, dropSteps(index, translation)); + }} + onStep={(direction) => move(microphone.uid, direction === "up" ? -1 : 1)} + onForget={() => forget(microphone.uid)} + /> + ); + })} + + + Voice input records from the first connected microphone in this list. New microphones + appear after you use voice input with them connected. + + + )} ); } function MicrophoneRow(props: { - readonly kind: MicrophoneKind; + readonly microphone: RememberedMicrophone; readonly position: number; readonly count: number; - readonly disabled: boolean; + readonly reorderable: boolean; + readonly editing: boolean; readonly offset: number; readonly lifted: boolean; readonly onHeight: (height: number) => void; @@ -133,9 +160,10 @@ function MicrophoneRow(props: { readonly onDragMove: (translation: number) => void; readonly onDragEnd: (translation: number, cancelled: boolean) => void; readonly onStep: (direction: "up" | "down") => void; + readonly onForget: () => void; }) { - const { lifted, offset } = props; - const microphone = MICROPHONES[props.kind]; + const { lifted, offset, microphone } = props; + const kind = KINDS[microphone.kind]; const style = useAnimatedStyle(() => ({ transform: [ { @@ -154,15 +182,32 @@ function MicrophoneRow(props: { "flex-row items-center gap-4 pl-4", props.position > 1 && !lifted && "border-t border-border-subtle", lifted && "bg-grouped-card shadow-md", + !props.reorderable && "pr-4", )} > + {props.editing ? ( + + + + ) : null} - {microphone.label} + {microphone.name} - {microphone.description} + {kind.label} - {props.disabled ? null : ( + {props.reorderable ? ( 1} canMoveDown={props.position < props.count} onStart={props.onDragStart} @@ -187,7 +232,7 @@ function MicrophoneRow(props: { onEnd={props.onDragEnd} onStep={props.onStep} /> - )} + ) : null} ); } diff --git a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx index 5b1607612e4b..cb1feeec45ed 100644 --- a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx +++ b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx @@ -1,4 +1,4 @@ -import { RegistryContext } from "@effect/atom-react"; +import { RegistryContext, useAtomMount } from "@effect/atom-react"; import { AudioModule, RecordingPresets, @@ -24,9 +24,9 @@ import { AppState, Platform } from "react-native"; import { AsyncResult } from "effect/reactivity"; import { useSharedValue } from "react-native-reanimated"; -import { preferredMicrophoneInput, resolveMicrophonePriority } from "../../lib/microphonePriority"; +import { preferredMicrophoneInput, rememberMicrophones } from "../../lib/microphonePriority"; import { getLocalVoiceTranscriber } from "../../native/voiceTranscription"; -import { mobilePreferencesAtom } from "../../state/preferences"; +import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences"; import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene"; import { VOICE_RECORDING_LIMIT_SECONDS, @@ -96,6 +96,9 @@ export function useGlobalVoiceInput() { function useVoiceInputRuntime() { const registry = use(RegistryContext); + // Dictation reads and records microphones without subscribing the provider to preferences. + useAtomMount(mobilePreferencesAtom); + useAtomMount(updateMobilePreferencesAtom); const [{ state, ownerKey, label }, setState] = useState({ state: INITIAL_STATE, ownerKey: null as string | null, @@ -134,12 +137,17 @@ function useVoiceInputRuntime() { // Read when recording starts so preference changes do not re-render every composer. selectInput: (inputs) => { const preferences = registry.get(mobilePreferencesAtom); - return preferredMicrophoneInput( - inputs, - resolveMicrophonePriority( - AsyncResult.isSuccess(preferences) ? preferences.value.microphonePriority : undefined, - ), - ); + if (!AsyncResult.isSuccess(preferences)) return null; + const known = preferences.value.microphones ?? []; + const microphones = rememberMicrophones(known, inputs); + if (microphones !== known) { + registry.set(updateMobilePreferencesAtom, { + transform: (current) => ({ + microphones: rememberMicrophones(current.microphones ?? [], inputs), + }), + }); + } + return preferredMicrophoneInput(inputs, microphones); }, }); recorderRef.current = recorder; diff --git a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts index 076d26cd6401..d0e3832fd94d 100644 --- a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts +++ b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts @@ -31,8 +31,8 @@ function createNativeRecorder(uri: string) { return { remove: () => listeners.delete(listener) }; }), getAvailableInputs: vi.fn((): ReadonlyArray => [ - { uid: "car", type: "CarAudio" }, - { uid: "built-in", type: "MicrophoneBuiltIn" }, + { uid: "car", name: "CarPlay", type: "CarAudio" }, + { uid: "built-in", name: "iPhone Microphone", type: "MicrophoneBuiltIn" }, ]), setInput: vi.fn((_uid: string) => undefined), release: vi.fn(), diff --git a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts index eae72b884ce2..678673e9910d 100644 --- a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts +++ b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts @@ -4,6 +4,7 @@ type Subscription = { remove(): void }; export interface VoiceRecordingInput { readonly uid: string; + readonly name: string; readonly type: string; } diff --git a/apps/mobile/src/lib/microphonePriority.test.ts b/apps/mobile/src/lib/microphonePriority.test.ts index 0970a60af533..85a3382db88d 100644 --- a/apps/mobile/src/lib/microphonePriority.test.ts +++ b/apps/mobile/src/lib/microphonePriority.test.ts @@ -1,43 +1,64 @@ import { describe, expect, it } from "vite-plus/test"; -import { - DEFAULT_MICROPHONE_PRIORITY, - preferredMicrophoneInput, - resolveMicrophonePriority, -} from "./microphonePriority"; - -const builtIn = { uid: "built-in", type: "MicrophoneBuiltIn" }; -const airPods = { uid: "airpods", type: "BluetoothHFP" }; -const carPlay = { uid: "car", type: "CarAudio" }; -const usb = { uid: "usb", type: "USBAudio" }; - -describe("resolveMicrophonePriority", () => { - it("keeps the saved order and appends kinds it does not mention", () => { - expect(resolveMicrophonePriority(["builtIn", "carPlay", "builtIn"])).toEqual([ - "builtIn", - "carPlay", - "wired", - "bluetooth", +import { preferredMicrophoneInput, rememberMicrophones } from "./microphonePriority"; + +const iPhone = { uid: "Built-In Microphone", name: "iPhone Microphone", type: "MicrophoneBuiltIn" }; +const airPods = { uid: "airpods", name: "Dak’s AirPods Pro", type: "BluetoothHFP" }; +const bluetoothCar = { uid: "civic", name: "Honda Civic", type: "BluetoothHFP" }; +const carPlay = { uid: "carplay", name: "CarPlay", type: "CarAudio" }; +const usb = { uid: "usb", name: "Shure MV7", type: "USBAudio" }; + +const uids = (microphones: ReadonlyArray<{ readonly uid: string }>) => + microphones.map((microphone) => microphone.uid); + +describe("rememberMicrophones", () => { + it("remembers new microphones in the default order, CarPlay below the iPhone", () => { + expect(uids(rememberMicrophones([], [carPlay, iPhone, airPods]))).toEqual([ + "airpods", + "Built-In Microphone", + "carplay", ]); - expect(resolveMicrophonePriority(undefined)).toEqual(DEFAULT_MICROPHONE_PRIORITY); }); -}); -describe("preferredMicrophoneInput", () => { - it("records from the device instead of CarPlay by default", () => { - const priority = resolveMicrophonePriority(undefined); - expect(preferredMicrophoneInput([carPlay, builtIn], priority)).toBe(builtIn); - expect(preferredMicrophoneInput([builtIn, airPods], priority)).toBe(airPods); + it("files a new device after the last one of its kind, keeping the user's order", () => { + const reordered = rememberMicrophones([], [iPhone, airPods]).toReversed(); + expect(uids(reordered)).toEqual(["Built-In Microphone", "airpods"]); + + expect(uids(rememberMicrophones(reordered, [bluetoothCar, iPhone]))).toEqual([ + "Built-In Microphone", + "airpods", + "civic", + ]); + expect(uids(rememberMicrophones(reordered, [usb]))).toEqual([ + "usb", + "Built-In Microphone", + "airpods", + ]); }); - it("follows the saved order", () => { - const priority = resolveMicrophonePriority(["builtIn"]); - expect(preferredMicrophoneInput([airPods, usb, builtIn], priority)).toBe(builtIn); - expect(preferredMicrophoneInput([airPods, usb], priority)).toBe(usb); + it("updates a renamed device in place and returns the same list when nothing changed", () => { + const remembered = rememberMicrophones([], [iPhone, airPods]); + expect(rememberMicrophones(remembered, [iPhone, airPods])).toBe(remembered); + expect(rememberMicrophones(remembered, [{ uid: "virtual", name: "x", type: "Virtual" }])).toBe( + remembered, + ); + + const renamed = rememberMicrophones(remembered, [{ ...airPods, name: "AirPods Max" }]); + expect(renamed.map((microphone) => microphone.name)).toEqual([ + "AirPods Max", + "iPhone Microphone", + ]); + }); +}); + +describe("preferredMicrophoneInput", () => { + it("records from the first connected remembered microphone", () => { + const remembered = rememberMicrophones([], [iPhone, airPods, bluetoothCar]).toReversed(); + expect(preferredMicrophoneInput([airPods, iPhone], remembered)).toBe(iPhone); + expect(preferredMicrophoneInput([airPods, bluetoothCar], remembered)).toBe(bluetoothCar); }); - it("leaves unknown inputs to iOS", () => { - const priority = resolveMicrophonePriority(undefined); - expect(preferredMicrophoneInput([{ uid: "x", type: "Virtual" }], priority)).toBeNull(); + it("leaves microphones it does not know to iOS", () => { + expect(preferredMicrophoneInput([usb], rememberMicrophones([], [iPhone]))).toBeNull(); }); }); diff --git a/apps/mobile/src/lib/microphonePriority.ts b/apps/mobile/src/lib/microphonePriority.ts index 43f83b7bad2f..c5eaae27874e 100644 --- a/apps/mobile/src/lib/microphonePriority.ts +++ b/apps/mobile/src/lib/microphonePriority.ts @@ -1,24 +1,35 @@ /** - * The kinds of microphone voice input can record from, in the order the user - * prefers them. iOS reports each input's port type; several port types share - * a kind. A car without CarPlay connects as a Bluetooth headset, so only - * CarPlay can be ranked on its own. + * Microphones voice input has recorded near, in the order the user prefers + * them. iOS lists only connected inputs, and only while a recording session + * is set up, so each microphone is remembered the first time dictation sees + * it and then appears in Settings. */ export const MICROPHONE_KINDS = ["wired", "bluetooth", "builtIn", "carPlay"] as const; export type MicrophoneKind = (typeof MICROPHONE_KINDS)[number]; -/** CarPlay ranks below the device microphone so dictation in the car avoids the car's microphone. */ -export const DEFAULT_MICROPHONE_PRIORITY: ReadonlyArray = MICROPHONE_KINDS; +export interface RememberedMicrophone { + /** iOS's port identifier, stable across reconnects of the same device. */ + readonly uid: string; + readonly name: string; + readonly kind: MicrophoneKind; +} + +interface MicrophoneInput { + readonly uid: string; + readonly name: string; + readonly type: string; +} -// Raw values of AVAudioSession.Port. +// Raw values of AVAudioSession.Port. A car without CarPlay connects as a +// Bluetooth headset, so it shares the Bluetooth kind. const KIND_BY_PORT_TYPE: Readonly> = { - MicrophoneBuiltIn: "builtIn", - BluetoothHFP: "bluetooth", - BluetoothLE: "bluetooth", MicrophoneWired: "wired", USBAudio: "wired", LineIn: "wired", + BluetoothHFP: "bluetooth", + BluetoothLE: "bluetooth", + MicrophoneBuiltIn: "builtIn", CarAudio: "carPlay", }; @@ -26,20 +37,55 @@ export function isMicrophoneKind(value: unknown): value is MicrophoneKind { return (MICROPHONE_KINDS as ReadonlyArray).includes(value); } -/** The saved order, with kinds it does not mention appended in their default order. */ -export function resolveMicrophonePriority( - saved: ReadonlyArray | undefined, -): ReadonlyArray { - return [...new Set([...(saved ?? []), ...DEFAULT_MICROPHONE_PRIORITY])]; +/** + * Adds connected microphones the list has not seen and refreshes renamed ones. + * A new microphone goes after the last one of its kind, so a preference shown + * for AirPods carries over to a Bluetooth car. A kind seen for the first time + * ranks by MICROPHONE_KINDS, which keeps CarPlay below the device microphone. + * Returns `remembered` itself when nothing changed. + */ +export function rememberMicrophones( + remembered: ReadonlyArray, + inputs: ReadonlyArray, +): ReadonlyArray { + const rank = (kind: MicrophoneKind) => MICROPHONE_KINDS.indexOf(kind); + // Copies, not ES2023 array methods: Hermes does not ship toSorted or toSpliced. + const next = [...remembered]; + let changed = false; + const seen = inputs + .flatMap((input) => { + const kind = KIND_BY_PORT_TYPE[input.type]; + return kind ? [{ uid: input.uid, name: input.name, kind }] : []; + }) + .sort((left, right) => rank(left.kind) - rank(right.kind)); + for (const microphone of seen) { + const index = next.findIndex((entry) => entry.uid === microphone.uid); + if (index !== -1) { + if (next[index]!.name !== microphone.name) { + next[index] = microphone; + changed = true; + } + continue; + } + let at = next.length; + for (let candidate = next.length - 1; candidate >= 0; candidate -= 1) { + if (rank(next[candidate]!.kind) <= rank(microphone.kind)) break; + at = candidate; + } + const lastOfKind = next.map((entry) => entry.kind).lastIndexOf(microphone.kind); + next.splice(lastOfKind !== -1 ? lastOfKind + 1 : at, 0, microphone); + changed = true; + } + return changed ? next : remembered; } -/** The available input of the highest-ranked kind, or null to leave the route to iOS. */ -export function preferredMicrophoneInput( +/** The connected input listed first, or null to leave the route to iOS. */ +export function preferredMicrophoneInput( inputs: ReadonlyArray, - priority: ReadonlyArray, + remembered: ReadonlyArray, ): Input | null { - for (const kind of priority) { - const input = inputs.find((candidate) => KIND_BY_PORT_TYPE[candidate.type] === kind); + for (const microphone of remembered) { + const input = inputs.find((candidate) => candidate.uid === microphone.uid); if (input) return input; } return null; diff --git a/apps/mobile/src/persistence/mobile-preferences.ts b/apps/mobile/src/persistence/mobile-preferences.ts index c53c806b06cb..d32b7f5bbfb6 100644 --- a/apps/mobile/src/persistence/mobile-preferences.ts +++ b/apps/mobile/src/persistence/mobile-preferences.ts @@ -8,7 +8,7 @@ import * as Semaphore from "effect/Semaphore"; import type { ProviderInstanceId, SidebarProjectGroupingMode } from "@t3tools/contracts"; import type { ComposerEnterBehavior } from "../lib/composerEnterBehavior"; import type { FollowUpBehavior } from "../lib/followUpBehavior"; -import { isMicrophoneKind, type MicrophoneKind } from "../lib/microphonePriority"; +import { isMicrophoneKind, type RememberedMicrophone } from "../lib/microphonePriority"; import { MOBILE_THEME_IDS, type MobileThemeId, type MobileThemeMode } from "../lib/mobileTheme"; import * as MobileDatabase from "./mobile-database"; import * as MobileSecureStorage from "./mobile-secure-storage"; @@ -37,8 +37,8 @@ export interface Preferences { * message sent during a running turn queues behind it or steers it. */ readonly followUpBehavior?: FollowUpBehavior; - /** The order voice input tries microphone kinds in, most preferred first. iOS only. */ - readonly microphonePriority?: ReadonlyArray; + /** Microphones voice input has seen, most preferred first. iOS only. */ + readonly microphones?: ReadonlyArray; /** @deprecated Kept temporarily so older OTA bundles retain the selected mode. */ readonly projectGroupingEnabled?: boolean; readonly projectGroupingMode?: SidebarProjectGroupingMode; @@ -110,7 +110,7 @@ function sanitizePreferences(parsed: Preferences): Preferences { collapsedProjectGroups?: readonly string[]; composerEnterBehavior?: ComposerEnterBehavior; followUpBehavior?: FollowUpBehavior; - microphonePriority?: ReadonlyArray; + microphones?: ReadonlyArray; projectGroupingEnabled?: boolean; projectGroupingMode?: SidebarProjectGroupingMode; planModeEnabled?: boolean; @@ -176,8 +176,15 @@ function sanitizePreferences(parsed: Preferences): Preferences { if (parsed.followUpBehavior === "queue" || parsed.followUpBehavior === "steer") { preferences.followUpBehavior = parsed.followUpBehavior; } - if (Array.isArray(parsed.microphonePriority)) { - preferences.microphonePriority = parsed.microphonePriority.filter(isMicrophoneKind); + if (Array.isArray(parsed.microphones)) { + preferences.microphones = parsed.microphones.filter( + (microphone) => + typeof microphone === "object" && + microphone !== null && + typeof microphone.uid === "string" && + typeof microphone.name === "string" && + isMicrophoneKind(microphone.kind), + ); } if (typeof parsed.projectGroupingEnabled === "boolean") { preferences.projectGroupingEnabled = parsed.projectGroupingEnabled; diff --git a/docs/user/composer.md b/docs/user/composer.md index 5c51d1ff2bd6..e59838cf64b6 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -165,8 +165,10 @@ recording and preserves your existing draft. While recording, the screen stays awake; it can sleep normally once recording stops. Voice input records from the first connected microphone in **Settings → Microphone**. -Drag the list to choose, for example, the iPhone microphone over AirPods or your car. -By default, CarPlay comes after the iPhone microphone. +A microphone joins that list the first time you use voice input with it connected, +such as your AirPods or your car. Drag the list to prefer, for example, the iPhone +microphone over either. New Bluetooth devices start below the ones you already +ranked, and CarPlay starts below the iPhone microphone. Transcription runs on your device. T3 Code deletes the temporary audio after transcription or cancellation; only the message text is sent when you submit. From 6b5ef5c56eecedc153f24a535419e3c20f793b1f Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 10 Oct 2026 12:42:38 -0700 Subject: [PATCH 3/4] feat(mobile): ask for microphone access from the Microphone settings Without microphone access the list stayed empty with no explanation. The screen now shows an Allow microphone access row while iOS can still ask, and Open Settings once it can't, rechecking when the app returns. Co-Authored-By: Claude Opus 5.5 --- .../SettingsMicrophoneRouteScreen.tsx | 63 ++++++++++++++++--- 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx index e8a448e3850e..ac4a3e420605 100644 --- a/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx @@ -1,7 +1,8 @@ import { useAtomSet, useAtomValue } from "@effect/atom-react"; import { AsyncResult } from "effect/reactivity"; -import { useState } from "react"; -import { Pressable, ScrollView, View } from "react-native"; +import { getRecordingPermissionsAsync, requestRecordingPermissionsAsync } from "expo-audio"; +import { useCallback, useEffect, useState } from "react"; +import { AppState, Linking, Pressable, ScrollView, View } from "react-native"; import Reanimated, { ReduceMotion, useAnimatedStyle, withTiming } from "react-native-reanimated"; import { useSafeAreaInsets } from "react-native-safe-area-context"; @@ -10,11 +11,40 @@ import { AppText as Text } from "../../components/AppText"; import { cn } from "../../lib/cn"; import type { MicrophoneKind, RememberedMicrophone } from "../../lib/microphonePriority"; import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences"; +import { SettingsActionRow } from "./components/SettingsActionRow"; import { SettingsDragHandle } from "./components/SettingsDragHandle"; import { SettingsSection } from "./components/SettingsSection"; const REMOVE_SIZE = 20; +type MicrophonePermission = "checking" | "granted" | "ask" | "denied"; + +/** Re-reads microphone access whenever the app returns from system Settings. */ +function useMicrophonePermission() { + const [permission, setPermission] = useState("checking"); + const apply = useCallback( + (response: { readonly granted: boolean; readonly canAskAgain: boolean }) => + setPermission(response.granted ? "granted" : response.canAskAgain ? "ask" : "denied"), + [], + ); + const refresh = useCallback( + () => void getRecordingPermissionsAsync().then(apply, () => setPermission("ask")), + [apply], + ); + useEffect(() => { + refresh(); + const subscription = AppState.addEventListener("change", (state) => { + if (state === "active") refresh(); + }); + return () => subscription.remove(); + }, [refresh]); + const request = useCallback( + () => void requestRecordingPermissionsAsync().then(apply, refresh), + [apply, refresh], + ); + return { permission, request }; +} + const KINDS: Record = { wired: { label: "Wired", icon: "cable.connector" }, bluetooth: { label: "Bluetooth", icon: "headphones" }, @@ -34,6 +64,7 @@ export function SettingsMicrophoneRouteScreen() { const microphones = AsyncResult.isSuccess(preferencesResult) ? (preferencesResult.value.microphones ?? []) : []; + const { permission, request: requestPermission } = useMicrophonePermission(); const [editing, setEditing] = useState(false); const [drag, setDrag] = useState<{ readonly uid: string; readonly translation: number } | null>( null, @@ -81,12 +112,30 @@ export function SettingsMicrophoneRouteScreen() { contentContainerClassName="gap-3 px-5 pt-4" contentContainerStyle={{ paddingBottom: Math.max(insets.bottom, 18) + 18 }} > + {permission === "ask" || permission === "denied" ? ( + <> + + void Linking.openSettings() + } + /> + + + {permission === "ask" + ? "Voice input needs microphone access before it can record or list your microphones." + : "Microphone access is off for T3 Code. Turn it on in Settings to use voice input."} + + + ) : null} {microphones.length === 0 ? ( - - {ready - ? "Microphones appear here after you use voice input with them connected." - : null} - + permission === "granted" && ready ? ( + + Microphones appear here after you use voice input with them connected. + + ) : null ) : ( <> Date: Sat, 10 Oct 2026 12:49:11 -0700 Subject: [PATCH 4/4] feat(mobile): rank microphone kinds and devices in one list Ranking only remembered devices left a device voice input had never seen to iOS, and gave no way to prefer "any other Bluetooth device" over specific AirPods. The Microphone order now mixes kind rows (built-in, other wired, other Bluetooth, CarPlay) with wired and Bluetooth devices listed by name. A kind row ranks its devices not listed by name, a new device is listed just above its kind, and saved device-only orders keep their ranking. Co-Authored-By: Claude Opus 5.5 --- .../SettingsMicrophoneRouteScreen.tsx | 218 ++++++++++-------- .../voice-input/VoiceInputProvider.tsx | 16 +- .../mobile/src/lib/microphonePriority.test.ts | 105 ++++++--- apps/mobile/src/lib/microphonePriority.ts | 128 ++++++---- .../src/persistence/mobile-preferences.ts | 23 +- docs/user/composer.md | 9 +- 6 files changed, 304 insertions(+), 195 deletions(-) diff --git a/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx index ac4a3e420605..9013258f3282 100644 --- a/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx @@ -9,7 +9,13 @@ import { useSafeAreaInsets } from "react-native-safe-area-context"; import { type AppSymbolName, SymbolView } from "../../components/AppSymbol"; import { AppText as Text } from "../../components/AppText"; import { cn } from "../../lib/cn"; -import type { MicrophoneKind, RememberedMicrophone } from "../../lib/microphonePriority"; +import { + isMicrophoneDevice, + type MicrophoneEntry, + type MicrophoneKind, + microphoneEntryKey, + microphoneOrder, +} from "../../lib/microphonePriority"; import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences"; import { SettingsActionRow } from "./components/SettingsActionRow"; import { SettingsDragHandle } from "./components/SettingsDragHandle"; @@ -45,40 +51,71 @@ function useMicrophonePermission() { return { permission, request }; } -const KINDS: Record = { - wired: { label: "Wired", icon: "cable.connector" }, - bluetooth: { label: "Bluetooth", icon: "headphones" }, - builtIn: { label: "Built-in", icon: "iphone" }, - carPlay: { label: "CarPlay", icon: "car" }, +const KINDS: Record< + MicrophoneKind, + { + /** Shown under a remembered device's name. */ + readonly label: string; + readonly row: string; + readonly rowDescription: string; + readonly icon: AppSymbolName; + } +> = { + wired: { + label: "Wired", + row: "Other wired", + rowDescription: "Headsets and USB microphones not listed", + icon: "cable.connector", + }, + bluetooth: { + label: "Bluetooth", + row: "Other Bluetooth", + rowDescription: "AirPods, headsets, and cars not listed", + icon: "headphones", + }, + builtIn: { + label: "Built-in", + row: "Built-in microphone", + rowDescription: "The microphone on this device", + icon: "iphone", + }, + carPlay: { + label: "CarPlay", + row: "CarPlay", + rowDescription: "The car’s microphone", + icon: "car", + }, }; /** - * Microphones voice input has seen, in the order it prefers them. iOS only. - * A microphone joins the list the first time it is connected during dictation. + * Microphone kinds and remembered devices, in the order voice input prefers + * them. iOS only. A device joins the list the first time it is connected + * during dictation; until then its kind's row ranks it. */ export function SettingsMicrophoneRouteScreen() { const insets = useSafeAreaInsets(); const preferencesResult = useAtomValue(mobilePreferencesAtom); const savePreferences = useAtomSet(updateMobilePreferencesAtom); const ready = AsyncResult.isSuccess(preferencesResult) && !preferencesResult.waiting; - const microphones = AsyncResult.isSuccess(preferencesResult) - ? (preferencesResult.value.microphones ?? []) - : []; + const entries = microphoneOrder( + AsyncResult.isSuccess(preferencesResult) ? preferencesResult.value.microphones : undefined, + ); + const hasDevices = entries.some(isMicrophoneDevice); const { permission, request: requestPermission } = useMicrophonePermission(); const [editing, setEditing] = useState(false); - const [drag, setDrag] = useState<{ readonly uid: string; readonly translation: number } | null>( + const [drag, setDrag] = useState<{ readonly key: string; readonly translation: number } | null>( null, ); const [rowHeight, setRowHeight] = useState(0); // Each drop remounts the rows so the new order and the cleared offsets land in one frame. const [drops, setDrops] = useState(0); - // Transforms apply to the stored list, so a microphone recorded meanwhile is kept. - const move = (uid: string, steps: number) => + // Transforms apply to the stored order, so a device recorded meanwhile is kept. + const move = (key: string, steps: number) => savePreferences({ transform: (current) => { - const next = [...(current.microphones ?? [])]; - const from = next.findIndex((microphone) => microphone.uid === uid); + const next = [...microphoneOrder(current.microphones)]; + const from = next.findIndex((entry) => microphoneEntryKey(entry) === key); const to = Math.min(next.length - 1, Math.max(0, from + steps)); if (from === -1 || from === to) return {}; const [moved] = next.splice(from, 1); @@ -86,21 +123,23 @@ export function SettingsMicrophoneRouteScreen() { return { microphones: next }; }, }); - const forget = (uid: string) => + const forget = (key: string) => savePreferences({ transform: (current) => ({ - microphones: (current.microphones ?? []).filter((microphone) => microphone.uid !== uid), + microphones: microphoneOrder(current.microphones).filter( + (entry) => microphoneEntryKey(entry) !== key, + ), }), }); // A lifted row takes a neighbour's slot once it has moved past half of that row. const dropSteps = (from: number, translation: number) => rowHeight === 0 ? 0 - : Math.min(microphones.length - 1, Math.max(0, from + Math.round(translation / rowHeight))) - + : Math.min(entries.length - 1, Math.max(0, from + Math.round(translation / rowHeight))) - from; const dragFrom = - drag === null ? -1 : microphones.findIndex((microphone) => microphone.uid === drag.uid); + drag === null ? -1 : entries.findIndex((entry) => microphoneEntryKey(entry) === drag.key); const dragTo = drag === null ? -1 : dragFrom + dropSteps(dragFrom, drag.translation); return ( @@ -130,74 +169,68 @@ export function SettingsMicrophoneRouteScreen() { ) : null} - {microphones.length === 0 ? ( - permission === "granted" && ready ? ( - - Microphones appear here after you use voice input with them connected. - - ) : null - ) : ( - <> - setEditing((value) => !value)} - className="px-2 py-1 active:opacity-70" - > - - {editing ? "Done" : "Edit"} - - - } - > - {microphones.map((microphone, index) => { - const shift = - dragFrom === -1 || index === dragFrom - ? 0 - : dragFrom < dragTo && index > dragFrom && index <= dragTo - ? -rowHeight - : dragFrom > dragTo && index < dragFrom && index >= dragTo - ? rowHeight - : 0; - return ( - 1} - editing={editing} - offset={index === dragFrom ? (drag?.translation ?? 0) : shift} - lifted={index === dragFrom} - onHeight={setRowHeight} - onDragStart={() => setDrag({ uid: microphone.uid, translation: 0 })} - onDragMove={(translation) => setDrag({ uid: microphone.uid, translation })} - onDragEnd={(translation, cancelled) => { - setDrag(null); - setDrops((count) => count + 1); - if (!cancelled) move(microphone.uid, dropSteps(index, translation)); - }} - onStep={(direction) => move(microphone.uid, direction === "up" ? -1 : 1)} - onForget={() => forget(microphone.uid)} - /> - ); - })} - - - Voice input records from the first connected microphone in this list. New microphones - appear after you use voice input with them connected. - - - )} + setEditing((value) => !value)} + className="px-2 py-1 active:opacity-70" + > + + {editing ? "Done" : "Edit"} + + + ) : undefined + } + > + {entries.map((entry, index) => { + const key = microphoneEntryKey(entry); + const shift = + dragFrom === -1 || index === dragFrom + ? 0 + : dragFrom < dragTo && index > dragFrom && index <= dragTo + ? -rowHeight + : dragFrom > dragTo && index < dragFrom && index >= dragTo + ? rowHeight + : 0; + return ( + setDrag({ key, translation: 0 })} + onDragMove={(translation) => setDrag({ key, translation })} + onDragEnd={(translation, cancelled) => { + setDrag(null); + setDrops((count) => count + 1); + if (!cancelled) move(key, dropSteps(index, translation)); + }} + onStep={(direction) => move(key, direction === "up" ? -1 : 1)} + onForget={isMicrophoneDevice(entry) ? () => forget(key) : undefined} + /> + ); + })} + + + Voice input records from the first connected microphone in this list. Wired and Bluetooth + devices are added by name after you use voice input with them connected; until then, the + Other row ranks them. + ); } function MicrophoneRow(props: { - readonly microphone: RememberedMicrophone; + readonly entry: MicrophoneEntry; readonly position: number; readonly count: number; readonly reorderable: boolean; @@ -209,10 +242,11 @@ function MicrophoneRow(props: { readonly onDragMove: (translation: number) => void; readonly onDragEnd: (translation: number, cancelled: boolean) => void; readonly onStep: (direction: "up" | "down") => void; - readonly onForget: () => void; + readonly onForget: (() => void) | undefined; }) { - const { lifted, offset, microphone } = props; - const kind = KINDS[microphone.kind]; + const { lifted, offset, entry } = props; + const kind = KINDS[entry.kind]; + const title = isMicrophoneDevice(entry) ? entry.name : kind.row; const style = useAnimatedStyle(() => ({ transform: [ { @@ -234,10 +268,10 @@ function MicrophoneRow(props: { !props.reorderable && "pr-4", )} > - {props.editing ? ( + {props.editing && props.onForget ? ( + ) : props.editing ? ( + ) : null} - {microphone.name} + {title} - {kind.label} + {isMicrophoneDevice(entry) ? kind.label : kind.rowDescription} {props.reorderable ? ( 1} canMoveDown={props.position < props.count} onStart={props.onDragStart} diff --git a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx index cb1feeec45ed..6dce91180700 100644 --- a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx +++ b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx @@ -24,7 +24,11 @@ import { AppState, Platform } from "react-native"; import { AsyncResult } from "effect/reactivity"; import { useSharedValue } from "react-native-reanimated"; -import { preferredMicrophoneInput, rememberMicrophones } from "../../lib/microphonePriority"; +import { + microphoneOrder, + preferredMicrophoneInput, + rememberMicrophones, +} from "../../lib/microphonePriority"; import { getLocalVoiceTranscriber } from "../../native/voiceTranscription"; import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences"; import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene"; @@ -138,16 +142,16 @@ function useVoiceInputRuntime() { selectInput: (inputs) => { const preferences = registry.get(mobilePreferencesAtom); if (!AsyncResult.isSuccess(preferences)) return null; - const known = preferences.value.microphones ?? []; - const microphones = rememberMicrophones(known, inputs); - if (microphones !== known) { + const known = microphoneOrder(preferences.value.microphones); + const order = rememberMicrophones(known, inputs); + if (order !== known) { registry.set(updateMobilePreferencesAtom, { transform: (current) => ({ - microphones: rememberMicrophones(current.microphones ?? [], inputs), + microphones: rememberMicrophones(microphoneOrder(current.microphones), inputs), }), }); } - return preferredMicrophoneInput(inputs, microphones); + return preferredMicrophoneInput(inputs, order); }, }); recorderRef.current = recorder; diff --git a/apps/mobile/src/lib/microphonePriority.test.ts b/apps/mobile/src/lib/microphonePriority.test.ts index 85a3382db88d..e92459cb314e 100644 --- a/apps/mobile/src/lib/microphonePriority.test.ts +++ b/apps/mobile/src/lib/microphonePriority.test.ts @@ -1,64 +1,93 @@ import { describe, expect, it } from "vite-plus/test"; -import { preferredMicrophoneInput, rememberMicrophones } from "./microphonePriority"; +import { + type MicrophoneEntry, + microphoneEntryKey, + microphoneOrder, + preferredMicrophoneInput, + rememberMicrophones, +} from "./microphonePriority"; const iPhone = { uid: "Built-In Microphone", name: "iPhone Microphone", type: "MicrophoneBuiltIn" }; -const airPods = { uid: "airpods", name: "Dak’s AirPods Pro", type: "BluetoothHFP" }; -const bluetoothCar = { uid: "civic", name: "Honda Civic", type: "BluetoothHFP" }; +const airPods = { uid: "airpods", name: "Julius’s AirPods Pro", type: "BluetoothHFP" }; +const bose = { uid: "bose", name: "Julius’s Bose QC", type: "BluetoothHFP" }; +const civic = { uid: "civic", name: "Honda Civic", type: "BluetoothHFP" }; const carPlay = { uid: "carplay", name: "CarPlay", type: "CarAudio" }; const usb = { uid: "usb", name: "Shure MV7", type: "USBAudio" }; -const uids = (microphones: ReadonlyArray<{ readonly uid: string }>) => - microphones.map((microphone) => microphone.uid); +const keys = (order: ReadonlyArray) => order.map(microphoneEntryKey); -describe("rememberMicrophones", () => { - it("remembers new microphones in the default order, CarPlay below the iPhone", () => { - expect(uids(rememberMicrophones([], [carPlay, iPhone, airPods]))).toEqual([ - "airpods", - "Built-In Microphone", - "carplay", +describe("microphoneOrder", () => { + it("lists every kind once, CarPlay below the built-in microphone by default", () => { + expect(keys(microphoneOrder())).toEqual([ + "kind:wired", + "kind:bluetooth", + "kind:builtIn", + "kind:carPlay", ]); }); - it("files a new device after the last one of its kind, keeping the user's order", () => { - const reordered = rememberMicrophones([], [iPhone, airPods]).toReversed(); - expect(uids(reordered)).toEqual(["Built-In Microphone", "airpods"]); - - expect(uids(rememberMicrophones(reordered, [bluetoothCar, iPhone]))).toEqual([ - "Built-In Microphone", + it("keeps a saved order and files missing kinds after their devices", () => { + const saved: MicrophoneEntry[] = [ + { kind: "builtIn" }, + { kind: "bluetooth", uid: "airpods", name: "AirPods" }, + { kind: "builtIn" }, + ]; + expect(keys(microphoneOrder(saved))).toEqual([ + "kind:wired", + "kind:builtIn", "airpods", - "civic", + "kind:bluetooth", + "kind:carPlay", ]); - expect(uids(rememberMicrophones(reordered, [usb]))).toEqual([ - "usb", - "Built-In Microphone", + }); +}); + +describe("rememberMicrophones", () => { + it("lists a new device just above its kind and leaves built-in and CarPlay as kinds", () => { + const order = microphoneOrder(); + expect(keys(rememberMicrophones(order, [iPhone, carPlay, airPods]))).toEqual([ + "kind:wired", "airpods", + "kind:bluetooth", + "kind:builtIn", + "kind:carPlay", ]); }); - it("updates a renamed device in place and returns the same list when nothing changed", () => { - const remembered = rememberMicrophones([], [iPhone, airPods]); - expect(rememberMicrophones(remembered, [iPhone, airPods])).toBe(remembered); - expect(rememberMicrophones(remembered, [{ uid: "virtual", name: "x", type: "Virtual" }])).toBe( - remembered, - ); + it("refreshes renamed devices and returns the same order when nothing changed", () => { + const order = rememberMicrophones(microphoneOrder(), [airPods]); + expect(rememberMicrophones(order, [airPods, iPhone])).toBe(order); - const renamed = rememberMicrophones(remembered, [{ ...airPods, name: "AirPods Max" }]); - expect(renamed.map((microphone) => microphone.name)).toEqual([ - "AirPods Max", - "iPhone Microphone", - ]); + const renamed = rememberMicrophones(order, [{ ...airPods, name: "AirPods Max" }]); + expect(renamed.find((entry) => microphoneEntryKey(entry) === "airpods")).toEqual({ + kind: "bluetooth", + uid: "airpods", + name: "AirPods Max", + }); }); }); describe("preferredMicrophoneInput", () => { - it("records from the first connected remembered microphone", () => { - const remembered = rememberMicrophones([], [iPhone, airPods, bluetoothCar]).toReversed(); - expect(preferredMicrophoneInput([airPods, iPhone], remembered)).toBe(iPhone); - expect(preferredMicrophoneInput([airPods, bluetoothCar], remembered)).toBe(bluetoothCar); + // Built-in, Bose, Wired, other Bluetooth, AirPods, CarPlay. + const order: MicrophoneEntry[] = [ + { kind: "builtIn" }, + { kind: "bluetooth", uid: "bose", name: bose.name }, + { kind: "wired" }, + { kind: "bluetooth" }, + { kind: "bluetooth", uid: "airpods", name: airPods.name }, + { kind: "carPlay" }, + ]; + + it("ranks unlisted devices of a kind at that kind, ahead of a device listed below it", () => { + expect(preferredMicrophoneInput([airPods, civic], order)).toBe(civic); + expect(preferredMicrophoneInput([airPods, bose], order)).toBe(bose); + expect(preferredMicrophoneInput([airPods, usb], order)).toBe(usb); + expect(preferredMicrophoneInput([airPods, carPlay], order)).toBe(airPods); + expect(preferredMicrophoneInput([airPods, iPhone], order)).toBe(iPhone); }); - it("leaves microphones it does not know to iOS", () => { - expect(preferredMicrophoneInput([usb], rememberMicrophones([], [iPhone]))).toBeNull(); + it("leaves inputs of unknown kinds to iOS", () => { + expect(preferredMicrophoneInput([{ uid: "x", name: "x", type: "Virtual" }], order)).toBeNull(); }); }); diff --git a/apps/mobile/src/lib/microphonePriority.ts b/apps/mobile/src/lib/microphonePriority.ts index c5eaae27874e..46acb36d2f33 100644 --- a/apps/mobile/src/lib/microphonePriority.ts +++ b/apps/mobile/src/lib/microphonePriority.ts @@ -1,20 +1,23 @@ /** - * Microphones voice input has recorded near, in the order the user prefers - * them. iOS lists only connected inputs, and only while a recording session - * is set up, so each microphone is remembered the first time dictation sees - * it and then appears in Settings. + * The order voice input picks a microphone in. An entry is either a kind, + * which covers every connected microphone of that kind not listed by name, or + * a device remembered the first time dictation saw it connected. iOS lists + * only connected inputs, and only while a recording session is set up, so a + * device cannot be listed before then. */ export const MICROPHONE_KINDS = ["wired", "bluetooth", "builtIn", "carPlay"] as const; export type MicrophoneKind = (typeof MICROPHONE_KINDS)[number]; -export interface RememberedMicrophone { +export interface MicrophoneDevice { + readonly kind: MicrophoneKind; /** iOS's port identifier, stable across reconnects of the same device. */ readonly uid: string; readonly name: string; - readonly kind: MicrophoneKind; } +export type MicrophoneEntry = { readonly kind: MicrophoneKind } | MicrophoneDevice; + interface MicrophoneInput { readonly uid: string; readonly name: string; @@ -22,7 +25,7 @@ interface MicrophoneInput { } // Raw values of AVAudioSession.Port. A car without CarPlay connects as a -// Bluetooth headset, so it shares the Bluetooth kind. +// Bluetooth headset, so it is a Bluetooth device. const KIND_BY_PORT_TYPE: Readonly> = { MicrophoneWired: "wired", USBAudio: "wired", @@ -33,59 +36,96 @@ const KIND_BY_PORT_TYPE: Readonly> = { CarAudio: "carPlay", }; +// Built-in and CarPlay are listed only as kinds: a phone has one built-in +// microphone, and CarPlay is the car. +const DEVICE_KINDS: ReadonlySet = new Set(["wired", "bluetooth"]); + +const rank = (kind: MicrophoneKind) => MICROPHONE_KINDS.indexOf(kind); + export function isMicrophoneKind(value: unknown): value is MicrophoneKind { return (MICROPHONE_KINDS as ReadonlyArray).includes(value); } +export function isMicrophoneDevice(entry: MicrophoneEntry): entry is MicrophoneDevice { + return "uid" in entry; +} + +export function microphoneEntryKey(entry: MicrophoneEntry): string { + return isMicrophoneDevice(entry) ? entry.uid : `kind:${entry.kind}`; +} + +/** + * The saved order with each kind listed once. A missing kind goes after the + * devices of that kind, else above the first kind ranked below it by default, + * which puts CarPlay below the built-in microphone. + */ +export function microphoneOrder( + saved: ReadonlyArray = [], +): ReadonlyArray { + const order: MicrophoneEntry[] = []; + const keys = new Set(); + for (const entry of saved) { + const row = + isMicrophoneDevice(entry) && DEVICE_KINDS.has(entry.kind) ? entry : { kind: entry.kind }; + const key = microphoneEntryKey(row); + if (keys.has(key)) continue; + keys.add(key); + order.push(row); + } + for (const kind of MICROPHONE_KINDS) { + if (keys.has(`kind:${kind}`)) continue; + const lastDevice = order.map((entry) => entry.kind).lastIndexOf(kind); + const below = order.findIndex( + (entry) => !isMicrophoneDevice(entry) && rank(entry.kind) > rank(kind), + ); + order.splice(lastDevice !== -1 ? lastDevice + 1 : below !== -1 ? below : order.length, 0, { + kind, + }); + } + return order; +} + /** - * Adds connected microphones the list has not seen and refreshes renamed ones. - * A new microphone goes after the last one of its kind, so a preference shown - * for AirPods carries over to a Bluetooth car. A kind seen for the first time - * ranks by MICROPHONE_KINDS, which keeps CarPlay below the device microphone. - * Returns `remembered` itself when nothing changed. + * Lists connected wired and Bluetooth microphones by name and refreshes + * renamed ones. A new device goes just above its kind, keeping the rank it had + * while unlisted. Returns `order` itself when nothing changed. */ export function rememberMicrophones( - remembered: ReadonlyArray, + order: ReadonlyArray, inputs: ReadonlyArray, -): ReadonlyArray { - const rank = (kind: MicrophoneKind) => MICROPHONE_KINDS.indexOf(kind); - // Copies, not ES2023 array methods: Hermes does not ship toSorted or toSpliced. - const next = [...remembered]; - let changed = false; - const seen = inputs - .flatMap((input) => { - const kind = KIND_BY_PORT_TYPE[input.type]; - return kind ? [{ uid: input.uid, name: input.name, kind }] : []; - }) - .sort((left, right) => rank(left.kind) - rank(right.kind)); - for (const microphone of seen) { - const index = next.findIndex((entry) => entry.uid === microphone.uid); +): ReadonlyArray { + let next: MicrophoneEntry[] | null = null; + for (const input of inputs) { + const kind = KIND_BY_PORT_TYPE[input.type]; + if (!kind || !DEVICE_KINDS.has(kind)) continue; + const current = next ?? order; + const index = current.findIndex((entry) => microphoneEntryKey(entry) === input.uid); + const existing = current[index]; + if (existing && isMicrophoneDevice(existing) && existing.name === input.name) continue; + next ??= [...order]; + const device = { kind, uid: input.uid, name: input.name }; if (index !== -1) { - if (next[index]!.name !== microphone.name) { - next[index] = microphone; - changed = true; - } + next[index] = device; continue; } - let at = next.length; - for (let candidate = next.length - 1; candidate >= 0; candidate -= 1) { - if (rank(next[candidate]!.kind) <= rank(microphone.kind)) break; - at = candidate; - } - const lastOfKind = next.map((entry) => entry.kind).lastIndexOf(microphone.kind); - next.splice(lastOfKind !== -1 ? lastOfKind + 1 : at, 0, microphone); - changed = true; + const kindRow = next.findIndex((entry) => microphoneEntryKey(entry) === `kind:${kind}`); + next.splice(kindRow === -1 ? next.length : kindRow, 0, device); } - return changed ? next : remembered; + return next ?? order; } -/** The connected input listed first, or null to leave the route to iOS. */ -export function preferredMicrophoneInput( +/** The connected input of the first entry that has one, or null to leave the route to iOS. */ +export function preferredMicrophoneInput( inputs: ReadonlyArray, - remembered: ReadonlyArray, + order: ReadonlyArray, ): Input | null { - for (const microphone of remembered) { - const input = inputs.find((candidate) => candidate.uid === microphone.uid); + const listed = new Set(order.filter(isMicrophoneDevice).map((device) => device.uid)); + for (const entry of order) { + const input = inputs.find((candidate) => + isMicrophoneDevice(entry) + ? candidate.uid === entry.uid + : KIND_BY_PORT_TYPE[candidate.type] === entry.kind && !listed.has(candidate.uid), + ); if (input) return input; } return null; diff --git a/apps/mobile/src/persistence/mobile-preferences.ts b/apps/mobile/src/persistence/mobile-preferences.ts index d32b7f5bbfb6..98f6b3d1e8d1 100644 --- a/apps/mobile/src/persistence/mobile-preferences.ts +++ b/apps/mobile/src/persistence/mobile-preferences.ts @@ -8,7 +8,7 @@ import * as Semaphore from "effect/Semaphore"; import type { ProviderInstanceId, SidebarProjectGroupingMode } from "@t3tools/contracts"; import type { ComposerEnterBehavior } from "../lib/composerEnterBehavior"; import type { FollowUpBehavior } from "../lib/followUpBehavior"; -import { isMicrophoneKind, type RememberedMicrophone } from "../lib/microphonePriority"; +import { isMicrophoneKind, type MicrophoneEntry } from "../lib/microphonePriority"; import { MOBILE_THEME_IDS, type MobileThemeId, type MobileThemeMode } from "../lib/mobileTheme"; import * as MobileDatabase from "./mobile-database"; import * as MobileSecureStorage from "./mobile-secure-storage"; @@ -37,8 +37,8 @@ export interface Preferences { * message sent during a running turn queues behind it or steers it. */ readonly followUpBehavior?: FollowUpBehavior; - /** Microphones voice input has seen, most preferred first. iOS only. */ - readonly microphones?: ReadonlyArray; + /** Microphone kinds and remembered devices, most preferred first. iOS only. */ + readonly microphones?: ReadonlyArray; /** @deprecated Kept temporarily so older OTA bundles retain the selected mode. */ readonly projectGroupingEnabled?: boolean; readonly projectGroupingMode?: SidebarProjectGroupingMode; @@ -110,7 +110,7 @@ function sanitizePreferences(parsed: Preferences): Preferences { collapsedProjectGroups?: readonly string[]; composerEnterBehavior?: ComposerEnterBehavior; followUpBehavior?: FollowUpBehavior; - microphones?: ReadonlyArray; + microphones?: ReadonlyArray; projectGroupingEnabled?: boolean; projectGroupingMode?: SidebarProjectGroupingMode; planModeEnabled?: boolean; @@ -177,14 +177,13 @@ function sanitizePreferences(parsed: Preferences): Preferences { preferences.followUpBehavior = parsed.followUpBehavior; } if (Array.isArray(parsed.microphones)) { - preferences.microphones = parsed.microphones.filter( - (microphone) => - typeof microphone === "object" && - microphone !== null && - typeof microphone.uid === "string" && - typeof microphone.name === "string" && - isMicrophoneKind(microphone.kind), - ); + preferences.microphones = parsed.microphones.flatMap((entry): MicrophoneEntry[] => { + if (typeof entry !== "object" || entry === null || !isMicrophoneKind(entry.kind)) return []; + if (!("uid" in entry)) return [{ kind: entry.kind }]; + return typeof entry.uid === "string" && typeof entry.name === "string" + ? [{ kind: entry.kind, uid: entry.uid, name: entry.name }] + : []; + }); } if (typeof parsed.projectGroupingEnabled === "boolean") { preferences.projectGroupingEnabled = parsed.projectGroupingEnabled; diff --git a/docs/user/composer.md b/docs/user/composer.md index e59838cf64b6..831a8ac9792d 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -165,10 +165,11 @@ recording and preserves your existing draft. While recording, the screen stays awake; it can sleep normally once recording stops. Voice input records from the first connected microphone in **Settings → Microphone**. -A microphone joins that list the first time you use voice input with it connected, -such as your AirPods or your car. Drag the list to prefer, for example, the iPhone -microphone over either. New Bluetooth devices start below the ones you already -ranked, and CarPlay starts below the iPhone microphone. +The list mixes kinds (built-in, other wired, other Bluetooth, CarPlay) with wired and +Bluetooth devices you have dictated with, listed by name. Drag to rank them together: +for example, put **Other Bluetooth** above your AirPods so a new headset is preferred +while the AirPods microphone stays low. By default, CarPlay comes after the built-in +microphone. Transcription runs on your device. T3 Code deletes the temporary audio after transcription or cancellation; only the message text is sent when you submit.