From ab0079c36fa8573f69d2a5a5b5c4c19777058d07 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:56:18 +0000 Subject: [PATCH 1/7] fix(mobile): keep dictation recording through a reconnect The host tied a phone's dictation to its socket, so any reconnect cancelled it, and the phone ended the recording as soon as the connection dropped. The host now owns the session by device and waits a grace period for the phone to come back. The phone keeps recording through the reconnect and keeps the text transcribed so far if the connection does not return. Fixes #35211 --- ...tation-chunk-speech.dictation.chunk-1.json | 12 +- .../src/hooks/mobile-dictation-audio-chunk.ts | 35 +++- ...ile-dictation-pending-audio-budget.test.ts | 8 +- .../mobile-dictation-pending-audio-budget.ts | 5 +- .../hooks/mobile-dictation-reconnect.test.tsx | 153 ++++++++++++++++++ mobile/src/hooks/use-mobile-dictation.ts | 11 +- .../bridge/bridge-audio-verbs.ts | 6 +- ...mobile-dictation-agent-chat-start.test.tsx | 48 +++--- .../mobile-dictation-mode-refresh.test.tsx | 1 + ...se-mobile-session-native-chat-dictation.ts | 10 +- ...e-runtime-close-intent.integration.test.ts | 1 - ...ime-request-connection.integration.test.ts | 4 - src/main/runtime/rpc/methods/speech.test.ts | 3 +- src/main/runtime/rpc/methods/speech.ts | 15 +- ...untime-mobile-dictation-controller.test.ts | 76 +++++++++ .../runtime-mobile-dictation-controller.ts | 94 ++++++----- ...untime-rpc-client-attach-placement.test.ts | 1 - .../runtime-rpc/runtime-rpc-lifecycle.ts | 1 - .../runtime-service-command-surface.ts | 2 - .../layout/terminal-client-disconnect.ts | 2 +- 20 files changed, 364 insertions(+), 124 deletions(-) create mode 100644 mobile/src/hooks/mobile-dictation-reconnect.test.tsx create mode 100644 src/main/runtime/runtime-mobile-dictation-controller.test.ts diff --git a/mobile/rpc-foundation/goldens/matrix-speech.dictation-chunk-speech.dictation.chunk-1.json b/mobile/rpc-foundation/goldens/matrix-speech.dictation-chunk-speech.dictation.chunk-1.json index 0291a80ae288..f136e2ff5239 100644 --- a/mobile/rpc-foundation/goldens/matrix-speech.dictation-chunk-speech.dictation.chunk-1.json +++ b/mobile/rpc-foundation/goldens/matrix-speech.dictation-chunk-speech.dictation.chunk-1.json @@ -277,10 +277,6 @@ "failures": ["Unknown method"], "pending": 0 }, - "28366801162a": { - "failures": ["transport failure"], - "pending": 0 - }, "88a268fa1b7b": { "name": "speech.dictation.chunk#1", "ordinal": 1, @@ -565,8 +561,8 @@ "settlements": { "chunk": "bc459c132276" }, - "state": "28366801162a", - "effects": ["19c961e90970"] + "state": "02cebe6f0062", + "effects": [] } }, { @@ -577,8 +573,8 @@ "settlements": { "chunk": "bc459c132276" }, - "state": "b5157118341f", - "effects": ["19c961e90970"] + "state": "02cebe6f0062", + "effects": [] } } ] diff --git a/mobile/src/hooks/mobile-dictation-audio-chunk.ts b/mobile/src/hooks/mobile-dictation-audio-chunk.ts index 12a8eac10542..3f326fd65d15 100644 --- a/mobile/src/hooks/mobile-dictation-audio-chunk.ts +++ b/mobile/src/hooks/mobile-dictation-audio-chunk.ts @@ -4,6 +4,9 @@ import { } from './mobile-dictation-pending-audio-budget' import { bytesToBase64 } from './mobile-dictation-session-state' import { dictationAudioChunkSend } from '../dictation/mobile-dictation-operations' +import { isRpcDeliveryUnknown } from '../transport/rpc-delivery-ambiguity' +import { rpcPayloadMember } from '../transport/rpc-reader-payload' +import { isLogicalClientCutoverError } from '../transport/stable-logical-rpc-client' import type { DictationCaptureChunk } from '../platform/dictation-capture-contract' import type { MobileDictationPendingAudioBudget } from './mobile-dictation-pending-audio-budget' import type { RpcClient } from '../transport/rpc-client' @@ -13,6 +16,7 @@ type MobileDictationAudioChunkQueue = { pendingAudioBudget: MobileDictationPendingAudioBudget shouldReleaseBudget: (dictationId: string) => boolean failActiveDictation: (dictationId: string, err: unknown) => void + noteTranscriptSoFar: (dictationId: string, text: string) => void } export function enqueueMobileDictationAudioChunk( @@ -30,16 +34,33 @@ export function enqueueMobileDictationAudioChunk( ) return } + const params = { + dictationId, + audioBase64: bytesToBase64(bytes), + sampleRate: MOBILE_DICTATION_PCM_SAMPLE_RATE + } const sendChunk = dictationAudioChunkSend - .request(client, { - dictationId, - audioBase64: bytesToBase64(bytes), - sampleRate: MOBILE_DICTATION_PCM_SAMPLE_RATE - }) + .request(client, params) + .catch((err: unknown) => + // A chunk still waiting when the connection moved never reached the host; send it on the new one. + isLogicalClientCutoverError(err) && !isRpcDeliveryUnknown(err) + ? dictationAudioChunkSend.request(client, params) + : Promise.reject(err) + ) .then((reply) => { - dictationAudioChunkSend.interpret(reply) + // Only newer hosts send the text so far; any other result still acknowledges the chunk. + const result = dictationAudioChunkSend.interpret(reply) + const text = result == null ? undefined : rpcPayloadMember(result, 'text') + if (typeof text === 'string') { + queue.noteTranscriptSoFar(dictationId, text) + } + }) + .catch((err) => { + // The host keeps the session through a reconnect; only audio in flight at the drop is lost. + if (!isRpcDeliveryUnknown(err)) { + queue.failActiveDictation(dictationId, err) + } }) - .catch((err) => queue.failActiveDictation(dictationId, err)) .finally(() => { if (queue.shouldReleaseBudget(dictationId)) { queue.pendingAudioBudget.release(byteLength) diff --git a/mobile/src/hooks/mobile-dictation-pending-audio-budget.test.ts b/mobile/src/hooks/mobile-dictation-pending-audio-budget.test.ts index 7f2102265463..bd6564258fcc 100644 --- a/mobile/src/hooks/mobile-dictation-pending-audio-budget.test.ts +++ b/mobile/src/hooks/mobile-dictation-pending-audio-budget.test.ts @@ -2,13 +2,13 @@ import { describe, expect, it } from 'vitest' import { MobileDictationPendingAudioBudget } from './mobile-dictation-pending-audio-budget' describe('MobileDictationPendingAudioBudget', () => { - it('caps pending raw PCM audio at five seconds', () => { + it('caps pending raw PCM audio at thirty seconds', () => { const budget = new MobileDictationPendingAudioBudget() - expect(budget.tryReserve(159_999)).toBe(true) + expect(budget.tryReserve(959_999)).toBe(true) expect(budget.tryReserve(1)).toBe(true) - expect(budget.pendingAudioBytes).toBe(160_000) + expect(budget.pendingAudioBytes).toBe(960_000) expect(budget.tryReserve(1)).toBe(false) - expect(budget.pendingAudioBytes).toBe(160_000) + expect(budget.pendingAudioBytes).toBe(960_000) }) it('releases completed chunks and clamps stale releases after reset', () => { diff --git a/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts b/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts index 7f814e493bc2..37dc70e268b9 100644 --- a/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts +++ b/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts @@ -1,9 +1,10 @@ export const MOBILE_DICTATION_PCM_SAMPLE_RATE = 16000 const PCM_BYTES_PER_SAMPLE = 2 -const MAX_PENDING_AUDIO_SECONDS = 5 +// Covers the host's reconnect grace (MOBILE_DICTATION_RECONNECT_GRACE_MS) so a reconnect keeps the speech queued behind it. +const MAX_PENDING_AUDIO_SECONDS = 30 -// Why: dictation chunk RPCs can wait through reconnect before timing out, so +// Why: dictation chunk RPCs wait through reconnect before timing out, so // cap retained raw microphone audio before it is expanded into base64 payloads. export const MOBILE_DICTATION_MAX_PENDING_AUDIO_BYTES = MOBILE_DICTATION_PCM_SAMPLE_RATE * PCM_BYTES_PER_SAMPLE * MAX_PENDING_AUDIO_SECONDS diff --git a/mobile/src/hooks/mobile-dictation-reconnect.test.tsx b/mobile/src/hooks/mobile-dictation-reconnect.test.tsx new file mode 100644 index 000000000000..381f517498f4 --- /dev/null +++ b/mobile/src/hooks/mobile-dictation-reconnect.test.tsx @@ -0,0 +1,153 @@ +// The host keeps a phone's dictation through a reconnect; the phone must not end it over the chunks the drop took with it. +import { createElement } from 'react' +import { act, create } from 'react-test-renderer' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + createFakeRpcClient, + type FakeRpcClient, + type SentRequest +} from '../mobile-web-shell/bridge-host-test-fakes' +import { markRpcDeliveryUnknown } from '../transport/rpc-delivery-ambiguity' +import { LogicalClientCutoverError } from '../transport/stable-logical-rpc-client' +import type { + DictationCapture, + DictationCaptureChunk +} from '../platform/dictation-capture-contract' + +const seam = vi.hoisted(() => ({ + chunkHandlers: new Set<(chunk: DictationCaptureChunk) => void>() +})) + +vi.mock('react-native', () => ({ + AppState: { currentState: 'active', addEventListener: () => ({ remove: () => {} }) }, + Platform: { OS: 'ios' } +})) + +vi.mock('../platform/dictation-capture', () => { + const capture: DictationCapture = { + open: async () => ({ ok: true }), + begin: () => true, + end: async () => {}, + release: () => {}, + onChunk: (handler) => { + seam.chunkHandlers.add(handler) + return { remove: () => seam.chunkHandlers.delete(handler) } + }, + onInterruption: () => ({ remove: () => {} }) + } + return { useDictationCapture: () => capture } +}) + +import { useMobileDictation, type UseMobileDictationResult } from './use-mobile-dictation' + +type Reply = { text?: string } | Error + +const held: { dictation: UseMobileDictationResult | null } = { dictation: null } + +function mount(rpc: FakeRpcClient, transcripts: string[], errors: string[]): void { + function Probe(): null { + held.dictation = useMobileDictation({ + client: rpc, + enabled: true, + onTranscript: (text) => transcripts.push(text), + onError: (err) => errors.push(err.message) + }) + return null + } + act(() => { + create(createElement(Probe)) + }) +} + +function dictation(): UseMobileDictationResult { + if (held.dictation === null) { + throw new Error('nothing mounted') + } + return held.dictation +} + +// Answer each chunk with the next scripted reply and everything else with success. +async function pump(rpc: FakeRpcClient, sent: SentRequest[], chunkReplies: Reply[]): Promise { + for (let round = 0; round < 8; round += 1) { + for (const request of rpc.requests.splice(0)) { + sent.push(request) + const reply: Reply = + request.method === 'speech.dictation.chunk' + ? (chunkReplies.shift() ?? {}) + : request.method === 'speech.dictation.finish' + ? { text: 'hello world' } + : {} + if (reply instanceof Error) { + request.reject(reply) + } else { + request.resolve({ id: 'desktop', ok: true, result: reply }) + } + } + await Promise.resolve() + await Promise.resolve() + } +} + +async function record(rpc: FakeRpcClient, sent: SentRequest[], chunkReplies: Reply[]) { + await act(async () => { + const started = dictation().start() + await pump(rpc, sent, []) + await started + }) + for (const reply of chunkReplies) { + await act(async () => { + for (const handler of seam.chunkHandlers) { + handler({ data: Uint8Array.from([1, 2]), droppedBytes: 0 }) + } + await pump(rpc, sent, [reply]) + }) + } +} + +const methodsOf = (sent: SentRequest[]) => sent.map((request) => request.method) + +beforeEach(() => { + seam.chunkHandlers.clear() + held.dictation = null +}) + +describe('dictation through a reconnect', () => { + it('keeps recording past chunks the drop lost, and resends ones that never left', async () => { + const rpc = createFakeRpcClient() + const sent: SentRequest[] = [] + const transcripts: string[] = [] + const errors: string[] = [] + mount(rpc, transcripts, errors) + + await record(rpc, sent, [ + { text: 'hello' }, + markRpcDeliveryUnknown(new Error('Connection interrupted')), + new LogicalClientCutoverError(new Error('Connection closed')) + ]) + + expect(dictation().status).toBe('recording') + // Each captured chunk went out once, and the one the cutover held back went out again. + expect(methodsOf(sent).filter((m) => m === 'speech.dictation.chunk')).toHaveLength(4) + await act(async () => { + const stopped = dictation().stop() + await pump(rpc, sent, []) + await stopped + }) + expect(methodsOf(sent)).not.toContain('speech.dictation.cancel') + expect(transcripts).toEqual(['hello world']) + expect(errors).toEqual([]) + }) + + it('keeps what was transcribed when the connection does not come back', async () => { + const rpc = createFakeRpcClient() + const sent: SentRequest[] = [] + const transcripts: string[] = [] + const errors: string[] = [] + mount(rpc, transcripts, errors) + + await record(rpc, sent, [{ text: 'hello' }, new Error('Connection closed')]) + + expect(transcripts).toEqual(['hello']) + expect(errors).toEqual(['Connection closed']) + }) +}) diff --git a/mobile/src/hooks/use-mobile-dictation.ts b/mobile/src/hooks/use-mobile-dictation.ts index 33fcb05d4a33..4a933506099d 100644 --- a/mobile/src/hooks/use-mobile-dictation.ts +++ b/mobile/src/hooks/use-mobile-dictation.ts @@ -43,6 +43,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil const pendingAudioBudgetRef = useRef(new MobileDictationPendingAudioBudget()) const acceptingChunksRef = useRef(false) const generationRef = useRef(0) + const transcriptSoFarRef = useRef<{ dictationId: string; text: string } | null>(null) useLayoutEffect(() => { // Native audio events can arrive before passive Effects flush, but refs @@ -91,6 +92,11 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil if (client && dictationId) { void dictationSessionCancel.request(client, { dictationId }).catch(() => undefined) } + // Keep what the host transcribed before the failure instead of dropping it with the error. + const kept = transcriptSoFarRef.current + if (kept?.dictationId === dictationId && kept.text) { + onTranscriptRef.current(kept.text) + } reportError(err) }, [closeDictationAudio, reportError] @@ -102,7 +108,10 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil pendingChunks: pendingChunksRef.current, pendingAudioBudget: pendingAudioBudgetRef.current, shouldReleaseBudget: (id: string) => activeIdRef.current === id, - failActiveDictation + failActiveDictation, + noteTranscriptSoFar: (dictationId: string, text: string) => { + transcriptSoFarRef.current = { dictationId, text } + } } const sub = capture.onChunk((chunk) => { const client = clientRef.current diff --git a/mobile/src/mobile-web-shell/bridge/bridge-audio-verbs.ts b/mobile/src/mobile-web-shell/bridge/bridge-audio-verbs.ts index 72424b049f51..d6ed1082889f 100644 --- a/mobile/src/mobile-web-shell/bridge/bridge-audio-verbs.ts +++ b/mobile/src/mobile-web-shell/bridge/bridge-audio-verbs.ts @@ -1,11 +1,11 @@ import { z } from 'zod' -import { MOBILE_DICTATION_MAX_PENDING_AUDIO_BYTES } from '../../hooks/mobile-dictation-pending-audio-budget' +import { MAX_DICTATION_AUDIO_CHUNK_BYTES } from '../../../../src/shared/rpc-contract/speech-params' // The shell owns microphone PCM and its wake lock; the page owns dictation state/RPCs. Pull from the ring through existing request/reply rather than adding a push frame. // Keep verb grant segments lowercase: a camel-cased audio.readChunk would fail manifest grammar and reject the whole bundle. -// Use the page’s pending-audio budget; a larger shell ring would hand the page bytes its own queue refuses. -export const BRIDGE_AUDIO_RING_MAX_BYTES = MOBILE_DICTATION_MAX_PENDING_AUDIO_BYTES +// A full ring drain goes to the host as one chunk, so stay within its chunk cap; shipped shells validate maxBytes against this. +export const BRIDGE_AUDIO_RING_MAX_BYTES = MAX_DICTATION_AUDIO_CHUNK_BYTES // Base64 uses four chars per three bytes; a full ring drain fits the message cap. export const BRIDGE_AUDIO_READ_MAX_BASE64_CHARS = Math.ceil(BRIDGE_AUDIO_RING_MAX_BYTES / 3) * 4 diff --git a/mobile/src/session/mobile-dictation-agent-chat-start.test.tsx b/mobile/src/session/mobile-dictation-agent-chat-start.test.tsx index ab9ab68e309a..2b8fd756c7a0 100644 --- a/mobile/src/session/mobile-dictation-agent-chat-start.test.tsx +++ b/mobile/src/session/mobile-dictation-agent-chat-start.test.tsx @@ -110,6 +110,7 @@ type SessionScope = Pick< | 'activeHandleRef' | 'flushPendingLiveInputBeforeExternalSend' | 'canSend' + | 'canCompose' | 'liveInputEnabled' | 'showToast' | 'resetLiveInputFocus' @@ -143,6 +144,11 @@ let terminalText = '' function Probe({ client, tab, connState, dictationEnabled }: ProbeOptions): null { const activeHandle = tab.type === 'terminal' ? tab.terminal : null const activeHandleRef = useRef(activeHandle) + const gate = resolveMobileTerminalInputGate({ + connState, + activeHandle, + activeSessionTabType: tab.type + }) const scope: SessionScope = { hostId: 'h1', worktreeId: 'w1', @@ -166,11 +172,8 @@ function Probe({ client, tab, connState, dictationEnabled }: ProbeOptions): null dictationRouteContextRef: useRef(null), activeHandleRef, flushPendingLiveInputBeforeExternalSend: async () => true, - canSend: resolveMobileTerminalInputGate({ - connState, - activeHandle, - activeSessionTabType: tab.type - }).canSend, + canSend: gate.canSend, + canCompose: gate.canCompose, liveInputEnabled: false, showToast: () => {}, resetLiveInputFocus: () => {} @@ -279,7 +282,8 @@ describe('dictation uses the active composer readiness', () => { expect(draft.text).toBe('Draft') }) - it('cancels an in-flight agent-chat start when the connection drops', async () => { + // The host keeps the session through a reconnect, so a drop is not the end of the input. + it('keeps a start that spans a connection drop, and records once it lands', async () => { const client = createFakeRpcClient() const options = { client, @@ -289,21 +293,14 @@ describe('dictation uses the active composer readiness', () => { } satisfies ProbeOptions await render(options) await act(async () => model().startDictation()) - const pendingStart = client.requests.shift() - expect(pendingStart?.method).toBe('speech.dictation.start') - await render({ ...options, connState: 'disconnected' }) - await reply(client, 'speech.dictation.cancel', {}) - await act(async () => - pendingStart?.resolve({ id: 'desktop', ok: true, result: { started: true } }) - ) - await reply(client, 'speech.dictation.cancel', {}) - expect(device.recording).toBe(false) - expect(draft.text).toBe('Draft') + await render({ ...options, connState: 'reconnecting' }) await render(options) - await start(client) + await reply(client, 'speech.dictation.start', { started: true }) + expect(model().dictation.status).toBe('recording') + expect(device.recording).toBe(true) }) - it('drops a late transcript after disconnecting, then starts again on reconnect', async () => { + it('delivers the transcript of a finish that spans a connection drop', async () => { const client = createFakeRpcClient() const options = { client, @@ -313,19 +310,14 @@ describe('dictation uses the active composer readiness', () => { } satisfies ProbeOptions await render(options) await start(client) + await render({ ...options, connState: 'reconnecting' }) + expect(model().dictation.status).toBe('recording') await act(async () => { void model().dictation.stop() }) - const pendingFinish = client.requests.shift() - expect(pendingFinish?.method).toBe('speech.dictation.finish') - await render({ ...options, connState: 'disconnected' }) - await reply(client, 'speech.dictation.cancel', {}) - await act(async () => - pendingFinish?.resolve({ id: 'desktop', ok: true, result: { text: 'late words' } }) - ) - expect(draft.text).toBe('Draft') - expect(device.recording).toBe(false) await render(options) - await start(client) + await reply(client, 'speech.dictation.finish', { text: 'words across the drop' }) + expect(client.requests).toEqual([]) + expect(draft.text).toBe('Draft words across the drop') }) }) diff --git a/mobile/src/session/mobile-dictation-mode-refresh.test.tsx b/mobile/src/session/mobile-dictation-mode-refresh.test.tsx index 5828f8394972..640123a7b5d5 100644 --- a/mobile/src/session/mobile-dictation-mode-refresh.test.tsx +++ b/mobile/src/session/mobile-dictation-mode-refresh.test.tsx @@ -142,6 +142,7 @@ function Probe() { deviceTokenRef, flushPendingLiveInputBeforeExternalSend: async () => true, canSend: env.connected, + canCompose: true, liveInputEnabled: false, showToast: () => {}, resetLiveInputFocus: () => {} diff --git a/mobile/src/session/use-mobile-session-native-chat-dictation.ts b/mobile/src/session/use-mobile-session-native-chat-dictation.ts index 48e53a7c450a..39e09bb6526f 100644 --- a/mobile/src/session/use-mobile-session-native-chat-dictation.ts +++ b/mobile/src/session/use-mobile-session-native-chat-dictation.ts @@ -51,6 +51,7 @@ export function useMobileSessionNativeChatDictation( activeSessionTab, flushPendingLiveInputBeforeExternalSend, canSend, + canCompose, liveInputEnabled, showToast, resetLiveInputFocus @@ -129,10 +130,11 @@ export function useMobileSessionNativeChatDictation( } const chatDictationDestination = useRef<((text: string) => boolean | void) | undefined>(undefined) - const dictationTargetReady = showNativeChat ? connState === 'connected' : canSend + // Starting needs the host, but a reconnect must not end a recording: the host keeps the session through it. + const dictationCanStart = showNativeChat ? connState === 'connected' : canSend const dictation = useMobileDictation({ client, - enabled: dictationTargetReady && dictationEnabled, + enabled: (showNativeChat || canCompose) && dictationEnabled, onTranscript: (text) => { // Route dictation to the visible chat composer locally; terminal mode retains live-input routing. if (showNativeChatRef.current || chatDictationDestination.current) { @@ -185,6 +187,9 @@ export function useMobileSessionNativeChatDictation( const dictationNeedsSetupRef = useRef(false) const startDictation = useCallback(() => { + if (!dictationCanStart) { + return + } if (dictationNeedsSetupRef.current) { // Explain setup before the OS microphone prompt: mic access cannot help until the host can transcribe. setShowDictationSetup(true) @@ -204,6 +209,7 @@ export function useMobileSessionNativeChatDictation( }, [ activeHandle, dictation, + dictationCanStart, liveInputTerminalHandles, reportDictationFailure, setShowDictationSetup diff --git a/src/main/runtime/remote-runtime-close-intent.integration.test.ts b/src/main/runtime/remote-runtime-close-intent.integration.test.ts index 59122de21aaf..bc2d4738a5df 100644 --- a/src/main/runtime/remote-runtime-close-intent.integration.test.ts +++ b/src/main/runtime/remote-runtime-close-intent.integration.test.ts @@ -49,7 +49,6 @@ it( getRuntimeId: () => 'close-intent-runtime-test', getStartedAt: () => 1, cleanupSubscriptionsForConnection: () => {}, - cancelMobileDictationForConnection: () => {}, onClientDisconnected: () => {}, listMobileSessionTabs, refuseUnattributedMobileSessionTabClose, diff --git a/src/main/runtime/remote-runtime-request-connection.integration.test.ts b/src/main/runtime/remote-runtime-request-connection.integration.test.ts index dd6fae879199..9f48eee6f2a1 100644 --- a/src/main/runtime/remote-runtime-request-connection.integration.test.ts +++ b/src/main/runtime/remote-runtime-request-connection.integration.test.ts @@ -49,7 +49,6 @@ describe('remote runtime request connection integration', () => { getRuntimeId: () => 'fetch-runtime-test', getStartedAt: () => 1, cleanupSubscriptionsForConnection: () => {}, - cancelMobileDictationForConnection: () => {}, onClientDisconnected: () => {}, listVisibleRepos: () => repos } as unknown as OrcaRuntimeService @@ -137,7 +136,6 @@ describe('remote runtime request connection integration', () => { subscriptionCleanups.get(id)?.() subscriptionCleanups.delete(id) }, - cancelMobileDictationForConnection: () => {}, onClientDisconnected: () => {}, showRepo: (selector: string) => { if (selector !== repo.id && selector !== `id:${repo.id}`) { @@ -305,7 +303,6 @@ describe('remote runtime request connection integration', () => { subscriptionCleanups.get(id)?.() subscriptionCleanups.delete(id) }, - cancelMobileDictationForConnection: () => {}, onClientDisconnected: () => {}, onClientEvent: (listener: (event: RuntimeClientEvent) => void) => { clientEventListeners.add(listener) @@ -553,7 +550,6 @@ describe('remote runtime request connection integration', () => { } } }, - cancelMobileDictationForConnection: () => {}, onClientDisconnected: () => {}, onClientEvent: (listener: (event: RuntimeClientEvent) => void) => { clientEventListeners.add(listener) diff --git a/src/main/runtime/rpc/methods/speech.test.ts b/src/main/runtime/rpc/methods/speech.test.ts index 2ef23c697621..d812febe0d1a 100644 --- a/src/main/runtime/rpc/methods/speech.test.ts +++ b/src/main/runtime/rpc/methods/speech.test.ts @@ -30,8 +30,7 @@ describe('speech RPC methods', () => { dictationId: 'dict-1', audioBase64: 'AAAA', sampleRate: 16_000, - clientId: undefined, - connectionId: undefined + clientId: undefined }) }) diff --git a/src/main/runtime/rpc/methods/speech.ts b/src/main/runtime/rpc/methods/speech.ts index 0d8da9b6de98..79bff553a38f 100644 --- a/src/main/runtime/rpc/methods/speech.ts +++ b/src/main/runtime/rpc/methods/speech.ts @@ -42,29 +42,28 @@ export const SPEECH_METHODS = [ name: 'speech.dictation.start', permission: 'workspace', params: DictationStart, - handler: async (params, { runtime, clientId, connectionId }) => - runtime.startMobileDictation({ ...params, clientId, connectionId }) + handler: async (params, { runtime, clientId }) => + runtime.startMobileDictation({ ...params, clientId }) }), defineMethod({ name: 'speech.dictation.chunk', permission: 'workspace', params: DictationChunk, - handler: (params, { runtime, clientId, connectionId }) => - runtime.feedMobileDictation({ ...params, clientId, connectionId }) + handler: (params, { runtime, clientId }) => runtime.feedMobileDictation({ ...params, clientId }) }), defineMethod({ name: 'speech.dictation.finish', permission: 'workspace', params: DictationHandle, - handler: async (params, { runtime, clientId, connectionId }) => - runtime.finishMobileDictation({ ...params, clientId, connectionId }) + handler: async (params, { runtime, clientId }) => + runtime.finishMobileDictation({ ...params, clientId }) }), defineMethod({ name: 'speech.dictation.cancel', permission: 'workspace', params: DictationHandle, - handler: async (params, { runtime, clientId, connectionId }) => - runtime.cancelMobileDictation({ ...params, clientId, connectionId }) + handler: async (params, { runtime, clientId }) => + runtime.cancelMobileDictation({ ...params, clientId }) }), defineMethod({ name: 'speech.openAiKey.status', diff --git a/src/main/runtime/runtime-mobile-dictation-controller.test.ts b/src/main/runtime/runtime-mobile-dictation-controller.test.ts new file mode 100644 index 000000000000..ebc7235e9e4b --- /dev/null +++ b/src/main/runtime/runtime-mobile-dictation-controller.test.ts @@ -0,0 +1,76 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { getDefaultVoiceSettings } from '../../shared/constants' +import { + setSpeechServiceFactories, + type SpeechServiceFactories +} from '../speech/speech-runtime-service' +import { + MOBILE_DICTATION_RECONNECT_GRACE_MS, + RuntimeMobileDictationController +} from './runtime-mobile-dictation-controller' +import type { RuntimeStore } from './runtime-store-contract' + +type Sink = (event: { type: string; text?: string }) => void + +let sink: Sink = () => {} +const stopDictation = vi.fn(async () => {}) + +beforeEach(() => { + vi.useFakeTimers() + stopDictation.mockClear() + const factories = { + createModelManager: () => ({ getModelState: async () => ({ status: 'ready' }) }), + createSttService: () => ({ + startDictation: async (_modelId: string, next: Sink) => { + sink = next + }, + feedAudio: () => {}, + stopDictation + }) + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the controller calls only getModelState, startDictation, feedAudio and stopDictation. + setSpeechServiceFactories(factories as unknown as SpeechServiceFactories) +}) + +afterEach(() => { + setSpeechServiceFactories(null) + vi.useRealTimers() +}) + +async function startPhoneDictation(): Promise { + const store = { getSettings: () => ({ voice: { ...getDefaultVoiceSettings(), enabled: true } }) } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the controller reads only getSettings. + const controller = new RuntimeMobileDictationController(() => store as unknown as RuntimeStore) + await controller.start({ dictationId: 'd-1', modelId: 'test-model', clientId: 'phone-a' }) + sink({ type: 'final', text: 'before the drop' }) + return controller +} + +const chunk = { dictationId: 'd-1', audioBase64: 'AAAA', sampleRate: 16_000, clientId: 'phone-a' } + +describe('mobile dictation across a phone reconnect', () => { + it('keeps the session when the phone comes back within the grace', async () => { + const controller = await startPhoneDictation() + + controller.cancelAfterClientDisconnectGrace('phone-a') + vi.advanceTimersByTime(MOBILE_DICTATION_RECONNECT_GRACE_MS - 1) + expect(controller.feed(chunk)).toEqual({ dictationId: 'd-1', text: 'before the drop' }) + await vi.advanceTimersByTimeAsync(MOBILE_DICTATION_RECONNECT_GRACE_MS) + sink({ type: 'final', text: 'after it' }) + + await expect(controller.finish({ dictationId: 'd-1', clientId: 'phone-a' })).resolves.toEqual({ + dictationId: 'd-1', + text: 'before the drop after it' + }) + }) + + it('releases the speech model when the phone never comes back', async () => { + const controller = await startPhoneDictation() + + controller.cancelAfterClientDisconnectGrace('phone-a') + await vi.advanceTimersByTimeAsync(MOBILE_DICTATION_RECONNECT_GRACE_MS) + + expect(stopDictation).toHaveBeenCalledOnce() + expect(() => controller.feed(chunk)).toThrow('dictation_stream_not_started') + }) +}) diff --git a/src/main/runtime/runtime-mobile-dictation-controller.ts b/src/main/runtime/runtime-mobile-dictation-controller.ts index b7da6af0acdc..6582e9790baf 100644 --- a/src/main/runtime/runtime-mobile-dictation-controller.ts +++ b/src/main/runtime/runtime-mobile-dictation-controller.ts @@ -7,15 +7,18 @@ import { } from '../speech/stt-worker-paths' import type { RuntimeStore } from './runtime-store-contract' +// Phones reconnect by redialling (network switch, Relay fallback); keep the session that long. +export const MOBILE_DICTATION_RECONNECT_GRACE_MS = 30_000 + type MobileDictationSession = { id: string owner: string - clientId?: string - connectionId?: string + clientId: string state: 'starting' | 'active' | 'closing' partialText: string finalTexts: string[] errors: string[] + reconnectGraceTimer?: ReturnType } export class RuntimeMobileDictationController { @@ -27,7 +30,6 @@ export class RuntimeMobileDictationController { dictationId: string modelId?: string clientId?: string - connectionId?: string }): Promise<{ dictationId: string; modelId: string }> { const store = this.requireStore() const voice = store.getSettings().voice ?? getDefaultVoiceSettings() @@ -49,6 +51,10 @@ export class RuntimeMobileDictationController { if (!params.clientId) { throw new Error('dictation_requires_mobile_client') } + if (this.session?.clientId === params.clientId && this.session.state === 'active') { + // A phone runs one dictation at a time, so its new start retires one it lost track of (app relaunch inside the grace). + await this.stopSession(this.session) + } if (this.session) { throw new Error('dictation_already_active') } @@ -58,7 +64,6 @@ export class RuntimeMobileDictationController { id: params.dictationId, owner, clientId: params.clientId, - connectionId: params.connectionId, state: 'starting', partialText: '', finalTexts: [], @@ -89,8 +94,7 @@ export class RuntimeMobileDictationController { audioBase64: string sampleRate: number clientId?: string - connectionId?: string - }): { dictationId: string } { + }): { dictationId: string; text: string } { const session = this.requireOwnedSession(params) if (session.state !== 'active') { throw new Error('dictation_stream_closing') @@ -104,13 +108,13 @@ export class RuntimeMobileDictationController { samples[i] = pcm.readInt16LE(i * 2) / 32768 } getSpeechSttService(this.requireStore()).feedAudio(samples, params.sampleRate, session.owner) - return { dictationId: params.dictationId } + // The text so far lets the phone keep it if the connection never comes back. + return { dictationId: params.dictationId, text: transcriptOf(session) } } async finish(params: { dictationId: string clientId?: string - connectionId?: string }): Promise<{ dictationId: string; text: string }> { const session = this.requireOwnedSession(params) session.state = 'closing' @@ -119,10 +123,7 @@ export class RuntimeMobileDictationController { if (session.errors.length > 0) { throw new Error(session.errors[0]) } - return { - dictationId: params.dictationId, - text: [...session.finalTexts, session.partialText].join(' ').trim() - } + return { dictationId: params.dictationId, text: transcriptOf(session) } } finally { if (this.session?.id === session.id) { this.session = null @@ -133,37 +134,25 @@ export class RuntimeMobileDictationController { async cancel(params: { dictationId: string clientId?: string - connectionId?: string }): Promise<{ dictationId: string }> { const session = this.session - if ( - session?.id === params.dictationId && - params.clientId && - session.clientId === params.clientId && - (!session.connectionId || session.connectionId === params.connectionId) - ) { - session.state = 'closing' - try { - await getSpeechSttService(this.requireStore()).stopDictation(session.owner) - } finally { - if (this.session?.id === session.id) { - this.session = null - } - } + if (session?.id === params.dictationId && session.clientId === params.clientId) { + await this.stopSession(session) } return { dictationId: params.dictationId } } - cancelForConnection(connectionId: string): void { - if (this.session?.connectionId === connectionId) { - this.cancelSession(this.session) - } - } - - cancelForClient(clientId: string): void { - if (this.session?.clientId === clientId) { - this.cancelSession(this.session) + // Cancel only if the phone has not come back within the grace; its next request clears this. + cancelAfterClientDisconnectGrace(clientId: string): void { + const session = this.session + if (session?.clientId !== clientId || session.state === 'closing') { + return } + clearTimeout(session.reconnectGraceTimer) + session.reconnectGraceTimer = setTimeout( + () => this.cancelSession(session), + MOBILE_DICTATION_RECONNECT_GRACE_MS + ) } private acceptEvent( @@ -187,36 +176,39 @@ export class RuntimeMobileDictationController { } } + // Owned by the phone, not its socket: a reconnect arrives on a new connection. private requireOwnedSession(params: { dictationId: string clientId?: string - connectionId?: string }): MobileDictationSession { const session = this.session if (!session || session.id !== params.dictationId) { throw new Error('dictation_stream_not_started') } - if (!params.clientId || session.clientId !== params.clientId) { - throw new Error('dictation_owner_mismatch') - } - if (session.connectionId && session.connectionId !== params.connectionId) { + if (session.clientId !== params.clientId) { throw new Error('dictation_owner_mismatch') } + clearTimeout(session.reconnectGraceTimer) return session } + private async stopSession(session: MobileDictationSession): Promise { + clearTimeout(session.reconnectGraceTimer) + session.state = 'closing' + try { + await getSpeechSttService(this.requireStore()).stopDictation(session.owner) + } finally { + if (this.session?.id === session.id) { + this.session = null + } + } + } + private cancelSession(session: MobileDictationSession): void { if (session.state === 'closing') { return } - session.state = 'closing' - void getSpeechSttService(this.requireStore()) - .stopDictation(session.owner) - .finally(() => { - if (this.session?.id === session.id) { - this.session = null - } - }) + void this.stopSession(session).catch(() => undefined) } private requireStore(): RuntimeStore { @@ -227,3 +219,7 @@ export class RuntimeMobileDictationController { return store } } + +function transcriptOf(session: MobileDictationSession): string { + return [...session.finalTexts, session.partialText].join(' ').trim() +} diff --git a/src/main/runtime/runtime-rpc-client-attach-placement.test.ts b/src/main/runtime/runtime-rpc-client-attach-placement.test.ts index 9cb9bfc34e7d..0a33d34260f7 100644 --- a/src/main/runtime/runtime-rpc-client-attach-placement.test.ts +++ b/src/main/runtime/runtime-rpc-client-attach-placement.test.ts @@ -19,7 +19,6 @@ function runtimeRecordingPlacement() { machineNameReady: async () => undefined, getStatus: () => ({ graphStatus: 'unavailable' }), cleanupSubscriptionsForConnection: () => {}, - cancelMobileDictationForConnection: () => {}, onClientDisconnected: () => {}, placeUnshownLiveTerminals } as unknown as OrcaRuntimeService diff --git a/src/main/runtime/runtime-rpc/runtime-rpc-lifecycle.ts b/src/main/runtime/runtime-rpc/runtime-rpc-lifecycle.ts index 8ad9e2e15d38..6f526959ff3e 100644 --- a/src/main/runtime/runtime-rpc/runtime-rpc-lifecycle.ts +++ b/src/main/runtime/runtime-rpc/runtime-rpc-lifecycle.ts @@ -269,7 +269,6 @@ export class RuntimeRpcLifecycle extends RuntimeRpcWebSocketDispatch { this.mobileDesktopRelay?.closePhoneConnection(socket.connectionId) // Stream cleanup is socket-scoped; disconnect state spans all transports for that device. this.runtime.cleanupSubscriptionsForConnection(socket.connectionId) - this.runtime.cancelMobileDictationForConnection(socket.connectionId) // Optional: runtime stand-ins without staged saves must still close cleanly. void this.runtime.stagedFileSaves?.abortConnection(socket.connectionId) this.binaryMessageRouter.deleteConnection(socket.connectionId) diff --git a/src/main/runtime/runtime-service-command-surface.ts b/src/main/runtime/runtime-service-command-surface.ts index aab4c0ebec21..9f708f7b8307 100644 --- a/src/main/runtime/runtime-service-command-surface.ts +++ b/src/main/runtime/runtime-service-command-surface.ts @@ -82,7 +82,6 @@ export type RuntimeServiceCommandSurface = { feedMobileDictation: RuntimeMobileDictationController['feed'] finishMobileDictation: RuntimeMobileDictationController['finish'] cancelMobileDictation: RuntimeMobileDictationController['cancel'] - cancelMobileDictationForConnection: RuntimeMobileDictationController['cancelForConnection'] getAllBrowserDrivers: RuntimeBrowserDriverController['getAll'] reclaimBrowserForDesktop: RuntimeBrowserDriverController['reclaimForDesktop'] notifyMessageArrived(handle: string, messageType?: string): void @@ -195,7 +194,6 @@ export function installRuntimeServiceCommandSurface( feedMobileDictation: dictation.feed.bind(dictation), finishMobileDictation: dictation.finish.bind(dictation), cancelMobileDictation: dictation.cancel.bind(dictation), - cancelMobileDictationForConnection: dictation.cancelForConnection.bind(dictation), getAllBrowserDrivers: browsers.getAll.bind(browsers), reclaimBrowserForDesktop: browsers.reclaimForDesktop.bind(browsers), waitForMessage: waiters.wait.bind(waiters), diff --git a/src/main/runtime/terminal/layout/terminal-client-disconnect.ts b/src/main/runtime/terminal/layout/terminal-client-disconnect.ts index 737e82623488..f593aea0a3f9 100644 --- a/src/main/runtime/terminal/layout/terminal-client-disconnect.ts +++ b/src/main/runtime/terminal/layout/terminal-client-disconnect.ts @@ -15,7 +15,7 @@ export abstract class OrcaRuntimeWithOnClientDisconnected extends OrcaRuntimeWit onClientDisconnected(clientId: string): void { this.revokeTerminalFileGrantsForClient(clientId) - this.mobileDictation.cancelForClient(clientId) + this.mobileDictation.cancelAfterClientDisconnectGrace(clientId) for (const [ptyId, entry] of this.pendingRestoreTimers) { if (entry.clientId === clientId) { From f0852a637dc704a3f835f0713f1f2495909d22a5 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:26:47 +0000 Subject: [PATCH 2/7] fix(mobile): bound dictation reconnect waits and release retired relay waiters --- ...tation-chunk-speech.dictation.chunk-1.json | 204 +++++++++--------- .../speech-audio-chunk-acknowledged.json | 6 +- .../src/hooks/mobile-dictation-audio-chunk.ts | 7 +- .../mobile-dictation-pending-audio-budget.ts | 6 +- .../session/MobileTerminalInputActions.tsx | 6 +- ...obile-dictation-disabled-controls.test.tsx | 34 +++ .../transport/mobile-relay-connect-waiters.ts | 51 +++++ .../mobile-relay-rpc-session.test.ts | 14 ++ .../src/transport/mobile-relay-rpc-session.ts | 29 +-- ...untime-mobile-dictation-controller.test.ts | 16 +- .../runtime-mobile-dictation-controller.ts | 44 ++-- .../layout/terminal-client-disconnect.ts | 1 - src/shared/rpc-contract/speech-params.ts | 3 + 13 files changed, 248 insertions(+), 173 deletions(-) create mode 100644 mobile/src/transport/mobile-relay-connect-waiters.ts diff --git a/mobile/rpc-foundation/goldens/matrix-speech.dictation-chunk-speech.dictation.chunk-1.json b/mobile/rpc-foundation/goldens/matrix-speech.dictation-chunk-speech.dictation.chunk-1.json index f136e2ff5239..8ba233197bf0 100644 --- a/mobile/rpc-foundation/goldens/matrix-speech.dictation-chunk-speech.dictation.chunk-1.json +++ b/mobile/rpc-foundation/goldens/matrix-speech.dictation-chunk-speech.dictation.chunk-1.json @@ -4,11 +4,7 @@ "family": "speech.dictation-chunk", "namedDeltas": [], "values": { - "02cebe6f0062": { - "failures": [], - "pending": 0 - }, - "0a3ec57e0321": { + "028afac3649e": { "name": "speech.dictation.chunk#1", "ordinal": 1, "args": [ @@ -27,7 +23,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -47,12 +43,27 @@ } } }, + "02cebe6f0062": { + "failures": [], + "pending": 0 + }, "12521d219054": { "name": "speech.dictation.chunk#1", "ordinal": 2, "json": "{\"id\":\"frame-1\",\"deviceToken\":\"recording-device\",\"method\":\"speech.dictation.chunk\",\"params\":{\"dictationId\":\"dictation-1\",\"audioBase64\":\"ACVKb5S53gM=\",\"sampleRate\":16000}}" }, - "16da7db6669b": { + "19c961e90970": { + "name": "dictation-failed", + "ordinal": 3, + "value": { + "id": "dictation-1" + } + }, + "2762d465637e": { + "failures": ["Unknown method"], + "pending": 0 + }, + "30ae620f58dc": { "name": "speech.dictation.chunk#1", "ordinal": 1, "args": [ @@ -71,25 +82,22 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], "settlement": { - "status": "fulfilled", + "status": "rejected", "startedAt": 0, "settledAt": 0, - "value": { - "id": "frame-1", - "ok": true, - "result": { - "error": "inner refused", - "ok": false - } + "error": { + "category": "Error", + "message": "transport failure", + "isRpcDeliveryUnknown": true } } }, - "182109450de8": { + "62cfcef21f01": { "name": "speech.dictation.chunk#1", "ordinal": 1, "args": [ @@ -108,7 +116,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -118,21 +126,11 @@ "settledAt": 0, "value": { "id": "frame-1", - "ok": true, - "result": { - "received": true - } + "ok": true } } }, - "19c961e90970": { - "name": "dictation-failed", - "ordinal": 3, - "value": { - "id": "dictation-1" - } - }, - "1ca8c123cb2d": { + "66dece39b5ba": { "name": "speech.dictation.chunk#1", "ordinal": 1, "args": [ @@ -151,22 +149,25 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], "settlement": { - "status": "rejected", + "status": "fulfilled", "startedAt": 0, "settledAt": 0, - "error": { - "category": "Error", - "message": "transport failure", - "isRpcDeliveryUnknown": true + "value": { + "error": { + "code": "refused", + "message": "outer refused" + }, + "id": "frame-1", + "ok": false } } }, - "1d0f84aa0730": { + "83a5bdf2ac1a": { "name": "speech.dictation.chunk#1", "ordinal": 1, "args": [ @@ -185,7 +186,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -194,16 +195,15 @@ "startedAt": 0, "settledAt": 0, "value": { - "error": { - "code": "refused", - "message": "outer refused" - }, "id": "frame-1", - "ok": false + "ok": true, + "result": { + "error": "refused" + } } } }, - "208244a4b935": { + "888a29b4fd97": { "name": "speech.dictation.chunk#1", "ordinal": 1, "args": [ @@ -222,7 +222,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -231,16 +231,16 @@ "startedAt": 0, "settledAt": 0, "value": { - "error": { - "code": "method_not_found", - "message": "Unknown method" - }, "id": "frame-1", - "ok": false + "ok": true, + "result": { + "error": "inner refused", + "ok": false + } } } }, - "25256657a918": { + "99956eef8fff": { "name": "speech.dictation.chunk#1", "ordinal": 1, "args": [ @@ -259,7 +259,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -269,15 +269,14 @@ "settledAt": 0, "value": { "id": "frame-1", - "ok": true + "ok": true, + "result": { + "received": true + } } } }, - "2762d465637e": { - "failures": ["Unknown method"], - "pending": 0 - }, - "88a268fa1b7b": { + "9d43743b1b10": { "name": "speech.dictation.chunk#1", "ordinal": 1, "args": [ @@ -296,7 +295,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -311,7 +310,28 @@ } } }, - "923922ffae75": { + "a2f842e44f38": { + "failures": ["outer refused"], + "pending": 0 + }, + "b5157118341f": { + "failures": [""], + "pending": 0 + }, + "bc459c132276": { + "status": "fulfilled", + "startedAt": 0, + "settledAt": 0, + "value": [ + { + "status": "fulfilled", + "value": { + "$rpc": "undefined" + } + } + ] + }, + "da1579f3254f": { "name": "speech.dictation.chunk#1", "ordinal": 1, "args": [ @@ -330,7 +350,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -339,19 +359,16 @@ "startedAt": 0, "settledAt": 0, "value": { + "error": { + "code": "method_not_found", + "message": "Unknown method" + }, "id": "frame-1", - "ok": true, - "result": { - "$rpc": "null" - } + "ok": false } } }, - "a2f842e44f38": { - "failures": ["outer refused"], - "pending": 0 - }, - "a5db8d0cb481": { + "f6a8a6f550f5": { "name": "speech.dictation.chunk#1", "ordinal": 1, "args": [ @@ -370,7 +387,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -388,24 +405,7 @@ } } }, - "b5157118341f": { - "failures": [""], - "pending": 0 - }, - "bc459c132276": { - "status": "fulfilled", - "startedAt": 0, - "settledAt": 0, - "value": [ - { - "status": "fulfilled", - "value": { - "$rpc": "undefined" - } - } - ] - }, - "e9501f2e1278": { + "f779babe2a18": { "name": "speech.dictation.chunk#1", "ordinal": 1, "args": [ @@ -424,7 +424,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -436,7 +436,7 @@ "id": "frame-1", "ok": true, "result": { - "error": "refused" + "$rpc": "null" } } } @@ -448,7 +448,7 @@ { "id": "speech-audio-chunk-acknowledged.normal:acknowledged", "observation": { - "sender": ["182109450de8"], + "sender": ["99956eef8fff"], "payloads": ["12521d219054"], "settlements": { "chunk": "bc459c132276" @@ -460,7 +460,7 @@ { "id": "speech-audio-chunk-acknowledged.result-absent:acknowledged", "observation": { - "sender": ["25256657a918"], + "sender": ["62cfcef21f01"], "payloads": ["12521d219054"], "settlements": { "chunk": "bc459c132276" @@ -472,7 +472,7 @@ { "id": "speech-audio-chunk-acknowledged.result-null:acknowledged", "observation": { - "sender": ["923922ffae75"], + "sender": ["f779babe2a18"], "payloads": ["12521d219054"], "settlements": { "chunk": "bc459c132276" @@ -484,7 +484,7 @@ { "id": "speech-audio-chunk-acknowledged.inner-ok-missing:acknowledged", "observation": { - "sender": ["e9501f2e1278"], + "sender": ["83a5bdf2ac1a"], "payloads": ["12521d219054"], "settlements": { "chunk": "bc459c132276" @@ -496,7 +496,7 @@ { "id": "speech-audio-chunk-acknowledged.inner-false-string-error:acknowledged", "observation": { - "sender": ["16da7db6669b"], + "sender": ["888a29b4fd97"], "payloads": ["12521d219054"], "settlements": { "chunk": "bc459c132276" @@ -508,7 +508,7 @@ { "id": "speech-audio-chunk-acknowledged.inner-false-object-error:acknowledged", "observation": { - "sender": ["0a3ec57e0321"], + "sender": ["028afac3649e"], "payloads": ["12521d219054"], "settlements": { "chunk": "bc459c132276" @@ -520,7 +520,7 @@ { "id": "speech-audio-chunk-acknowledged.outer-refused:acknowledged", "observation": { - "sender": ["1d0f84aa0730"], + "sender": ["66dece39b5ba"], "payloads": ["12521d219054"], "settlements": { "chunk": "bc459c132276" @@ -532,7 +532,7 @@ { "id": "speech-audio-chunk-acknowledged.outer-refused-no-message:acknowledged", "observation": { - "sender": ["a5db8d0cb481"], + "sender": ["f6a8a6f550f5"], "payloads": ["12521d219054"], "settlements": { "chunk": "bc459c132276" @@ -544,7 +544,7 @@ { "id": "speech-audio-chunk-acknowledged.method-not-found:acknowledged", "observation": { - "sender": ["208244a4b935"], + "sender": ["da1579f3254f"], "payloads": ["12521d219054"], "settlements": { "chunk": "bc459c132276" @@ -556,7 +556,7 @@ { "id": "speech-audio-chunk-acknowledged.transport-rejection:acknowledged", "observation": { - "sender": ["1ca8c123cb2d"], + "sender": ["30ae620f58dc"], "payloads": ["12521d219054"], "settlements": { "chunk": "bc459c132276" @@ -568,7 +568,7 @@ { "id": "speech-audio-chunk-acknowledged.transport-rejection-no-message:acknowledged", "observation": { - "sender": ["88a268fa1b7b"], + "sender": ["9d43743b1b10"], "payloads": ["12521d219054"], "settlements": { "chunk": "bc459c132276" diff --git a/mobile/rpc-foundation/goldens/speech-audio-chunk-acknowledged.json b/mobile/rpc-foundation/goldens/speech-audio-chunk-acknowledged.json index 42b1624f28bd..0f3839c8ed0b 100644 --- a/mobile/rpc-foundation/goldens/speech-audio-chunk-acknowledged.json +++ b/mobile/rpc-foundation/goldens/speech-audio-chunk-acknowledged.json @@ -13,7 +13,7 @@ "ordinal": 2, "json": "{\"id\":\"frame-1\",\"deviceToken\":\"recording-device\",\"method\":\"speech.dictation.chunk\",\"params\":{\"dictationId\":\"dictation-1\",\"audioBase64\":\"ACVKb5S53gM=\",\"sampleRate\":16000}}" }, - "182109450de8": { + "99956eef8fff": { "name": "speech.dictation.chunk#1", "ordinal": 1, "args": [ @@ -32,7 +32,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -69,7 +69,7 @@ { "id": "acknowledged", "observation": { - "sender": ["182109450de8"], + "sender": ["99956eef8fff"], "payloads": ["12521d219054"], "settlements": { "chunk": "bc459c132276" diff --git a/mobile/src/hooks/mobile-dictation-audio-chunk.ts b/mobile/src/hooks/mobile-dictation-audio-chunk.ts index 3f326fd65d15..044551f19f9c 100644 --- a/mobile/src/hooks/mobile-dictation-audio-chunk.ts +++ b/mobile/src/hooks/mobile-dictation-audio-chunk.ts @@ -4,6 +4,7 @@ import { } from './mobile-dictation-pending-audio-budget' import { bytesToBase64 } from './mobile-dictation-session-state' import { dictationAudioChunkSend } from '../dictation/mobile-dictation-operations' +import { MOBILE_DICTATION_RECONNECT_GRACE_MS } from '../../../src/shared/rpc-contract/speech-params' import { isRpcDeliveryUnknown } from '../transport/rpc-delivery-ambiguity' import { rpcPayloadMember } from '../transport/rpc-reader-payload' import { isLogicalClientCutoverError } from '../transport/stable-logical-rpc-client' @@ -39,12 +40,14 @@ export function enqueueMobileDictationAudioChunk( audioBase64: bytesToBase64(bytes), sampleRate: MOBILE_DICTATION_PCM_SAMPLE_RATE } + // Wait for a reconnect only as long as the host keeps the session; past that, fail and keep the text so far. + const options = { timeoutMs: MOBILE_DICTATION_RECONNECT_GRACE_MS } const sendChunk = dictationAudioChunkSend - .request(client, params) + .request(client, params, options) .catch((err: unknown) => // A chunk still waiting when the connection moved never reached the host; send it on the new one. isLogicalClientCutoverError(err) && !isRpcDeliveryUnknown(err) - ? dictationAudioChunkSend.request(client, params) + ? dictationAudioChunkSend.request(client, params, options) : Promise.reject(err) ) .then((reply) => { diff --git a/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts b/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts index 37dc70e268b9..a94a6b35d5e3 100644 --- a/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts +++ b/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts @@ -1,8 +1,10 @@ +import { MOBILE_DICTATION_RECONNECT_GRACE_MS } from '../../../src/shared/rpc-contract/speech-params' + export const MOBILE_DICTATION_PCM_SAMPLE_RATE = 16000 const PCM_BYTES_PER_SAMPLE = 2 -// Covers the host's reconnect grace (MOBILE_DICTATION_RECONNECT_GRACE_MS) so a reconnect keeps the speech queued behind it. -const MAX_PENDING_AUDIO_SECONDS = 30 +// Hold the speech captured while a reconnect is pending, for as long as the host keeps the session. +const MAX_PENDING_AUDIO_SECONDS = MOBILE_DICTATION_RECONNECT_GRACE_MS / 1000 // Why: dictation chunk RPCs wait through reconnect before timing out, so // cap retained raw microphone audio before it is expanded into base64 payloads. diff --git a/mobile/src/session/MobileTerminalInputActions.tsx b/mobile/src/session/MobileTerminalInputActions.tsx index ffd650651a62..c9c30a0a8929 100644 --- a/mobile/src/session/MobileTerminalInputActions.tsx +++ b/mobile/src/session/MobileTerminalInputActions.tsx @@ -45,6 +45,8 @@ export function MobileTerminalInputActions({ }: MobileTerminalInputActionsProps) { useMobileColorScheme() const dictationActive = dictation.isStarting || dictation.isRecording + // Dictation outlives a reconnect, so its stop and cancel must too; only starting needs the connection. + const micDisabled = !canSend && !dictationActive && !dictation.isProcessing return ( <> renderer.unmount()) }) + +it('keeps the microphone able to stop a recording while the connection is down', async () => { + const idle = { isStarting: false, isRecording: false, isProcessing: false } + const props = { + canSend: false, + isAttaching: false, + dictation: { ...idle, isRecording: true }, + dictationMode: 'toggle', + buttonStyle: {}, + activeButtonStyle: {}, + disabledButtonStyle: {}, + onAttachImage: vi.fn(), + onAttachFile: vi.fn(), + onDictationToggle: vi.fn(), + onDictationPressIn: vi.fn(), + onDictationPressOut: vi.fn(), + onDictationCancel: vi.fn() + } + let renderer!: ReturnType + await act(async () => { + renderer = create(createElement(MobileTerminalInputActions, props)) + }) + const mic = () => + renderer.root.find( + (node) => + String(node.type) === 'Pressable' && node.props.accessibilityLabel !== 'Attach a photo' + ) + expect(mic().props.disabled).toBe(false) + await act(async () => + renderer.update(createElement(MobileTerminalInputActions, { ...props, dictation: idle })) + ) + expect(mic().props.disabled).toBe(true) + await act(async () => renderer.unmount()) +}) diff --git a/mobile/src/transport/mobile-relay-connect-waiters.ts b/mobile/src/transport/mobile-relay-connect-waiters.ts new file mode 100644 index 000000000000..8015d74534fe --- /dev/null +++ b/mobile/src/transport/mobile-relay-connect-waiters.ts @@ -0,0 +1,51 @@ +import type { ConnectionState } from './types' + +type RelaySessionStateSource = { + getState(): ConnectionState + onStateChange(listener: (state: ConnectionState) => void): () => void +} + +// Requests issued before a relay session connects wait here until it connects, fails, times out or is retired. +export class MobileRelayConnectWaiters { + private readonly releases = new Set<() => void>() + + constructor(private readonly session: RelaySessionStateSource) {} + + wait(timeoutMs: number): Promise { + if (this.session.getState() === 'connected') { + return Promise.resolve() + } + return new Promise((resolve, reject) => { + const settle = (error: Error | null): void => { + clearTimeout(timer) + unsubscribe() + this.releases.delete(release) + if (error) { + reject(error) + } else { + resolve() + } + } + const release = () => settle(new Error('relay session closed')) + this.releases.add(release) + const unsubscribe = this.session.onStateChange((next) => { + if (next === 'connected') { + settle(null) + } else if (next === 'disconnected' || next === 'auth-failed') { + settle(new Error(`relay session ${next}`)) + } + }) + const timer = setTimeout( + () => settle(new Error('relay session connection timed out')), + timeoutMs + ) + }) + } + + // A failed session already published disconnected, so requests issued since are still waiting; retiring it must release them to the replacement. + releaseAll(): void { + for (const release of this.releases) { + release() + } + } +} diff --git a/mobile/src/transport/mobile-relay-rpc-session.test.ts b/mobile/src/transport/mobile-relay-rpc-session.test.ts index 0f58b87b020d..b58e0f0415e6 100644 --- a/mobile/src/transport/mobile-relay-rpc-session.test.ts +++ b/mobile/src/transport/mobile-relay-rpc-session.test.ts @@ -326,6 +326,20 @@ describe('mobile relay RPC session', () => { expect(session.getState()).toBe('disconnected') }) + it('releases requests waiting on a failed session once it is retired, unsent', async () => { + const { session } = await authenticateSession() + fakes.linkOptions!.onError(new Error('relay lost')) + vi.useFakeTimers() + const settled = vi.fn() + session.sendRequest('status.get').catch(settled) + session.close() + // No timer advances: the request must not sit out its connect timeout. + await vi.waitFor(() => expect(settled).toHaveBeenCalledOnce()) + + expect(isRpcDeliveryUnknown(settled.mock.calls[0]![0])).toBe(false) + expect(fakes.sendText).not.toHaveBeenCalled() + }) + it('keeps a synchronous pre-write relay failure definite', async () => { const { session } = await authenticateSession() fakes.sendText.mockImplementationOnce(() => { diff --git a/mobile/src/transport/mobile-relay-rpc-session.ts b/mobile/src/transport/mobile-relay-rpc-session.ts index 911bb59f889d..9ac1d9717632 100644 --- a/mobile/src/transport/mobile-relay-rpc-session.ts +++ b/mobile/src/transport/mobile-relay-rpc-session.ts @@ -16,6 +16,7 @@ import { openRpcRequestBudget, resolvePostConnectRequestTimeout } from './rpc-re import { isRpcResponse } from './rpc-response-shape' import { RelayDialStageTracker, type RelayDialStageSource } from './relay-dial-stage' import { RelayPendingRequests } from './relay-pending-requests' +import { MobileRelayConnectWaiters } from './mobile-relay-connect-waiters' import { RpcSessionLivenessWatchdog } from './rpc-session-liveness-watchdog' import { settleMobileRuntimeCapabilities } from './mobile-runtime-capability-negotiation' import type { RelayHostCloseReason } from '../../../src/shared/relay-host-close-reason' @@ -138,6 +139,7 @@ export function connectMobileRelayRpcSession(args: { } }, close() { + connectWaiters.releaseAll() if (closed) { return } @@ -157,6 +159,7 @@ export function connectMobileRelayRpcSession(args: { probeAfterBackground: () => state === 'connected' && livenessWatchdog.probeNow(livenessIdentity, RELAY_RESUME_PROBE_MS) } + const connectWaiters = new MobileRelayConnectWaiters(client) const livenessWatchdog = new RpcSessionLivenessWatchdog({ transport: 'relay', idleProbeMs: null, @@ -265,31 +268,7 @@ export function connectMobileRelayRpcSession(args: { } function waitForConnected(timeoutMs = requestTimeoutMs): Promise { - if (state === 'connected') { - return Promise.resolve() - } - return new Promise((resolve, reject) => { - let timer: ReturnType | null = null - const unsubscribe = client.onStateChange((next) => { - if (next === 'connected') { - finish() - resolve() - } else if (next === 'disconnected' || next === 'auth-failed') { - finish() - reject(new Error(`relay session ${next}`)) - } - }) - timer = setTimeout(() => { - finish() - reject(new Error('relay session connection timed out')) - }, timeoutMs) - function finish(): void { - if (timer) { - clearTimeout(timer) - } - unsubscribe() - } - }) + return connectWaiters.wait(timeoutMs) } function publishState(next: ConnectionState): void { diff --git a/src/main/runtime/runtime-mobile-dictation-controller.test.ts b/src/main/runtime/runtime-mobile-dictation-controller.test.ts index ebc7235e9e4b..bbecb880ba92 100644 --- a/src/main/runtime/runtime-mobile-dictation-controller.test.ts +++ b/src/main/runtime/runtime-mobile-dictation-controller.test.ts @@ -4,10 +4,8 @@ import { setSpeechServiceFactories, type SpeechServiceFactories } from '../speech/speech-runtime-service' -import { - MOBILE_DICTATION_RECONNECT_GRACE_MS, - RuntimeMobileDictationController -} from './runtime-mobile-dictation-controller' +import { MOBILE_DICTATION_RECONNECT_GRACE_MS } from '../../shared/rpc-contract/speech-params' +import { RuntimeMobileDictationController } from './runtime-mobile-dictation-controller' import type { RuntimeStore } from './runtime-store-contract' type Sink = (event: { type: string; text?: string }) => void @@ -49,13 +47,12 @@ async function startPhoneDictation(): Promise const chunk = { dictationId: 'd-1', audioBase64: 'AAAA', sampleRate: 16_000, clientId: 'phone-a' } describe('mobile dictation across a phone reconnect', () => { - it('keeps the session when the phone comes back within the grace', async () => { + it('keeps the session while the phone keeps reaching it, whatever connection it uses', async () => { const controller = await startPhoneDictation() - controller.cancelAfterClientDisconnectGrace('phone-a') - vi.advanceTimersByTime(MOBILE_DICTATION_RECONNECT_GRACE_MS - 1) + await vi.advanceTimersByTimeAsync(MOBILE_DICTATION_RECONNECT_GRACE_MS - 1) expect(controller.feed(chunk)).toEqual({ dictationId: 'd-1', text: 'before the drop' }) - await vi.advanceTimersByTimeAsync(MOBILE_DICTATION_RECONNECT_GRACE_MS) + await vi.advanceTimersByTimeAsync(MOBILE_DICTATION_RECONNECT_GRACE_MS - 1) sink({ type: 'final', text: 'after it' }) await expect(controller.finish({ dictationId: 'd-1', clientId: 'phone-a' })).resolves.toEqual({ @@ -64,10 +61,9 @@ describe('mobile dictation across a phone reconnect', () => { }) }) - it('releases the speech model when the phone never comes back', async () => { + it('releases the speech model when the phone goes quiet for the grace', async () => { const controller = await startPhoneDictation() - controller.cancelAfterClientDisconnectGrace('phone-a') await vi.advanceTimersByTimeAsync(MOBILE_DICTATION_RECONNECT_GRACE_MS) expect(stopDictation).toHaveBeenCalledOnce() diff --git a/src/main/runtime/runtime-mobile-dictation-controller.ts b/src/main/runtime/runtime-mobile-dictation-controller.ts index 6582e9790baf..ab194fe3d1c4 100644 --- a/src/main/runtime/runtime-mobile-dictation-controller.ts +++ b/src/main/runtime/runtime-mobile-dictation-controller.ts @@ -1,4 +1,5 @@ import { getDefaultVoiceSettings } from '../../shared/constants' +import { MOBILE_DICTATION_RECONNECT_GRACE_MS } from '../../shared/rpc-contract/speech-params' import { getSpeechModelManager, getSpeechSttService } from '../speech/speech-runtime-service' import { getCatalogModel, isLocalSpeechModel } from '../speech/model-catalog' import { @@ -7,9 +8,6 @@ import { } from '../speech/stt-worker-paths' import type { RuntimeStore } from './runtime-store-contract' -// Phones reconnect by redialling (network switch, Relay fallback); keep the session that long. -export const MOBILE_DICTATION_RECONNECT_GRACE_MS = 30_000 - type MobileDictationSession = { id: string owner: string @@ -18,7 +16,7 @@ type MobileDictationSession = { partialText: string finalTexts: string[] errors: string[] - reconnectGraceTimer?: ReturnType + abandonTimer?: ReturnType } export class RuntimeMobileDictationController { @@ -80,6 +78,7 @@ export class RuntimeMobileDictationController { throw new Error('dictation_canceled') } this.session.state = 'active' + this.armAbandonTimer(this.session) } catch (error) { if (this.session?.id === params.dictationId) { this.session = null @@ -117,6 +116,7 @@ export class RuntimeMobileDictationController { clientId?: string }): Promise<{ dictationId: string; text: string }> { const session = this.requireOwnedSession(params) + clearTimeout(session.abandonTimer) session.state = 'closing' try { await getSpeechSttService(this.requireStore()).stopDictation(session.owner) @@ -142,19 +142,6 @@ export class RuntimeMobileDictationController { return { dictationId: params.dictationId } } - // Cancel only if the phone has not come back within the grace; its next request clears this. - cancelAfterClientDisconnectGrace(clientId: string): void { - const session = this.session - if (session?.clientId !== clientId || session.state === 'closing') { - return - } - clearTimeout(session.reconnectGraceTimer) - session.reconnectGraceTimer = setTimeout( - () => this.cancelSession(session), - MOBILE_DICTATION_RECONNECT_GRACE_MS - ) - } - private acceptEvent( dictationId: string, event: { type: string; text?: string; error?: string } @@ -177,6 +164,16 @@ export class RuntimeMobileDictationController { } // Owned by the phone, not its socket: a reconnect arrives on a new connection. + // The phone streams audio continuously, so silence this long means it is gone (killed, lost its cancel, never reconnected). + private armAbandonTimer(session: MobileDictationSession): void { + clearTimeout(session.abandonTimer) + session.abandonTimer = setTimeout(() => { + if (session.state === 'active') { + void this.stopSession(session).catch(() => undefined) + } + }, MOBILE_DICTATION_RECONNECT_GRACE_MS) + } + private requireOwnedSession(params: { dictationId: string clientId?: string @@ -188,12 +185,14 @@ export class RuntimeMobileDictationController { if (session.clientId !== params.clientId) { throw new Error('dictation_owner_mismatch') } - clearTimeout(session.reconnectGraceTimer) + if (session.state === 'active') { + this.armAbandonTimer(session) + } return session } private async stopSession(session: MobileDictationSession): Promise { - clearTimeout(session.reconnectGraceTimer) + clearTimeout(session.abandonTimer) session.state = 'closing' try { await getSpeechSttService(this.requireStore()).stopDictation(session.owner) @@ -204,13 +203,6 @@ export class RuntimeMobileDictationController { } } - private cancelSession(session: MobileDictationSession): void { - if (session.state === 'closing') { - return - } - void this.stopSession(session).catch(() => undefined) - } - private requireStore(): RuntimeStore { const store = this.getStore() if (!store) { diff --git a/src/main/runtime/terminal/layout/terminal-client-disconnect.ts b/src/main/runtime/terminal/layout/terminal-client-disconnect.ts index f593aea0a3f9..e0753045be2d 100644 --- a/src/main/runtime/terminal/layout/terminal-client-disconnect.ts +++ b/src/main/runtime/terminal/layout/terminal-client-disconnect.ts @@ -15,7 +15,6 @@ export abstract class OrcaRuntimeWithOnClientDisconnected extends OrcaRuntimeWit onClientDisconnected(clientId: string): void { this.revokeTerminalFileGrantsForClient(clientId) - this.mobileDictation.cancelAfterClientDisconnectGrace(clientId) for (const [ptyId, entry] of this.pendingRestoreTimers) { if (entry.clientId === clientId) { diff --git a/src/shared/rpc-contract/speech-params.ts b/src/shared/rpc-contract/speech-params.ts index a6970b34c50e..043585c42f1a 100644 --- a/src/shared/rpc-contract/speech-params.ts +++ b/src/shared/rpc-contract/speech-params.ts @@ -9,6 +9,9 @@ export const PCM_BYTES_PER_SAMPLE = 2 export const MAX_DICTATION_AUDIO_SECONDS = 5 +// The host drops a phone dictation it hasn't heard from for this long, so the phone waits no longer for a reconnect. +export const MOBILE_DICTATION_RECONNECT_GRACE_MS = 30_000 + export const MAX_DICTATION_AUDIO_CHUNK_BYTES = DICTATION_SAMPLE_RATE * PCM_BYTES_PER_SAMPLE * MAX_DICTATION_AUDIO_SECONDS From 58e8859618061692f331fa98678595251f4731da Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:13:17 +0000 Subject: [PATCH 3/7] fix(mobile): resend an unsent dictation finish after a connection move and send the text so far only when it changes --- .../src/hooks/mobile-dictation-audio-chunk.ts | 84 +++++++++++++++---- .../hooks/mobile-dictation-reconnect.test.tsx | 16 ++-- mobile/src/hooks/use-mobile-dictation.ts | 56 +++++-------- .../dictation-capture-bridge-budget.test.tsx | 8 +- ...mobile-dictation-session-outcomes.test.tsx | 1 + .../adapters/dictation-mount-adapters.ts | 9 +- ...untime-mobile-dictation-controller.test.ts | 1 + .../runtime-mobile-dictation-controller.ts | 15 +++- 8 files changed, 126 insertions(+), 64 deletions(-) diff --git a/mobile/src/hooks/mobile-dictation-audio-chunk.ts b/mobile/src/hooks/mobile-dictation-audio-chunk.ts index 044551f19f9c..ab58e7f57e85 100644 --- a/mobile/src/hooks/mobile-dictation-audio-chunk.ts +++ b/mobile/src/hooks/mobile-dictation-audio-chunk.ts @@ -8,16 +8,70 @@ import { MOBILE_DICTATION_RECONNECT_GRACE_MS } from '../../../src/shared/rpc-con import { isRpcDeliveryUnknown } from '../transport/rpc-delivery-ambiguity' import { rpcPayloadMember } from '../transport/rpc-reader-payload' import { isLogicalClientCutoverError } from '../transport/stable-logical-rpc-client' -import type { DictationCaptureChunk } from '../platform/dictation-capture-contract' +import type { + DictationCapture, + DictationCaptureChunk +} from '../platform/dictation-capture-contract' import type { MobileDictationPendingAudioBudget } from './mobile-dictation-pending-audio-budget' import type { RpcClient } from '../transport/rpc-client' type MobileDictationAudioChunkQueue = { pendingChunks: Set> pendingAudioBudget: MobileDictationPendingAudioBudget - shouldReleaseBudget: (dictationId: string) => boolean + isActive: (dictationId: string) => boolean failActiveDictation: (dictationId: string, err: unknown) => void - noteTranscriptSoFar: (dictationId: string, text: string) => void + transcriptSoFar: MobileDictationTranscriptSoFar +} + +// Hosts that keep dictation through a reconnect report the text so far; a failure hands it over instead of dropping it. +export class MobileDictationTranscriptSoFar { + private kept: { dictationId: string; text: string } | null = null + + note(dictationId: string, text: string): void { + this.kept = { dictationId, text } + } + + take(dictationId: string): string { + const kept = this.kept + this.kept = null + return kept?.dictationId === dictationId ? kept.text : '' + } +} + +// A request still waiting when the connection moved never reached the host; send it again on the new one. +export function sendAcrossConnectionMove(send: () => Promise): Promise { + return send().catch((err: unknown) => + isLogicalClientCutoverError(err) && !isRpcDeliveryUnknown(err) ? send() : Promise.reject(err) + ) +} + +export function subscribeMobileDictationAudioChunks( + capture: Pick, + source: { + getClient: () => RpcClient | null + getActiveId: () => string | null + isAccepting: () => boolean + }, + queue: Omit +): { remove: () => void } { + // Reuse wiring on the microphone hot path instead of allocating per chunk. + const chunkQueue = { ...queue, isActive: (id: string) => source.getActiveId() === id } + return capture.onChunk((chunk) => { + const client = source.getClient() + const dictationId = source.getActiveId() + if (!client || !dictationId || !source.isAccepting()) { + return + } + if (chunk.droppedBytes > 0) { + // Page capture loss and budget overflow both mean the page fell behind the microphone; report the same error. Native capture never drops here. + queue.failActiveDictation( + dictationId, + new Error(MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE) + ) + return + } + enqueueMobileDictationAudioChunk(client, dictationId, chunk, chunkQueue) + }) } export function enqueueMobileDictationAudioChunk( @@ -40,22 +94,18 @@ export function enqueueMobileDictationAudioChunk( audioBase64: bytesToBase64(bytes), sampleRate: MOBILE_DICTATION_PCM_SAMPLE_RATE } - // Wait for a reconnect only as long as the host keeps the session; past that, fail and keep the text so far. - const options = { timeoutMs: MOBILE_DICTATION_RECONNECT_GRACE_MS } - const sendChunk = dictationAudioChunkSend - .request(client, params, options) - .catch((err: unknown) => - // A chunk still waiting when the connection moved never reached the host; send it on the new one. - isLogicalClientCutoverError(err) && !isRpcDeliveryUnknown(err) - ? dictationAudioChunkSend.request(client, params, options) - : Promise.reject(err) - ) + const sendChunk = sendAcrossConnectionMove(() => + // Wait for a reconnect only as long as the host keeps the session; past that, fail and keep the text so far. + dictationAudioChunkSend.request(client, params, { + timeoutMs: MOBILE_DICTATION_RECONNECT_GRACE_MS + }) + ) .then((reply) => { - // Only newer hosts send the text so far; any other result still acknowledges the chunk. + // Only newer hosts send the text, and only when it changed; any other result still acknowledges the chunk. const result = dictationAudioChunkSend.interpret(reply) const text = result == null ? undefined : rpcPayloadMember(result, 'text') - if (typeof text === 'string') { - queue.noteTranscriptSoFar(dictationId, text) + if (typeof text === 'string' && queue.isActive(dictationId)) { + queue.transcriptSoFar.note(dictationId, text) } }) .catch((err) => { @@ -65,7 +115,7 @@ export function enqueueMobileDictationAudioChunk( } }) .finally(() => { - if (queue.shouldReleaseBudget(dictationId)) { + if (queue.isActive(dictationId)) { queue.pendingAudioBudget.release(byteLength) } queue.pendingChunks.delete(sendChunk) diff --git a/mobile/src/hooks/mobile-dictation-reconnect.test.tsx b/mobile/src/hooks/mobile-dictation-reconnect.test.tsx index 381f517498f4..466ba538b0b9 100644 --- a/mobile/src/hooks/mobile-dictation-reconnect.test.tsx +++ b/mobile/src/hooks/mobile-dictation-reconnect.test.tsx @@ -66,8 +66,13 @@ function dictation(): UseMobileDictationResult { return held.dictation } -// Answer each chunk with the next scripted reply and everything else with success. -async function pump(rpc: FakeRpcClient, sent: SentRequest[], chunkReplies: Reply[]): Promise { +// Answer chunks and finishes with the next scripted reply, and everything else with success. +async function pump( + rpc: FakeRpcClient, + sent: SentRequest[], + chunkReplies: Reply[], + finishReplies: Reply[] = [] +): Promise { for (let round = 0; round < 8; round += 1) { for (const request of rpc.requests.splice(0)) { sent.push(request) @@ -75,7 +80,7 @@ async function pump(rpc: FakeRpcClient, sent: SentRequest[], chunkReplies: Reply request.method === 'speech.dictation.chunk' ? (chunkReplies.shift() ?? {}) : request.method === 'speech.dictation.finish' - ? { text: 'hello world' } + ? (finishReplies.shift() ?? { text: 'hello world' }) : {} if (reply instanceof Error) { request.reject(reply) @@ -112,7 +117,7 @@ beforeEach(() => { }) describe('dictation through a reconnect', () => { - it('keeps recording past chunks the drop lost, and resends ones that never left', async () => { + it('keeps recording past chunks the drop lost, and resends requests that never left', async () => { const rpc = createFakeRpcClient() const sent: SentRequest[] = [] const transcripts: string[] = [] @@ -130,9 +135,10 @@ describe('dictation through a reconnect', () => { expect(methodsOf(sent).filter((m) => m === 'speech.dictation.chunk')).toHaveLength(4) await act(async () => { const stopped = dictation().stop() - await pump(rpc, sent, []) + await pump(rpc, sent, [], [new LogicalClientCutoverError(new Error('Connection closed'))]) await stopped }) + expect(methodsOf(sent).filter((m) => m === 'speech.dictation.finish')).toHaveLength(2) expect(methodsOf(sent)).not.toContain('speech.dictation.cancel') expect(transcripts).toEqual(['hello world']) expect(errors).toEqual([]) diff --git a/mobile/src/hooks/use-mobile-dictation.ts b/mobile/src/hooks/use-mobile-dictation.ts index 4a933506099d..af2ae0c5a205 100644 --- a/mobile/src/hooks/use-mobile-dictation.ts +++ b/mobile/src/hooks/use-mobile-dictation.ts @@ -1,10 +1,11 @@ import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react' import { useDictationCapture } from '../platform/dictation-capture' +import { MobileDictationPendingAudioBudget } from './mobile-dictation-pending-audio-budget' import { - MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE, - MobileDictationPendingAudioBudget -} from './mobile-dictation-pending-audio-budget' -import { enqueueMobileDictationAudioChunk } from './mobile-dictation-audio-chunk' + MobileDictationTranscriptSoFar, + sendAcrossConnectionMove, + subscribeMobileDictationAudioChunks +} from './mobile-dictation-audio-chunk' import { DICTATION_FINISH_TIMEOUT_MS, MOBILE_DICTATION_INPUT_CLOSED_ERROR_MESSAGE, @@ -43,7 +44,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil const pendingAudioBudgetRef = useRef(new MobileDictationPendingAudioBudget()) const acceptingChunksRef = useRef(false) const generationRef = useRef(0) - const transcriptSoFarRef = useRef<{ dictationId: string; text: string } | null>(null) + const transcriptSoFarRef = useRef(new MobileDictationTranscriptSoFar()) useLayoutEffect(() => { // Native audio events can arrive before passive Effects flush, but refs @@ -92,10 +93,9 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil if (client && dictationId) { void dictationSessionCancel.request(client, { dictationId }).catch(() => undefined) } - // Keep what the host transcribed before the failure instead of dropping it with the error. - const kept = transcriptSoFarRef.current - if (kept?.dictationId === dictationId && kept.text) { - onTranscriptRef.current(kept.text) + const kept = transcriptSoFarRef.current.take(dictationId) + if (kept) { + onTranscriptRef.current(kept) } reportError(err) }, @@ -103,31 +103,19 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil ) useEffect(() => { - // Reuse wiring on the microphone hot path instead of allocating per chunk. - const audioChunkQueue = { + const source = { + getClient: () => clientRef.current, + getActiveId: () => activeIdRef.current, + isAccepting: () => enabledRef.current && acceptingChunksRef.current + } + const sub = subscribeMobileDictationAudioChunks(capture, source, { pendingChunks: pendingChunksRef.current, pendingAudioBudget: pendingAudioBudgetRef.current, - shouldReleaseBudget: (id: string) => activeIdRef.current === id, failActiveDictation, - noteTranscriptSoFar: (dictationId: string, text: string) => { - transcriptSoFarRef.current = { dictationId, text } - } - } - const sub = capture.onChunk((chunk) => { - const client = clientRef.current - const dictationId = activeIdRef.current - if (!client || !dictationId || !enabledRef.current || !acceptingChunksRef.current) { - return - } - if (chunk.droppedBytes > 0) { - // Page capture loss and budget overflow both mean the page fell behind the microphone; report the same error. Native capture never drops here. - failActiveDictation(dictationId, new Error(MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE)) - return - } - enqueueMobileDictationAudioChunk(client, dictationId, chunk, audioChunkQueue) + transcriptSoFar: transcriptSoFarRef.current }) return () => sub.remove() - }, [capture, failActiveDictation, reportError]) + }, [capture, failActiveDictation]) const start = useCallback(async () => { const client = clientRef.current @@ -230,10 +218,12 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil return } const finished = dictationSessionFinish.interpret( - await dictationSessionFinish.request( - client, - { dictationId }, - { timeoutMs: DICTATION_FINISH_TIMEOUT_MS } + await sendAcrossConnectionMove(() => + dictationSessionFinish.request( + client, + { dictationId }, + { timeoutMs: DICTATION_FINISH_TIMEOUT_MS } + ) ) ) if ( diff --git a/mobile/src/platform/dictation-capture-bridge-budget.test.tsx b/mobile/src/platform/dictation-capture-bridge-budget.test.tsx index ddf6ac6dee49..6a79c27f10d5 100644 --- a/mobile/src/platform/dictation-capture-bridge-budget.test.tsx +++ b/mobile/src/platform/dictation-capture-bridge-budget.test.tsx @@ -21,7 +21,10 @@ import { type BridgePortPair } from '../mobile-web-shell/bridge/bridge-port-pair-test-harness' import { MobileDictationPendingAudioBudget } from '../hooks/mobile-dictation-pending-audio-budget' -import { enqueueMobileDictationAudioChunk } from '../hooks/mobile-dictation-audio-chunk' +import { + enqueueMobileDictationAudioChunk, + MobileDictationTranscriptSoFar +} from '../hooks/mobile-dictation-audio-chunk' import { createNativeAudioCapture, type NativeAudioEngine } from './native-audio' import { createPageDictationCapture } from './dictation-capture.web' import { @@ -98,7 +101,8 @@ async function runDictation( enqueueMobileDictationAudioChunk(pair.client, 'dictation-1', chunk, { pendingChunks, pendingAudioBudget, - shouldReleaseBudget: () => true, + isActive: () => true, + transcriptSoFar: new MobileDictationTranscriptSoFar(), failActiveDictation: (_id, error) => { refusals.push(error instanceof Error ? error.message : String(error)) } diff --git a/mobile/src/session/mobile-dictation-session-outcomes.test.tsx b/mobile/src/session/mobile-dictation-session-outcomes.test.tsx index 9fcd61639727..916a06a06378 100644 --- a/mobile/src/session/mobile-dictation-session-outcomes.test.tsx +++ b/mobile/src/session/mobile-dictation-session-outcomes.test.tsx @@ -126,6 +126,7 @@ function mount( activeHandleRef: useRef('t1'), flushPendingLiveInputBeforeExternalSend: async () => true, canSend: true, + canCompose: true, liveInputEnabled: options.liveInput, showToast: (message: string) => { seen.toasts.push(message) diff --git a/mobile/src/test-support/rpc-recording/adapters/dictation-mount-adapters.ts b/mobile/src/test-support/rpc-recording/adapters/dictation-mount-adapters.ts index 5d23e6a7b416..a32c512d1ef7 100644 --- a/mobile/src/test-support/rpc-recording/adapters/dictation-mount-adapters.ts +++ b/mobile/src/test-support/rpc-recording/adapters/dictation-mount-adapters.ts @@ -1,6 +1,7 @@ import type { MountAdapter } from '../recording-scenario' import { hookMount, performHookAction } from '../hook-mount' import type { operationModuleLoader } from '../operation-module-loader' +import type * as AudioChunkModule from '../../../hooks/mobile-dictation-audio-chunk' const DICTATION_ID = 'dictation-1' const MODEL_ID = 'whisper-small' @@ -82,9 +83,10 @@ export function dictationMountAdapters( } }, 'speech.audio-chunk': ({ client, effect }) => { - const enqueue = modules.load( + const chunks = modules.load( 'mobile/src/hooks/mobile-dictation-audio-chunk.ts' - ).enqueueMobileDictationAudioChunk + ) + const enqueue = chunks.enqueueMobileDictationAudioChunk const budget = modules.load< typeof import('../../../hooks/mobile-dictation-pending-audio-budget') >('mobile/src/hooks/mobile-dictation-pending-audio-budget.ts') @@ -105,7 +107,8 @@ export function dictationMountAdapters( { pendingChunks, pendingAudioBudget, - shouldReleaseBudget: () => true, + isActive: () => true, + transcriptSoFar: new chunks.MobileDictationTranscriptSoFar(), failActiveDictation: (id: string, error: unknown) => { failures.push(error instanceof Error ? error.message : String(error)) effect('dictation-failed', { id }) diff --git a/src/main/runtime/runtime-mobile-dictation-controller.test.ts b/src/main/runtime/runtime-mobile-dictation-controller.test.ts index bbecb880ba92..5561274c3fec 100644 --- a/src/main/runtime/runtime-mobile-dictation-controller.test.ts +++ b/src/main/runtime/runtime-mobile-dictation-controller.test.ts @@ -52,6 +52,7 @@ describe('mobile dictation across a phone reconnect', () => { await vi.advanceTimersByTimeAsync(MOBILE_DICTATION_RECONNECT_GRACE_MS - 1) expect(controller.feed(chunk)).toEqual({ dictationId: 'd-1', text: 'before the drop' }) + expect(controller.feed(chunk)).toEqual({ dictationId: 'd-1' }) await vi.advanceTimersByTimeAsync(MOBILE_DICTATION_RECONNECT_GRACE_MS - 1) sink({ type: 'final', text: 'after it' }) diff --git a/src/main/runtime/runtime-mobile-dictation-controller.ts b/src/main/runtime/runtime-mobile-dictation-controller.ts index ab194fe3d1c4..bb7eb89f4fcb 100644 --- a/src/main/runtime/runtime-mobile-dictation-controller.ts +++ b/src/main/runtime/runtime-mobile-dictation-controller.ts @@ -16,6 +16,7 @@ type MobileDictationSession = { partialText: string finalTexts: string[] errors: string[] + reportedText: string abandonTimer?: ReturnType } @@ -65,7 +66,8 @@ export class RuntimeMobileDictationController { state: 'starting', partialText: '', finalTexts: [], - errors: [] + errors: [], + reportedText: '' } try { await getSpeechSttService(store).startDictation( @@ -93,7 +95,7 @@ export class RuntimeMobileDictationController { audioBase64: string sampleRate: number clientId?: string - }): { dictationId: string; text: string } { + }): { dictationId: string; text?: string } { const session = this.requireOwnedSession(params) if (session.state !== 'active') { throw new Error('dictation_stream_closing') @@ -107,8 +109,13 @@ export class RuntimeMobileDictationController { samples[i] = pcm.readInt16LE(i * 2) / 32768 } getSpeechSttService(this.requireStore()).feedAudio(samples, params.sampleRate, session.owner) - // The text so far lets the phone keep it if the connection never comes back. - return { dictationId: params.dictationId, text: transcriptOf(session) } + // The text so far lets the phone keep it if the connection never comes back; only changes are sent, as chunks arrive every few ms. + const text = transcriptOf(session) + if (text === session.reportedText) { + return { dictationId: params.dictationId } + } + session.reportedText = text + return { dictationId: params.dictationId, text } } async finish(params: { From 66520de9dc83efb5d355bea8b18b330623151ad0 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:33:45 +0000 Subject: [PATCH 4/7] fix(mobile): hold dictation requests through a suspended relay and bound the start wait --- ...ion-session-speech.dictation.finish-1.json | 88 +++--- ...tion-session-speech.dictation.start-1.json | 256 +++++++++--------- ...ation-start-speech.dictation.cancel-1.json | 48 ++-- ...tation-start-speech.dictation.start-1.json | 252 ++++++++--------- .../speech-desktop-start-fulfilled.json | 6 +- ...speech-desktop-start-recording-failed.json | 6 +- .../speech-desktop-start-superseded.json | 6 +- .../speech-dictation-session-cancelled.json | 6 +- .../speech-dictation-session-transcript.json | 6 +- .../src/hooks/mobile-dictation-audio-chunk.ts | 32 ++- .../hooks/mobile-dictation-desktop-start.ts | 8 +- .../hooks/mobile-dictation-reconnect.test.tsx | 28 +- mobile/src/hooks/use-mobile-dictation.ts | 2 +- .../stable-logical-rpc-client.test.ts | 18 ++ .../transport/stable-logical-rpc-client.ts | 16 +- 15 files changed, 429 insertions(+), 349 deletions(-) diff --git a/mobile/rpc-foundation/goldens/matrix-speech.dictation-session-speech.dictation.finish-1.json b/mobile/rpc-foundation/goldens/matrix-speech.dictation-session-speech.dictation.finish-1.json index 7c0ace696177..cbddc145a524 100644 --- a/mobile/rpc-foundation/goldens/matrix-speech.dictation-session-speech.dictation.finish-1.json +++ b/mobile/rpc-foundation/goldens/matrix-speech.dictation-session-speech.dictation.finish-1.json @@ -377,13 +377,13 @@ "startedAt": 0 } }, - "aca401ac072e": { - "name": "speech.dictation.start#1", - "ordinal": 1, + "b41f74be4c3d": { + "name": "speech.dictation.finish#1", + "ordinal": 3, "args": [ { "name": "method", - "value": "speech.dictation.start" + "value": "speech.dictation.finish" }, { "name": "params", @@ -394,7 +394,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 75000 } } ], @@ -403,21 +403,34 @@ "startedAt": 0, "settledAt": 0, "value": { - "id": "frame-1", - "ok": true, - "result": { - "started": true - } + "error": { + "code": "method_not_found", + "message": "Unknown method" + }, + "id": "frame-2", + "ok": false } } }, - "b41f74be4c3d": { - "name": "speech.dictation.finish#1", - "ordinal": 3, + "b432c878c8da": { + "error": "Cannot read properties of undefined (reading 'text')", + "status": "error", + "transcripts": [] + }, + "bca75e8024b4": { + "name": "dictation-error", + "ordinal": 6, + "value": { + "message": "Unknown method" + } + }, + "bd004568e24c": { + "name": "speech.dictation.start#1", + "ordinal": 1, "args": [ { "name": "method", - "value": "speech.dictation.finish" + "value": "speech.dictation.start" }, { "name": "params", @@ -428,7 +441,7 @@ { "name": "options", "value": { - "timeoutMs": 75000 + "timeoutMs": 30000 } } ], @@ -437,27 +450,14 @@ "startedAt": 0, "settledAt": 0, "value": { - "error": { - "code": "method_not_found", - "message": "Unknown method" - }, - "id": "frame-2", - "ok": false + "id": "frame-1", + "ok": true, + "result": { + "started": true + } } } }, - "b432c878c8da": { - "error": "Cannot read properties of undefined (reading 'text')", - "status": "error", - "transcripts": [] - }, - "bca75e8024b4": { - "name": "dictation-error", - "ordinal": 6, - "value": { - "message": "Unknown method" - } - }, "cd49b339a081": { "name": "dictation-error", "ordinal": 6, @@ -559,7 +559,7 @@ { "id": "speech-dictation-session-transcript.normal:transcribed", "observation": { - "sender": ["aca401ac072e", "4728ad10f92b"], + "sender": ["bd004568e24c", "4728ad10f92b"], "payloads": ["859d310e63d6", "1bdc94d60271"], "settlements": { "mount": "eb79a9b3682a", @@ -573,7 +573,7 @@ { "id": "speech-dictation-session-transcript.result-absent:transcribed", "observation": { - "sender": ["aca401ac072e", "88f795a14a21", "a7dbb6b2864f"], + "sender": ["bd004568e24c", "88f795a14a21", "a7dbb6b2864f"], "payloads": ["859d310e63d6", "1bdc94d60271", "6c3d0754a1ff"], "settlements": { "mount": "eb79a9b3682a", @@ -587,7 +587,7 @@ { "id": "speech-dictation-session-transcript.result-null:transcribed", "observation": { - "sender": ["aca401ac072e", "349e2cfd1a1f", "a7dbb6b2864f"], + "sender": ["bd004568e24c", "349e2cfd1a1f", "a7dbb6b2864f"], "payloads": ["859d310e63d6", "1bdc94d60271", "6c3d0754a1ff"], "settlements": { "mount": "eb79a9b3682a", @@ -601,7 +601,7 @@ { "id": "speech-dictation-session-transcript.inner-ok-missing:transcribed", "observation": { - "sender": ["aca401ac072e", "07d3e2ed39d8"], + "sender": ["bd004568e24c", "07d3e2ed39d8"], "payloads": ["859d310e63d6", "1bdc94d60271"], "settlements": { "mount": "eb79a9b3682a", @@ -615,7 +615,7 @@ { "id": "speech-dictation-session-transcript.inner-false-string-error:transcribed", "observation": { - "sender": ["aca401ac072e", "f4cf5aad78c0"], + "sender": ["bd004568e24c", "f4cf5aad78c0"], "payloads": ["859d310e63d6", "1bdc94d60271"], "settlements": { "mount": "eb79a9b3682a", @@ -629,7 +629,7 @@ { "id": "speech-dictation-session-transcript.inner-false-object-error:transcribed", "observation": { - "sender": ["aca401ac072e", "74617082bf69"], + "sender": ["bd004568e24c", "74617082bf69"], "payloads": ["859d310e63d6", "1bdc94d60271"], "settlements": { "mount": "eb79a9b3682a", @@ -643,7 +643,7 @@ { "id": "speech-dictation-session-transcript.outer-refused:transcribed", "observation": { - "sender": ["aca401ac072e", "a46c0f6c95b2", "a7dbb6b2864f"], + "sender": ["bd004568e24c", "a46c0f6c95b2", "a7dbb6b2864f"], "payloads": ["859d310e63d6", "1bdc94d60271", "6c3d0754a1ff"], "settlements": { "mount": "eb79a9b3682a", @@ -657,7 +657,7 @@ { "id": "speech-dictation-session-transcript.outer-refused-no-message:transcribed", "observation": { - "sender": ["aca401ac072e", "9fce1a80008e", "a7dbb6b2864f"], + "sender": ["bd004568e24c", "9fce1a80008e", "a7dbb6b2864f"], "payloads": ["859d310e63d6", "1bdc94d60271", "6c3d0754a1ff"], "settlements": { "mount": "eb79a9b3682a", @@ -671,7 +671,7 @@ { "id": "speech-dictation-session-transcript.method-not-found:transcribed", "observation": { - "sender": ["aca401ac072e", "b41f74be4c3d", "a7dbb6b2864f"], + "sender": ["bd004568e24c", "b41f74be4c3d", "a7dbb6b2864f"], "payloads": ["859d310e63d6", "1bdc94d60271", "6c3d0754a1ff"], "settlements": { "mount": "eb79a9b3682a", @@ -685,7 +685,7 @@ { "id": "speech-dictation-session-transcript.transport-rejection:transcribed", "observation": { - "sender": ["aca401ac072e", "cf10282da8e0", "a7dbb6b2864f"], + "sender": ["bd004568e24c", "cf10282da8e0", "a7dbb6b2864f"], "payloads": ["859d310e63d6", "1bdc94d60271", "6c3d0754a1ff"], "settlements": { "mount": "eb79a9b3682a", @@ -699,7 +699,7 @@ { "id": "speech-dictation-session-transcript.transport-rejection-no-message:transcribed", "observation": { - "sender": ["aca401ac072e", "6d6bb34ce2ab", "a7dbb6b2864f"], + "sender": ["bd004568e24c", "6d6bb34ce2ab", "a7dbb6b2864f"], "payloads": ["859d310e63d6", "1bdc94d60271", "6c3d0754a1ff"], "settlements": { "mount": "eb79a9b3682a", diff --git a/mobile/rpc-foundation/goldens/matrix-speech.dictation-session-speech.dictation.start-1.json b/mobile/rpc-foundation/goldens/matrix-speech.dictation-session-speech.dictation.start-1.json index 0455728f889d..e0cfcc85f189 100644 --- a/mobile/rpc-foundation/goldens/matrix-speech.dictation-session-speech.dictation.start-1.json +++ b/mobile/rpc-foundation/goldens/matrix-speech.dictation-session-speech.dictation.start-1.json @@ -4,12 +4,7 @@ "family": "speech.dictation-session", "namedDeltas": [], "values": { - "1bdc94d60271": { - "name": "speech.dictation.finish#1", - "ordinal": 4, - "json": "{\"id\":\"frame-2\",\"deviceToken\":\"recording-device\",\"method\":\"speech.dictation.finish\",\"params\":{\"dictationId\":\"mobile-dictation-1767225600000-8ig2henseon\"}}" - }, - "2104f46e44bd": { + "031fac8ed0c3": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -26,28 +21,31 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], "settlement": { - "status": "rejected", + "status": "fulfilled", "startedAt": 0, "settledAt": 0, - "error": { - "category": "Error", - "message": "", - "isRpcDeliveryUnknown": true + "value": { + "id": "frame-1", + "ok": true, + "result": { + "error": "inner refused", + "ok": false + } } } }, - "355b3eaef1fb": { - "name": "speech.dictation.cancel#1", - "ordinal": 3, + "062d9c83659b": { + "name": "speech.dictation.start#1", + "ordinal": 1, "args": [ { "name": "method", - "value": "speech.dictation.cancel" + "value": "speech.dictation.start" }, { "name": "params", @@ -58,22 +56,31 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], "settlement": { - "status": "pending", - "startedAt": 0 + "status": "fulfilled", + "startedAt": 0, + "settledAt": 0, + "value": { + "error": { + "code": "method_not_found", + "message": "Unknown method" + }, + "id": "frame-1", + "ok": false + } } }, - "4728ad10f92b": { - "name": "speech.dictation.finish#1", - "ordinal": 3, + "104a53ec5972": { + "name": "speech.dictation.start#1", + "ordinal": 1, "args": [ { "name": "method", - "value": "speech.dictation.finish" + "value": "speech.dictation.start" }, { "name": "params", @@ -84,24 +91,22 @@ { "name": "options", "value": { - "timeoutMs": 75000 + "timeoutMs": 30000 } } ], "settlement": { - "status": "fulfilled", + "status": "rejected", "startedAt": 0, "settledAt": 0, - "value": { - "id": "frame-2", - "ok": true, - "result": { - "text": " hello world " - } + "error": { + "category": "Error", + "message": "transport failure", + "isRpcDeliveryUnknown": true } } }, - "5655ac21ce64": { + "136c60a2a62f": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -118,7 +123,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -128,23 +133,22 @@ "settledAt": 0, "value": { "id": "frame-1", - "ok": true, - "result": { - "error": { - "message": "inner refused" - }, - "ok": false - } + "ok": true } } }, - "60ac03a80551": { - "name": "speech.dictation.start#1", - "ordinal": 1, + "1bdc94d60271": { + "name": "speech.dictation.finish#1", + "ordinal": 4, + "json": "{\"id\":\"frame-2\",\"deviceToken\":\"recording-device\",\"method\":\"speech.dictation.finish\",\"params\":{\"dictationId\":\"mobile-dictation-1767225600000-8ig2henseon\"}}" + }, + "355b3eaef1fb": { + "name": "speech.dictation.cancel#1", + "ordinal": 3, "args": [ { "name": "method", - "value": "speech.dictation.start" + "value": "speech.dictation.cancel" }, { "name": "params", @@ -160,26 +164,17 @@ } ], "settlement": { - "status": "fulfilled", - "startedAt": 0, - "settledAt": 0, - "value": { - "error": { - "code": "refused", - "message": "outer refused" - }, - "id": "frame-1", - "ok": false - } + "status": "pending", + "startedAt": 0 } }, - "624d174d4477": { - "name": "speech.dictation.start#1", - "ordinal": 1, + "4728ad10f92b": { + "name": "speech.dictation.finish#1", + "ordinal": 3, "args": [ { "name": "method", - "value": "speech.dictation.start" + "value": "speech.dictation.finish" }, { "name": "params", @@ -190,22 +185,24 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 75000 } } ], "settlement": { - "status": "rejected", + "status": "fulfilled", "startedAt": 0, "settledAt": 0, - "error": { - "category": "Error", - "message": "transport failure", - "isRpcDeliveryUnknown": true + "value": { + "id": "frame-2", + "ok": true, + "result": { + "text": " hello world " + } } } }, - "68f011596870": { + "6d8188e86d2a": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -222,7 +219,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -231,8 +228,12 @@ "startedAt": 0, "settledAt": 0, "value": { + "error": { + "code": "refused", + "message": "" + }, "id": "frame-1", - "ok": true + "ok": false } } }, @@ -252,7 +253,7 @@ "status": "idle", "transcripts": ["hello world"] }, - "aca401ac072e": { + "a8dc23386147": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -269,7 +270,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -281,12 +282,12 @@ "id": "frame-1", "ok": true, "result": { - "started": true + "error": "refused" } } } }, - "b05828b4ea8e": { + "b7421e659b44": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -303,7 +304,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -314,14 +315,14 @@ "value": { "error": { "code": "refused", - "message": "" + "message": "outer refused" }, "id": "frame-1", "ok": false } } }, - "baf7ef332fbf": { + "bd004568e24c": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -338,7 +339,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -350,26 +351,12 @@ "id": "frame-1", "ok": true, "result": { - "error": "inner refused", - "ok": false + "started": true } } } }, - "eb79a9b3682a": { - "status": "fulfilled", - "startedAt": 0, - "settledAt": 0, - "value": { - "$rpc": "undefined" - } - }, - "ebdac948a90d": { - "name": "speech.dictation.cancel#1", - "ordinal": 4, - "json": "{\"id\":\"frame-2\",\"deviceToken\":\"recording-device\",\"method\":\"speech.dictation.cancel\",\"params\":{\"dictationId\":\"mobile-dictation-1767225600000-8ig2henseon\"}}" - }, - "eebe8f57e84a": { + "c9d7f3d355eb": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -386,31 +373,22 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], "settlement": { - "status": "fulfilled", + "status": "rejected", "startedAt": 0, "settledAt": 0, - "value": { - "id": "frame-1", - "ok": true, - "result": { - "$rpc": "null" - } + "error": { + "category": "Error", + "message": "", + "isRpcDeliveryUnknown": true } } }, - "f6a74c428142": { - "error": { - "$rpc": "null" - }, - "status": "starting", - "transcripts": [] - }, - "fa7299197bf0": { + "e05190caf800": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -427,7 +405,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -439,12 +417,35 @@ "id": "frame-1", "ok": true, "result": { - "error": "refused" + "error": { + "message": "inner refused" + }, + "ok": false } } } }, - "fcb906b12a66": { + "eb79a9b3682a": { + "status": "fulfilled", + "startedAt": 0, + "settledAt": 0, + "value": { + "$rpc": "undefined" + } + }, + "ebdac948a90d": { + "name": "speech.dictation.cancel#1", + "ordinal": 4, + "json": "{\"id\":\"frame-2\",\"deviceToken\":\"recording-device\",\"method\":\"speech.dictation.cancel\",\"params\":{\"dictationId\":\"mobile-dictation-1767225600000-8ig2henseon\"}}" + }, + "f6a74c428142": { + "error": { + "$rpc": "null" + }, + "status": "starting", + "transcripts": [] + }, + "fb132fa96492": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -461,7 +462,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -470,12 +471,11 @@ "startedAt": 0, "settledAt": 0, "value": { - "error": { - "code": "method_not_found", - "message": "Unknown method" - }, "id": "frame-1", - "ok": false + "ok": true, + "result": { + "$rpc": "null" + } } } } @@ -486,7 +486,7 @@ { "id": "speech-dictation-session-transcript.normal:transcribed", "observation": { - "sender": ["aca401ac072e", "4728ad10f92b"], + "sender": ["bd004568e24c", "4728ad10f92b"], "payloads": ["859d310e63d6", "1bdc94d60271"], "settlements": { "mount": "eb79a9b3682a", @@ -500,7 +500,7 @@ { "id": "speech-dictation-session-transcript.result-absent:transcribed", "observation": { - "sender": ["68f011596870", "4728ad10f92b"], + "sender": ["136c60a2a62f", "4728ad10f92b"], "payloads": ["859d310e63d6", "1bdc94d60271"], "settlements": { "mount": "eb79a9b3682a", @@ -514,7 +514,7 @@ { "id": "speech-dictation-session-transcript.result-null:transcribed", "observation": { - "sender": ["eebe8f57e84a", "4728ad10f92b"], + "sender": ["fb132fa96492", "4728ad10f92b"], "payloads": ["859d310e63d6", "1bdc94d60271"], "settlements": { "mount": "eb79a9b3682a", @@ -528,7 +528,7 @@ { "id": "speech-dictation-session-transcript.inner-ok-missing:transcribed", "observation": { - "sender": ["fa7299197bf0", "4728ad10f92b"], + "sender": ["a8dc23386147", "4728ad10f92b"], "payloads": ["859d310e63d6", "1bdc94d60271"], "settlements": { "mount": "eb79a9b3682a", @@ -542,7 +542,7 @@ { "id": "speech-dictation-session-transcript.inner-false-string-error:transcribed", "observation": { - "sender": ["baf7ef332fbf", "4728ad10f92b"], + "sender": ["031fac8ed0c3", "4728ad10f92b"], "payloads": ["859d310e63d6", "1bdc94d60271"], "settlements": { "mount": "eb79a9b3682a", @@ -556,7 +556,7 @@ { "id": "speech-dictation-session-transcript.inner-false-object-error:transcribed", "observation": { - "sender": ["5655ac21ce64", "4728ad10f92b"], + "sender": ["e05190caf800", "4728ad10f92b"], "payloads": ["859d310e63d6", "1bdc94d60271"], "settlements": { "mount": "eb79a9b3682a", @@ -570,7 +570,7 @@ { "id": "speech-dictation-session-transcript.outer-refused:transcribed", "observation": { - "sender": ["60ac03a80551", "355b3eaef1fb"], + "sender": ["b7421e659b44", "355b3eaef1fb"], "payloads": ["859d310e63d6", "ebdac948a90d"], "settlements": { "mount": "eb79a9b3682a", @@ -584,7 +584,7 @@ { "id": "speech-dictation-session-transcript.outer-refused-no-message:transcribed", "observation": { - "sender": ["b05828b4ea8e", "355b3eaef1fb"], + "sender": ["6d8188e86d2a", "355b3eaef1fb"], "payloads": ["859d310e63d6", "ebdac948a90d"], "settlements": { "mount": "eb79a9b3682a", @@ -598,7 +598,7 @@ { "id": "speech-dictation-session-transcript.method-not-found:transcribed", "observation": { - "sender": ["fcb906b12a66", "355b3eaef1fb"], + "sender": ["062d9c83659b", "355b3eaef1fb"], "payloads": ["859d310e63d6", "ebdac948a90d"], "settlements": { "mount": "eb79a9b3682a", @@ -612,7 +612,7 @@ { "id": "speech-dictation-session-transcript.transport-rejection:transcribed", "observation": { - "sender": ["624d174d4477", "355b3eaef1fb"], + "sender": ["104a53ec5972", "355b3eaef1fb"], "payloads": ["859d310e63d6", "ebdac948a90d"], "settlements": { "mount": "eb79a9b3682a", @@ -626,7 +626,7 @@ { "id": "speech-dictation-session-transcript.transport-rejection-no-message:transcribed", "observation": { - "sender": ["2104f46e44bd", "355b3eaef1fb"], + "sender": ["c9d7f3d355eb", "355b3eaef1fb"], "payloads": ["859d310e63d6", "ebdac948a90d"], "settlements": { "mount": "eb79a9b3682a", diff --git a/mobile/rpc-foundation/goldens/matrix-speech.dictation-start-speech.dictation.cancel-1.json b/mobile/rpc-foundation/goldens/matrix-speech.dictation-start-speech.dictation.cancel-1.json index ef37367d2b4d..ca0bfb1a4294 100644 --- a/mobile/rpc-foundation/goldens/matrix-speech.dictation-start-speech.dictation.cancel-1.json +++ b/mobile/rpc-foundation/goldens/matrix-speech.dictation-start-speech.dictation.cancel-1.json @@ -105,13 +105,13 @@ } } }, - "3b5708754539": { - "name": "speech.dictation.start#1", - "ordinal": 1, + "46564194b0d9": { + "name": "speech.dictation.cancel#1", + "ordinal": 3, "args": [ { "name": "method", - "value": "speech.dictation.start" + "value": "speech.dictation.cancel" }, { "name": "params", @@ -131,21 +131,21 @@ "startedAt": 0, "settledAt": 0, "value": { - "id": "frame-1", + "id": "frame-2", "ok": true, "result": { - "started": true + "error": "refused" } } } }, - "46564194b0d9": { - "name": "speech.dictation.cancel#1", - "ordinal": 3, + "4c42677aca21": { + "name": "speech.dictation.start#1", + "ordinal": 1, "args": [ { "name": "method", - "value": "speech.dictation.cancel" + "value": "speech.dictation.start" }, { "name": "params", @@ -156,7 +156,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -165,10 +165,10 @@ "startedAt": 0, "settledAt": 0, "value": { - "id": "frame-2", + "id": "frame-1", "ok": true, "result": { - "error": "refused" + "started": true } } } @@ -450,7 +450,7 @@ { "id": "speech-desktop-start-superseded.normal:stale-start-cancelled", "observation": { - "sender": ["3b5708754539", "73faa32df517"], + "sender": ["4c42677aca21", "73faa32df517"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -463,7 +463,7 @@ { "id": "speech-desktop-start-superseded.result-absent:stale-start-cancelled", "observation": { - "sender": ["3b5708754539", "e59c2b1cff33"], + "sender": ["4c42677aca21", "e59c2b1cff33"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -476,7 +476,7 @@ { "id": "speech-desktop-start-superseded.result-null:stale-start-cancelled", "observation": { - "sender": ["3b5708754539", "02d3c28ecfb6"], + "sender": ["4c42677aca21", "02d3c28ecfb6"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -489,7 +489,7 @@ { "id": "speech-desktop-start-superseded.inner-ok-missing:stale-start-cancelled", "observation": { - "sender": ["3b5708754539", "46564194b0d9"], + "sender": ["4c42677aca21", "46564194b0d9"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -502,7 +502,7 @@ { "id": "speech-desktop-start-superseded.inner-false-string-error:stale-start-cancelled", "observation": { - "sender": ["3b5708754539", "1710d94f25d0"], + "sender": ["4c42677aca21", "1710d94f25d0"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -515,7 +515,7 @@ { "id": "speech-desktop-start-superseded.inner-false-object-error:stale-start-cancelled", "observation": { - "sender": ["3b5708754539", "cc661ef3e67e"], + "sender": ["4c42677aca21", "cc661ef3e67e"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -528,7 +528,7 @@ { "id": "speech-desktop-start-superseded.outer-refused:stale-start-cancelled", "observation": { - "sender": ["3b5708754539", "5f51c1b28c86"], + "sender": ["4c42677aca21", "5f51c1b28c86"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -541,7 +541,7 @@ { "id": "speech-desktop-start-superseded.outer-refused-no-message:stale-start-cancelled", "observation": { - "sender": ["3b5708754539", "dbc1fdbc29d3"], + "sender": ["4c42677aca21", "dbc1fdbc29d3"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -554,7 +554,7 @@ { "id": "speech-desktop-start-superseded.method-not-found:stale-start-cancelled", "observation": { - "sender": ["3b5708754539", "927c61fd29b5"], + "sender": ["4c42677aca21", "927c61fd29b5"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -567,7 +567,7 @@ { "id": "speech-desktop-start-superseded.transport-rejection:stale-start-cancelled", "observation": { - "sender": ["3b5708754539", "f3ac8ed5616e"], + "sender": ["4c42677aca21", "f3ac8ed5616e"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -580,7 +580,7 @@ { "id": "speech-desktop-start-superseded.transport-rejection-no-message:stale-start-cancelled", "observation": { - "sender": ["3b5708754539", "25f8db2d01b6"], + "sender": ["4c42677aca21", "25f8db2d01b6"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", diff --git a/mobile/rpc-foundation/goldens/matrix-speech.dictation-start-speech.dictation.start-1.json b/mobile/rpc-foundation/goldens/matrix-speech.dictation-start-speech.dictation.start-1.json index 1f50752b0dfd..a7860b36568f 100644 --- a/mobile/rpc-foundation/goldens/matrix-speech.dictation-start-speech.dictation.start-1.json +++ b/mobile/rpc-foundation/goldens/matrix-speech.dictation-start-speech.dictation.start-1.json @@ -4,7 +4,7 @@ "family": "speech.dictation-start", "namedDeltas": [], "values": { - "00ab3d9734bb": { + "02e3f6a398b9": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -21,7 +21,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -30,12 +30,16 @@ "startedAt": 0, "settledAt": 0, "value": { + "error": { + "code": "refused", + "message": "" + }, "id": "frame-1", - "ok": true + "ok": false } } }, - "0ec7faed0e3d": { + "0ac1d1294a4a": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -52,22 +56,24 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], "settlement": { - "status": "rejected", + "status": "fulfilled", "startedAt": 0, "settledAt": 0, - "error": { - "category": "Error", - "message": "transport failure", - "isRpcDeliveryUnknown": true + "value": { + "id": "frame-1", + "ok": true, + "result": { + "$rpc": "null" + } } } }, - "38b3ced9bf9c": { + "106b709a0579": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -84,24 +90,22 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], "settlement": { - "status": "fulfilled", + "status": "rejected", "startedAt": 0, "settledAt": 0, - "value": { - "id": "frame-1", - "ok": true, - "result": { - "error": "refused" - } + "error": { + "category": "Error", + "message": "", + "isRpcDeliveryUnknown": true } } }, - "3b5708754539": { + "4c42677aca21": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -118,7 +122,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -135,13 +139,13 @@ } } }, - "3c4f020193c4": { - "name": "speech.dictation.start#1", - "ordinal": 1, + "73faa32df517": { + "name": "speech.dictation.cancel#1", + "ordinal": 3, "args": [ { "name": "method", - "value": "speech.dictation.start" + "value": "speech.dictation.cancel" }, { "name": "params", @@ -161,16 +165,21 @@ "startedAt": 0, "settledAt": 0, "value": { - "error": { - "code": "method_not_found", - "message": "Unknown method" - }, - "id": "frame-1", - "ok": false + "id": "frame-2", + "ok": true, + "result": { + "cancelled": true + } } } }, - "50620ee89249": { + "7ed3d39f0607": { + "status": "fulfilled", + "startedAt": 0, + "settledAt": 0, + "value": false + }, + "9c6249e03654": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -187,22 +196,25 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], "settlement": { - "status": "rejected", + "status": "fulfilled", "startedAt": 0, "settledAt": 0, - "error": { - "category": "Error", - "message": "", - "isRpcDeliveryUnknown": true + "value": { + "error": { + "code": "refused", + "message": "outer refused" + }, + "id": "frame-1", + "ok": false } } }, - "55019d971687": { + "a1030fe3abd4": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -219,25 +231,32 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], "settlement": { - "status": "fulfilled", + "status": "rejected", "startedAt": 0, "settledAt": 0, - "value": { - "id": "frame-1", - "ok": true, - "result": { - "error": "inner refused", - "ok": false - } + "error": { + "category": "Error", + "message": "transport failure", + "isRpcDeliveryUnknown": true } } }, - "5bd3d1bd4863": { + "a7ddd021010a": { + "name": "speech.dictation.start#1", + "ordinal": 2, + "json": "{\"id\":\"frame-1\",\"deviceToken\":\"recording-device\",\"method\":\"speech.dictation.start\",\"params\":{\"dictationId\":\"dictation-1\"}}" + }, + "ce9cbc8837fa": { + "name": "speech.dictation.cancel#1", + "ordinal": 4, + "json": "{\"id\":\"frame-2\",\"deviceToken\":\"recording-device\",\"method\":\"speech.dictation.cancel\",\"params\":{\"dictationId\":\"dictation-1\"}}" + }, + "ced9bd128f52": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -254,7 +273,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -263,22 +282,39 @@ "startedAt": 0, "settledAt": 0, "value": { - "error": { - "code": "refused", - "message": "outer refused" - }, "id": "frame-1", - "ok": false + "ok": true, + "result": { + "error": { + "message": "inner refused" + }, + "ok": false + } } } }, - "73faa32df517": { - "name": "speech.dictation.cancel#1", - "ordinal": 3, + "e0fcd8f8c1a9": { + "activeId": { + "$rpc": "null" + }, + "idle": false, + "started": false + }, + "eb79a9b3682a": { + "status": "fulfilled", + "startedAt": 0, + "settledAt": 0, + "value": { + "$rpc": "undefined" + } + }, + "f010ef5b6bfc": { + "name": "speech.dictation.start#1", + "ordinal": 1, "args": [ { "name": "method", - "value": "speech.dictation.cancel" + "value": "speech.dictation.start" }, { "name": "params", @@ -289,7 +325,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -298,21 +334,15 @@ "startedAt": 0, "settledAt": 0, "value": { - "id": "frame-2", + "id": "frame-1", "ok": true, "result": { - "cancelled": true + "error": "refused" } } } }, - "7ed3d39f0607": { - "status": "fulfilled", - "startedAt": 0, - "settledAt": 0, - "value": false - }, - "a72f6bbdd0b3": { + "f2166355786a": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -329,7 +359,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -338,20 +368,16 @@ "startedAt": 0, "settledAt": 0, "value": { + "error": { + "code": "method_not_found", + "message": "Unknown method" + }, "id": "frame-1", - "ok": true, - "result": { - "$rpc": "null" - } + "ok": false } } }, - "a7ddd021010a": { - "name": "speech.dictation.start#1", - "ordinal": 2, - "json": "{\"id\":\"frame-1\",\"deviceToken\":\"recording-device\",\"method\":\"speech.dictation.start\",\"params\":{\"dictationId\":\"dictation-1\"}}" - }, - "ba48be88195e": { + "f2eeb517059d": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -368,7 +394,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -377,21 +403,16 @@ "startedAt": 0, "settledAt": 0, "value": { - "error": { - "code": "refused", - "message": "" - }, "id": "frame-1", - "ok": false + "ok": true, + "result": { + "error": "inner refused", + "ok": false + } } } }, - "ce9cbc8837fa": { - "name": "speech.dictation.cancel#1", - "ordinal": 4, - "json": "{\"id\":\"frame-2\",\"deviceToken\":\"recording-device\",\"method\":\"speech.dictation.cancel\",\"params\":{\"dictationId\":\"dictation-1\"}}" - }, - "ceca67daa178": { + "fe8f7e77754e": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -408,7 +429,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -418,30 +439,9 @@ "settledAt": 0, "value": { "id": "frame-1", - "ok": true, - "result": { - "error": { - "message": "inner refused" - }, - "ok": false - } + "ok": true } } - }, - "e0fcd8f8c1a9": { - "activeId": { - "$rpc": "null" - }, - "idle": false, - "started": false - }, - "eb79a9b3682a": { - "status": "fulfilled", - "startedAt": 0, - "settledAt": 0, - "value": { - "$rpc": "undefined" - } } }, "recording": { @@ -450,7 +450,7 @@ { "id": "speech-desktop-start-superseded.normal:stale-start-cancelled", "observation": { - "sender": ["3b5708754539", "73faa32df517"], + "sender": ["4c42677aca21", "73faa32df517"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -463,7 +463,7 @@ { "id": "speech-desktop-start-superseded.result-absent:stale-start-cancelled", "observation": { - "sender": ["00ab3d9734bb", "73faa32df517"], + "sender": ["fe8f7e77754e", "73faa32df517"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -476,7 +476,7 @@ { "id": "speech-desktop-start-superseded.result-null:stale-start-cancelled", "observation": { - "sender": ["a72f6bbdd0b3", "73faa32df517"], + "sender": ["0ac1d1294a4a", "73faa32df517"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -489,7 +489,7 @@ { "id": "speech-desktop-start-superseded.inner-ok-missing:stale-start-cancelled", "observation": { - "sender": ["38b3ced9bf9c", "73faa32df517"], + "sender": ["f010ef5b6bfc", "73faa32df517"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -502,7 +502,7 @@ { "id": "speech-desktop-start-superseded.inner-false-string-error:stale-start-cancelled", "observation": { - "sender": ["55019d971687", "73faa32df517"], + "sender": ["f2eeb517059d", "73faa32df517"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -515,7 +515,7 @@ { "id": "speech-desktop-start-superseded.inner-false-object-error:stale-start-cancelled", "observation": { - "sender": ["ceca67daa178", "73faa32df517"], + "sender": ["ced9bd128f52", "73faa32df517"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -528,7 +528,7 @@ { "id": "speech-desktop-start-superseded.outer-refused:stale-start-cancelled", "observation": { - "sender": ["5bd3d1bd4863", "73faa32df517"], + "sender": ["9c6249e03654", "73faa32df517"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -541,7 +541,7 @@ { "id": "speech-desktop-start-superseded.outer-refused-no-message:stale-start-cancelled", "observation": { - "sender": ["ba48be88195e", "73faa32df517"], + "sender": ["02e3f6a398b9", "73faa32df517"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -554,7 +554,7 @@ { "id": "speech-desktop-start-superseded.method-not-found:stale-start-cancelled", "observation": { - "sender": ["3c4f020193c4", "73faa32df517"], + "sender": ["f2166355786a", "73faa32df517"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -567,7 +567,7 @@ { "id": "speech-desktop-start-superseded.transport-rejection:stale-start-cancelled", "observation": { - "sender": ["0ec7faed0e3d", "73faa32df517"], + "sender": ["a1030fe3abd4", "73faa32df517"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -580,7 +580,7 @@ { "id": "speech-desktop-start-superseded.transport-rejection-no-message:stale-start-cancelled", "observation": { - "sender": ["50620ee89249", "73faa32df517"], + "sender": ["106b709a0579", "73faa32df517"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", diff --git a/mobile/rpc-foundation/goldens/speech-desktop-start-fulfilled.json b/mobile/rpc-foundation/goldens/speech-desktop-start-fulfilled.json index 9aabbf4419c3..4a7e325614f3 100644 --- a/mobile/rpc-foundation/goldens/speech-desktop-start-fulfilled.json +++ b/mobile/rpc-foundation/goldens/speech-desktop-start-fulfilled.json @@ -4,7 +4,7 @@ "family": "speech.dictation-start", "namedDeltas": [], "values": { - "3b5708754539": { + "4c42677aca21": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -21,7 +21,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -61,7 +61,7 @@ { "id": "recording", "observation": { - "sender": ["3b5708754539"], + "sender": ["4c42677aca21"], "payloads": ["a7ddd021010a"], "settlements": { "start": "84e5ca07cb7a" diff --git a/mobile/rpc-foundation/goldens/speech-desktop-start-recording-failed.json b/mobile/rpc-foundation/goldens/speech-desktop-start-recording-failed.json index ca5159beb865..d016eea4acd4 100644 --- a/mobile/rpc-foundation/goldens/speech-desktop-start-recording-failed.json +++ b/mobile/rpc-foundation/goldens/speech-desktop-start-recording-failed.json @@ -45,7 +45,7 @@ } } }, - "3b5708754539": { + "4c42677aca21": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -62,7 +62,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -111,7 +111,7 @@ { "id": "rolled-back", "observation": { - "sender": ["3b5708754539", "217153ab0a32"], + "sender": ["4c42677aca21", "217153ab0a32"], "payloads": ["a7ddd021010a", "d6e61920633f"], "settlements": { "start": "7a657475cacc" diff --git a/mobile/rpc-foundation/goldens/speech-desktop-start-superseded.json b/mobile/rpc-foundation/goldens/speech-desktop-start-superseded.json index ec8e3ea12c46..bde938c3cd61 100644 --- a/mobile/rpc-foundation/goldens/speech-desktop-start-superseded.json +++ b/mobile/rpc-foundation/goldens/speech-desktop-start-superseded.json @@ -4,7 +4,7 @@ "family": "speech.dictation-start", "namedDeltas": [], "values": { - "3b5708754539": { + "4c42677aca21": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -21,7 +21,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -110,7 +110,7 @@ { "id": "stale-start-cancelled", "observation": { - "sender": ["3b5708754539", "73faa32df517"], + "sender": ["4c42677aca21", "73faa32df517"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", diff --git a/mobile/rpc-foundation/goldens/speech-dictation-session-cancelled.json b/mobile/rpc-foundation/goldens/speech-dictation-session-cancelled.json index 7db1cd929bb7..d0fa0f48749d 100644 --- a/mobile/rpc-foundation/goldens/speech-dictation-session-cancelled.json +++ b/mobile/rpc-foundation/goldens/speech-dictation-session-cancelled.json @@ -50,7 +50,7 @@ "ordinal": 2, "json": "{\"id\":\"frame-1\",\"deviceToken\":\"recording-device\",\"method\":\"speech.dictation.start\",\"params\":{\"dictationId\":\"mobile-dictation-1767225600000-8ig2henseon\"}}" }, - "aca401ac072e": { + "bd004568e24c": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -67,7 +67,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -104,7 +104,7 @@ { "id": "cancelled", "observation": { - "sender": ["aca401ac072e", "422e902e66a5"], + "sender": ["bd004568e24c", "422e902e66a5"], "payloads": ["859d310e63d6", "ebdac948a90d"], "settlements": { "mount": "eb79a9b3682a", diff --git a/mobile/rpc-foundation/goldens/speech-dictation-session-transcript.json b/mobile/rpc-foundation/goldens/speech-dictation-session-transcript.json index fe2495757163..570afbc56d91 100644 --- a/mobile/rpc-foundation/goldens/speech-dictation-session-transcript.json +++ b/mobile/rpc-foundation/goldens/speech-dictation-session-transcript.json @@ -55,7 +55,7 @@ "status": "idle", "transcripts": ["hello world"] }, - "aca401ac072e": { + "bd004568e24c": { "name": "speech.dictation.start#1", "ordinal": 1, "args": [ @@ -72,7 +72,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -104,7 +104,7 @@ { "id": "transcribed", "observation": { - "sender": ["aca401ac072e", "4728ad10f92b"], + "sender": ["bd004568e24c", "4728ad10f92b"], "payloads": ["859d310e63d6", "1bdc94d60271"], "settlements": { "mount": "eb79a9b3682a", diff --git a/mobile/src/hooks/mobile-dictation-audio-chunk.ts b/mobile/src/hooks/mobile-dictation-audio-chunk.ts index ab58e7f57e85..cc08e02b5407 100644 --- a/mobile/src/hooks/mobile-dictation-audio-chunk.ts +++ b/mobile/src/hooks/mobile-dictation-audio-chunk.ts @@ -7,7 +7,11 @@ import { dictationAudioChunkSend } from '../dictation/mobile-dictation-operation import { MOBILE_DICTATION_RECONNECT_GRACE_MS } from '../../../src/shared/rpc-contract/speech-params' import { isRpcDeliveryUnknown } from '../transport/rpc-delivery-ambiguity' import { rpcPayloadMember } from '../transport/rpc-reader-payload' -import { isLogicalClientCutoverError } from '../transport/stable-logical-rpc-client' +import { waitForRpcClientReconnected } from '../transport/rpc-client-reconnect-wait' +import { + isLogicalClientCutoverError, + LogicalClientSuspendedError +} from '../transport/stable-logical-rpc-client' import type { DictationCapture, DictationCaptureChunk @@ -38,11 +42,25 @@ export class MobileDictationTranscriptSoFar { } } -// A request still waiting when the connection moved never reached the host; send it again on the new one. -export function sendAcrossConnectionMove(send: () => Promise): Promise { - return send().catch((err: unknown) => - isLogicalClientCutoverError(err) && !isRpcDeliveryUnknown(err) ? send() : Promise.reject(err) - ) +// A request refused or still waiting while the connection moved never reached the host; send it again on the new one, within the host's grace. +export async function sendAcrossConnectionMove( + client: RpcClient, + send: () => Promise +): Promise { + try { + return await send() + } catch (err) { + if (isLogicalClientCutoverError(err) && !isRpcDeliveryUnknown(err)) { + return send() + } + if ( + err instanceof LogicalClientSuspendedError && + (await waitForRpcClientReconnected(client, MOBILE_DICTATION_RECONNECT_GRACE_MS)) + ) { + return send() + } + throw err + } } export function subscribeMobileDictationAudioChunks( @@ -94,7 +112,7 @@ export function enqueueMobileDictationAudioChunk( audioBase64: bytesToBase64(bytes), sampleRate: MOBILE_DICTATION_PCM_SAMPLE_RATE } - const sendChunk = sendAcrossConnectionMove(() => + const sendChunk = sendAcrossConnectionMove(client, () => // Wait for a reconnect only as long as the host keeps the session; past that, fail and keep the text so far. dictationAudioChunkSend.request(client, params, { timeoutMs: MOBILE_DICTATION_RECONNECT_GRACE_MS diff --git a/mobile/src/hooks/mobile-dictation-desktop-start.ts b/mobile/src/hooks/mobile-dictation-desktop-start.ts index 9790d0fd3890..7d827140cea9 100644 --- a/mobile/src/hooks/mobile-dictation-desktop-start.ts +++ b/mobile/src/hooks/mobile-dictation-desktop-start.ts @@ -3,6 +3,7 @@ import { dictationSessionCancel, dictationSessionStart } from '../dictation/mobile-dictation-operations' +import { MOBILE_DICTATION_RECONNECT_GRACE_MS } from '../../../src/shared/rpc-contract/speech-params' import type { RpcClient } from '../transport/rpc-client' type StartMobileDictationDesktopSessionOptions = { @@ -51,7 +52,12 @@ export async function startMobileDictationDesktopSession( const { client, dictationId } = options try { - const reply = await dictationSessionStart.request(client, { dictationId }) + // The mic is already open, so a start stalled by a drop must give up rather than hold it. + const reply = await dictationSessionStart.request( + client, + { dictationId }, + { timeoutMs: MOBILE_DICTATION_RECONNECT_GRACE_MS } + ) dictationSessionStart.interpret(reply) } catch (err) { const wasCurrent = isCurrentStart(options) diff --git a/mobile/src/hooks/mobile-dictation-reconnect.test.tsx b/mobile/src/hooks/mobile-dictation-reconnect.test.tsx index 466ba538b0b9..ec766df540d1 100644 --- a/mobile/src/hooks/mobile-dictation-reconnect.test.tsx +++ b/mobile/src/hooks/mobile-dictation-reconnect.test.tsx @@ -8,7 +8,11 @@ import { type SentRequest } from '../mobile-web-shell/bridge-host-test-fakes' import { markRpcDeliveryUnknown } from '../transport/rpc-delivery-ambiguity' -import { LogicalClientCutoverError } from '../transport/stable-logical-rpc-client' +import { + LogicalClientCutoverError, + LogicalClientSuspendedError +} from '../transport/stable-logical-rpc-client' +import type { ConnectionState } from '../transport/types' import type { DictationCapture, DictationCaptureChunk @@ -144,6 +148,28 @@ describe('dictation through a reconnect', () => { expect(errors).toEqual([]) }) + it('holds a chunk refused while recovery dials a replacement, and sends it once connected', async () => { + let state: ConnectionState = 'connected' + const rpc = createFakeRpcClient({ getState: () => state }) + const sent: SentRequest[] = [] + const transcripts: string[] = [] + const errors: string[] = [] + mount(rpc, transcripts, errors) + + state = 'disconnected' + await record(rpc, sent, [new LogicalClientSuspendedError()]) + expect(methodsOf(sent).filter((m) => m === 'speech.dictation.chunk')).toHaveLength(1) + await act(async () => { + state = 'connected' + rpc.pushState('connected') + await pump(rpc, sent, []) + }) + + expect(methodsOf(sent).filter((m) => m === 'speech.dictation.chunk')).toHaveLength(2) + expect(dictation().status).toBe('recording') + expect(errors).toEqual([]) + }) + it('keeps what was transcribed when the connection does not come back', async () => { const rpc = createFakeRpcClient() const sent: SentRequest[] = [] diff --git a/mobile/src/hooks/use-mobile-dictation.ts b/mobile/src/hooks/use-mobile-dictation.ts index af2ae0c5a205..1cf3b3111efa 100644 --- a/mobile/src/hooks/use-mobile-dictation.ts +++ b/mobile/src/hooks/use-mobile-dictation.ts @@ -218,7 +218,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil return } const finished = dictationSessionFinish.interpret( - await sendAcrossConnectionMove(() => + await sendAcrossConnectionMove(client, () => dictationSessionFinish.request( client, { dictationId }, diff --git a/mobile/src/transport/stable-logical-rpc-client.test.ts b/mobile/src/transport/stable-logical-rpc-client.test.ts index f6bd60ba87f9..61482e0ad227 100644 --- a/mobile/src/transport/stable-logical-rpc-client.test.ts +++ b/mobile/src/transport/stable-logical-rpc-client.test.ts @@ -6,6 +6,7 @@ import { isRpcDeliveryUnknown, markRpcDeliveryUnknown } from './rpc-delivery-amb import { createStableLogicalRpcClient, LogicalClientCutoverError, + LogicalClientSuspendedError, type MobileConnectionPath } from './stable-logical-rpc-client' @@ -169,6 +170,23 @@ describe('stable logical RPC client', () => { await expect(client.sendRequest('status.get')).resolves.toEqual(success('next')) }) + it('reports requests the suspended session never sent as suspended, and written ones as ambiguous', async () => { + const session = new FakeSession('connected') + const unsent = deferred() + const written = deferred() + session.sendRequest.mockReturnValueOnce(unsent.promise).mockReturnValueOnce(written.promise) + const client = createStableLogicalRpcClient(session, 'relay') + const waiting = client.sendRequest('speech.dictation.chunk').catch((error: unknown) => error) + const inFlight = client.sendRequest('speech.dictation.chunk').catch((error: unknown) => error) + + client.suspendActiveSession() + unsent.reject(new Error('relay session closed')) + written.reject(markRpcDeliveryUnknown(new Error('Client closed'))) + + expect(await waiting).toBeInstanceOf(LogicalClientSuspendedError) + expect(isRpcDeliveryUnknown(await inFlight)).toBe(true) + }) + it('preserves delivery ambiguity without replaying a mutation after relay replacement', async () => { const session = new FakeSession('connected') const replacement = new FakeSession('connected') diff --git a/mobile/src/transport/stable-logical-rpc-client.ts b/mobile/src/transport/stable-logical-rpc-client.ts index c00def9cacb7..d1949c2ed49d 100644 --- a/mobile/src/transport/stable-logical-rpc-client.ts +++ b/mobile/src/transport/stable-logical-rpc-client.ts @@ -21,6 +21,13 @@ export class LogicalClientCutoverError extends Error { } } +// A suspended client refuses before anything is written, while recovery dials a replacement. +export class LogicalClientSuspendedError extends Error { + constructor() { + super('Client suspended') + } +} + // Why: instanceof can miss across bundle copies, so also match by message. export function isLogicalClientCutoverError(error: unknown): boolean { return ( @@ -87,7 +94,7 @@ export function createStableLogicalRpcClient( return Promise.reject(new Error('Client closed')) } if (suspended) { - return Promise.reject(new Error('Client suspended')) + return Promise.reject(new LogicalClientSuspendedError()) } const requestGeneration = generation const session = activeSession @@ -103,7 +110,12 @@ export function createStableLogicalRpcClient( (error: unknown) => { // Keep the physical error as cause: only that layer knows whether the frame reached the wire. reject( - requestGeneration !== generation ? new LogicalClientCutoverError(error) : error + requestGeneration !== generation + ? new LogicalClientCutoverError(error) + : // Suspension retires the session under requests it never sent. + suspended && !isRpcDeliveryUnknown(error) + ? new LogicalClientSuspendedError() + : error ) } ) From ff239a8a91fe0ee351669860907547bfdf6bd448 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:41:00 +0000 Subject: [PATCH 5/7] fix(mobile): recognise a suspended relay client across the page bridge --- mobile/src/hooks/mobile-dictation-audio-chunk.ts | 8 +++----- .../src/hooks/mobile-dictation-reconnect.test.tsx | 8 +++----- .../transport/logical-client-suspended-error.ts | 14 ++++++++++++++ .../transport/stable-logical-rpc-client.test.ts | 2 +- mobile/src/transport/stable-logical-rpc-client.ts | 8 +------- 5 files changed, 22 insertions(+), 18 deletions(-) create mode 100644 mobile/src/transport/logical-client-suspended-error.ts diff --git a/mobile/src/hooks/mobile-dictation-audio-chunk.ts b/mobile/src/hooks/mobile-dictation-audio-chunk.ts index cc08e02b5407..e8c9df5d62c6 100644 --- a/mobile/src/hooks/mobile-dictation-audio-chunk.ts +++ b/mobile/src/hooks/mobile-dictation-audio-chunk.ts @@ -8,10 +8,8 @@ import { MOBILE_DICTATION_RECONNECT_GRACE_MS } from '../../../src/shared/rpc-con import { isRpcDeliveryUnknown } from '../transport/rpc-delivery-ambiguity' import { rpcPayloadMember } from '../transport/rpc-reader-payload' import { waitForRpcClientReconnected } from '../transport/rpc-client-reconnect-wait' -import { - isLogicalClientCutoverError, - LogicalClientSuspendedError -} from '../transport/stable-logical-rpc-client' +import { isLogicalClientCutoverError } from '../transport/stable-logical-rpc-client' +import { isLogicalClientSuspendedError } from '../transport/logical-client-suspended-error' import type { DictationCapture, DictationCaptureChunk @@ -54,7 +52,7 @@ export async function sendAcrossConnectionMove( return send() } if ( - err instanceof LogicalClientSuspendedError && + isLogicalClientSuspendedError(err) && (await waitForRpcClientReconnected(client, MOBILE_DICTATION_RECONNECT_GRACE_MS)) ) { return send() diff --git a/mobile/src/hooks/mobile-dictation-reconnect.test.tsx b/mobile/src/hooks/mobile-dictation-reconnect.test.tsx index ec766df540d1..de777adac85d 100644 --- a/mobile/src/hooks/mobile-dictation-reconnect.test.tsx +++ b/mobile/src/hooks/mobile-dictation-reconnect.test.tsx @@ -8,10 +8,7 @@ import { type SentRequest } from '../mobile-web-shell/bridge-host-test-fakes' import { markRpcDeliveryUnknown } from '../transport/rpc-delivery-ambiguity' -import { - LogicalClientCutoverError, - LogicalClientSuspendedError -} from '../transport/stable-logical-rpc-client' +import { LogicalClientCutoverError } from '../transport/stable-logical-rpc-client' import type { ConnectionState } from '../transport/types' import type { DictationCapture, @@ -157,7 +154,8 @@ describe('dictation through a reconnect', () => { mount(rpc, transcripts, errors) state = 'disconnected' - await record(rpc, sent, [new LogicalClientSuspendedError()]) + // The OTA page sees the shell's error rebuilt as another class with the same message. + await record(rpc, sent, [new Error('Client suspended')]) expect(methodsOf(sent).filter((m) => m === 'speech.dictation.chunk')).toHaveLength(1) await act(async () => { state = 'connected' diff --git a/mobile/src/transport/logical-client-suspended-error.ts b/mobile/src/transport/logical-client-suspended-error.ts new file mode 100644 index 000000000000..dea40f896ee4 --- /dev/null +++ b/mobile/src/transport/logical-client-suspended-error.ts @@ -0,0 +1,14 @@ +// A suspended client refuses before anything is written, while recovery dials a replacement. +export class LogicalClientSuspendedError extends Error { + constructor() { + super('Client suspended') + } +} + +// Why: errors crossing the page bridge are rebuilt as other classes, so also match by message. +export function isLogicalClientSuspendedError(error: unknown): boolean { + return ( + error instanceof LogicalClientSuspendedError || + (error instanceof Error && error.message === 'Client suspended') + ) +} diff --git a/mobile/src/transport/stable-logical-rpc-client.test.ts b/mobile/src/transport/stable-logical-rpc-client.test.ts index 61482e0ad227..e20866d615e2 100644 --- a/mobile/src/transport/stable-logical-rpc-client.test.ts +++ b/mobile/src/transport/stable-logical-rpc-client.test.ts @@ -6,9 +6,9 @@ import { isRpcDeliveryUnknown, markRpcDeliveryUnknown } from './rpc-delivery-amb import { createStableLogicalRpcClient, LogicalClientCutoverError, - LogicalClientSuspendedError, type MobileConnectionPath } from './stable-logical-rpc-client' +import { LogicalClientSuspendedError } from './logical-client-suspended-error' class FakeSession implements RpcClient { readonly sendRequest = diff --git a/mobile/src/transport/stable-logical-rpc-client.ts b/mobile/src/transport/stable-logical-rpc-client.ts index d1949c2ed49d..fab0b476d826 100644 --- a/mobile/src/transport/stable-logical-rpc-client.ts +++ b/mobile/src/transport/stable-logical-rpc-client.ts @@ -9,6 +9,7 @@ import { projectMobileRpcRequestParams } from './mobile-rpc-request-projection' import { LogicalClientConnectionPath } from './logical-client-connection-path' import type { RelayHostReachability } from './relay-host-reachability' import { isRpcDeliveryUnknown, markRpcDeliveryUnknown } from './rpc-delivery-ambiguity' +import { LogicalClientSuspendedError } from './logical-client-suspended-error' export type MobileConnectionPath = 'lan' | 'tailscale' | 'relay' @@ -21,13 +22,6 @@ export class LogicalClientCutoverError extends Error { } } -// A suspended client refuses before anything is written, while recovery dials a replacement. -export class LogicalClientSuspendedError extends Error { - constructor() { - super('Client suspended') - } -} - // Why: instanceof can miss across bundle copies, so also match by message. export function isLogicalClientCutoverError(error: unknown): boolean { return ( From 65097ee2182513ea1ea722f3e3b8ec8be5a91fde Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:44:03 +0000 Subject: [PATCH 6/7] refactor(dictation): refresh one abandon timer, report text by version and share the reconnect wait --- .../src/hooks/mobile-dictation-audio-chunk.ts | 19 +++++-- .../mobile-dictation-pending-audio-budget.ts | 11 ++-- ...b-check-details-paired.integration.test.ts | 1 - .../runtime-mobile-dictation-controller.ts | 54 ++++++++----------- 4 files changed, 44 insertions(+), 41 deletions(-) diff --git a/mobile/src/hooks/mobile-dictation-audio-chunk.ts b/mobile/src/hooks/mobile-dictation-audio-chunk.ts index e8c9df5d62c6..1ce5a1c74858 100644 --- a/mobile/src/hooks/mobile-dictation-audio-chunk.ts +++ b/mobile/src/hooks/mobile-dictation-audio-chunk.ts @@ -51,16 +51,27 @@ export async function sendAcrossConnectionMove( if (isLogicalClientCutoverError(err) && !isRpcDeliveryUnknown(err)) { return send() } - if ( - isLogicalClientSuspendedError(err) && - (await waitForRpcClientReconnected(client, MOBILE_DICTATION_RECONNECT_GRACE_MS)) - ) { + if (isLogicalClientSuspendedError(err) && (await waitForReconnect(client))) { return send() } throw err } } +// Every chunk refused during one outage shares a single wait instead of a listener and timer each. +const reconnectWaits = new WeakMap>() + +function waitForReconnect(client: RpcClient): Promise { + let wait = reconnectWaits.get(client) + if (!wait) { + wait = waitForRpcClientReconnected(client, MOBILE_DICTATION_RECONNECT_GRACE_MS).finally(() => + reconnectWaits.delete(client) + ) + reconnectWaits.set(client, wait) + } + return wait +} + export function subscribeMobileDictationAudioChunks( capture: Pick, source: { diff --git a/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts b/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts index a94a6b35d5e3..c90d0f313c5f 100644 --- a/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts +++ b/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts @@ -1,14 +1,17 @@ -import { MOBILE_DICTATION_RECONNECT_GRACE_MS } from '../../../src/shared/rpc-contract/speech-params' +import { + DICTATION_SAMPLE_RATE, + MOBILE_DICTATION_RECONNECT_GRACE_MS, + PCM_BYTES_PER_SAMPLE +} from '../../../src/shared/rpc-contract/speech-params' -export const MOBILE_DICTATION_PCM_SAMPLE_RATE = 16000 +export const MOBILE_DICTATION_PCM_SAMPLE_RATE = DICTATION_SAMPLE_RATE -const PCM_BYTES_PER_SAMPLE = 2 // Hold the speech captured while a reconnect is pending, for as long as the host keeps the session. const MAX_PENDING_AUDIO_SECONDS = MOBILE_DICTATION_RECONNECT_GRACE_MS / 1000 // Why: dictation chunk RPCs wait through reconnect before timing out, so // cap retained raw microphone audio before it is expanded into base64 payloads. -export const MOBILE_DICTATION_MAX_PENDING_AUDIO_BYTES = +const MOBILE_DICTATION_MAX_PENDING_AUDIO_BYTES = MOBILE_DICTATION_PCM_SAMPLE_RATE * PCM_BYTES_PER_SAMPLE * MAX_PENDING_AUDIO_SECONDS export const MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE = diff --git a/src/main/runtime/github-check-details-paired.integration.test.ts b/src/main/runtime/github-check-details-paired.integration.test.ts index 6bc0625afaff..795db7e1e878 100644 --- a/src/main/runtime/github-check-details-paired.integration.test.ts +++ b/src/main/runtime/github-check-details-paired.integration.test.ts @@ -112,7 +112,6 @@ describe('paired check details over real encrypted loopback transport', () => { getRuntimeId: () => 'paired-check-host', getStartedAt: () => 1, cleanupSubscriptionsForConnection: () => {}, - cancelMobileDictationForConnection: () => {}, onClientDisconnected: () => {}, getRepoPRCheckDetails: (...args: Parameters) => query.getRepoPRCheckDetails(...args).finally(() => { diff --git a/src/main/runtime/runtime-mobile-dictation-controller.ts b/src/main/runtime/runtime-mobile-dictation-controller.ts index bb7eb89f4fcb..1535a2780dbd 100644 --- a/src/main/runtime/runtime-mobile-dictation-controller.ts +++ b/src/main/runtime/runtime-mobile-dictation-controller.ts @@ -16,7 +16,9 @@ type MobileDictationSession = { partialText: string finalTexts: string[] errors: string[] - reportedText: string + // Bumped by each transcript event; chunks arrive far more often, so replies carry text only after a bump. + textVersion: number + reportedVersion: number abandonTimer?: ReturnType } @@ -67,7 +69,8 @@ export class RuntimeMobileDictationController { partialText: '', finalTexts: [], errors: [], - reportedText: '' + textVersion: 0, + reportedVersion: 0 } try { await getSpeechSttService(store).startDictation( @@ -80,7 +83,12 @@ export class RuntimeMobileDictationController { throw new Error('dictation_canceled') } this.session.state = 'active' - this.armAbandonTimer(this.session) + const session = this.session + // Owned by the phone, not its socket; it streams continuously, so this long without a request means it is gone. + session.abandonTimer = setTimeout( + () => void this.stopSession(session).catch(() => undefined), + MOBILE_DICTATION_RECONNECT_GRACE_MS + ) } catch (error) { if (this.session?.id === params.dictationId) { this.session = null @@ -109,13 +117,12 @@ export class RuntimeMobileDictationController { samples[i] = pcm.readInt16LE(i * 2) / 32768 } getSpeechSttService(this.requireStore()).feedAudio(samples, params.sampleRate, session.owner) - // The text so far lets the phone keep it if the connection never comes back; only changes are sent, as chunks arrive every few ms. - const text = transcriptOf(session) - if (text === session.reportedText) { + // The text so far lets the phone keep it if the connection never comes back. + if (session.reportedVersion === session.textVersion) { return { dictationId: params.dictationId } } - session.reportedText = text - return { dictationId: params.dictationId, text } + session.reportedVersion = session.textVersion + return { dictationId: params.dictationId, text: transcriptOf(session) } } async finish(params: { @@ -123,19 +130,11 @@ export class RuntimeMobileDictationController { clientId?: string }): Promise<{ dictationId: string; text: string }> { const session = this.requireOwnedSession(params) - clearTimeout(session.abandonTimer) - session.state = 'closing' - try { - await getSpeechSttService(this.requireStore()).stopDictation(session.owner) - if (session.errors.length > 0) { - throw new Error(session.errors[0]) - } - return { dictationId: params.dictationId, text: transcriptOf(session) } - } finally { - if (this.session?.id === session.id) { - this.session = null - } + await this.stopSession(session) + if (session.errors.length > 0) { + throw new Error(session.errors[0]) } + return { dictationId: params.dictationId, text: transcriptOf(session) } } async cancel(params: { @@ -159,28 +158,19 @@ export class RuntimeMobileDictationController { } if (event.type === 'partial') { session.partialText = event.text ?? '' + session.textVersion += 1 } else if (event.type === 'final') { const text = event.text?.trim() if (text) { session.finalTexts.push(text) session.partialText = '' + session.textVersion += 1 } } else if (event.type === 'error') { session.errors.push(event.error ?? 'Speech worker error') } } - // Owned by the phone, not its socket: a reconnect arrives on a new connection. - // The phone streams audio continuously, so silence this long means it is gone (killed, lost its cancel, never reconnected). - private armAbandonTimer(session: MobileDictationSession): void { - clearTimeout(session.abandonTimer) - session.abandonTimer = setTimeout(() => { - if (session.state === 'active') { - void this.stopSession(session).catch(() => undefined) - } - }, MOBILE_DICTATION_RECONNECT_GRACE_MS) - } - private requireOwnedSession(params: { dictationId: string clientId?: string @@ -193,7 +183,7 @@ export class RuntimeMobileDictationController { throw new Error('dictation_owner_mismatch') } if (session.state === 'active') { - this.armAbandonTimer(session) + session.abandonTimer?.refresh() } return session } From 4249db68f40d4e9c132ea6212e2f363c85dac541 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:04:03 +0000 Subject: [PATCH 7/7] fix(mobile): resend dictation text each second and never wait offline for a cancel --- ...ion-session-speech.dictation.finish-1.json | 66 +++--- ...tion-session-speech.dictation.start-1.json | 62 ++--- ...ation-start-speech.dictation.cancel-1.json | 224 +++++++++--------- ...tation-start-speech.dictation.start-1.json | 38 +-- ...speech-desktop-start-recording-failed.json | 30 +-- .../speech-desktop-start-superseded.json | 18 +- .../speech-dictation-session-cancelled.json | 70 +++--- .../mobile-dictation-desktop-start.test.ts | 40 ++-- .../hooks/mobile-dictation-desktop-start.ts | 16 +- mobile/src/hooks/use-mobile-dictation.ts | 16 +- ...untime-mobile-dictation-controller.test.ts | 3 + .../runtime-mobile-dictation-controller.ts | 12 +- 12 files changed, 310 insertions(+), 285 deletions(-) diff --git a/mobile/rpc-foundation/goldens/matrix-speech.dictation-session-speech.dictation.finish-1.json b/mobile/rpc-foundation/goldens/matrix-speech.dictation-session-speech.dictation.finish-1.json index cbddc145a524..2a94882464f0 100644 --- a/mobile/rpc-foundation/goldens/matrix-speech.dictation-session-speech.dictation.finish-1.json +++ b/mobile/rpc-foundation/goldens/matrix-speech.dictation-session-speech.dictation.finish-1.json @@ -262,6 +262,32 @@ } } }, + "893e5291c492": { + "name": "speech.dictation.cancel#1", + "ordinal": 5, + "args": [ + { + "name": "method", + "value": "speech.dictation.cancel" + }, + { + "name": "params", + "value": { + "dictationId": "mobile-dictation-1767225600000-8ig2henseon" + } + }, + { + "name": "options", + "value": { + "failWhenDisconnected": true + } + } + ], + "settlement": { + "status": "pending", + "startedAt": 0 + } + }, "97af9f7912fc": { "name": "dictation-error", "ordinal": 6, @@ -351,32 +377,6 @@ } } }, - "a7dbb6b2864f": { - "name": "speech.dictation.cancel#1", - "ordinal": 5, - "args": [ - { - "name": "method", - "value": "speech.dictation.cancel" - }, - { - "name": "params", - "value": { - "dictationId": "mobile-dictation-1767225600000-8ig2henseon" - } - }, - { - "name": "options", - "value": { - "$rpc": "absent" - } - } - ], - "settlement": { - "status": "pending", - "startedAt": 0 - } - }, "b41f74be4c3d": { "name": "speech.dictation.finish#1", "ordinal": 3, @@ -573,7 +573,7 @@ { "id": "speech-dictation-session-transcript.result-absent:transcribed", "observation": { - "sender": ["bd004568e24c", "88f795a14a21", "a7dbb6b2864f"], + "sender": ["bd004568e24c", "88f795a14a21", "893e5291c492"], "payloads": ["859d310e63d6", "1bdc94d60271", "6c3d0754a1ff"], "settlements": { "mount": "eb79a9b3682a", @@ -587,7 +587,7 @@ { "id": "speech-dictation-session-transcript.result-null:transcribed", "observation": { - "sender": ["bd004568e24c", "349e2cfd1a1f", "a7dbb6b2864f"], + "sender": ["bd004568e24c", "349e2cfd1a1f", "893e5291c492"], "payloads": ["859d310e63d6", "1bdc94d60271", "6c3d0754a1ff"], "settlements": { "mount": "eb79a9b3682a", @@ -643,7 +643,7 @@ { "id": "speech-dictation-session-transcript.outer-refused:transcribed", "observation": { - "sender": ["bd004568e24c", "a46c0f6c95b2", "a7dbb6b2864f"], + "sender": ["bd004568e24c", "a46c0f6c95b2", "893e5291c492"], "payloads": ["859d310e63d6", "1bdc94d60271", "6c3d0754a1ff"], "settlements": { "mount": "eb79a9b3682a", @@ -657,7 +657,7 @@ { "id": "speech-dictation-session-transcript.outer-refused-no-message:transcribed", "observation": { - "sender": ["bd004568e24c", "9fce1a80008e", "a7dbb6b2864f"], + "sender": ["bd004568e24c", "9fce1a80008e", "893e5291c492"], "payloads": ["859d310e63d6", "1bdc94d60271", "6c3d0754a1ff"], "settlements": { "mount": "eb79a9b3682a", @@ -671,7 +671,7 @@ { "id": "speech-dictation-session-transcript.method-not-found:transcribed", "observation": { - "sender": ["bd004568e24c", "b41f74be4c3d", "a7dbb6b2864f"], + "sender": ["bd004568e24c", "b41f74be4c3d", "893e5291c492"], "payloads": ["859d310e63d6", "1bdc94d60271", "6c3d0754a1ff"], "settlements": { "mount": "eb79a9b3682a", @@ -685,7 +685,7 @@ { "id": "speech-dictation-session-transcript.transport-rejection:transcribed", "observation": { - "sender": ["bd004568e24c", "cf10282da8e0", "a7dbb6b2864f"], + "sender": ["bd004568e24c", "cf10282da8e0", "893e5291c492"], "payloads": ["859d310e63d6", "1bdc94d60271", "6c3d0754a1ff"], "settlements": { "mount": "eb79a9b3682a", @@ -699,7 +699,7 @@ { "id": "speech-dictation-session-transcript.transport-rejection-no-message:transcribed", "observation": { - "sender": ["bd004568e24c", "6d6bb34ce2ab", "a7dbb6b2864f"], + "sender": ["bd004568e24c", "6d6bb34ce2ab", "893e5291c492"], "payloads": ["859d310e63d6", "1bdc94d60271", "6c3d0754a1ff"], "settlements": { "mount": "eb79a9b3682a", diff --git a/mobile/rpc-foundation/goldens/matrix-speech.dictation-session-speech.dictation.start-1.json b/mobile/rpc-foundation/goldens/matrix-speech.dictation-session-speech.dictation.start-1.json index e0cfcc85f189..b259a3754c5b 100644 --- a/mobile/rpc-foundation/goldens/matrix-speech.dictation-session-speech.dictation.start-1.json +++ b/mobile/rpc-foundation/goldens/matrix-speech.dictation-session-speech.dictation.start-1.json @@ -142,32 +142,6 @@ "ordinal": 4, "json": "{\"id\":\"frame-2\",\"deviceToken\":\"recording-device\",\"method\":\"speech.dictation.finish\",\"params\":{\"dictationId\":\"mobile-dictation-1767225600000-8ig2henseon\"}}" }, - "355b3eaef1fb": { - "name": "speech.dictation.cancel#1", - "ordinal": 3, - "args": [ - { - "name": "method", - "value": "speech.dictation.cancel" - }, - { - "name": "params", - "value": { - "dictationId": "mobile-dictation-1767225600000-8ig2henseon" - } - }, - { - "name": "options", - "value": { - "$rpc": "absent" - } - } - ], - "settlement": { - "status": "pending", - "startedAt": 0 - } - }, "4728ad10f92b": { "name": "speech.dictation.finish#1", "ordinal": 3, @@ -242,6 +216,32 @@ "ordinal": 2, "json": "{\"id\":\"frame-1\",\"deviceToken\":\"recording-device\",\"method\":\"speech.dictation.start\",\"params\":{\"dictationId\":\"mobile-dictation-1767225600000-8ig2henseon\"}}" }, + "85d47816bfc5": { + "name": "speech.dictation.cancel#1", + "ordinal": 3, + "args": [ + { + "name": "method", + "value": "speech.dictation.cancel" + }, + { + "name": "params", + "value": { + "dictationId": "mobile-dictation-1767225600000-8ig2henseon" + } + }, + { + "name": "options", + "value": { + "failWhenDisconnected": true + } + } + ], + "settlement": { + "status": "pending", + "startedAt": 0 + } + }, "9270aeb7d9c6": { "status": "pending", "startedAt": 0 @@ -570,7 +570,7 @@ { "id": "speech-dictation-session-transcript.outer-refused:transcribed", "observation": { - "sender": ["b7421e659b44", "355b3eaef1fb"], + "sender": ["b7421e659b44", "85d47816bfc5"], "payloads": ["859d310e63d6", "ebdac948a90d"], "settlements": { "mount": "eb79a9b3682a", @@ -584,7 +584,7 @@ { "id": "speech-dictation-session-transcript.outer-refused-no-message:transcribed", "observation": { - "sender": ["6d8188e86d2a", "355b3eaef1fb"], + "sender": ["6d8188e86d2a", "85d47816bfc5"], "payloads": ["859d310e63d6", "ebdac948a90d"], "settlements": { "mount": "eb79a9b3682a", @@ -598,7 +598,7 @@ { "id": "speech-dictation-session-transcript.method-not-found:transcribed", "observation": { - "sender": ["062d9c83659b", "355b3eaef1fb"], + "sender": ["062d9c83659b", "85d47816bfc5"], "payloads": ["859d310e63d6", "ebdac948a90d"], "settlements": { "mount": "eb79a9b3682a", @@ -612,7 +612,7 @@ { "id": "speech-dictation-session-transcript.transport-rejection:transcribed", "observation": { - "sender": ["104a53ec5972", "355b3eaef1fb"], + "sender": ["104a53ec5972", "85d47816bfc5"], "payloads": ["859d310e63d6", "ebdac948a90d"], "settlements": { "mount": "eb79a9b3682a", @@ -626,7 +626,7 @@ { "id": "speech-dictation-session-transcript.transport-rejection-no-message:transcribed", "observation": { - "sender": ["c9d7f3d355eb", "355b3eaef1fb"], + "sender": ["c9d7f3d355eb", "85d47816bfc5"], "payloads": ["859d310e63d6", "ebdac948a90d"], "settlements": { "mount": "eb79a9b3682a", diff --git a/mobile/rpc-foundation/goldens/matrix-speech.dictation-start-speech.dictation.cancel-1.json b/mobile/rpc-foundation/goldens/matrix-speech.dictation-start-speech.dictation.cancel-1.json index ca0bfb1a4294..3b20c5a8e330 100644 --- a/mobile/rpc-foundation/goldens/matrix-speech.dictation-start-speech.dictation.cancel-1.json +++ b/mobile/rpc-foundation/goldens/matrix-speech.dictation-start-speech.dictation.cancel-1.json @@ -4,7 +4,7 @@ "family": "speech.dictation-start", "namedDeltas": [], "values": { - "02d3c28ecfb6": { + "0b8b17cc7b58": { "name": "speech.dictation.cancel#1", "ordinal": 3, "args": [ @@ -21,41 +21,7 @@ { "name": "options", "value": { - "$rpc": "absent" - } - } - ], - "settlement": { - "status": "fulfilled", - "startedAt": 0, - "settledAt": 0, - "value": { - "id": "frame-2", - "ok": true, - "result": { - "$rpc": "null" - } - } - } - }, - "1710d94f25d0": { - "name": "speech.dictation.cancel#1", - "ordinal": 3, - "args": [ - { - "name": "method", - "value": "speech.dictation.cancel" - }, - { - "name": "params", - "value": { - "dictationId": "dictation-1" - } - }, - { - "name": "options", - "value": { - "$rpc": "absent" + "failWhenDisconnected": true } } ], @@ -64,16 +30,16 @@ "startedAt": 0, "settledAt": 0, "value": { + "error": { + "code": "refused", + "message": "outer refused" + }, "id": "frame-2", - "ok": true, - "result": { - "error": "inner refused", - "ok": false - } + "ok": false } } }, - "25f8db2d01b6": { + "2e3b79ca68ce": { "name": "speech.dictation.cancel#1", "ordinal": 3, "args": [ @@ -90,7 +56,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "failWhenDisconnected": true } } ], @@ -105,7 +71,7 @@ } } }, - "46564194b0d9": { + "32caef6aa162": { "name": "speech.dictation.cancel#1", "ordinal": 3, "args": [ @@ -122,7 +88,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "failWhenDisconnected": true } } ], @@ -132,10 +98,7 @@ "settledAt": 0, "value": { "id": "frame-2", - "ok": true, - "result": { - "error": "refused" - } + "ok": true } } }, @@ -173,7 +136,7 @@ } } }, - "5f51c1b28c86": { + "5699810cc2a2": { "name": "speech.dictation.cancel#1", "ordinal": 3, "args": [ @@ -190,7 +153,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "failWhenDisconnected": true } } ], @@ -199,16 +162,18 @@ "startedAt": 0, "settledAt": 0, "value": { - "error": { - "code": "refused", - "message": "outer refused" - }, "id": "frame-2", - "ok": false + "ok": true, + "result": { + "error": { + "message": "inner refused" + }, + "ok": false + } } } }, - "73faa32df517": { + "6f5a48ebce0e": { "name": "speech.dictation.cancel#1", "ordinal": 3, "args": [ @@ -225,7 +190,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "failWhenDisconnected": true } } ], @@ -237,7 +202,7 @@ "id": "frame-2", "ok": true, "result": { - "cancelled": true + "$rpc": "null" } } } @@ -248,7 +213,7 @@ "settledAt": 0, "value": false }, - "927c61fd29b5": { + "9ab91ee194b8": { "name": "speech.dictation.cancel#1", "ordinal": 3, "args": [ @@ -265,7 +230,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "failWhenDisconnected": true } } ], @@ -274,12 +239,11 @@ "startedAt": 0, "settledAt": 0, "value": { - "error": { - "code": "method_not_found", - "message": "Unknown method" - }, "id": "frame-2", - "ok": false + "ok": true, + "result": { + "cancelled": true + } } } }, @@ -288,7 +252,7 @@ "ordinal": 2, "json": "{\"id\":\"frame-1\",\"deviceToken\":\"recording-device\",\"method\":\"speech.dictation.start\",\"params\":{\"dictationId\":\"dictation-1\"}}" }, - "cc661ef3e67e": { + "b20560e60f90": { "name": "speech.dictation.cancel#1", "ordinal": 3, "args": [ @@ -305,7 +269,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "failWhenDisconnected": true } } ], @@ -314,23 +278,16 @@ "startedAt": 0, "settledAt": 0, "value": { + "error": { + "code": "method_not_found", + "message": "Unknown method" + }, "id": "frame-2", - "ok": true, - "result": { - "error": { - "message": "inner refused" - }, - "ok": false - } + "ok": false } } }, - "ce9cbc8837fa": { - "name": "speech.dictation.cancel#1", - "ordinal": 4, - "json": "{\"id\":\"frame-2\",\"deviceToken\":\"recording-device\",\"method\":\"speech.dictation.cancel\",\"params\":{\"dictationId\":\"dictation-1\"}}" - }, - "dbc1fdbc29d3": { + "b90df4b1e290": { "name": "speech.dictation.cancel#1", "ordinal": 3, "args": [ @@ -347,7 +304,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "failWhenDisconnected": true } } ], @@ -365,14 +322,12 @@ } } }, - "e0fcd8f8c1a9": { - "activeId": { - "$rpc": "null" - }, - "idle": false, - "started": false + "ce9cbc8837fa": { + "name": "speech.dictation.cancel#1", + "ordinal": 4, + "json": "{\"id\":\"frame-2\",\"deviceToken\":\"recording-device\",\"method\":\"speech.dictation.cancel\",\"params\":{\"dictationId\":\"dictation-1\"}}" }, - "e59c2b1cff33": { + "d0501bb1dd67": { "name": "speech.dictation.cancel#1", "ordinal": 3, "args": [ @@ -389,7 +344,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "failWhenDisconnected": true } } ], @@ -399,19 +354,15 @@ "settledAt": 0, "value": { "id": "frame-2", - "ok": true + "ok": true, + "result": { + "error": "inner refused", + "ok": false + } } } }, - "eb79a9b3682a": { - "status": "fulfilled", - "startedAt": 0, - "settledAt": 0, - "value": { - "$rpc": "undefined" - } - }, - "f3ac8ed5616e": { + "d80561ce52f2": { "name": "speech.dictation.cancel#1", "ordinal": 3, "args": [ @@ -428,7 +379,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "failWhenDisconnected": true } } ], @@ -442,6 +393,55 @@ "isRpcDeliveryUnknown": true } } + }, + "df38de088787": { + "name": "speech.dictation.cancel#1", + "ordinal": 3, + "args": [ + { + "name": "method", + "value": "speech.dictation.cancel" + }, + { + "name": "params", + "value": { + "dictationId": "dictation-1" + } + }, + { + "name": "options", + "value": { + "failWhenDisconnected": true + } + } + ], + "settlement": { + "status": "fulfilled", + "startedAt": 0, + "settledAt": 0, + "value": { + "id": "frame-2", + "ok": true, + "result": { + "error": "refused" + } + } + } + }, + "e0fcd8f8c1a9": { + "activeId": { + "$rpc": "null" + }, + "idle": false, + "started": false + }, + "eb79a9b3682a": { + "status": "fulfilled", + "startedAt": 0, + "settledAt": 0, + "value": { + "$rpc": "undefined" + } } }, "recording": { @@ -450,7 +450,7 @@ { "id": "speech-desktop-start-superseded.normal:stale-start-cancelled", "observation": { - "sender": ["4c42677aca21", "73faa32df517"], + "sender": ["4c42677aca21", "9ab91ee194b8"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -463,7 +463,7 @@ { "id": "speech-desktop-start-superseded.result-absent:stale-start-cancelled", "observation": { - "sender": ["4c42677aca21", "e59c2b1cff33"], + "sender": ["4c42677aca21", "32caef6aa162"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -476,7 +476,7 @@ { "id": "speech-desktop-start-superseded.result-null:stale-start-cancelled", "observation": { - "sender": ["4c42677aca21", "02d3c28ecfb6"], + "sender": ["4c42677aca21", "6f5a48ebce0e"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -489,7 +489,7 @@ { "id": "speech-desktop-start-superseded.inner-ok-missing:stale-start-cancelled", "observation": { - "sender": ["4c42677aca21", "46564194b0d9"], + "sender": ["4c42677aca21", "df38de088787"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -502,7 +502,7 @@ { "id": "speech-desktop-start-superseded.inner-false-string-error:stale-start-cancelled", "observation": { - "sender": ["4c42677aca21", "1710d94f25d0"], + "sender": ["4c42677aca21", "d0501bb1dd67"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -515,7 +515,7 @@ { "id": "speech-desktop-start-superseded.inner-false-object-error:stale-start-cancelled", "observation": { - "sender": ["4c42677aca21", "cc661ef3e67e"], + "sender": ["4c42677aca21", "5699810cc2a2"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -528,7 +528,7 @@ { "id": "speech-desktop-start-superseded.outer-refused:stale-start-cancelled", "observation": { - "sender": ["4c42677aca21", "5f51c1b28c86"], + "sender": ["4c42677aca21", "0b8b17cc7b58"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -541,7 +541,7 @@ { "id": "speech-desktop-start-superseded.outer-refused-no-message:stale-start-cancelled", "observation": { - "sender": ["4c42677aca21", "dbc1fdbc29d3"], + "sender": ["4c42677aca21", "b90df4b1e290"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -554,7 +554,7 @@ { "id": "speech-desktop-start-superseded.method-not-found:stale-start-cancelled", "observation": { - "sender": ["4c42677aca21", "927c61fd29b5"], + "sender": ["4c42677aca21", "b20560e60f90"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -567,7 +567,7 @@ { "id": "speech-desktop-start-superseded.transport-rejection:stale-start-cancelled", "observation": { - "sender": ["4c42677aca21", "f3ac8ed5616e"], + "sender": ["4c42677aca21", "d80561ce52f2"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -580,7 +580,7 @@ { "id": "speech-desktop-start-superseded.transport-rejection-no-message:stale-start-cancelled", "observation": { - "sender": ["4c42677aca21", "25f8db2d01b6"], + "sender": ["4c42677aca21", "2e3b79ca68ce"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", diff --git a/mobile/rpc-foundation/goldens/matrix-speech.dictation-start-speech.dictation.start-1.json b/mobile/rpc-foundation/goldens/matrix-speech.dictation-start-speech.dictation.start-1.json index a7860b36568f..ab98d6156c90 100644 --- a/mobile/rpc-foundation/goldens/matrix-speech.dictation-start-speech.dictation.start-1.json +++ b/mobile/rpc-foundation/goldens/matrix-speech.dictation-start-speech.dictation.start-1.json @@ -139,7 +139,13 @@ } } }, - "73faa32df517": { + "7ed3d39f0607": { + "status": "fulfilled", + "startedAt": 0, + "settledAt": 0, + "value": false + }, + "9ab91ee194b8": { "name": "speech.dictation.cancel#1", "ordinal": 3, "args": [ @@ -156,7 +162,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "failWhenDisconnected": true } } ], @@ -173,12 +179,6 @@ } } }, - "7ed3d39f0607": { - "status": "fulfilled", - "startedAt": 0, - "settledAt": 0, - "value": false - }, "9c6249e03654": { "name": "speech.dictation.start#1", "ordinal": 1, @@ -450,7 +450,7 @@ { "id": "speech-desktop-start-superseded.normal:stale-start-cancelled", "observation": { - "sender": ["4c42677aca21", "73faa32df517"], + "sender": ["4c42677aca21", "9ab91ee194b8"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -463,7 +463,7 @@ { "id": "speech-desktop-start-superseded.result-absent:stale-start-cancelled", "observation": { - "sender": ["fe8f7e77754e", "73faa32df517"], + "sender": ["fe8f7e77754e", "9ab91ee194b8"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -476,7 +476,7 @@ { "id": "speech-desktop-start-superseded.result-null:stale-start-cancelled", "observation": { - "sender": ["0ac1d1294a4a", "73faa32df517"], + "sender": ["0ac1d1294a4a", "9ab91ee194b8"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -489,7 +489,7 @@ { "id": "speech-desktop-start-superseded.inner-ok-missing:stale-start-cancelled", "observation": { - "sender": ["f010ef5b6bfc", "73faa32df517"], + "sender": ["f010ef5b6bfc", "9ab91ee194b8"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -502,7 +502,7 @@ { "id": "speech-desktop-start-superseded.inner-false-string-error:stale-start-cancelled", "observation": { - "sender": ["f2eeb517059d", "73faa32df517"], + "sender": ["f2eeb517059d", "9ab91ee194b8"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -515,7 +515,7 @@ { "id": "speech-desktop-start-superseded.inner-false-object-error:stale-start-cancelled", "observation": { - "sender": ["ced9bd128f52", "73faa32df517"], + "sender": ["ced9bd128f52", "9ab91ee194b8"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -528,7 +528,7 @@ { "id": "speech-desktop-start-superseded.outer-refused:stale-start-cancelled", "observation": { - "sender": ["9c6249e03654", "73faa32df517"], + "sender": ["9c6249e03654", "9ab91ee194b8"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -541,7 +541,7 @@ { "id": "speech-desktop-start-superseded.outer-refused-no-message:stale-start-cancelled", "observation": { - "sender": ["02e3f6a398b9", "73faa32df517"], + "sender": ["02e3f6a398b9", "9ab91ee194b8"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -554,7 +554,7 @@ { "id": "speech-desktop-start-superseded.method-not-found:stale-start-cancelled", "observation": { - "sender": ["f2166355786a", "73faa32df517"], + "sender": ["f2166355786a", "9ab91ee194b8"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -567,7 +567,7 @@ { "id": "speech-desktop-start-superseded.transport-rejection:stale-start-cancelled", "observation": { - "sender": ["a1030fe3abd4", "73faa32df517"], + "sender": ["a1030fe3abd4", "9ab91ee194b8"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", @@ -580,7 +580,7 @@ { "id": "speech-desktop-start-superseded.transport-rejection-no-message:stale-start-cancelled", "observation": { - "sender": ["106b709a0579", "73faa32df517"], + "sender": ["106b709a0579", "9ab91ee194b8"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", diff --git a/mobile/rpc-foundation/goldens/speech-desktop-start-recording-failed.json b/mobile/rpc-foundation/goldens/speech-desktop-start-recording-failed.json index d016eea4acd4..0ebe3392aec1 100644 --- a/mobile/rpc-foundation/goldens/speech-desktop-start-recording-failed.json +++ b/mobile/rpc-foundation/goldens/speech-desktop-start-recording-failed.json @@ -11,13 +11,13 @@ "idle": true, "started": "unstarted" }, - "217153ab0a32": { - "name": "speech.dictation.cancel#1", - "ordinal": 4, + "4c42677aca21": { + "name": "speech.dictation.start#1", + "ordinal": 1, "args": [ { "name": "method", - "value": "speech.dictation.cancel" + "value": "speech.dictation.start" }, { "name": "params", @@ -28,7 +28,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "timeoutMs": 30000 } } ], @@ -37,21 +37,21 @@ "startedAt": 0, "settledAt": 0, "value": { - "id": "frame-2", + "id": "frame-1", "ok": true, "result": { - "cancelled": true + "started": true } } } }, - "4c42677aca21": { - "name": "speech.dictation.start#1", - "ordinal": 1, + "4c97e2d0e296": { + "name": "speech.dictation.cancel#1", + "ordinal": 4, "args": [ { "name": "method", - "value": "speech.dictation.start" + "value": "speech.dictation.cancel" }, { "name": "params", @@ -62,7 +62,7 @@ { "name": "options", "value": { - "timeoutMs": 30000 + "failWhenDisconnected": true } } ], @@ -71,10 +71,10 @@ "startedAt": 0, "settledAt": 0, "value": { - "id": "frame-1", + "id": "frame-2", "ok": true, "result": { - "started": true + "cancelled": true } } } @@ -111,7 +111,7 @@ { "id": "rolled-back", "observation": { - "sender": ["4c42677aca21", "217153ab0a32"], + "sender": ["4c42677aca21", "4c97e2d0e296"], "payloads": ["a7ddd021010a", "d6e61920633f"], "settlements": { "start": "7a657475cacc" diff --git a/mobile/rpc-foundation/goldens/speech-desktop-start-superseded.json b/mobile/rpc-foundation/goldens/speech-desktop-start-superseded.json index bde938c3cd61..7b5aad5f7771 100644 --- a/mobile/rpc-foundation/goldens/speech-desktop-start-superseded.json +++ b/mobile/rpc-foundation/goldens/speech-desktop-start-superseded.json @@ -38,7 +38,13 @@ } } }, - "73faa32df517": { + "7ed3d39f0607": { + "status": "fulfilled", + "startedAt": 0, + "settledAt": 0, + "value": false + }, + "9ab91ee194b8": { "name": "speech.dictation.cancel#1", "ordinal": 3, "args": [ @@ -55,7 +61,7 @@ { "name": "options", "value": { - "$rpc": "absent" + "failWhenDisconnected": true } } ], @@ -72,12 +78,6 @@ } } }, - "7ed3d39f0607": { - "status": "fulfilled", - "startedAt": 0, - "settledAt": 0, - "value": false - }, "a7ddd021010a": { "name": "speech.dictation.start#1", "ordinal": 2, @@ -110,7 +110,7 @@ { "id": "stale-start-cancelled", "observation": { - "sender": ["4c42677aca21", "73faa32df517"], + "sender": ["4c42677aca21", "9ab91ee194b8"], "payloads": ["a7ddd021010a", "ce9cbc8837fa"], "settlements": { "supersede": "eb79a9b3682a", diff --git a/mobile/rpc-foundation/goldens/speech-dictation-session-cancelled.json b/mobile/rpc-foundation/goldens/speech-dictation-session-cancelled.json index d0fa0f48749d..c69d15c66033 100644 --- a/mobile/rpc-foundation/goldens/speech-dictation-session-cancelled.json +++ b/mobile/rpc-foundation/goldens/speech-dictation-session-cancelled.json @@ -4,40 +4,6 @@ "family": "speech.dictation-session", "namedDeltas": [], "values": { - "422e902e66a5": { - "name": "speech.dictation.cancel#1", - "ordinal": 3, - "args": [ - { - "name": "method", - "value": "speech.dictation.cancel" - }, - { - "name": "params", - "value": { - "dictationId": "mobile-dictation-1767225600000-8ig2henseon" - } - }, - { - "name": "options", - "value": { - "$rpc": "absent" - } - } - ], - "settlement": { - "status": "fulfilled", - "startedAt": 0, - "settledAt": 0, - "value": { - "id": "frame-2", - "ok": true, - "result": { - "cancelled": true - } - } - } - }, "6b76435b6f3e": { "error": { "$rpc": "null" @@ -96,6 +62,40 @@ "name": "speech.dictation.cancel#1", "ordinal": 4, "json": "{\"id\":\"frame-2\",\"deviceToken\":\"recording-device\",\"method\":\"speech.dictation.cancel\",\"params\":{\"dictationId\":\"mobile-dictation-1767225600000-8ig2henseon\"}}" + }, + "ed7a0ff994cd": { + "name": "speech.dictation.cancel#1", + "ordinal": 3, + "args": [ + { + "name": "method", + "value": "speech.dictation.cancel" + }, + { + "name": "params", + "value": { + "dictationId": "mobile-dictation-1767225600000-8ig2henseon" + } + }, + { + "name": "options", + "value": { + "failWhenDisconnected": true + } + } + ], + "settlement": { + "status": "fulfilled", + "startedAt": 0, + "settledAt": 0, + "value": { + "id": "frame-2", + "ok": true, + "result": { + "cancelled": true + } + } + } } }, "recording": { @@ -104,7 +104,7 @@ { "id": "cancelled", "observation": { - "sender": ["bd004568e24c", "422e902e66a5"], + "sender": ["bd004568e24c", "ed7a0ff994cd"], "payloads": ["859d310e63d6", "ebdac948a90d"], "settlements": { "mount": "eb79a9b3682a", diff --git a/mobile/src/hooks/mobile-dictation-desktop-start.test.ts b/mobile/src/hooks/mobile-dictation-desktop-start.test.ts index 9049ac725a9a..4a6871b758dc 100644 --- a/mobile/src/hooks/mobile-dictation-desktop-start.test.ts +++ b/mobile/src/hooks/mobile-dictation-desktop-start.test.ts @@ -72,9 +72,11 @@ describe('startMobileDictationDesktopSession', () => { expect(harness.setIdle).not.toHaveBeenCalled() expect(harness.getActiveId()).toBe('dictation-b') expect(harness.commitRecordingStart).not.toHaveBeenCalled() - expect(harness.sendRequest).toHaveBeenCalledWith('speech.dictation.cancel', { - dictationId: 'dictation-a' - }) + expect(harness.sendRequest).toHaveBeenCalledWith( + 'speech.dictation.cancel', + { dictationId: 'dictation-a' }, + { failWhenDisconnected: true } + ) }) it('returns to idle when a disable makes the start stale', async () => { @@ -112,9 +114,11 @@ describe('startMobileDictationDesktopSession', () => { // The hook opened capture before requesting a session; refusal must release it and the screen because no outer start catch does so. expect(harness.rollbackRecordingStart).toHaveBeenCalledOnce() - expect(harness.sendRequest).toHaveBeenCalledWith('speech.dictation.cancel', { - dictationId: 'dictation-a' - }) + expect(harness.sendRequest).toHaveBeenCalledWith( + 'speech.dictation.cancel', + { dictationId: 'dictation-a' }, + { failWhenDisconnected: true } + ) }) it('leaves the capture alone when the failure is no longer the current start', async () => { @@ -135,9 +139,11 @@ describe('startMobileDictationDesktopSession', () => { // Capture has no start identity; rolling back stale A would stop B’s microphone and release B’s screen. expect(harness.rollbackRecordingStart).not.toHaveBeenCalled() // A still owns its desktop session and must cancel that alone. - expect(harness.sendRequest).toHaveBeenCalledWith('speech.dictation.cancel', { - dictationId: 'dictation-a' - }) + expect(harness.sendRequest).toHaveBeenCalledWith( + 'speech.dictation.cancel', + { dictationId: 'dictation-a' }, + { failWhenDisconnected: true } + ) }) it('commits recording before returning a current start to the hook', async () => { @@ -160,9 +166,11 @@ describe('startMobileDictationDesktopSession', () => { 'Audio focus request failed' ) - expect(harness.sendRequest).toHaveBeenCalledWith('speech.dictation.cancel', { - dictationId: 'dictation-a' - }) + expect(harness.sendRequest).toHaveBeenCalledWith( + 'speech.dictation.cancel', + { dictationId: 'dictation-a' }, + { failWhenDisconnected: true } + ) expect(harness.getActiveId()).toBeNull() expect(harness.setIdle).toHaveBeenCalledOnce() expect(harness.rollbackRecordingStart).toHaveBeenCalledOnce() @@ -175,9 +183,11 @@ describe('startMobileDictationDesktopSession', () => { 'Failed to start microphone recording' ) - expect(harness.sendRequest).toHaveBeenCalledWith('speech.dictation.cancel', { - dictationId: 'dictation-a' - }) + expect(harness.sendRequest).toHaveBeenCalledWith( + 'speech.dictation.cancel', + { dictationId: 'dictation-a' }, + { failWhenDisconnected: true } + ) expect(harness.getActiveId()).toBeNull() expect(harness.setIdle).toHaveBeenCalledOnce() expect(harness.rollbackRecordingStart).toHaveBeenCalledOnce() diff --git a/mobile/src/hooks/mobile-dictation-desktop-start.ts b/mobile/src/hooks/mobile-dictation-desktop-start.ts index 7d827140cea9..79cdf356c9d5 100644 --- a/mobile/src/hooks/mobile-dictation-desktop-start.ts +++ b/mobile/src/hooks/mobile-dictation-desktop-start.ts @@ -19,6 +19,16 @@ type StartMobileDictationDesktopSessionOptions = { rollbackRecordingStart: () => void } +// Offline there is nothing to cancel against; the host drops a session its phone stops reaching, so never wait for a reconnect. +export function cancelDesktopDictation(client: RpcClient, dictationId: string): Promise { + return dictationSessionCancel + .request(client, { dictationId }, { failWhenDisconnected: true }) + .then( + () => undefined, + () => undefined + ) +} + function isCurrentStart(options: StartMobileDictationDesktopSessionOptions): boolean { return isCurrentMobileDictationStart( options.getCurrentGeneration(), @@ -43,7 +53,7 @@ async function cancelStaleStart(options: StartMobileDictationDesktopSessionOptio const { client, dictationId } = options options.clearActiveId(dictationId) setIdleIfGenerationCurrent(options) - await dictationSessionCancel.request(client, { dictationId }).catch(() => undefined) + await cancelDesktopDictation(client, dictationId) } export async function startMobileDictationDesktopSession( @@ -70,7 +80,7 @@ export async function startMobileDictationDesktopSession( } } options.clearActiveId(dictationId) - await dictationSessionCancel.request(client, { dictationId }).catch(() => undefined) + await cancelDesktopDictation(client, dictationId) // Awaited cleanup may overlap a newer start; stale work must not reset or // report over the replacement session. const shouldReport = wasCurrent && canReportStartFailure(options) @@ -102,7 +112,7 @@ export async function startMobileDictationDesktopSession( // Continue releasing independently owned resources after native audio failure. } options.clearActiveId(dictationId) - await dictationSessionCancel.request(client, { dictationId }).catch(() => undefined) + await cancelDesktopDictation(client, dictationId) const shouldReport = wasCurrent && canReportStartFailure(options) setIdleIfGenerationCurrent(options) if (!shouldReport) { diff --git a/mobile/src/hooks/use-mobile-dictation.ts b/mobile/src/hooks/use-mobile-dictation.ts index 1cf3b3111efa..e4de0b23a176 100644 --- a/mobile/src/hooks/use-mobile-dictation.ts +++ b/mobile/src/hooks/use-mobile-dictation.ts @@ -13,11 +13,11 @@ import { createMobileDictationId, isCurrentMobileDictationFinish } from './mobile-dictation-session-state' -import { startMobileDictationDesktopSession } from './mobile-dictation-desktop-start' import { - dictationSessionCancel, - dictationSessionFinish -} from '../dictation/mobile-dictation-operations' + cancelDesktopDictation, + startMobileDictationDesktopSession +} from './mobile-dictation-desktop-start' +import { dictationSessionFinish } from '../dictation/mobile-dictation-operations' import { rpcPayloadMember } from '../transport/rpc-reader-payload' import type { DictationStatus, @@ -91,7 +91,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil activeIdRef.current = null closeDictationAudio() if (client && dictationId) { - void dictationSessionCancel.request(client, { dictationId }).catch(() => undefined) + void cancelDesktopDictation(client, dictationId) } const kept = transcriptSoFarRef.current.take(dictationId) if (kept) { @@ -268,7 +268,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil activeIdRef.current = null closeDictationAudio() if (client && dictationId) { - await dictationSessionCancel.request(client, { dictationId }).catch(() => undefined) + await cancelDesktopDictation(client, dictationId) } // The cancel round trip may overlap a new recording; stale completion must neither report over it nor reset it. if (generationRef.current !== generation) { @@ -312,9 +312,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil closeDictationAudio() capture.release() if (clientRef.current && dictationId) { - void dictationSessionCancel - .request(clientRef.current, { dictationId }) - .catch(() => undefined) + void cancelDesktopDictation(clientRef.current, dictationId) } } }, [capture, closeDictationAudio]) diff --git a/src/main/runtime/runtime-mobile-dictation-controller.test.ts b/src/main/runtime/runtime-mobile-dictation-controller.test.ts index 5561274c3fec..e13664b3e877 100644 --- a/src/main/runtime/runtime-mobile-dictation-controller.test.ts +++ b/src/main/runtime/runtime-mobile-dictation-controller.test.ts @@ -53,6 +53,9 @@ describe('mobile dictation across a phone reconnect', () => { await vi.advanceTimersByTimeAsync(MOBILE_DICTATION_RECONNECT_GRACE_MS - 1) expect(controller.feed(chunk)).toEqual({ dictationId: 'd-1', text: 'before the drop' }) expect(controller.feed(chunk)).toEqual({ dictationId: 'd-1' }) + // A reply carrying the text may be lost at a drop, so it is sent again a second later. + await vi.advanceTimersByTimeAsync(1000) + expect(controller.feed(chunk)).toEqual({ dictationId: 'd-1', text: 'before the drop' }) await vi.advanceTimersByTimeAsync(MOBILE_DICTATION_RECONNECT_GRACE_MS - 1) sink({ type: 'final', text: 'after it' }) diff --git a/src/main/runtime/runtime-mobile-dictation-controller.ts b/src/main/runtime/runtime-mobile-dictation-controller.ts index 1535a2780dbd..99d3c1708704 100644 --- a/src/main/runtime/runtime-mobile-dictation-controller.ts +++ b/src/main/runtime/runtime-mobile-dictation-controller.ts @@ -16,9 +16,10 @@ type MobileDictationSession = { partialText: string finalTexts: string[] errors: string[] - // Bumped by each transcript event; chunks arrive far more often, so replies carry text only after a bump. + // Bumped by each transcript event; chunks arrive far more often, so replies carry text only after a bump or a second. textVersion: number reportedVersion: number + reportedAt: number abandonTimer?: ReturnType } @@ -70,7 +71,8 @@ export class RuntimeMobileDictationController { finalTexts: [], errors: [], textVersion: 0, - reportedVersion: 0 + reportedVersion: 0, + reportedAt: 0 } try { await getSpeechSttService(store).startDictation( @@ -117,11 +119,13 @@ export class RuntimeMobileDictationController { samples[i] = pcm.readInt16LE(i * 2) / 32768 } getSpeechSttService(this.requireStore()).feedAudio(samples, params.sampleRate, session.owner) - // The text so far lets the phone keep it if the connection never comes back. - if (session.reportedVersion === session.textVersion) { + // The text so far lets the phone keep it if the connection never comes back; resent each second in case a reply was lost. + const now = Date.now() + if (session.reportedVersion === session.textVersion && now - session.reportedAt < 1000) { return { dictationId: params.dictationId } } session.reportedVersion = session.textVersion + session.reportedAt = now return { dictationId: params.dictationId, text: transcriptOf(session) } }