From d1430ee9119b8a7189e02e5d5336b383bdd1b866 Mon Sep 17 00:00:00 2001 From: "t3-code[bot]" <269035359+t3-code[bot]@users.noreply.github.com> Date: Sun, 23 Aug 2026 16:44:44 +0000 Subject: [PATCH 1/9] feat(desktop): add local voice input Co-authored-by: Utkarsh Patil <73941998+UtkarshUsername@users.noreply.github.com> --- apps/desktop/package.json | 4 +- apps/desktop/src/ipc/DesktopIpcHandlers.ts | 7 + apps/desktop/src/ipc/channels.ts | 6 + apps/desktop/src/ipc/methods/speech.ts | 128 +++++++++++ apps/desktop/src/preload.ts | 15 ++ .../src/speech/DesktopMicrophoneCapture.ts | 99 ++++++++ .../speech/DesktopSpeechController.test.ts | 65 ++++++ .../src/speech/DesktopSpeechController.ts | 181 +++++++++++++++ .../src/speech/DesktopTranscriptionBackend.ts | 39 ++++ apps/desktop/src/speech/speechModel.test.ts | 72 ++++++ apps/desktop/src/speech/speechModel.ts | 106 +++++++++ apps/web/src/components/chat/ChatComposer.tsx | 27 +++ .../components/chat/ComposerSpeechButton.tsx | 69 ++++++ .../components/settings/SettingsPanels.tsx | 40 ++++ apps/web/src/speech/speechInsertion.test.ts | 10 + apps/web/src/speech/speechInsertion.ts | 9 + apps/web/src/speech/useDesktopSpeechInput.ts | 93 ++++++++ docs/README.md | 1 + docs/user/voice-input.md | 13 ++ packages/contracts/src/index.ts | 1 + packages/contracts/src/ipc.ts | 10 + packages/contracts/src/speech.test.ts | 26 +++ packages/contracts/src/speech.ts | 51 +++++ pnpm-lock.yaml | 211 ++++++++++++++++++ pnpm-workspace.yaml | 1 + scripts/build-desktop-artifact.test.ts | 18 +- scripts/build-desktop-artifact.ts | 12 + 27 files changed, 1307 insertions(+), 7 deletions(-) create mode 100644 apps/desktop/src/ipc/methods/speech.ts create mode 100644 apps/desktop/src/speech/DesktopMicrophoneCapture.ts create mode 100644 apps/desktop/src/speech/DesktopSpeechController.test.ts create mode 100644 apps/desktop/src/speech/DesktopSpeechController.ts create mode 100644 apps/desktop/src/speech/DesktopTranscriptionBackend.ts create mode 100644 apps/desktop/src/speech/speechModel.test.ts create mode 100644 apps/desktop/src/speech/speechModel.ts create mode 100644 apps/web/src/components/chat/ComposerSpeechButton.tsx create mode 100644 apps/web/src/speech/speechInsertion.test.ts create mode 100644 apps/web/src/speech/speechInsertion.ts create mode 100644 apps/web/src/speech/useDesktopSpeechInput.ts create mode 100644 docs/user/voice-input.md create mode 100644 packages/contracts/src/speech.test.ts create mode 100644 packages/contracts/src/speech.ts diff --git a/apps/desktop/package.json b/apps/desktop/package.json index a34a55f16acf..45a03eae1ed0 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -15,6 +15,7 @@ "@clerk/electron": "catalog:", "@clerk/electron-passkeys": "catalog:", "@effect/platform-node": "catalog:", + "@picovoice/pvrecorder-node": "1.2.9", "@t3tools/client-runtime": "workspace:*", "@t3tools/contracts": "workspace:*", "@t3tools/shared": "workspace:*", @@ -25,7 +26,8 @@ "electron-store": "^8.2.0", "electron-updater": "^6.6.2", "playwright-core": "1.60.0", - "react-grab": "^0.1.32" + "react-grab": "^0.1.32", + "transcribe-cpp": "0.2.1" }, "devDependencies": { "@effect/vitest": "catalog:", diff --git a/apps/desktop/src/ipc/DesktopIpcHandlers.ts b/apps/desktop/src/ipc/DesktopIpcHandlers.ts index 8e8317db7971..3af5e758d940 100644 --- a/apps/desktop/src/ipc/DesktopIpcHandlers.ts +++ b/apps/desktop/src/ipc/DesktopIpcHandlers.ts @@ -45,11 +45,13 @@ import { showContextMenu, } from "./methods/window.ts"; import * as PreviewIpc from "./methods/preview.ts"; +import * as SpeechIpc from "./methods/speech.ts"; import { getWslState, setWslBackendEnabled, setWslDistro, setWslOnly } from "./methods/wsl.ts"; export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers")(function* () { const ipc = yield* DesktopIpc.DesktopIpc; yield* PreviewIpc.installPreviewEventForwarding(); + yield* SpeechIpc.installSpeechEventForwarding(); yield* ipc.handleSync(getAppBranding); yield* ipc.handleSync(getSystemLocale); @@ -94,6 +96,11 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers" yield* ipc.handle(downloadUpdate); yield* ipc.handle(installUpdate); yield* ipc.handle(checkForUpdate); + yield* ipc.handle(SpeechIpc.getSpeechStatus); + yield* ipc.handle(SpeechIpc.startSpeech); + yield* ipc.handle(SpeechIpc.stopSpeech); + yield* ipc.handle(SpeechIpc.cancelSpeech); + yield* ipc.handle(SpeechIpc.removeSpeechModelMethod); for (const previewMethod of PreviewIpc.methods) { yield* ipc.handle(previewMethod); } diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index c4ef82ec8cb7..50bb40d48a2e 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -83,3 +83,9 @@ export const PREVIEW_RECORDING_SAVE_CHANNEL = "desktop:preview-recording-save"; export const PREVIEW_RECORDING_FRAME_CHANNEL = "desktop:preview-recording-frame"; export const PREVIEW_STATE_CHANGE_CHANNEL = "desktop:preview-state-change"; export const PREVIEW_POINTER_EVENT_CHANNEL = "desktop:preview-pointer-event"; +export const SPEECH_GET_STATUS_CHANNEL = "desktop:speech-get-status"; +export const SPEECH_START_CHANNEL = "desktop:speech-start"; +export const SPEECH_STOP_CHANNEL = "desktop:speech-stop"; +export const SPEECH_CANCEL_CHANNEL = "desktop:speech-cancel"; +export const SPEECH_REMOVE_MODEL_CHANNEL = "desktop:speech-remove-model"; +export const SPEECH_EVENT_CHANNEL = "desktop:speech-event"; diff --git a/apps/desktop/src/ipc/methods/speech.ts b/apps/desktop/src/ipc/methods/speech.ts new file mode 100644 index 000000000000..a4bf735f1a66 --- /dev/null +++ b/apps/desktop/src/ipc/methods/speech.ts @@ -0,0 +1,128 @@ +// @effect-diagnostics nodeBuiltinImport:off - platform tuple and Electron app paths define native availability. +import { DesktopSpeechStatusSchema, type DesktopSpeechEvent } from "@t3tools/contracts"; +import { HostProcessArchitecture, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import { app } from "electron"; +import * as NodePath from "node:path"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import * as ElectronWindow from "../../electron/ElectronWindow.ts"; +import { DesktopMicrophoneCapture } from "../../speech/DesktopMicrophoneCapture.ts"; +import { DesktopSpeechController } from "../../speech/DesktopSpeechController.ts"; +import { DesktopTranscriptionBackend } from "../../speech/DesktopTranscriptionBackend.ts"; +import { + SPEECH_MODEL, + downloadVerifiedModel, + isSpeechModelReady, + removeSpeechModel, + speechModelPath, +} from "../../speech/speechModel.ts"; +import * as IpcChannels from "../channels.ts"; +import * as DesktopIpc from "../DesktopIpc.ts"; + +let controller: DesktopSpeechController | undefined; +let emitEvent: (event: DesktopSpeechEvent) => void = () => undefined; +let availability: { supported: boolean; reason?: string } = { + supported: false, + reason: "voice input is still starting", +}; + +function support(platform: string, architecture: string): { supported: boolean; reason?: string } { + if (platform === "win32" && architecture === "arm64") { + return { supported: false, reason: "voice input is not available on Windows arm64 yet" }; + } + const tuple = `${platform}-${architecture}`; + const supported = new Set([ + "darwin-arm64", + "darwin-x64", + "win32-x64", + "linux-x64", + "linux-arm64", + ]).has(tuple); + return supported + ? { supported: true } + : { supported: false, reason: `voice input is not available on ${tuple}` }; +} + +function speechModelDirectory(): string { + return NodePath.join(app.getPath("userData"), "speech", "models"); +} + +function getController(): DesktopSpeechController { + if (controller) return controller; + const directory = speechModelDirectory(); + controller = new DesktopSpeechController({ + supported: availability.supported, + ...(availability.reason ? { unsupportedReason: availability.reason } : {}), + modelPath: speechModelPath(directory), + modelReady: () => isSpeechModelReady(directory), + downloadModel: (onProgress) => + downloadVerifiedModel({ + directory, + filename: SPEECH_MODEL.filename, + url: SPEECH_MODEL.url, + size: SPEECH_MODEL.size, + sha256: SPEECH_MODEL.sha256, + onProgress, + }), + removeModel: () => removeSpeechModel(directory), + createCapture: () => + new DesktopMicrophoneCapture((level, elapsedMs) => + emitEvent({ type: "level", level, elapsedMs }), + ), + createBackend: (modelPath) => new DesktopTranscriptionBackend(modelPath), + emit: (event) => emitEvent(event), + }); + return controller; +} + +function invoke( + operation: (value: DesktopSpeechController) => Promise, +): Effect.Effect { + return Effect.promise(() => operation(getController())); +} + +export const installSpeechEventForwarding = Effect.fn("desktop.ipc.speech.events")(function* () { + const windows = yield* ElectronWindow.ElectronWindow; + const platform = yield* HostProcessPlatform; + const architecture = yield* HostProcessArchitecture; + availability = support(platform, architecture); + emitEvent = (event) => { + Effect.runFork(windows.sendAll(IpcChannels.SPEECH_EVENT_CHANNEL, event)); + }; +}); + +export const getSpeechStatus = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.SPEECH_GET_STATUS_CHANNEL, + payload: Schema.Void, + result: DesktopSpeechStatusSchema, + handler: () => invoke((value) => value.getStatus()), +}); + +export const startSpeech = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.SPEECH_START_CHANNEL, + payload: Schema.Void, + result: DesktopSpeechStatusSchema, + handler: () => invoke((value) => value.start()), +}); + +export const stopSpeech = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.SPEECH_STOP_CHANNEL, + payload: Schema.Void, + result: DesktopSpeechStatusSchema, + handler: () => invoke((value) => value.stop()), +}); + +export const cancelSpeech = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.SPEECH_CANCEL_CHANNEL, + payload: Schema.Void, + result: DesktopSpeechStatusSchema, + handler: () => invoke((value) => value.cancel()), +}); + +export const removeSpeechModelMethod = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.SPEECH_REMOVE_MODEL_CHANNEL, + payload: Schema.Void, + result: DesktopSpeechStatusSchema, + handler: () => invoke((value) => value.removeModel()), +}); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 407c7c3ef498..003900ff60a9 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -164,6 +164,21 @@ contextBridge.exposeInMainWorld("desktopBridge", { ipcRenderer.removeListener(IpcChannels.UPDATE_STATE_CHANNEL, wrappedListener); }; }, + speech: { + getStatus: () => ipcRenderer.invoke(IpcChannels.SPEECH_GET_STATUS_CHANNEL), + start: () => ipcRenderer.invoke(IpcChannels.SPEECH_START_CHANNEL), + stop: () => ipcRenderer.invoke(IpcChannels.SPEECH_STOP_CHANNEL), + cancel: () => ipcRenderer.invoke(IpcChannels.SPEECH_CANCEL_CHANNEL), + removeModel: () => ipcRenderer.invoke(IpcChannels.SPEECH_REMOVE_MODEL_CHANNEL), + onEvent: (listener) => { + const wrappedListener = (_event: Electron.IpcRendererEvent, speechEvent: unknown) => { + if (typeof speechEvent !== "object" || speechEvent === null) return; + listener(speechEvent as Parameters[0]); + }; + ipcRenderer.on(IpcChannels.SPEECH_EVENT_CHANNEL, wrappedListener); + return () => ipcRenderer.removeListener(IpcChannels.SPEECH_EVENT_CHANNEL, wrappedListener); + }, + }, preview: { createTab: (tabId, defaults) => ipcRenderer.invoke(IpcChannels.PREVIEW_CREATE_TAB_CHANNEL, { diff --git a/apps/desktop/src/speech/DesktopMicrophoneCapture.ts b/apps/desktop/src/speech/DesktopMicrophoneCapture.ts new file mode 100644 index 000000000000..3fc81ba0a78a --- /dev/null +++ b/apps/desktop/src/speech/DesktopMicrophoneCapture.ts @@ -0,0 +1,99 @@ +// @effect-diagnostics globalDate:off - native microphone callback is outside Effect runtime. +import { PvRecorder } from "@picovoice/pvrecorder-node"; + +const SAMPLE_RATE = 16_000; +const FRAME_LENGTH = 512; + +export class DesktopMicrophoneCapture { + private readonly onLevel: (level: number, elapsedMs: number) => void; + private recorder: PvRecorder | undefined; + private frames: Int16Array[] = []; + private readLoop: Promise | undefined; + private stopping = false; + private readError: Error | undefined; + private startedAt = 0; + private lastLevelAt = 0; + + constructor(onLevel: (level: number, elapsedMs: number) => void) { + this.onLevel = onLevel; + } + + start(): void { + if (this.recorder) throw new Error("microphone capture is already active"); + const recorder = new PvRecorder(FRAME_LENGTH, -1); + try { + if (recorder.sampleRate !== SAMPLE_RATE) { + throw new Error( + `microphone reported ${recorder.sampleRate} Hz; expected ${SAMPLE_RATE} Hz`, + ); + } + this.frames = []; + this.stopping = false; + this.readError = undefined; + this.startedAt = Date.now(); + this.lastLevelAt = 0; + recorder.start(); + this.recorder = recorder; + this.readLoop = this.readFrames(recorder); + } catch (error) { + recorder.release(); + throw error; + } + } + + async stop(): Promise { + const recorder = this.recorder; + if (!recorder) throw new Error("microphone capture is not active"); + this.stopping = true; + try { + if (recorder.isRecording) recorder.stop(); + await this.readLoop; + } finally { + recorder.release(); + this.recorder = undefined; + this.readLoop = undefined; + } + if (this.readError) throw this.readError; + const sampleCount = this.frames.reduce((total, frame) => total + frame.length, 0); + if (sampleCount === 0) throw new Error("no microphone audio was captured"); + const pcm = new Float32Array(sampleCount); + let offset = 0; + for (const frame of this.frames) { + for (let index = 0; index < frame.length; index += 1) { + pcm[offset + index] = (frame[index] ?? 0) / 32_768; + } + offset += frame.length; + } + this.frames = []; + return pcm; + } + + async cancel(): Promise { + if (!this.recorder) return; + await this.stop().catch(() => undefined); + this.frames = []; + } + + private async readFrames(recorder: PvRecorder): Promise { + try { + while (!this.stopping && recorder.isRecording) { + const frame = await recorder.read(); + if (this.stopping) continue; + this.frames.push(frame); + const now = Date.now(); + if (now - this.lastLevelAt >= 100) { + this.lastLevelAt = now; + let energy = 0; + for (const sample of frame) energy += (sample / 32_768) ** 2; + this.onLevel( + Math.min(1, Math.sqrt(energy / Math.max(1, frame.length)) * 8), + now - this.startedAt, + ); + } + } + } catch (error) { + if (!this.stopping) + this.readError = error instanceof Error ? error : new Error(String(error)); + } + } +} diff --git a/apps/desktop/src/speech/DesktopSpeechController.test.ts b/apps/desktop/src/speech/DesktopSpeechController.test.ts new file mode 100644 index 000000000000..06047c0c820d --- /dev/null +++ b/apps/desktop/src/speech/DesktopSpeechController.test.ts @@ -0,0 +1,65 @@ +// @effect-diagnostics globalTimers:off - this test exercises the controller's native timer boundary. +import { describe, expect, it, vi } from "vite-plus/test"; + +import { DesktopSpeechController } from "./DesktopSpeechController.ts"; + +function makeController(maxRecordingMs?: number) { + const events: unknown[] = []; + const capture = { + start: vi.fn(), + stop: vi.fn().mockResolvedValue(new Float32Array([0.25, -0.25])), + cancel: vi.fn().mockResolvedValue(undefined), + }; + const backend = { + prepare: vi.fn().mockResolvedValue(undefined), + transcribe: vi.fn().mockResolvedValue("hello from speech"), + dispose: vi.fn().mockResolvedValue(undefined), + }; + const controller = new DesktopSpeechController({ + supported: true, + modelReady: vi.fn().mockResolvedValue(false), + downloadModel: vi.fn().mockImplementation(async (onProgress) => { + onProgress(5, 10); + return "/tmp/model.gguf"; + }), + removeModel: vi.fn().mockResolvedValue(undefined), + createCapture: () => capture, + createBackend: () => backend, + emit: (event) => events.push(event), + ...(maxRecordingMs === undefined ? {} : { maxRecordingMs }), + }); + return { controller, capture, backend, events }; +} + +describe("DesktopSpeechController", () => { + it("downloads, records, transcribes, and emits the final transcript", async () => { + const { controller, capture, backend, events } = makeController(); + + expect(await controller.start()).toMatchObject({ supported: true, state: "recording" }); + expect(await controller.stop()).toMatchObject({ supported: true, state: "ready" }); + + expect(capture.start).toHaveBeenCalledOnce(); + expect(backend.prepare).toHaveBeenCalledOnce(); + expect(backend.transcribe).toHaveBeenCalledWith(new Float32Array([0.25, -0.25])); + expect(events).toContainEqual({ type: "download-progress", downloaded: 5, total: 10 }); + expect(events).toContainEqual({ type: "transcript", text: "hello from speech" }); + }); + + it("cancels recording without transcribing", async () => { + const { controller, capture, backend, events } = makeController(); + + await controller.start(); + expect(await controller.cancel()).toMatchObject({ supported: true, state: "ready" }); + + expect(capture.cancel).toHaveBeenCalledOnce(); + expect(backend.transcribe).not.toHaveBeenCalled(); + expect(events.some((event) => (event as { type?: string }).type === "transcript")).toBe(false); + }); + + it("automatically stops a recording at the duration limit", async () => { + const { controller, backend } = makeController(1); + await controller.start(); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(backend.transcribe).toHaveBeenCalledOnce(); + }); +}); diff --git a/apps/desktop/src/speech/DesktopSpeechController.ts b/apps/desktop/src/speech/DesktopSpeechController.ts new file mode 100644 index 000000000000..09eb6d85e585 --- /dev/null +++ b/apps/desktop/src/speech/DesktopSpeechController.ts @@ -0,0 +1,181 @@ +// @effect-diagnostics globalTimers:off - this timer bounds native microphone capture. +import type { DesktopSpeechEvent, DesktopSpeechStatus } from "@t3tools/contracts"; + +type Capture = { + start(): void; + stop(): Promise; + cancel(): Promise; +}; + +type Backend = { + prepare(): Promise; + transcribe(pcm: Float32Array): Promise; + dispose(): Promise; +}; + +type ControllerOptions = { + supported: boolean; + unsupportedReason?: string; + modelPath?: string; + modelReady(): Promise; + downloadModel(onProgress: (downloaded: number, total: number) => void): Promise; + removeModel(): Promise; + createCapture(): Capture; + createBackend(modelPath: string): Backend; + emit(event: DesktopSpeechEvent): void; + maxRecordingMs?: number; +}; + +export class DesktopSpeechController { + private readonly options: ControllerOptions; + private capture: Capture | undefined; + private backend: Backend | undefined; + private backendReady: Promise | undefined; + private modelPath: string | undefined; + private state: + | "missing-model" + | "downloading" + | "ready" + | "recording" + | "transcribing" + | "error" = "missing-model"; + private operation: Promise | undefined; + private recordingTimer: ReturnType | undefined; + + constructor(options: ControllerOptions) { + this.options = options; + this.modelPath = options.modelPath; + } + + async getStatus(): Promise { + if (!this.options.supported) { + return { supported: false, reason: this.options.unsupportedReason ?? "unsupported platform" }; + } + if (this.state === "missing-model" && (await this.options.modelReady())) this.state = "ready"; + return { supported: true, state: this.state }; + } + + start(): Promise { + return this.exclusive(async () => { + const initial = await this.getStatus(); + if (!initial.supported) return initial; + if (this.capture) throw new Error("voice input is already recording"); + + if (!(await this.options.modelReady())) { + this.setState("downloading"); + this.modelPath = await this.options.downloadModel((downloaded, total) => + this.options.emit({ type: "download-progress", downloaded, total }), + ); + } + if (!this.modelPath) this.modelPath = this.options.modelPath; + if (!this.modelPath) throw new Error("speech model path is unavailable"); + + const capture = this.options.createCapture(); + const backend = this.options.createBackend(this.modelPath); + this.capture = capture; + this.backend = backend; + this.backendReady = backend.prepare(); + try { + capture.start(); + } catch (error) { + this.capture = undefined; + this.backend = undefined; + this.backendReady = undefined; + await backend.dispose().catch(() => undefined); + throw error; + } + this.setState("recording"); + this.recordingTimer = setTimeout( + () => void this.stop(), + this.options.maxRecordingMs ?? 120_000, + ); + this.recordingTimer.unref?.(); + return { supported: true, state: "recording" }; + }); + } + + stop(): Promise { + return this.exclusive(async () => { + const capture = this.capture; + const backend = this.backend; + const ready = this.backendReady; + if (!capture || !backend || !ready) return this.getStatus(); + this.capture = undefined; + this.clearRecordingTimer(); + this.setState("transcribing"); + try { + const pcm = await capture.stop(); + await ready; + const text = (await backend.transcribe(pcm)).trim(); + if (text) this.options.emit({ type: "transcript", text }); + this.setState("ready"); + return { supported: true, state: "ready" }; + } finally { + this.backend = undefined; + this.backendReady = undefined; + await backend.dispose().catch(() => undefined); + } + }); + } + + cancel(): Promise { + return this.exclusive(async () => { + const capture = this.capture; + const backend = this.backend; + this.capture = undefined; + this.clearRecordingTimer(); + this.backend = undefined; + this.backendReady = undefined; + await capture?.cancel().catch(() => undefined); + await backend?.dispose().catch(() => undefined); + this.setState( + this.modelPath || (await this.options.modelReady()) ? "ready" : "missing-model", + ); + return { supported: true, state: this.state }; + }); + } + + removeModel(): Promise { + return this.exclusive(async () => { + if (this.capture) throw new Error("stop voice input before removing its model"); + await this.options.removeModel(); + this.modelPath = undefined; + this.setState("missing-model"); + return { supported: true, state: "missing-model" }; + }); + } + + async shutdown(): Promise { + await this.cancel().catch(() => undefined); + } + + private setState(state: typeof this.state): void { + this.state = state; + this.options.emit({ type: "status", status: { supported: true, state } }); + } + + private clearRecordingTimer(): void { + if (this.recordingTimer) clearTimeout(this.recordingTimer); + this.recordingTimer = undefined; + } + + private exclusive(task: () => Promise): Promise { + if (this.operation) return this.operation; + const operation = task() + .catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + this.state = "error"; + this.options.emit({ type: "error", message }); + this.options.emit({ + type: "status", + status: { supported: true, state: "error", message }, + }); + return { supported: true, state: "error", message } as const; + }) + .finally(() => { + if (this.operation === operation) this.operation = undefined; + }); + this.operation = operation; + return operation; + } +} diff --git a/apps/desktop/src/speech/DesktopTranscriptionBackend.ts b/apps/desktop/src/speech/DesktopTranscriptionBackend.ts new file mode 100644 index 000000000000..656235cda38b --- /dev/null +++ b/apps/desktop/src/speech/DesktopTranscriptionBackend.ts @@ -0,0 +1,39 @@ +import type { TranscribeModel } from "transcribe-cpp"; + +export class DesktopTranscriptionBackend { + private readonly modelPath: string; + private model: TranscribeModel | undefined; + private loading: Promise | undefined; + + constructor(modelPath: string) { + this.modelPath = modelPath; + } + + async prepare(): Promise { + if (this.model) return; + this.loading ??= import("transcribe-cpp") + .then(({ TranscribeModel }) => TranscribeModel.load(this.modelPath)) + .then((model) => { + this.model = model; + return model; + }); + await this.loading; + } + + async transcribe(pcm: Float32Array): Promise { + if (pcm.length === 0) throw new Error("no audio samples were provided"); + await this.prepare(); + const result = await this.model!.transcribe(pcm, { + timestamps: "none", + language: "en", + }); + return result.text.trim(); + } + + async dispose(): Promise { + await this.loading?.catch(() => undefined); + this.model?.dispose(); + this.model = undefined; + this.loading = undefined; + } +} diff --git a/apps/desktop/src/speech/speechModel.test.ts b/apps/desktop/src/speech/speechModel.test.ts new file mode 100644 index 000000000000..625a78e10175 --- /dev/null +++ b/apps/desktop/src/speech/speechModel.test.ts @@ -0,0 +1,72 @@ +// @effect-diagnostics nodeBuiltinImport:off - model downloader tests use local Node fixtures. +import { afterEach, describe, expect, it } from "vite-plus/test"; +import * as NodeFSP from "node:fs/promises"; +import * as NodeHttp from "node:http"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import { downloadVerifiedModel } from "./speechModel.ts"; + +const directories: string[] = []; + +afterEach(async () => { + await Promise.all( + directories.splice(0).map((path) => NodeFSP.rm(path, { recursive: true, force: true })), + ); +}); + +describe("downloadVerifiedModel", () => { + it("publishes bytes only after size and sha256 verification", async () => { + const directory = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-speech-model-")); + directories.push(directory); + const bytes = Buffer.from("verified model bytes"); + const server = NodeHttp.createServer((_request, response) => response.end(bytes)); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("missing test server address"); + + try { + const path = await downloadVerifiedModel({ + directory, + filename: "model.gguf", + url: `http://127.0.0.1:${address.port}/model.gguf`, + size: bytes.length, + sha256: "03cfa25d83f5eaa1faac98ed6ceaaf0e7afe3c273a1e1502c2714ebe10b8263e", + }); + + expect(await NodeFSP.readFile(path)).toEqual(bytes); + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } + }); + + it("removes partial bytes when verification fails", async () => { + const directory = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-speech-model-")); + directories.push(directory); + const server = NodeHttp.createServer((_request, response) => response.end("corrupt")); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("missing test server address"); + + try { + await expect( + downloadVerifiedModel({ + directory, + filename: "model.gguf", + url: `http://127.0.0.1:${address.port}/model.gguf`, + size: 7, + sha256: "0".repeat(64), + }), + ).rejects.toThrow("verification failed"); + await expect(NodeFSP.stat(NodePath.join(directory, "model.gguf"))).rejects.toMatchObject({ + code: "ENOENT", + }); + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } + }); +}); diff --git a/apps/desktop/src/speech/speechModel.ts b/apps/desktop/src/speech/speechModel.ts new file mode 100644 index 000000000000..b4a0d1e22ab4 --- /dev/null +++ b/apps/desktop/src/speech/speechModel.ts @@ -0,0 +1,106 @@ +// @effect-diagnostics nodeBuiltinImport:off globalFetch:off globalDate:off - low-level streaming download boundary. +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; +import * as NodeStream from "node:stream"; +import * as NodeStreamPromises from "node:stream/promises"; +import type * as NodeStreamWeb from "node:stream/web"; + +export const SPEECH_MODEL = { + id: "moonshine-streaming-tiny", + name: "Moonshine Streaming Tiny", + filename: "moonshine-streaming-tiny-Q8_0.gguf", + size: 50_462_816, + sha256: "930e4622ad3a24158b91406c30c977fa6a26b34cb32d6ac3e57cfb23383a869e", + revision: "85ddff612fa3a2cf40b2f745abcfa90ef82f293b", + url: "https://huggingface.co/handy-computer/moonshine-streaming-tiny-gguf/resolve/85ddff612fa3a2cf40b2f745abcfa90ef82f293b/moonshine-streaming-tiny-Q8_0.gguf", +} as const; + +type DownloadInput = { + directory: string; + filename: string; + url: string; + size: number; + sha256: string; + signal?: AbortSignal; + onProgress?: (downloaded: number, total: number) => void; +}; + +async function hasExpectedSize(path: string, size: number): Promise { + return NodeFSP.stat(path).then( + (value) => value.size === size, + () => false, + ); +} + +export async function downloadVerifiedModel(input: DownloadInput): Promise { + const { directory, filename, url, size, sha256, signal, onProgress } = input; + const finalPath = NodePath.join(directory, filename); + signal?.throwIfAborted(); + await NodeFSP.mkdir(directory, { recursive: true }); + if (await hasExpectedSize(finalPath, size)) { + onProgress?.(size, size); + return finalPath; + } + + const partialPath = `${finalPath}.${process.pid}.${NodeCrypto.randomUUID()}.part`; + const response = await fetch(url, { + redirect: "follow", + ...(signal ? { signal } : {}), + }); + if (!response.ok || !response.body) { + throw new Error(`speech model download failed with status ${response.status}`); + } + + const digest = NodeCrypto.createHash("sha256"); + let downloaded = 0; + let lastReport = 0; + const progress = new NodeStream.Transform({ + transform(chunk: Buffer, _encoding, callback) { + downloaded += chunk.length; + digest.update(chunk); + const now = Date.now(); + if (now - lastReport >= 100 || downloaded === size) { + lastReport = now; + onProgress?.(downloaded, size); + } + callback(null, chunk); + }, + }); + + onProgress?.(0, size); + try { + await NodeStreamPromises.pipeline( + NodeStream.Readable.fromWeb(response.body as NodeStreamWeb.ReadableStream), + progress, + NodeFS.createWriteStream(partialPath, { mode: 0o600 }), + { signal }, + ); + signal?.throwIfAborted(); + const partialSize = (await NodeFSP.stat(partialPath)).size; + const actualSha256 = digest.digest("hex"); + if (partialSize !== size || actualSha256 !== sha256) { + throw new Error("speech model verification failed; partial bytes were removed"); + } + await NodeFSP.rm(finalPath, { force: true }); + await NodeFSP.rename(partialPath, finalPath); + onProgress?.(size, size); + return finalPath; + } catch (error) { + await NodeFSP.rm(partialPath, { force: true }).catch(() => undefined); + throw error; + } +} + +export async function removeSpeechModel(directory: string): Promise { + await NodeFSP.rm(NodePath.join(directory, SPEECH_MODEL.filename), { force: true }); +} + +export function speechModelPath(directory: string): string { + return NodePath.join(directory, SPEECH_MODEL.filename); +} + +export async function isSpeechModelReady(directory: string): Promise { + return hasExpectedSize(speechModelPath(directory), SPEECH_MODEL.size); +} diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index f06a9658225f..fc424be57a0b 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -123,6 +123,9 @@ import { submitComposerDraft, } from "./composerSubmission"; import { ComposerPromptLengthValidation } from "./ComposerPromptLengthValidation"; +import { ComposerSpeechButton } from "./ComposerSpeechButton"; +import { formatSpeechInsertion } from "../../speech/speechInsertion"; +import { useDesktopSpeechInput } from "../../speech/useDesktopSpeechInput"; type ComposerCommandMenuPosition = { bottom: number; @@ -1719,6 +1722,17 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) }; }, [composerCursor, composerTerminalContexts, promptRef]); + const insertSpeechTranscript = useCallback( + (text: string) => { + const snapshot = readComposerSnapshot(); + const replacement = formatSpeechInsertion(snapshot.value, snapshot.expandedCursor, text); + if (!replacement) return; + applyPromptReplacement(snapshot.expandedCursor, snapshot.expandedCursor, replacement); + }, + [applyPromptReplacement, readComposerSnapshot], + ); + const speechInput = useDesktopSpeechInput(insertSpeechTranscript); + const resolveActiveComposerTrigger = useCallback((): { snapshot: { value: string; cursor: number; expandedCursor: number }; trigger: ComposerTrigger | null; @@ -3383,6 +3397,19 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) > {showMobilePendingAnswerActions ? null : inlineTasksBadge} {showMobilePendingAnswerActions ? null : inlineStashBadge} + {speechInput.available ? ( + 0 + } + onStart={() => void speechInput.start()} + onStop={() => void speechInput.stop()} + onCancel={() => void speechInput.cancel()} + /> + ) : null} + + {recording ? ( + <> + + + + ) : busy ? ( + + ) : state === "error" ? ( + + ) : ( + + )} + + } + /> + + {label} + {recording ? " · esc to cancel" : ""} + + + ); +} diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index 582dc9f6cb94..bc9704630374 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -6,6 +6,7 @@ import { useAtomValue } from "@effect/atom-react"; import { type BackgroundActivityProfile, type DesktopUpdateChannel, + type DesktopSpeechStatus, ProviderDriverKind, type ScopedThreadRef, type SidebarProjectGroupingMode, @@ -1852,6 +1853,15 @@ export function GeneralSettingsPanel() { const settings = usePrimarySettings(); const updateSettings = useUpdatePrimarySettings(); const [backgroundActivityDialogOpen, setBackgroundActivityDialogOpen] = useState(false); + const [speechStatus, setSpeechStatus] = useState(null); + useEffect(() => { + const speech = window.desktopBridge?.speech; + if (!speech) return; + void speech.getStatus().then(setSpeechStatus); + return speech.onEvent((event) => { + if (event.type === "status") setSpeechStatus(event.status); + }); + }, []); const lastEnabledProjectGroupingMode = useRef( readLastEnabledProjectGroupingMode(), ); @@ -1904,6 +1914,36 @@ export function GeneralSettingsPanel() { return ( + {window.desktopBridge?.speech && speechStatus?.supported ? ( + Download on first use + ) : ( + + ) + } + /> + ) : null} { + it("adds boundary spaces without changing transcript punctuation", () => { + expect(formatSpeechInsertion("helloworld", 5, "new text")).toBe(" new text "); + expect(formatSpeechInsertion("hello ", 6, "world.")).toBe("world."); + expect(formatSpeechInsertion("", 0, " hello ")).toBe("hello"); + }); +}); diff --git a/apps/web/src/speech/speechInsertion.ts b/apps/web/src/speech/speechInsertion.ts new file mode 100644 index 000000000000..aaf4eb77dd62 --- /dev/null +++ b/apps/web/src/speech/speechInsertion.ts @@ -0,0 +1,9 @@ +export function formatSpeechInsertion(value: string, cursor: number, transcript: string): string { + const text = transcript.trim(); + if (!text) return ""; + const before = value[cursor - 1]; + const after = value[cursor]; + const prefix = before && !/\s/.test(before) && !/^\p{P}/u.test(text) ? " " : ""; + const suffix = after && !/\s/.test(after) ? " " : ""; + return `${prefix}${text}${suffix}`; +} diff --git a/apps/web/src/speech/useDesktopSpeechInput.ts b/apps/web/src/speech/useDesktopSpeechInput.ts new file mode 100644 index 000000000000..7695759a9487 --- /dev/null +++ b/apps/web/src/speech/useDesktopSpeechInput.ts @@ -0,0 +1,93 @@ +import type { DesktopSpeechStatus } from "@t3tools/contracts"; +import { useCallback, useEffect, useRef, useState } from "react"; + +import { ensureLocalApi } from "../localApi"; + +export function useDesktopSpeechInput(onTranscript: (text: string) => void) { + const bridge = typeof window === "undefined" ? undefined : window.desktopBridge?.speech; + const [status, setStatus] = useState(null); + const [progress, setProgress] = useState<{ downloaded: number; total: number } | null>(null); + const [level, setLevel] = useState(0); + const activeRef = useRef(false); + const transcriptRef = useRef(onTranscript); + transcriptRef.current = onTranscript; + + useEffect(() => { + if (!bridge) return; + let disposed = false; + void bridge.getStatus().then((next) => { + if (!disposed) setStatus(next); + }); + const unsubscribe = bridge.onEvent((event) => { + if (event.type === "status") { + setStatus(event.status); + activeRef.current = event.status.supported && event.status.state === "recording"; + } else if (event.type === "download-progress") { + setProgress({ downloaded: event.downloaded, total: event.total }); + } else if (event.type === "level") { + setLevel(event.level); + } else if (event.type === "transcript") { + transcriptRef.current(event.text); + } else if (event.type === "error") { + setStatus({ supported: true, state: "error", message: event.message }); + } + }); + return () => { + disposed = true; + unsubscribe(); + if (activeRef.current) void bridge.cancel(); + }; + }, [bridge]); + + useEffect(() => { + if (!bridge || !status?.supported || status.state !== "recording") return; + const timeout = window.setTimeout(() => void bridge.stop(), 120_000); + const onKeyDown = (event: KeyboardEvent) => { + if (event.key !== "Escape") return; + event.preventDefault(); + event.stopPropagation(); + void bridge.cancel(); + }; + window.addEventListener("keydown", onKeyDown, true); + return () => { + window.clearTimeout(timeout); + window.removeEventListener("keydown", onKeyDown, true); + }; + }, [bridge, status]); + + const start = useCallback(async () => { + if (!bridge) return; + if (status?.supported && status.state === "missing-model") { + const confirmed = await ensureLocalApi().dialogs.confirm( + "Download a 48 MiB English speech model? Voice input is processed locally and microphone audio is not saved.", + ); + if (!confirmed) return; + } + setProgress(null); + const next = await bridge.start(); + activeRef.current = next.supported && next.state === "recording"; + setStatus(next); + }, [bridge, status]); + + const stop = useCallback(async () => { + if (!bridge) return; + activeRef.current = false; + setStatus(await bridge.stop()); + }, [bridge]); + + const cancel = useCallback(async () => { + if (!bridge) return; + activeRef.current = false; + setStatus(await bridge.cancel()); + }, [bridge]); + + return { + available: bridge !== undefined && status?.supported !== false, + status, + progress, + level, + start, + stop, + cancel, + }; +} diff --git a/docs/README.md b/docs/README.md index 622d81064387..043952a6a8cc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,6 +5,7 @@ - [Install and first run](./user/install.md) - [Permission modes](./user/permission-modes.md) - [Keyboard shortcuts](./user/keybindings.md) +- [Local voice input](./user/voice-input.md) - [Organizing threads](./user/thread-sidebar.md) - [Review usage](./user/usage.md) - [Customize a project icon](./user/project-settings.md) diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md new file mode 100644 index 000000000000..4a9ffc171c89 --- /dev/null +++ b/docs/user/voice-input.md @@ -0,0 +1,13 @@ +# Local voice input + +Local voice input is available in the T3 Code desktop app on macOS, Windows x64, and Linux. It is not available in the browser or mobile apps yet. + +Select the microphone button beside Send to begin recording. Select the stop button to transcribe, or press Escape to discard the recording. T3 Code inserts the result at the current composer cursor and never sends it automatically. + +The first recording asks before downloading Moonshine Streaming Tiny, a 48 MiB English speech model. T3 Code verifies the model before using it. Speech processing runs on the local computer, and microphone audio is kept in memory only for the current recording. + +To remove the downloaded model, open Settings, then General, and select Remove model under Local voice input. + +On macOS, grant microphone access to T3 Code when prompted. If access was previously denied, enable it under System Settings, Privacy & Security, Microphone. + +Windows arm64 is not supported because the transcription runtime does not currently publish a compatible native package. diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index c6daef8687ba..ca8e1818a174 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -18,6 +18,7 @@ export * from "./settings.ts"; export * from "./git.ts"; export * from "./vcs.ts"; export * from "./sourceControl.ts"; +export * from "./speech.ts"; export * from "./pullRequest.ts"; export * from "./orchestration.ts"; export * from "./t3ProjectFile.ts"; diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index e753596f3d33..d05acd2120d3 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -26,6 +26,7 @@ import type { } from "./review.ts"; import type { FilesystemBrowseInput, FilesystemBrowseResult } from "./filesystem.ts"; import type { AssetCreateUrlInput, AssetCreateUrlResult } from "./assets.ts"; +import type { DesktopSpeechEvent, DesktopSpeechStatus } from "./speech.ts"; import type { ProjectListEntriesInput, ProjectListEntriesResult, @@ -1145,6 +1146,15 @@ export interface DesktopBridge { downloadUpdate: () => Promise; installUpdate: () => Promise; onUpdateState: (listener: (state: DesktopUpdateState) => void) => () => void; + /** Desktop-local speech input. Absent in older desktop shells and regular browsers. */ + speech?: { + getStatus: () => Promise; + start: () => Promise; + stop: () => Promise; + cancel: () => Promise; + removeModel: () => Promise; + onEvent: (listener: (event: DesktopSpeechEvent) => void) => () => void; + }; /** * Desktop-only preview surface. Present iff the renderer is hosted by the * Electron desktop build; web builds have `preview === undefined`. diff --git a/packages/contracts/src/speech.test.ts b/packages/contracts/src/speech.test.ts new file mode 100644 index 000000000000..7d97ae9b32ae --- /dev/null +++ b/packages/contracts/src/speech.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vite-plus/test"; +import * as Schema from "effect/Schema"; + +import { DesktopSpeechEventSchema, DesktopSpeechStatusSchema } from "./speech.ts"; + +const decodeStatus = Schema.decodeUnknownSync(DesktopSpeechStatusSchema); +const decodeEvent = Schema.decodeUnknownSync(DesktopSpeechEventSchema); + +describe("desktop speech contracts", () => { + it("accepts a ready status", () => { + expect(decodeStatus({ supported: true, state: "ready" })).toEqual({ + supported: true, + state: "ready", + }); + }); + + it("rejects negative model progress", () => { + expect(() => + decodeEvent({ + type: "download-progress", + downloaded: -1, + total: 10, + }), + ).toThrow(); + }); +}); diff --git a/packages/contracts/src/speech.ts b/packages/contracts/src/speech.ts new file mode 100644 index 000000000000..a844d46009b9 --- /dev/null +++ b/packages/contracts/src/speech.ts @@ -0,0 +1,51 @@ +import * as Schema from "effect/Schema"; +import { NonNegativeInt } from "./baseSchemas.ts"; + +export const DesktopSpeechStateSchema = Schema.Literals([ + "missing-model", + "downloading", + "ready", + "recording", + "transcribing", + "error", +]); +export type DesktopSpeechState = typeof DesktopSpeechStateSchema.Type; + +export const DesktopSpeechStatusSchema = Schema.Union([ + Schema.Struct({ + supported: Schema.Literal(false), + reason: Schema.String, + }), + Schema.Struct({ + supported: Schema.Literal(true), + state: DesktopSpeechStateSchema, + message: Schema.optionalKey(Schema.String), + }), +]); +export type DesktopSpeechStatus = typeof DesktopSpeechStatusSchema.Type; + +export const DesktopSpeechEventSchema = Schema.Union([ + Schema.Struct({ + type: Schema.Literal("status"), + status: DesktopSpeechStatusSchema, + }), + Schema.Struct({ + type: Schema.Literal("download-progress"), + downloaded: NonNegativeInt, + total: NonNegativeInt, + }), + Schema.Struct({ + type: Schema.Literal("level"), + level: Schema.Number, + elapsedMs: NonNegativeInt, + }), + Schema.Struct({ + type: Schema.Literal("transcript"), + text: Schema.String, + }), + Schema.Struct({ + type: Schema.Literal("error"), + message: Schema.String, + }), +]); +export type DesktopSpeechEvent = typeof DesktopSpeechEventSchema.Type; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4b550bebb15e..12f34898633f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -130,6 +130,9 @@ importers: '@effect/platform-node': specifier: 4.0.0-beta.103 version: 4.0.0-beta.103(bufferutil@4.1.0)(effect@4.0.0-beta.103(patch_hash=af36b7948b6f9c56623074662b51dade5699880c1a7c71245de73e13c3185fb6))(ioredis@5.11.0)(utf-8-validate@6.0.6) + '@picovoice/pvrecorder-node': + specifier: 1.2.9 + version: 1.2.9 '@t3tools/client-runtime': specifier: workspace:* version: link:../../packages/client-runtime @@ -163,6 +166,9 @@ importers: react-grab: specifier: ^0.1.32 version: 0.1.44(react@19.2.6) + transcribe-cpp: + specifier: 0.2.1 + version: 0.2.1 devDependencies: '@effect/vitest': specifier: 4.0.0-beta.103 @@ -2939,6 +2945,81 @@ packages: resolution: {integrity: sha512-hloP58zRVCRSpgDxmqCWJNlizAlUgJFqG2ypq79DCvyv9tHjRYMDOcPFjzfl/A1/YxDvRCZz8wvZvmapQnKwFQ==} engines: {node: '>=12'} + '@koromix/koffi-darwin-arm64@3.1.6': + resolution: {integrity: sha512-8FHyXGCZN7/iQf4f7W5BRysmtdlAFvSx6FpmX4u6wmkZiX/2e9hIRdGLiZYlHGudlcA18UmXB/cMiyhJ7fJkzA==} + cpu: [arm64] + os: [darwin] + + '@koromix/koffi-darwin-x64@3.1.6': + resolution: {integrity: sha512-uzx/jqFQuSHqgg1zaRidTBTCfj8Y9M0SDTO8HeoI9s9fJhiJ1mbB9TTwJO5c2hiMnuWg2m1byczC8BaIH6cG/w==} + cpu: [x64] + os: [darwin] + + '@koromix/koffi-freebsd-arm64@3.1.6': + resolution: {integrity: sha512-PjpTVrsCK5YTtixOw7VsseYXJOyoY6k0qBt+bf0T9h3wyV06y73rALsorFDDEoYpLUBZO7R6EIMs6CpUrEkNTQ==} + cpu: [arm64] + os: [freebsd] + + '@koromix/koffi-freebsd-ia32@3.1.6': + resolution: {integrity: sha512-ETYwL820HtFwYoOVzgyvmFmzTHRo9DJtGYTxa5Nb7ajaa5ldCum0jbmUJ3PMECxFTLxD5Q6PYZ8Xbp101bGeSg==} + cpu: [ia32] + os: [freebsd] + + '@koromix/koffi-freebsd-x64@3.1.6': + resolution: {integrity: sha512-BkqxkNXhAAT9toU2stvLwx1iKHPDx7h08NCICyBbjYEXkCAEr84igTkpE5V3XJ9xZZ2gKll7VvdhxorHtHUqZw==} + cpu: [x64] + os: [freebsd] + + '@koromix/koffi-linux-arm64@3.1.6': + resolution: {integrity: sha512-cM4XPm9ljbCrcPgXjzFYjDNxDUvvuR7TCYaEoo1AKjwZT/vmWhu2xN3pomfbsHh6aVn80SFA3enufQnaETW1rQ==} + cpu: [arm64] + os: [linux] + + '@koromix/koffi-linux-ia32@3.1.6': + resolution: {integrity: sha512-l1SVTpO10iaQt8slbowJpzK4fbwQZ7ufj9tmCyAcIwWUpyAbPS83mJMctU72If6N9/gCS2wuRqwnYB2uPLLhLg==} + cpu: [ia32] + os: [linux] + + '@koromix/koffi-linux-loong64@3.1.6': + resolution: {integrity: sha512-KpTJpMSbIdCVFU26ynt0xy4x15h+y6AwPJxj2+iVxJhCzJf4oisCPc0YH2VnutuLV2nVzSrFm7sL/WSOqPgkXw==} + cpu: [loong64] + os: [linux] + + '@koromix/koffi-linux-riscv64@3.1.6': + resolution: {integrity: sha512-YdFNpsywnXiYOYQlDAatf7TJLnspbGXdmfwIZhf82kbKSflYUsq4tI6NmoUOzM89oIWDGpYqd4Hz3xL7tsyXsw==} + cpu: [riscv64] + os: [linux] + + '@koromix/koffi-linux-x64@3.1.6': + resolution: {integrity: sha512-Xx5mpr9VcaMCXfvbqIiLIWIL9Iuu6F4r3iMXg7+zZCqYUFZPFwJgiDQBLxctHv2OYgIfAoaaHMW0GC1cJkHfbA==} + cpu: [x64] + os: [linux] + + '@koromix/koffi-openbsd-ia32@3.1.6': + resolution: {integrity: sha512-39Np4QTxhTlTT6RRveIeP+TnbzrwuDJ0UMyHxEZ+oGtzmb9GqWhl9T1oyehG6v/O+c4BffafG2NwLkCZ7tDKWw==} + cpu: [ia32] + os: [openbsd] + + '@koromix/koffi-openbsd-x64@3.1.6': + resolution: {integrity: sha512-3EynGn3ycQRqaMWGmUJ0tdtuQdStByqSy/tJ0ZGKWizbMGdFAE73YpgLsyd8BDvwnKWytVG/OLNb5nDHpfd9Dg==} + cpu: [x64] + os: [openbsd] + + '@koromix/koffi-win32-arm64@3.1.6': + resolution: {integrity: sha512-27FdPPRtT4xbO9bsd2OZa95M5YQ7bcJ8QjCRO57UUMI21REfkDegjqKwqo/CFlugxXlJf5IYtG2rq4BEYIrvxg==} + cpu: [arm64] + os: [win32] + + '@koromix/koffi-win32-ia32@3.1.6': + resolution: {integrity: sha512-5mVelLKVDup4eoxZOpCzCyMPxoctsg+Qe4J9O5BP4KbBEdqoOEqaNEBBRgNzXcdr2g+GGfmIUo+oVR1NvIdiJw==} + cpu: [ia32] + os: [win32] + + '@koromix/koffi-win32-x64@3.1.6': + resolution: {integrity: sha512-lPKjAaHz0aoiZXT/wDVqH+joR5y3lCZj1s9Bk5qx/DGRq+0MK8Ib8VoqiBOrJ69NG5AsJSvn0tQDcSqfRgLmBQ==} + cpu: [x64] + os: [win32] + '@legendapp/list@3.3.5': resolution: {integrity: sha512-XTsLYtpg41SVb5uLBYA+YcDSA3w0tgoPq/W8ZggQ2tx+3lrC/rf+ehTP9KYHea9oFaZIuePAgzACs5/auVMJlQ==} peerDependencies: @@ -3560,6 +3641,10 @@ packages: resolution: {integrity: sha512-ODOov0sGMJMf3jPonOkgGqPknTsu+DdQ7kD++gz8aI+aFMOMHFbWAA2taqXXVTdP+OTOQR/znGvSpmkeI0WTYQ==} engines: {node: '>=14.18.0'} + '@picovoice/pvrecorder-node@1.2.9': + resolution: {integrity: sha512-e398ue4G8YihrnnyTpnVePvOiBEM5+R4D+ChoyyeC8J8KFBMuhGrZwrdmpIPqDqCjCA+YqO2mFaicHstVhEpcg==} + engines: {node: '>=18.0.0'} + '@pierre/diffs@1.3.0-beta.10': resolution: {integrity: sha512-efyFM9GRfI6WkmHJP0CnZBopuM8yCwGqIKbZHoe1D5PV15VDkr7Vpi8EZt40AYrN1km//utQtYhHDSwt2KwjSg==} peerDependencies: @@ -4740,6 +4825,33 @@ packages: peerDependencies: '@testing-library/dom': '>=7.21.4' + '@transcribe-cpp/darwin-arm64-metal@0.2.1': + resolution: {integrity: sha512-StvvbrSGNDwMN8UC03muvZkECdchyW8CRjtpj3eP1Wi9JGfxQcUE8PRCH1raDMdAorp0L9lmXMnNKt/aMIo+mA==} + cpu: [arm64] + os: [darwin] + + '@transcribe-cpp/darwin-x64-cpu@0.2.1': + resolution: {integrity: sha512-dk+snKC7ENo4nQfSwjkWW9Ptv2aSmJA02LRDX26FERc+K6bL9R7eiQst+JET/JfT1sVn8UESnrBOF1oBKyxUJQ==} + cpu: [x64] + os: [darwin] + + '@transcribe-cpp/linux-arm64-cpu-vulkan@0.2.1': + resolution: {integrity: sha512-c/wSkZfbQmsjqHnu2AXZT81uUpHPJW1Hxu35xp8hZiHpVbUw94zds0xbBtY8c66Nw5dFU2Y+XwhgEDil28cXqA==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@transcribe-cpp/linux-x64-cpu-vulkan@0.2.1': + resolution: {integrity: sha512-0sCMolWceToa9QYN5AizAn0cCserEPM7hRf1zETHKY4UQgVZzeRaFJETLrz6YLhgEiIRpr9d7osV8mp7O4e/Ww==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@transcribe-cpp/win32-x64-cpu-vulkan@0.2.1': + resolution: {integrity: sha512-wHaKJY2XRjL/J2p6YNvLE1Oy+b0MejdyHmx1PvksRQssIivkWSITdHMpd6VseokJsYO+pKj+OwqVrIycJas0yw==} + cpu: [x64] + os: [win32] + '@tybys/wasm-util@0.10.3': resolution: {integrity: sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==} @@ -5148,10 +5260,12 @@ packages: '@xmldom/xmldom@0.8.13': resolution: {integrity: sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==} engines: {node: '>=10.0.0'} + deprecated: this version has critical issues, please update to the latest version '@xmldom/xmldom@0.9.10': resolution: {integrity: sha512-A9gOqLdi6cV4ibazAjcQufGj0B1y/vDqYrcuP6d/6x8P27gRS8643Dj9o1dEKtB6O7fwxb2FgBmJS2mX7gpvdw==} engines: {node: '>=14.6'} + deprecated: this version has critical issues, please update to the latest version '@yuuang/ffi-rs-android-arm64@1.3.2': resolution: {integrity: sha512-eDYLT0kVBkp7e2BwdRDmt6N1rkeDPUHDefk3ZX0/nok+GLsqfy1WBoSL3Yg7HVXN1EyW8OBVc2uK8Zq8HbmaSA==} @@ -7527,6 +7641,9 @@ packages: resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} engines: {node: '>=6'} + koffi@3.1.6: + resolution: {integrity: sha512-ln60chEb3o7Du1ayjwl6BFiNN1wZK+3cTM2wWGiHLEzCY/FdTIN1ER5VWDwHq7J/j4tSnnrHaH5ABS1EO6+6ag==} + kubernetes-types@1.30.0: resolution: {integrity: sha512-Dew1okvhM/SQcIa2rcgujNndZwU8VnSapDgdxlYoB84ZlpAD43U6KLAFqYo17ykSFGHNPrg0qry0bP+GJd9v7Q==} @@ -9791,6 +9908,10 @@ packages: resolution: {integrity: sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==} engines: {node: '>=18'} + transcribe-cpp@0.2.1: + resolution: {integrity: sha512-Vr5saZu2fI9L8pch2WujCZbmz25OjXp96CbAxwXd6DJUgxqSZmnCaDlNB5MtYCd6AgstocoZWayYQxXumks3Gg==} + engines: {node: '>=22'} + trim-lines@3.0.1: resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} @@ -12946,6 +13067,51 @@ snapshots: dependencies: jsbi: 4.3.2 + '@koromix/koffi-darwin-arm64@3.1.6': + optional: true + + '@koromix/koffi-darwin-x64@3.1.6': + optional: true + + '@koromix/koffi-freebsd-arm64@3.1.6': + optional: true + + '@koromix/koffi-freebsd-ia32@3.1.6': + optional: true + + '@koromix/koffi-freebsd-x64@3.1.6': + optional: true + + '@koromix/koffi-linux-arm64@3.1.6': + optional: true + + '@koromix/koffi-linux-ia32@3.1.6': + optional: true + + '@koromix/koffi-linux-loong64@3.1.6': + optional: true + + '@koromix/koffi-linux-riscv64@3.1.6': + optional: true + + '@koromix/koffi-linux-x64@3.1.6': + optional: true + + '@koromix/koffi-openbsd-ia32@3.1.6': + optional: true + + '@koromix/koffi-openbsd-x64@3.1.6': + optional: true + + '@koromix/koffi-win32-arm64@3.1.6': + optional: true + + '@koromix/koffi-win32-ia32@3.1.6': + optional: true + + '@koromix/koffi-win32-x64@3.1.6': + optional: true + '@legendapp/list@3.3.5(patch_hash=6befc76c7f590a0b0915b531386ce7e3bbb364612868e1f04e4ac84f60a39ab5)(react-dom@19.2.3(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)': dependencies: react: 19.2.3 @@ -13537,6 +13703,8 @@ snapshots: tslib: 2.8.1 webcrypto-core: 1.9.2 + '@picovoice/pvrecorder-node@1.2.9': {} + '@pierre/diffs@1.3.0-beta.10(patch_hash=7ef7cb0cbabb17c15cdb137554068b36f15f6f5265e73fb51452aa3380db91aa)(@shikijs/themes@4.2.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': dependencies: '@pierre/theme': 1.1.0 @@ -14867,6 +15035,21 @@ snapshots: dependencies: '@testing-library/dom': 10.4.1 + '@transcribe-cpp/darwin-arm64-metal@0.2.1': + optional: true + + '@transcribe-cpp/darwin-x64-cpu@0.2.1': + optional: true + + '@transcribe-cpp/linux-arm64-cpu-vulkan@0.2.1': + optional: true + + '@transcribe-cpp/linux-x64-cpu-vulkan@0.2.1': + optional: true + + '@transcribe-cpp/win32-x64-cpu-vulkan@0.2.1': + optional: true + '@tybys/wasm-util@0.10.3': dependencies: tslib: 2.8.1 @@ -18165,6 +18348,24 @@ snapshots: kleur@4.1.5: {} + koffi@3.1.6: + optionalDependencies: + '@koromix/koffi-darwin-arm64': 3.1.6 + '@koromix/koffi-darwin-x64': 3.1.6 + '@koromix/koffi-freebsd-arm64': 3.1.6 + '@koromix/koffi-freebsd-ia32': 3.1.6 + '@koromix/koffi-freebsd-x64': 3.1.6 + '@koromix/koffi-linux-arm64': 3.1.6 + '@koromix/koffi-linux-ia32': 3.1.6 + '@koromix/koffi-linux-loong64': 3.1.6 + '@koromix/koffi-linux-riscv64': 3.1.6 + '@koromix/koffi-linux-x64': 3.1.6 + '@koromix/koffi-openbsd-ia32': 3.1.6 + '@koromix/koffi-openbsd-x64': 3.1.6 + '@koromix/koffi-win32-arm64': 3.1.6 + '@koromix/koffi-win32-ia32': 3.1.6 + '@koromix/koffi-win32-x64': 3.1.6 + kubernetes-types@1.30.0: {} lan-network@0.2.1: {} @@ -21038,6 +21239,16 @@ snapshots: dependencies: punycode: 2.3.1 + transcribe-cpp@0.2.1: + dependencies: + koffi: 3.1.6 + optionalDependencies: + '@transcribe-cpp/darwin-arm64-metal': 0.2.1 + '@transcribe-cpp/darwin-x64-cpu': 0.2.1 + '@transcribe-cpp/linux-arm64-cpu-vulkan': 0.2.1 + '@transcribe-cpp/linux-x64-cpu-vulkan': 0.2.1 + '@transcribe-cpp/win32-x64-cpu-vulkan': 0.2.1 + trim-lines@3.0.1: {} trough@2.2.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 0a4d3cc7cb50..9400495386ac 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -15,6 +15,7 @@ allowBuilds: electron: true electron-winstaller: false esbuild: true + koffi: true msgpackr-extract: true msw: false node-pty: true diff --git a/scripts/build-desktop-artifact.test.ts b/scripts/build-desktop-artifact.test.ts index d1c0d54589df..30e5dd34e905 100644 --- a/scripts/build-desktop-artifact.test.ts +++ b/scripts/build-desktop-artifact.test.ts @@ -440,12 +440,14 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { undefined, ); - // All platforms keep app.asar fully packed; Windows ships the server - // tree as the hand-packed server.asar sidecar in extraResources instead - // of unpacking thousands of loose files at install time. - assert.notProperty(mac, "asarUnpack"); - assert.notProperty(linux, "asarUnpack"); - assert.notProperty(win, "asarUnpack"); + assert.deepStrictEqual(mac.asarUnpack, [ + "**/node_modules/transcribe-cpp/**/*", + "**/node_modules/@transcribe-cpp/**/*", + "**/node_modules/koffi/**/*", + "**/node_modules/@picovoice/pvrecorder-node/**/*", + ]); + assert.deepStrictEqual(linux.asarUnpack, mac.asarUnpack); + assert.deepStrictEqual(win.asarUnpack, mac.asarUnpack); assert.deepStrictEqual(win.extraResources, [ { from: "apps/desktop/prod-resources/resource-monitor", @@ -478,6 +480,10 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { iconSize: 80, iconTextSize: 12, }); + assert.deepStrictEqual((mac.mac as Record).extendInfo, { + NSMicrophoneUsageDescription: + "T3 Code uses your microphone for local voice input. Audio is processed on this device.", + }); // Linux must register the renderer schemes so the generated .desktop // entry advertises MimeType=x-scheme-handler/t3code; for OAuth deep links. assert.deepStrictEqual((linux.linux as Record).protocols, [ diff --git a/scripts/build-desktop-artifact.ts b/scripts/build-desktop-artifact.ts index 3abe682b51a1..d50999391483 100644 --- a/scripts/build-desktop-artifact.ts +++ b/scripts/build-desktop-artifact.ts @@ -1066,6 +1066,8 @@ ${associatedDomains} com.apple.security.cs.disable-library-validation + com.apple.security.device.audio-input + `; @@ -2062,6 +2064,12 @@ export const createBuildConfig = Effect.fn("createBuildConfig")(function* ( artifactName: "T3-Code-${version}-${arch}.${ext}", electronLanguages: [...DESKTOP_ELECTRON_LANGUAGES], files: [...DESKTOP_FILE_EXCLUSIONS], + asarUnpack: [ + "**/node_modules/transcribe-cpp/**/*", + "**/node_modules/@transcribe-cpp/**/*", + "**/node_modules/koffi/**/*", + "**/node_modules/@picovoice/pvrecorder-node/**/*", + ], directories: { buildResources: "apps/desktop/resources", }, @@ -2092,6 +2100,10 @@ export const createBuildConfig = Effect.fn("createBuildConfig")(function* ( target: target === "dmg" ? [target, "zip"] : [target], icon: "icon.icns", category: "public.app-category.developer-tools", + extendInfo: { + NSMicrophoneUsageDescription: + "T3 Code uses your microphone for local voice input. Audio is processed on this device.", + }, protocols: [ { name: "T3 Code", From 3ea876e3f50f9c546054433fb0052c10629bac47 Mon Sep 17 00:00:00 2001 From: "t3-code[bot]" <269035359+t3-code[bot]@users.noreply.github.com> Date: Sun, 23 Aug 2026 16:56:40 +0000 Subject: [PATCH 2/9] fix(desktop): align speech service and controls Co-authored-by: Utkarsh Patil <73941998+UtkarshUsername@users.noreply.github.com> --- apps/desktop/src/ipc/methods/speech.ts | 112 +++---------- apps/desktop/src/main.ts | 2 + apps/desktop/src/speech/DesktopSpeech.ts | 149 ++++++++++++++++++ .../components/chat/ComposerSpeechButton.tsx | 100 +++++++----- .../components/settings/SettingsPanels.tsx | 2 +- .../src/components/settings/settingsSearch.ts | 6 + apps/web/src/speech/useDesktopSpeechInput.ts | 7 +- 7 files changed, 247 insertions(+), 131 deletions(-) create mode 100644 apps/desktop/src/speech/DesktopSpeech.ts diff --git a/apps/desktop/src/ipc/methods/speech.ts b/apps/desktop/src/ipc/methods/speech.ts index a4bf735f1a66..67d557048b35 100644 --- a/apps/desktop/src/ipc/methods/speech.ts +++ b/apps/desktop/src/ipc/methods/speech.ts @@ -1,128 +1,64 @@ -// @effect-diagnostics nodeBuiltinImport:off - platform tuple and Electron app paths define native availability. -import { DesktopSpeechStatusSchema, type DesktopSpeechEvent } from "@t3tools/contracts"; -import { HostProcessArchitecture, HostProcessPlatform } from "@t3tools/shared/hostProcess"; -import { app } from "electron"; -import * as NodePath from "node:path"; +import { DesktopSpeechStatusSchema } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; import * as ElectronWindow from "../../electron/ElectronWindow.ts"; -import { DesktopMicrophoneCapture } from "../../speech/DesktopMicrophoneCapture.ts"; -import { DesktopSpeechController } from "../../speech/DesktopSpeechController.ts"; -import { DesktopTranscriptionBackend } from "../../speech/DesktopTranscriptionBackend.ts"; -import { - SPEECH_MODEL, - downloadVerifiedModel, - isSpeechModelReady, - removeSpeechModel, - speechModelPath, -} from "../../speech/speechModel.ts"; +import * as DesktopSpeech from "../../speech/DesktopSpeech.ts"; import * as IpcChannels from "../channels.ts"; import * as DesktopIpc from "../DesktopIpc.ts"; -let controller: DesktopSpeechController | undefined; -let emitEvent: (event: DesktopSpeechEvent) => void = () => undefined; -let availability: { supported: boolean; reason?: string } = { - supported: false, - reason: "voice input is still starting", -}; - -function support(platform: string, architecture: string): { supported: boolean; reason?: string } { - if (platform === "win32" && architecture === "arm64") { - return { supported: false, reason: "voice input is not available on Windows arm64 yet" }; - } - const tuple = `${platform}-${architecture}`; - const supported = new Set([ - "darwin-arm64", - "darwin-x64", - "win32-x64", - "linux-x64", - "linux-arm64", - ]).has(tuple); - return supported - ? { supported: true } - : { supported: false, reason: `voice input is not available on ${tuple}` }; -} - -function speechModelDirectory(): string { - return NodePath.join(app.getPath("userData"), "speech", "models"); -} - -function getController(): DesktopSpeechController { - if (controller) return controller; - const directory = speechModelDirectory(); - controller = new DesktopSpeechController({ - supported: availability.supported, - ...(availability.reason ? { unsupportedReason: availability.reason } : {}), - modelPath: speechModelPath(directory), - modelReady: () => isSpeechModelReady(directory), - downloadModel: (onProgress) => - downloadVerifiedModel({ - directory, - filename: SPEECH_MODEL.filename, - url: SPEECH_MODEL.url, - size: SPEECH_MODEL.size, - sha256: SPEECH_MODEL.sha256, - onProgress, - }), - removeModel: () => removeSpeechModel(directory), - createCapture: () => - new DesktopMicrophoneCapture((level, elapsedMs) => - emitEvent({ type: "level", level, elapsedMs }), - ), - createBackend: (modelPath) => new DesktopTranscriptionBackend(modelPath), - emit: (event) => emitEvent(event), - }); - return controller; -} - -function invoke( - operation: (value: DesktopSpeechController) => Promise, -): Effect.Effect { - return Effect.promise(() => operation(getController())); -} - export const installSpeechEventForwarding = Effect.fn("desktop.ipc.speech.events")(function* () { const windows = yield* ElectronWindow.ElectronWindow; - const platform = yield* HostProcessPlatform; - const architecture = yield* HostProcessArchitecture; - availability = support(platform, architecture); - emitEvent = (event) => { - Effect.runFork(windows.sendAll(IpcChannels.SPEECH_EVENT_CHANNEL, event)); - }; + const speech = yield* DesktopSpeech.DesktopSpeech; + yield* speech.subscribe((event) => windows.sendAll(IpcChannels.SPEECH_EVENT_CHANNEL, event)); }); export const getSpeechStatus = DesktopIpc.makeIpcMethod({ channel: IpcChannels.SPEECH_GET_STATUS_CHANNEL, payload: Schema.Void, result: DesktopSpeechStatusSchema, - handler: () => invoke((value) => value.getStatus()), + handler: Effect.fn("desktop.ipc.speech.getStatus")(function* () { + const speech = yield* DesktopSpeech.DesktopSpeech; + return yield* speech.getStatus; + }), }); export const startSpeech = DesktopIpc.makeIpcMethod({ channel: IpcChannels.SPEECH_START_CHANNEL, payload: Schema.Void, result: DesktopSpeechStatusSchema, - handler: () => invoke((value) => value.start()), + handler: Effect.fn("desktop.ipc.speech.start")(function* () { + const speech = yield* DesktopSpeech.DesktopSpeech; + return yield* speech.start; + }), }); export const stopSpeech = DesktopIpc.makeIpcMethod({ channel: IpcChannels.SPEECH_STOP_CHANNEL, payload: Schema.Void, result: DesktopSpeechStatusSchema, - handler: () => invoke((value) => value.stop()), + handler: Effect.fn("desktop.ipc.speech.stop")(function* () { + const speech = yield* DesktopSpeech.DesktopSpeech; + return yield* speech.stop; + }), }); export const cancelSpeech = DesktopIpc.makeIpcMethod({ channel: IpcChannels.SPEECH_CANCEL_CHANNEL, payload: Schema.Void, result: DesktopSpeechStatusSchema, - handler: () => invoke((value) => value.cancel()), + handler: Effect.fn("desktop.ipc.speech.cancel")(function* () { + const speech = yield* DesktopSpeech.DesktopSpeech; + return yield* speech.cancel; + }), }); export const removeSpeechModelMethod = DesktopIpc.makeIpcMethod({ channel: IpcChannels.SPEECH_REMOVE_MODEL_CHANNEL, payload: Schema.Void, result: DesktopSpeechStatusSchema, - handler: () => invoke((value) => value.removeModel()), + handler: Effect.fn("desktop.ipc.speech.removeModel")(function* () { + const speech = yield* DesktopSpeech.DesktopSpeech; + return yield* speech.removeModel; + }), }); diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index 14caeed8a9a1..23e8c4d3e160 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -56,6 +56,7 @@ import * as DesktopSshEnvironment from "./ssh/DesktopSshEnvironment.ts"; import * as DesktopSshPasswordPrompts from "./ssh/DesktopSshPasswordPrompts.ts"; import * as DesktopState from "./app/DesktopState.ts"; import * as DesktopTelemetryPublisher from "./telemetry/DesktopTelemetryPublisher.ts"; +import * as DesktopSpeech from "./speech/DesktopSpeech.ts"; import * as DesktopUpdates from "./updates/DesktopUpdates.ts"; import * as BrowserSession from "./preview/BrowserSession.ts"; import * as PreviewManager from "./preview/Manager.ts"; @@ -188,6 +189,7 @@ const desktopApplicationLayer = Layer.mergeAll( DesktopLinuxUrlHandler.layer, DesktopShellEnvironment.layer, desktopSshLayer, + DesktopSpeech.layer, ).pipe( Layer.provideMerge(DesktopUpdates.layer), Layer.provideMerge(desktopWslBackendLayer), diff --git a/apps/desktop/src/speech/DesktopSpeech.ts b/apps/desktop/src/speech/DesktopSpeech.ts new file mode 100644 index 000000000000..365f9d62e579 --- /dev/null +++ b/apps/desktop/src/speech/DesktopSpeech.ts @@ -0,0 +1,149 @@ +// @effect-diagnostics nodeBuiltinImport:off - Electron app paths define local model storage. +import type { DesktopSpeechEvent, DesktopSpeechStatus } from "@t3tools/contracts"; +import { HostProcessArchitecture, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import { app } from "electron"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Ref from "effect/Ref"; + +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as NodePath from "node:path"; + +import { DesktopMicrophoneCapture } from "./DesktopMicrophoneCapture.ts"; +import { DesktopSpeechController } from "./DesktopSpeechController.ts"; +import { DesktopTranscriptionBackend } from "./DesktopTranscriptionBackend.ts"; +import { + SPEECH_MODEL, + downloadVerifiedModel, + isSpeechModelReady, + removeSpeechModel, + speechModelPath, +} from "./speechModel.ts"; + +type SpeechEventListener = (event: DesktopSpeechEvent) => Effect.Effect; + +export class DesktopSpeechOperationError extends Schema.TaggedErrorClass()( + "DesktopSpeechOperationError", + { + operation: Schema.String, + cause: Schema.Defect(), + }, +) { + override get message(): string { + return `Desktop speech ${this.operation} failed.`; + } +} + +export class DesktopSpeech extends Context.Service< + DesktopSpeech, + { + readonly getStatus: Effect.Effect; + readonly start: Effect.Effect; + readonly stop: Effect.Effect; + readonly cancel: Effect.Effect; + readonly removeModel: Effect.Effect; + readonly subscribe: (listener: SpeechEventListener) => Effect.Effect; + } +>()("@t3tools/desktop/speech/DesktopSpeech") {} + +function support( + platform: string, + architecture: string, +): { + supported: boolean; + reason?: string; +} { + if (platform === "win32" && architecture === "arm64") { + return { supported: false, reason: "voice input is not available on Windows arm64 yet" }; + } + const tuple = `${platform}-${architecture}`; + const supported = new Set([ + "darwin-arm64", + "darwin-x64", + "win32-x64", + "linux-x64", + "linux-arm64", + ]).has(tuple); + return supported + ? { supported: true } + : { supported: false, reason: `voice input is not available on ${tuple}` }; +} + +function attempt( + operation: string, + run: () => Promise, +): Effect.Effect { + return Effect.tryPromise({ + try: run, + catch: (cause) => new DesktopSpeechOperationError({ operation, cause }), + }); +} + +export const make = Effect.gen(function* () { + const platform = yield* HostProcessPlatform; + const architecture = yield* HostProcessArchitecture; + const availability = support(platform, architecture); + const directory = NodePath.join(app.getPath("userData"), "speech", "models"); + const listeners = yield* Ref.make>(new Set()); + const context = yield* Effect.context(); + const runFork = Effect.runForkWith(context); + + const emit = (event: DesktopSpeechEvent): void => { + runFork( + Ref.get(listeners).pipe( + Effect.flatMap((current) => + Effect.forEach(current, (listener) => listener(event), { discard: true }), + ), + ), + ); + }; + + const controller = new DesktopSpeechController({ + supported: availability.supported, + ...(availability.reason ? { unsupportedReason: availability.reason } : {}), + modelPath: speechModelPath(directory), + modelReady: () => isSpeechModelReady(directory), + downloadModel: (onProgress) => + downloadVerifiedModel({ + directory, + filename: SPEECH_MODEL.filename, + url: SPEECH_MODEL.url, + size: SPEECH_MODEL.size, + sha256: SPEECH_MODEL.sha256, + onProgress, + }), + removeModel: () => removeSpeechModel(directory), + createCapture: () => + new DesktopMicrophoneCapture((level, elapsedMs) => emit({ type: "level", level, elapsedMs })), + createBackend: (modelPath) => new DesktopTranscriptionBackend(modelPath), + emit, + }); + + yield* Effect.addFinalizer(() => + attempt("shutdown", () => controller.shutdown()).pipe(Effect.orDie), + ); + + const subscribe = (listener: SpeechEventListener): Effect.Effect => + Effect.acquireRelease( + Ref.update(listeners, (current) => new Set([...current, listener])), + () => + Ref.update(listeners, (current) => { + const next = new Set(current); + next.delete(listener); + return next; + }), + ).pipe(Effect.asVoid); + + return DesktopSpeech.of({ + getStatus: attempt("get status", () => controller.getStatus()), + start: attempt("start", () => controller.start()), + stop: attempt("stop", () => controller.stop()), + cancel: attempt("cancel", () => controller.cancel()), + removeModel: attempt("remove model", () => controller.removeModel()), + subscribe, + }); +}); + +export const layer = Layer.effect(DesktopSpeech, make); diff --git a/apps/web/src/components/chat/ComposerSpeechButton.tsx b/apps/web/src/components/chat/ComposerSpeechButton.tsx index 2c8428af3197..a4ac241a2a55 100644 --- a/apps/web/src/components/chat/ComposerSpeechButton.tsx +++ b/apps/web/src/components/chat/ComposerSpeechButton.tsx @@ -1,8 +1,10 @@ import type { DesktopSpeechStatus } from "@t3tools/contracts"; -import { LoaderCircleIcon, MicIcon, SquareIcon, XIcon } from "lucide-react"; +import { MicIcon, SquareIcon, XIcon } from "lucide-react"; import { Button } from "../ui/button"; +import { Spinner } from "../ui/spinner"; import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip"; +import { cn } from "~/lib/utils"; export function ComposerSpeechButton(props: { status: DesktopSpeechStatus | null; @@ -17,6 +19,7 @@ export function ComposerSpeechButton(props: { const state = props.status?.supported ? props.status.state : "missing-model"; const recording = state === "recording"; const busy = state === "downloading" || state === "transcribing"; + const inactive = props.disabled || busy; const label = recording ? "Stop and transcribe" : state === "downloading" @@ -28,42 +31,63 @@ export function ComposerSpeechButton(props: { : "Start voice input"; return ( - - - {recording ? ( - <> - - - - ) : busy ? ( - - ) : state === "error" ? ( - - ) : ( - - )} - - } - /> - - {label} - {recording ? " · esc to cancel" : ""} - - +
+ + { + if (inactive) return; + if (recording) props.onStop(); + else props.onStart(); + }} + className={cn("relative", inactive && "cursor-not-allowed opacity-64")} + > + {recording ? ( + <> + + + + ) : busy ? ( + + ) : state === "error" ? ( + + ) : ( + + )} + + } + /> + {label} + + {recording ? ( + + + + + } + /> + Discard voice input + + ) : null} +
); } diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index bc9704630374..bd5e055ed4ad 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -1916,7 +1916,7 @@ export function GeneralSettingsPanel() { {window.desktopBridge?.speech && speechStatus?.supported ? ( void) { if (!bridge || !status?.supported || status.state !== "recording") return; const timeout = window.setTimeout(() => void bridge.stop(), 120_000); const onKeyDown = (event: KeyboardEvent) => { - if (event.key !== "Escape") return; + if (event.key !== "Escape" || event.defaultPrevented) return; event.preventDefault(); - event.stopPropagation(); void bridge.cancel(); }; - window.addEventListener("keydown", onKeyDown, true); + window.addEventListener("keydown", onKeyDown); return () => { window.clearTimeout(timeout); - window.removeEventListener("keydown", onKeyDown, true); + window.removeEventListener("keydown", onKeyDown); }; }, [bridge, status]); From 22db4c56b1ab74a8e8be5159f47009e6a3a2d3de Mon Sep 17 00:00:00 2001 From: "t3-code[bot]" <269035359+t3-code[bot]@users.noreply.github.com> Date: Sun, 23 Aug 2026 17:03:49 +0000 Subject: [PATCH 3/9] fix(desktop): harden speech cancellation and models Co-authored-by: Utkarsh Patil <73941998+UtkarshUsername@users.noreply.github.com> --- .../speech/DesktopSpeechController.test.ts | 49 ++++++++++++++++--- .../src/speech/DesktopSpeechController.ts | 44 ++++++++++++----- apps/desktop/src/speech/speechModel.test.ts | 1 + apps/desktop/src/speech/speechModel.ts | 15 +++++- apps/web/src/components/chat/ChatComposer.tsx | 5 +- apps/web/src/speech/useDesktopSpeechInput.ts | 25 ++++++++-- 6 files changed, 113 insertions(+), 26 deletions(-) diff --git a/apps/desktop/src/speech/DesktopSpeechController.test.ts b/apps/desktop/src/speech/DesktopSpeechController.test.ts index 06047c0c820d..b5bc804bbd1d 100644 --- a/apps/desktop/src/speech/DesktopSpeechController.test.ts +++ b/apps/desktop/src/speech/DesktopSpeechController.test.ts @@ -5,6 +5,7 @@ import { DesktopSpeechController } from "./DesktopSpeechController.ts"; function makeController(maxRecordingMs?: number) { const events: unknown[] = []; + let modelReady = false; const capture = { start: vi.fn(), stop: vi.fn().mockResolvedValue(new Float32Array([0.25, -0.25])), @@ -15,20 +16,23 @@ function makeController(maxRecordingMs?: number) { transcribe: vi.fn().mockResolvedValue("hello from speech"), dispose: vi.fn().mockResolvedValue(undefined), }; + const downloadModel = vi.fn().mockImplementation(async (onProgress) => { + onProgress(5, 10); + modelReady = true; + return "/tmp/model.gguf"; + }); const controller = new DesktopSpeechController({ supported: true, - modelReady: vi.fn().mockResolvedValue(false), - downloadModel: vi.fn().mockImplementation(async (onProgress) => { - onProgress(5, 10); - return "/tmp/model.gguf"; - }), + modelPath: "/tmp/model.gguf", + modelReady: vi.fn().mockImplementation(async () => modelReady), + downloadModel, removeModel: vi.fn().mockResolvedValue(undefined), createCapture: () => capture, createBackend: () => backend, emit: (event) => events.push(event), ...(maxRecordingMs === undefined ? {} : { maxRecordingMs }), }); - return { controller, capture, backend, events }; + return { controller, capture, backend, downloadModel, events }; } describe("DesktopSpeechController", () => { @@ -62,4 +66,37 @@ describe("DesktopSpeechController", () => { await new Promise((resolve) => setTimeout(resolve, 10)); expect(backend.transcribe).toHaveBeenCalledOnce(); }); + + it("suppresses a transcript when cancellation arrives during stop", async () => { + const { controller, backend, events } = makeController(); + let resolveTranscription!: (text: string) => void; + backend.transcribe.mockImplementation( + () => new Promise((resolve) => (resolveTranscription = resolve)), + ); + await controller.start(); + + const stopping = controller.stop(); + await vi.waitFor(() => expect(backend.transcribe).toHaveBeenCalledOnce()); + const cancelling = controller.cancel(); + resolveTranscription("discard this text"); + await Promise.all([stopping, cancelling]); + + expect(events).not.toContainEqual({ type: "transcript", text: "discard this text" }); + }); + + it("does not open the microphone when cancelled during model download", async () => { + const { controller, capture, downloadModel } = makeController(); + let finishDownload!: (path: string) => void; + downloadModel.mockImplementation( + () => new Promise((resolve) => (finishDownload = resolve)), + ); + + const starting = controller.start(); + await vi.waitFor(() => expect(downloadModel).toHaveBeenCalledOnce()); + const cancelling = controller.cancel(); + finishDownload("/tmp/model.gguf"); + await Promise.all([starting, cancelling]); + + expect(capture.start).not.toHaveBeenCalled(); + }); }); diff --git a/apps/desktop/src/speech/DesktopSpeechController.ts b/apps/desktop/src/speech/DesktopSpeechController.ts index 09eb6d85e585..38246d327835 100644 --- a/apps/desktop/src/speech/DesktopSpeechController.ts +++ b/apps/desktop/src/speech/DesktopSpeechController.ts @@ -13,10 +13,12 @@ type Backend = { dispose(): Promise; }; +type BackendPreparation = { readonly ok: true } | { readonly ok: false; readonly error: unknown }; + type ControllerOptions = { supported: boolean; unsupportedReason?: string; - modelPath?: string; + modelPath: string; modelReady(): Promise; downloadModel(onProgress: (downloaded: number, total: number) => void): Promise; removeModel(): Promise; @@ -30,7 +32,7 @@ export class DesktopSpeechController { private readonly options: ControllerOptions; private capture: Capture | undefined; private backend: Backend | undefined; - private backendReady: Promise | undefined; + private backendReady: Promise | undefined; private modelPath: string | undefined; private state: | "missing-model" @@ -41,10 +43,10 @@ export class DesktopSpeechController { | "error" = "missing-model"; private operation: Promise | undefined; private recordingTimer: ReturnType | undefined; + private cancelRequested = false; constructor(options: ControllerOptions) { this.options = options; - this.modelPath = options.modelPath; } async getStatus(): Promise { @@ -57,24 +59,33 @@ export class DesktopSpeechController { start(): Promise { return this.exclusive(async () => { + this.cancelRequested = false; const initial = await this.getStatus(); if (!initial.supported) return initial; if (this.capture) throw new Error("voice input is already recording"); - if (!(await this.options.modelReady())) { + if (await this.options.modelReady()) { + this.modelPath = this.options.modelPath; + } else { this.setState("downloading"); this.modelPath = await this.options.downloadModel((downloaded, total) => this.options.emit({ type: "download-progress", downloaded, total }), ); } - if (!this.modelPath) this.modelPath = this.options.modelPath; + if (this.cancelRequested) { + this.setState((await this.options.modelReady()) ? "ready" : "missing-model"); + return { supported: true, state: this.state }; + } if (!this.modelPath) throw new Error("speech model path is unavailable"); const capture = this.options.createCapture(); const backend = this.options.createBackend(this.modelPath); this.capture = capture; this.backend = backend; - this.backendReady = backend.prepare(); + this.backendReady = backend.prepare().then( + () => ({ ok: true }), + (error: unknown) => ({ ok: false, error }), + ); try { capture.start(); } catch (error) { @@ -105,9 +116,14 @@ export class DesktopSpeechController { this.setState("transcribing"); try { const pcm = await capture.stop(); - await ready; + const preparation = await ready; + if (!preparation.ok) throw preparation.error; + if (this.cancelRequested) { + this.setState("ready"); + return { supported: true, state: "ready" }; + } const text = (await backend.transcribe(pcm)).trim(); - if (text) this.options.emit({ type: "transcript", text }); + if (!this.cancelRequested && text) this.options.emit({ type: "transcript", text }); this.setState("ready"); return { supported: true, state: "ready" }; } finally { @@ -119,6 +135,7 @@ export class DesktopSpeechController { } cancel(): Promise { + this.cancelRequested = true; return this.exclusive(async () => { const capture = this.capture; const backend = this.backend; @@ -128,9 +145,9 @@ export class DesktopSpeechController { this.backendReady = undefined; await capture?.cancel().catch(() => undefined); await backend?.dispose().catch(() => undefined); - this.setState( - this.modelPath || (await this.options.modelReady()) ? "ready" : "missing-model", - ); + this.modelPath = (await this.options.modelReady()) ? this.options.modelPath : undefined; + this.setState(this.modelPath ? "ready" : "missing-model"); + this.cancelRequested = false; return { supported: true, state: this.state }; }); } @@ -160,8 +177,9 @@ export class DesktopSpeechController { } private exclusive(task: () => Promise): Promise { - if (this.operation) return this.operation; - const operation = task() + const previous = this.operation; + const operation = (previous ?? Promise.resolve()) + .then(task) .catch((error: unknown) => { const message = error instanceof Error ? error.message : String(error); this.state = "error"; diff --git a/apps/desktop/src/speech/speechModel.test.ts b/apps/desktop/src/speech/speechModel.test.ts index 625a78e10175..a1f00d061e39 100644 --- a/apps/desktop/src/speech/speechModel.test.ts +++ b/apps/desktop/src/speech/speechModel.test.ts @@ -20,6 +20,7 @@ describe("downloadVerifiedModel", () => { const directory = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-speech-model-")); directories.push(directory); const bytes = Buffer.from("verified model bytes"); + await NodeFSP.writeFile(NodePath.join(directory, "model.gguf"), Buffer.alloc(bytes.length)); const server = NodeHttp.createServer((_request, response) => response.end(bytes)); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); const address = server.address(); diff --git a/apps/desktop/src/speech/speechModel.ts b/apps/desktop/src/speech/speechModel.ts index b4a0d1e22ab4..5feb56710f29 100644 --- a/apps/desktop/src/speech/speechModel.ts +++ b/apps/desktop/src/speech/speechModel.ts @@ -34,12 +34,23 @@ async function hasExpectedSize(path: string, size: number): Promise { ); } +async function hasExpectedDigest(path: string, size: number, sha256: string): Promise { + if (!(await hasExpectedSize(path, size))) return false; + const digest = NodeCrypto.createHash("sha256"); + try { + for await (const chunk of NodeFS.createReadStream(path)) digest.update(chunk); + return digest.digest("hex") === sha256; + } catch { + return false; + } +} + export async function downloadVerifiedModel(input: DownloadInput): Promise { const { directory, filename, url, size, sha256, signal, onProgress } = input; const finalPath = NodePath.join(directory, filename); signal?.throwIfAborted(); await NodeFSP.mkdir(directory, { recursive: true }); - if (await hasExpectedSize(finalPath, size)) { + if (await hasExpectedDigest(finalPath, size, sha256)) { onProgress?.(size, size); return finalPath; } @@ -102,5 +113,5 @@ export function speechModelPath(directory: string): string { } export async function isSpeechModelReady(directory: string): Promise { - return hasExpectedSize(speechModelPath(directory), SPEECH_MODEL.size); + return hasExpectedDigest(speechModelPath(directory), SPEECH_MODEL.size, SPEECH_MODEL.sha256); } diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index fc424be57a0b..2a3159bcd31e 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -1731,7 +1731,10 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) }, [applyPromptReplacement, readComposerSnapshot], ); - const speechInput = useDesktopSpeechInput(insertSpeechTranscript); + const speechInput = useDesktopSpeechInput( + insertSpeechTranscript, + JSON.stringify(composerDraftTarget), + ); const resolveActiveComposerTrigger = useCallback((): { snapshot: { value: string; cursor: number; expandedCursor: number }; diff --git a/apps/web/src/speech/useDesktopSpeechInput.ts b/apps/web/src/speech/useDesktopSpeechInput.ts index 130364e7f523..73274a6ec1f9 100644 --- a/apps/web/src/speech/useDesktopSpeechInput.ts +++ b/apps/web/src/speech/useDesktopSpeechInput.ts @@ -3,14 +3,17 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { ensureLocalApi } from "../localApi"; -export function useDesktopSpeechInput(onTranscript: (text: string) => void) { +export function useDesktopSpeechInput(onTranscript: (text: string) => void, ownerKey: string) { const bridge = typeof window === "undefined" ? undefined : window.desktopBridge?.speech; const [status, setStatus] = useState(null); const [progress, setProgress] = useState<{ downloaded: number; total: number } | null>(null); const [level, setLevel] = useState(0); const activeRef = useRef(false); const transcriptRef = useRef(onTranscript); + const currentOwnerRef = useRef(ownerKey); + const recordingOwnerRef = useRef(null); transcriptRef.current = onTranscript; + currentOwnerRef.current = ownerKey; useEffect(() => { if (!bridge) return; @@ -27,7 +30,10 @@ export function useDesktopSpeechInput(onTranscript: (text: string) => void) { } else if (event.type === "level") { setLevel(event.level); } else if (event.type === "transcript") { - transcriptRef.current(event.text); + if (recordingOwnerRef.current === currentOwnerRef.current) { + transcriptRef.current(event.text); + } + recordingOwnerRef.current = null; } else if (event.type === "error") { setStatus({ supported: true, state: "error", message: event.message }); } @@ -35,10 +41,19 @@ export function useDesktopSpeechInput(onTranscript: (text: string) => void) { return () => { disposed = true; unsubscribe(); + recordingOwnerRef.current = null; if (activeRef.current) void bridge.cancel(); }; }, [bridge]); + useEffect(() => { + if (!bridge || recordingOwnerRef.current === null) return; + if (recordingOwnerRef.current === ownerKey) return; + recordingOwnerRef.current = null; + activeRef.current = false; + void bridge.cancel(); + }, [bridge, ownerKey]); + useEffect(() => { if (!bridge || !status?.supported || status.state !== "recording") return; const timeout = window.setTimeout(() => void bridge.stop(), 120_000); @@ -56,17 +71,18 @@ export function useDesktopSpeechInput(onTranscript: (text: string) => void) { const start = useCallback(async () => { if (!bridge) return; - if (status?.supported && status.state === "missing-model") { + if (!status || (status.supported && status.state === "missing-model")) { const confirmed = await ensureLocalApi().dialogs.confirm( "Download a 48 MiB English speech model? Voice input is processed locally and microphone audio is not saved.", ); if (!confirmed) return; } setProgress(null); + recordingOwnerRef.current = ownerKey; const next = await bridge.start(); activeRef.current = next.supported && next.state === "recording"; setStatus(next); - }, [bridge, status]); + }, [bridge, ownerKey, status]); const stop = useCallback(async () => { if (!bridge) return; @@ -77,6 +93,7 @@ export function useDesktopSpeechInput(onTranscript: (text: string) => void) { const cancel = useCallback(async () => { if (!bridge) return; activeRef.current = false; + recordingOwnerRef.current = null; setStatus(await bridge.cancel()); }, [bridge]); From b76c225f9fbd92bbd0a663b63ceaa51600eab91a Mon Sep 17 00:00:00 2001 From: "t3-code[bot]" <269035359+t3-code[bot]@users.noreply.github.com> Date: Sun, 23 Aug 2026 17:15:26 +0000 Subject: [PATCH 4/9] fix(desktop): close speech input edge cases Co-authored-by: Utkarsh Patil <73941998+UtkarshUsername@users.noreply.github.com> --- .../speech/DesktopSpeechController.test.ts | 9 ++++++ .../src/speech/DesktopSpeechController.ts | 3 +- apps/desktop/src/speech/speechModel.test.ts | 31 +++++++++++++++++++ apps/desktop/src/speech/speechModel.ts | 9 ++++++ .../components/chat/ComposerSpeechButton.tsx | 4 +-- apps/web/src/speech/useDesktopSpeechInput.ts | 11 ++++++- 6 files changed, 62 insertions(+), 5 deletions(-) diff --git a/apps/desktop/src/speech/DesktopSpeechController.test.ts b/apps/desktop/src/speech/DesktopSpeechController.test.ts index b5bc804bbd1d..e63b8364215f 100644 --- a/apps/desktop/src/speech/DesktopSpeechController.test.ts +++ b/apps/desktop/src/speech/DesktopSpeechController.test.ts @@ -60,6 +60,15 @@ describe("DesktopSpeechController", () => { expect(events.some((event) => (event as { type?: string }).type === "transcript")).toBe(false); }); + it("keeps the active recording when start is requested twice", async () => { + const { controller, capture, events } = makeController(); + await controller.start(); + expect(await controller.start()).toMatchObject({ supported: true, state: "recording" }); + expect(capture.start).toHaveBeenCalledOnce(); + expect(events.some((event) => (event as { type?: string }).type === "error")).toBe(false); + await controller.cancel(); + }); + it("automatically stops a recording at the duration limit", async () => { const { controller, backend } = makeController(1); await controller.start(); diff --git a/apps/desktop/src/speech/DesktopSpeechController.ts b/apps/desktop/src/speech/DesktopSpeechController.ts index 38246d327835..36be44c63d83 100644 --- a/apps/desktop/src/speech/DesktopSpeechController.ts +++ b/apps/desktop/src/speech/DesktopSpeechController.ts @@ -59,10 +59,9 @@ export class DesktopSpeechController { start(): Promise { return this.exclusive(async () => { - this.cancelRequested = false; const initial = await this.getStatus(); if (!initial.supported) return initial; - if (this.capture) throw new Error("voice input is already recording"); + if (this.capture) return { supported: true, state: this.state }; if (await this.options.modelReady()) { this.modelPath = this.options.modelPath; diff --git a/apps/desktop/src/speech/speechModel.test.ts b/apps/desktop/src/speech/speechModel.test.ts index a1f00d061e39..b6a9b451c5b3 100644 --- a/apps/desktop/src/speech/speechModel.test.ts +++ b/apps/desktop/src/speech/speechModel.test.ts @@ -70,4 +70,35 @@ describe("downloadVerifiedModel", () => { ); } }); + + it("stops an oversized response before publishing it", async () => { + const directory = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-speech-model-")); + directories.push(directory); + const server = NodeHttp.createServer((_request, response) => { + response.write("12345678"); + response.end(); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("missing test server address"); + + try { + await expect( + downloadVerifiedModel({ + directory, + filename: "model.gguf", + url: `http://127.0.0.1:${address.port}/model.gguf`, + size: 7, + sha256: "0".repeat(64), + }), + ).rejects.toThrow(/exceeded expected size|size mismatch/); + await expect(NodeFSP.stat(NodePath.join(directory, "model.gguf"))).rejects.toMatchObject({ + code: "ENOENT", + }); + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } + }); }); diff --git a/apps/desktop/src/speech/speechModel.ts b/apps/desktop/src/speech/speechModel.ts index 5feb56710f29..c36b175e746c 100644 --- a/apps/desktop/src/speech/speechModel.ts +++ b/apps/desktop/src/speech/speechModel.ts @@ -63,6 +63,11 @@ export async function downloadVerifiedModel(input: DownloadInput): Promise size) { + callback(new Error("speech model download exceeded expected size")); + return; + } digest.update(chunk); const now = Date.now(); if (now - lastReport >= 100 || downloaded === size) { diff --git a/apps/web/src/components/chat/ComposerSpeechButton.tsx b/apps/web/src/components/chat/ComposerSpeechButton.tsx index a4ac241a2a55..91b0b066268e 100644 --- a/apps/web/src/components/chat/ComposerSpeechButton.tsx +++ b/apps/web/src/components/chat/ComposerSpeechButton.tsx @@ -19,7 +19,7 @@ export function ComposerSpeechButton(props: { const state = props.status?.supported ? props.status.state : "missing-model"; const recording = state === "recording"; const busy = state === "downloading" || state === "transcribing"; - const inactive = props.disabled || busy; + const inactive = !recording && (props.disabled || busy); const label = recording ? "Stop and transcribe" : state === "downloading" @@ -54,7 +54,7 @@ export function ComposerSpeechButton(props: { diff --git a/apps/web/src/speech/useDesktopSpeechInput.ts b/apps/web/src/speech/useDesktopSpeechInput.ts index 73274a6ec1f9..0e46850c289c 100644 --- a/apps/web/src/speech/useDesktopSpeechInput.ts +++ b/apps/web/src/speech/useDesktopSpeechInput.ts @@ -2,6 +2,7 @@ import type { DesktopSpeechStatus } from "@t3tools/contracts"; import { useCallback, useEffect, useRef, useState } from "react"; import { ensureLocalApi } from "../localApi"; +import { toastManager } from "../components/ui/toast"; export function useDesktopSpeechInput(onTranscript: (text: string) => void, ownerKey: string) { const bridge = typeof window === "undefined" ? undefined : window.desktopBridge?.speech; @@ -32,6 +33,12 @@ export function useDesktopSpeechInput(onTranscript: (text: string) => void, owne } else if (event.type === "transcript") { if (recordingOwnerRef.current === currentOwnerRef.current) { transcriptRef.current(event.text); + } else { + toastManager.add({ + type: "info", + title: "Voice input finished in another draft", + description: event.text, + }); } recordingOwnerRef.current = null; } else if (event.type === "error") { @@ -49,10 +56,11 @@ export function useDesktopSpeechInput(onTranscript: (text: string) => void, owne useEffect(() => { if (!bridge || recordingOwnerRef.current === null) return; if (recordingOwnerRef.current === ownerKey) return; + if (status?.supported && status.state === "transcribing") return; recordingOwnerRef.current = null; activeRef.current = false; void bridge.cancel(); - }, [bridge, ownerKey]); + }, [bridge, ownerKey, status]); useEffect(() => { if (!bridge || !status?.supported || status.state !== "recording") return; @@ -87,6 +95,7 @@ export function useDesktopSpeechInput(onTranscript: (text: string) => void, owne const stop = useCallback(async () => { if (!bridge) return; activeRef.current = false; + setStatus({ supported: true, state: "transcribing" }); setStatus(await bridge.stop()); }, [bridge]); From e59f77c7281935a916f226184ceeca315072ae61 Mon Sep 17 00:00:00 2001 From: "t3-code[bot]" <269035359+t3-code[bot]@users.noreply.github.com> Date: Sun, 23 Aug 2026 17:21:51 +0000 Subject: [PATCH 5/9] fix(desktop): resolve speech data path from app identity Co-authored-by: Utkarsh Patil <73941998+UtkarshUsername@users.noreply.github.com> --- apps/desktop/src/speech/DesktopSpeech.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/desktop/src/speech/DesktopSpeech.ts b/apps/desktop/src/speech/DesktopSpeech.ts index 365f9d62e579..5c32d5df62d7 100644 --- a/apps/desktop/src/speech/DesktopSpeech.ts +++ b/apps/desktop/src/speech/DesktopSpeech.ts @@ -1,7 +1,7 @@ -// @effect-diagnostics nodeBuiltinImport:off - Electron app paths define local model storage. +// @effect-diagnostics nodeBuiltinImport:off - native model storage uses the resolved desktop data path. import type { DesktopSpeechEvent, DesktopSpeechStatus } from "@t3tools/contracts"; import { HostProcessArchitecture, HostProcessPlatform } from "@t3tools/shared/hostProcess"; -import { app } from "electron"; + import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -11,6 +11,7 @@ import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as NodePath from "node:path"; +import * as DesktopAppIdentity from "../app/DesktopAppIdentity.ts"; import { DesktopMicrophoneCapture } from "./DesktopMicrophoneCapture.ts"; import { DesktopSpeechController } from "./DesktopSpeechController.ts"; import { DesktopTranscriptionBackend } from "./DesktopTranscriptionBackend.ts"; @@ -84,8 +85,9 @@ function attempt( export const make = Effect.gen(function* () { const platform = yield* HostProcessPlatform; const architecture = yield* HostProcessArchitecture; + const appIdentity = yield* DesktopAppIdentity.DesktopAppIdentity; const availability = support(platform, architecture); - const directory = NodePath.join(app.getPath("userData"), "speech", "models"); + const directory = NodePath.join(yield* appIdentity.resolveUserDataPath, "speech", "models"); const listeners = yield* Ref.make>(new Set()); const context = yield* Effect.context(); const runFork = Effect.runForkWith(context); From 7c038f5d7001b4abe68e58571c03b3b4e58601ef Mon Sep 17 00:00:00 2001 From: "t3-code[bot]" <269035359+t3-code[bot]@users.noreply.github.com> Date: Sun, 23 Aug 2026 17:32:22 +0000 Subject: [PATCH 6/9] fix(desktop): inject speech model http client Co-authored-by: Utkarsh Patil <73941998+UtkarshUsername@users.noreply.github.com> --- apps/desktop/src/speech/DesktopSpeech.ts | 17 +++++++++++++++ apps/desktop/src/speech/speechModel.test.ts | 15 ++++++++++++- apps/desktop/src/speech/speechModel.ts | 24 ++++++++++++--------- 3 files changed, 45 insertions(+), 11 deletions(-) diff --git a/apps/desktop/src/speech/DesktopSpeech.ts b/apps/desktop/src/speech/DesktopSpeech.ts index 5c32d5df62d7..71f6d4a89e95 100644 --- a/apps/desktop/src/speech/DesktopSpeech.ts +++ b/apps/desktop/src/speech/DesktopSpeech.ts @@ -9,6 +9,8 @@ import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; +import { HttpClient, HttpClientResponse } from "effect/unstable/http"; import * as NodePath from "node:path"; import * as DesktopAppIdentity from "../app/DesktopAppIdentity.ts"; @@ -86,11 +88,25 @@ export const make = Effect.gen(function* () { const platform = yield* HostProcessPlatform; const architecture = yield* HostProcessArchitecture; const appIdentity = yield* DesktopAppIdentity.DesktopAppIdentity; + const httpClient = yield* HttpClient.HttpClient; const availability = support(platform, architecture); const directory = NodePath.join(yield* appIdentity.resolveUserDataPath, "speech", "models"); const listeners = yield* Ref.make>(new Set()); const context = yield* Effect.context(); const runFork = Effect.runForkWith(context); + const runPromise = Effect.runPromiseWith(context); + + const requestModel = async (url: string, signal?: AbortSignal) => { + const response = await runPromise( + httpClient.get(url).pipe(Effect.flatMap(HttpClientResponse.filterStatusOk)), + signal ? { signal } : undefined, + ); + return { + status: response.status, + headers: response.headers, + body: Stream.toAsyncIterableWith(response.stream, Context.empty()), + }; + }; const emit = (event: DesktopSpeechEvent): void => { runFork( @@ -114,6 +130,7 @@ export const make = Effect.gen(function* () { url: SPEECH_MODEL.url, size: SPEECH_MODEL.size, sha256: SPEECH_MODEL.sha256, + request: requestModel, onProgress, }), removeModel: () => removeSpeechModel(directory), diff --git a/apps/desktop/src/speech/speechModel.test.ts b/apps/desktop/src/speech/speechModel.test.ts index b6a9b451c5b3..b7cdcdf3ff2f 100644 --- a/apps/desktop/src/speech/speechModel.test.ts +++ b/apps/desktop/src/speech/speechModel.test.ts @@ -1,4 +1,4 @@ -// @effect-diagnostics nodeBuiltinImport:off - model downloader tests use local Node fixtures. +// @effect-diagnostics nodeBuiltinImport:off globalFetch:off - model downloader tests use local Node fixtures. import { afterEach, describe, expect, it } from "vite-plus/test"; import * as NodeFSP from "node:fs/promises"; import * as NodeHttp from "node:http"; @@ -9,6 +9,16 @@ import { downloadVerifiedModel } from "./speechModel.ts"; const directories: string[] = []; +async function request(url: string, signal?: AbortSignal) { + const response = await fetch(url, { ...(signal ? { signal } : {}) }); + if (!response.body) throw new Error("test response has no body"); + return { + status: response.status, + headers: Object.fromEntries(response.headers.entries()), + body: response.body as AsyncIterable, + }; +} + afterEach(async () => { await Promise.all( directories.splice(0).map((path) => NodeFSP.rm(path, { recursive: true, force: true })), @@ -33,6 +43,7 @@ describe("downloadVerifiedModel", () => { url: `http://127.0.0.1:${address.port}/model.gguf`, size: bytes.length, sha256: "03cfa25d83f5eaa1faac98ed6ceaaf0e7afe3c273a1e1502c2714ebe10b8263e", + request, }); expect(await NodeFSP.readFile(path)).toEqual(bytes); @@ -59,6 +70,7 @@ describe("downloadVerifiedModel", () => { url: `http://127.0.0.1:${address.port}/model.gguf`, size: 7, sha256: "0".repeat(64), + request, }), ).rejects.toThrow("verification failed"); await expect(NodeFSP.stat(NodePath.join(directory, "model.gguf"))).rejects.toMatchObject({ @@ -90,6 +102,7 @@ describe("downloadVerifiedModel", () => { url: `http://127.0.0.1:${address.port}/model.gguf`, size: 7, sha256: "0".repeat(64), + request, }), ).rejects.toThrow(/exceeded expected size|size mismatch/); await expect(NodeFSP.stat(NodePath.join(directory, "model.gguf"))).rejects.toMatchObject({ diff --git a/apps/desktop/src/speech/speechModel.ts b/apps/desktop/src/speech/speechModel.ts index c36b175e746c..53d307b8747a 100644 --- a/apps/desktop/src/speech/speechModel.ts +++ b/apps/desktop/src/speech/speechModel.ts @@ -1,11 +1,10 @@ -// @effect-diagnostics nodeBuiltinImport:off globalFetch:off globalDate:off - low-level streaming download boundary. +// @effect-diagnostics nodeBuiltinImport:off globalDate:off - low-level streaming download boundary. import * as NodeCrypto from "node:crypto"; import * as NodeFS from "node:fs"; import * as NodeFSP from "node:fs/promises"; import * as NodePath from "node:path"; import * as NodeStream from "node:stream"; import * as NodeStreamPromises from "node:stream/promises"; -import type * as NodeStreamWeb from "node:stream/web"; export const SPEECH_MODEL = { id: "moonshine-streaming-tiny", @@ -23,6 +22,14 @@ type DownloadInput = { url: string; size: number; sha256: string; + request: ( + url: string, + signal?: AbortSignal, + ) => Promise<{ + status: number; + headers: Readonly>; + body: AsyncIterable; + }>; signal?: AbortSignal; onProgress?: (downloaded: number, total: number) => void; }; @@ -46,7 +53,7 @@ async function hasExpectedDigest(path: string, size: number, sha256: string): Pr } export async function downloadVerifiedModel(input: DownloadInput): Promise { - const { directory, filename, url, size, sha256, signal, onProgress } = input; + const { directory, filename, url, size, sha256, request, signal, onProgress } = input; const finalPath = NodePath.join(directory, filename); signal?.throwIfAborted(); await NodeFSP.mkdir(directory, { recursive: true }); @@ -56,14 +63,11 @@ export async function downloadVerifiedModel(input: DownloadInput): Promise= 300) { throw new Error(`speech model download failed with status ${response.status}`); } - const contentLengthHeader = response.headers.get("content-length"); + const contentLengthHeader = response.headers["content-length"]; const contentLength = contentLengthHeader === null ? null : Number(contentLengthHeader); if (contentLength !== null && Number.isFinite(contentLength) && contentLength !== size) { throw new Error(`speech model download size mismatch: expected ${size}, got ${contentLength}`); @@ -92,7 +96,7 @@ export async function downloadVerifiedModel(input: DownloadInput): Promise), + NodeStream.Readable.from(response.body), progress, NodeFS.createWriteStream(partialPath, { mode: 0o600 }), { signal }, From 5d92c0a0eebf51a44b70ccf035b28fa4d1fa5380 Mon Sep 17 00:00:00 2001 From: "t3-code[bot]" <269035359+t3-code[bot]@users.noreply.github.com> Date: Sun, 23 Aug 2026 17:39:05 +0000 Subject: [PATCH 7/9] fix(web): align voice input feedback Co-authored-by: Utkarsh Patil <73941998+UtkarshUsername@users.noreply.github.com> --- apps/web/src/components/chat/ComposerSpeechButton.tsx | 10 ++++++---- apps/web/src/components/settings/SettingsPanels.tsx | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/web/src/components/chat/ComposerSpeechButton.tsx b/apps/web/src/components/chat/ComposerSpeechButton.tsx index 91b0b066268e..4aa0aa374080 100644 --- a/apps/web/src/components/chat/ComposerSpeechButton.tsx +++ b/apps/web/src/components/chat/ComposerSpeechButton.tsx @@ -1,5 +1,5 @@ import type { DesktopSpeechStatus } from "@t3tools/contracts"; -import { MicIcon, SquareIcon, XIcon } from "lucide-react"; +import { CircleAlertIcon, MicIcon, SquareIcon, XIcon } from "lucide-react"; import { Button } from "../ui/button"; import { Spinner } from "../ui/spinner"; @@ -38,7 +38,9 @@ export function ComposerSpeechButton(props: {