feat(desktop): add local voice input

apps/desktop/package.json

@@ -15,6 +15,7 @@
     "@clerk/electron": "catalog:",
     "@clerk/electron-passkeys": "catalog:",
     "@effect/platform-node": "catalog:",
+    "@picovoice/pvrecorder-node": "1.2.9",
     "@t3tools/client-runtime": "workspace:*",
     "@t3tools/contracts": "workspace:*",
     "@t3tools/shared": "workspace:*",
@@ -25,7 +26,8 @@
     "electron-store": "^8.2.0",
     "electron-updater": "^6.6.2",
     "playwright-core": "1.60.0",
-    "react-grab": "^0.1.32"
+    "react-grab": "^0.1.32",
+    "transcribe-cpp": "0.2.1"
   },
   "devDependencies": {
     "@effect/vitest": "catalog:",

apps/desktop/src/ipc/DesktopIpcHandlers.ts

@@ -45,11 +45,13 @@ import {
   showContextMenu,
 } from "./methods/window.ts";
 import * as PreviewIpc from "./methods/preview.ts";
+import * as SpeechIpc from "./methods/speech.ts";
 import { getWslState, setWslBackendEnabled, setWslDistro, setWslOnly } from "./methods/wsl.ts";
 
 export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers")(function* () {
   const ipc = yield* DesktopIpc.DesktopIpc;
   yield* PreviewIpc.installPreviewEventForwarding();
+  yield* SpeechIpc.installSpeechEventForwarding();
 
   yield* ipc.handleSync(getAppBranding);
   yield* ipc.handleSync(getSystemLocale);
@@ -94,6 +96,11 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers"
   yield* ipc.handle(downloadUpdate);
   yield* ipc.handle(installUpdate);
   yield* ipc.handle(checkForUpdate);
+  yield* ipc.handle(SpeechIpc.getSpeechStatus);
+  yield* ipc.handle(SpeechIpc.startSpeech);
+  yield* ipc.handle(SpeechIpc.stopSpeech);
+  yield* ipc.handle(SpeechIpc.cancelSpeech);
+  yield* ipc.handle(SpeechIpc.removeSpeechModelMethod);
   for (const previewMethod of PreviewIpc.methods) {
     yield* ipc.handle(previewMethod);
   }

apps/desktop/src/ipc/channels.ts

@@ -83,3 +83,9 @@ export const PREVIEW_RECORDING_SAVE_CHANNEL = "desktop:preview-recording-save";
 export const PREVIEW_RECORDING_FRAME_CHANNEL = "desktop:preview-recording-frame";
 export const PREVIEW_STATE_CHANGE_CHANNEL = "desktop:preview-state-change";
 export const PREVIEW_POINTER_EVENT_CHANNEL = "desktop:preview-pointer-event";
+export const SPEECH_GET_STATUS_CHANNEL = "desktop:speech-get-status";
+export const SPEECH_START_CHANNEL = "desktop:speech-start";
+export const SPEECH_STOP_CHANNEL = "desktop:speech-stop";
+export const SPEECH_CANCEL_CHANNEL = "desktop:speech-cancel";
+export const SPEECH_REMOVE_MODEL_CHANNEL = "desktop:speech-remove-model";
+export const SPEECH_EVENT_CHANNEL = "desktop:speech-event";

apps/desktop/src/ipc/methods/speech.ts

@@ -0,0 +1,64 @@
+import { DesktopSpeechStatusSchema } from "@t3tools/contracts";
+import * as Effect from "effect/Effect";
+import * as Schema from "effect/Schema";
+
+import * as ElectronWindow from "../../electron/ElectronWindow.ts";
+import * as DesktopSpeech from "../../speech/DesktopSpeech.ts";
+import * as IpcChannels from "../channels.ts";
+import * as DesktopIpc from "../DesktopIpc.ts";
+
+export const installSpeechEventForwarding = Effect.fn("desktop.ipc.speech.events")(function* () {
+  const windows = yield* ElectronWindow.ElectronWindow;
+  const speech = yield* DesktopSpeech.DesktopSpeech;
+  yield* speech.subscribe((event) => windows.sendAll(IpcChannels.SPEECH_EVENT_CHANNEL, event));
+});
+
+export const getSpeechStatus = DesktopIpc.makeIpcMethod({
+  channel: IpcChannels.SPEECH_GET_STATUS_CHANNEL,
+  payload: Schema.Void,
+  result: DesktopSpeechStatusSchema,
+  handler: Effect.fn("desktop.ipc.speech.getStatus")(function* () {
+    const speech = yield* DesktopSpeech.DesktopSpeech;
+    return yield* speech.getStatus;
+  }),
+});
+
+export const startSpeech = DesktopIpc.makeIpcMethod({
+  channel: IpcChannels.SPEECH_START_CHANNEL,
+  payload: Schema.Void,
+  result: DesktopSpeechStatusSchema,
+  handler: Effect.fn("desktop.ipc.speech.start")(function* () {
+    const speech = yield* DesktopSpeech.DesktopSpeech;
+    return yield* speech.start;
+  }),
+});
+
+export const stopSpeech = DesktopIpc.makeIpcMethod({
+  channel: IpcChannels.SPEECH_STOP_CHANNEL,
+  payload: Schema.Void,
+  result: DesktopSpeechStatusSchema,
+  handler: Effect.fn("desktop.ipc.speech.stop")(function* () {
+    const speech = yield* DesktopSpeech.DesktopSpeech;
+    return yield* speech.stop;
+  }),
+});
+
+export const cancelSpeech = DesktopIpc.makeIpcMethod({
+  channel: IpcChannels.SPEECH_CANCEL_CHANNEL,
+  payload: Schema.Void,
+  result: DesktopSpeechStatusSchema,
+  handler: Effect.fn("desktop.ipc.speech.cancel")(function* () {
+    const speech = yield* DesktopSpeech.DesktopSpeech;
+    return yield* speech.cancel;
+  }),
+});
+
+export const removeSpeechModelMethod = DesktopIpc.makeIpcMethod({
+  channel: IpcChannels.SPEECH_REMOVE_MODEL_CHANNEL,
+  payload: Schema.Void,
+  result: DesktopSpeechStatusSchema,
+  handler: Effect.fn("desktop.ipc.speech.removeModel")(function* () {
+    const speech = yield* DesktopSpeech.DesktopSpeech;
+    return yield* speech.removeModel;
+  }),
+});

apps/desktop/src/main.ts

@@ -56,6 +56,7 @@ import * as DesktopSshEnvironment from "./ssh/DesktopSshEnvironment.ts";
 import * as DesktopSshPasswordPrompts from "./ssh/DesktopSshPasswordPrompts.ts";
 import * as DesktopState from "./app/DesktopState.ts";
 import * as DesktopTelemetryPublisher from "./telemetry/DesktopTelemetryPublisher.ts";
+import * as DesktopSpeech from "./speech/DesktopSpeech.ts";
 import * as DesktopUpdates from "./updates/DesktopUpdates.ts";
 import * as BrowserSession from "./preview/BrowserSession.ts";
 import * as PreviewManager from "./preview/Manager.ts";
@@ -188,6 +189,7 @@ const desktopApplicationLayer = Layer.mergeAll(
   DesktopLinuxUrlHandler.layer,
   DesktopShellEnvironment.layer,
   desktopSshLayer,
+  DesktopSpeech.layer,
 ).pipe(
   Layer.provideMerge(DesktopUpdates.layer),
   Layer.provideMerge(desktopWslBackendLayer),

apps/desktop/src/preload.ts

@@ -164,6 +164,21 @@ contextBridge.exposeInMainWorld("desktopBridge", {
       ipcRenderer.removeListener(IpcChannels.UPDATE_STATE_CHANNEL, wrappedListener);
     };
   },
+  speech: {
+    getStatus: () => ipcRenderer.invoke(IpcChannels.SPEECH_GET_STATUS_CHANNEL),
+    start: () => ipcRenderer.invoke(IpcChannels.SPEECH_START_CHANNEL),
+    stop: () => ipcRenderer.invoke(IpcChannels.SPEECH_STOP_CHANNEL),
+    cancel: () => ipcRenderer.invoke(IpcChannels.SPEECH_CANCEL_CHANNEL),
+    removeModel: () => ipcRenderer.invoke(IpcChannels.SPEECH_REMOVE_MODEL_CHANNEL),
+    onEvent: (listener) => {
+      const wrappedListener = (_event: Electron.IpcRendererEvent, speechEvent: unknown) => {
+        if (typeof speechEvent !== "object" || speechEvent === null) return;
+        listener(speechEvent as Parameters<typeof listener>[0]);
+      };
+      ipcRenderer.on(IpcChannels.SPEECH_EVENT_CHANNEL, wrappedListener);
+      return () => ipcRenderer.removeListener(IpcChannels.SPEECH_EVENT_CHANNEL, wrappedListener);
+    },
+  },
   preview: {
     createTab: (tabId, defaults) =>
       ipcRenderer.invoke(IpcChannels.PREVIEW_CREATE_TAB_CHANNEL, {

apps/desktop/src/speech/DesktopMicrophoneCapture.ts

@@ -0,0 +1,99 @@
+// @effect-diagnostics globalDate:off - native microphone callback is outside Effect runtime.
+import { PvRecorder } from "@picovoice/pvrecorder-node";
+
+const SAMPLE_RATE = 16_000;
+const FRAME_LENGTH = 512;
+
+export class DesktopMicrophoneCapture {
+  private readonly onLevel: (level: number, elapsedMs: number) => void;
+  private recorder: PvRecorder | undefined;
+  private frames: Int16Array[] = [];
+  private readLoop: Promise<void> | undefined;
+  private stopping = false;
+  private readError: Error | undefined;
+  private startedAt = 0;
+  private lastLevelAt = 0;
+
+  constructor(onLevel: (level: number, elapsedMs: number) => void) {
+    this.onLevel = onLevel;
+  }
+
+  start(): void {
+    if (this.recorder) throw new Error("microphone capture is already active");
+    const recorder = new PvRecorder(FRAME_LENGTH, -1);
+    try {
+      if (recorder.sampleRate !== SAMPLE_RATE) {
+        throw new Error(
+          `microphone reported ${recorder.sampleRate} Hz; expected ${SAMPLE_RATE} Hz`,
+        );
+      }
+      this.frames = [];
+      this.stopping = false;
+      this.readError = undefined;
+      this.startedAt = Date.now();
+      this.lastLevelAt = 0;
+      recorder.start();
+      this.recorder = recorder;
+      this.readLoop = this.readFrames(recorder);
+    } catch (error) {
+      recorder.release();
+      throw error;
+    }
+  }
+
+  async stop(): Promise<Float32Array> {
+    const recorder = this.recorder;
+    if (!recorder) throw new Error("microphone capture is not active");
+    this.stopping = true;
+    try {
+      if (recorder.isRecording) recorder.stop();
+      await this.readLoop;
+    } finally {
+      recorder.release();
+      this.recorder = undefined;
+      this.readLoop = undefined;
+    }
+    if (this.readError) throw this.readError;
+    const sampleCount = this.frames.reduce((total, frame) => total + frame.length, 0);
+    if (sampleCount === 0) throw new Error("no microphone audio was captured");
+    const pcm = new Float32Array(sampleCount);
+    let offset = 0;
+    for (const frame of this.frames) {
+      for (let index = 0; index < frame.length; index += 1) {
+        pcm[offset + index] = (frame[index] ?? 0) / 32_768;
+      }
+      offset += frame.length;
+    }
+    this.frames = [];
+    return pcm;
+  }
+
+  async cancel(): Promise<void> {
+    if (!this.recorder) return;
+    await this.stop().catch(() => undefined);
+    this.frames = [];
+  }
+
+  private async readFrames(recorder: PvRecorder): Promise<void> {
+    try {
+      while (!this.stopping && recorder.isRecording) {
+        const frame = await recorder.read();
+        if (this.stopping) continue;
+        this.frames.push(frame);
+        const now = Date.now();
+        if (now - this.lastLevelAt >= 100) {
+          this.lastLevelAt = now;
+          let energy = 0;
+          for (const sample of frame) energy += (sample / 32_768) ** 2;
+          this.onLevel(
+            Math.min(1, Math.sqrt(energy / Math.max(1, frame.length)) * 8),
+            now - this.startedAt,
+          );
+        }
+      }
+    } catch (error) {
+      if (!this.stopping)
+        this.readError = error instanceof Error ? error : new Error(String(error));
+    }
+  }
+}