From 0f07f03666f74f05d464bcb4e19df9f58d0311aa Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 27 Oct 2026 13:53:49 +0100 Subject: [PATCH 1/2] feat(ai-components): add VoiceGlow and VoiceWave voice-agent visualizations Port the voice visualizations from the Android, iOS and web AI components, so the platforms look and move alike. - VoiceGlow (a Skia shader, ported from Android's AGSL glow) and VoiceWave (a Skia path), both driven by one motion driver that runs as worklets on the UI thread. They take `state`, `colors`, `expressivity` and `audioLevel` (a SharedValue), name themselves "Voice agent" with the state as the value, honour Reduce Motion, and react to taps and hover. - New `voiceVisualization` theme keys (colors, backdropColor). - Dictation emits the microphone level (`AIDictationOnLevel`): from the recognizer's RMS on Android and from the audio tap, in dBFS, on iOS. `useDictationAudioLevel()` exposes it. - iOS dictation now restores the audio session it found once it ends; its `.record` category used to mute all playback in the app after the first dictation. - Export `useDictationState` and `useDictationTranscript`. - ExpoChatGPTSample: a hands-free voice mode (dictation in, expo-speech out), the send path shared through `useSendAIMessage`, and the demo backend called with the `openai` platform. --- .../components/VoiceMode/VoiceMode.tsx | 105 ++++++ .../components/VoiceMode/index.ts | 1 + .../VoiceMode/useVoiceConversation.ts | 144 ++++++++ .../hooks/useSendAIMessage.ts | 35 ++ examples/ExpoChatGPTSample/http/requests.ts | 9 +- examples/ExpoChatGPTSample/package.json | 1 + .../ExpoChatGPTSample/screens/ChatContent.tsx | 30 +- package/jest-setup.tsx | 56 +++- .../dictation/AIDictationController.kt | 25 +- package/shared-native/ios/AIDictation.mm | 13 +- .../shared-native/ios/AIDictationModule.swift | 68 +++- package/src/ai-components/README.md | 51 ++- .../__tests__/VoiceVisualization.test.tsx | 199 +++++++++++ .../__tests__/dictation.test.tsx | 35 +- .../__tests__/voiceDriver.test.ts | 317 ++++++++++++++++++ .../__tests__/voicePalette.test.ts | 124 +++++++ package/src/ai-components/index.ts | 4 +- .../src/ai-components/native/AIDictation.ts | 11 + package/src/ai-components/store/index.ts | 2 + package/src/ai-components/theme.ts | 16 + .../transcription/useDictationAudioLevel.ts | 38 +++ .../voice-visualization/VoiceGlow.tsx | 79 +++++ .../VoiceVisualizationView.tsx | 174 ++++++++++ .../voice-visualization/VoiceWave.tsx | 67 ++++ .../voice-visualization/bounce.ts | 61 ++++ .../voice-visualization/driver.ts | 240 +++++++++++++ .../voice-visualization/glowShader.ts | 227 +++++++++++++ .../voice-visualization/index.ts | 3 + .../ai-components/voice-visualization/math.ts | 73 ++++ .../voice-visualization/palette.ts | 59 ++++ .../voice-visualization/simplex.ts | 92 +++++ .../voice-visualization/styles.ts | 42 +++ .../voice-visualization/types.ts | 38 +++ .../voice-visualization/useReduceMotion.ts | 34 ++ .../useVoiceVisualization.ts | 154 +++++++++ .../voice-visualization/waveGeometry.ts | 137 ++++++++ yarn.lock | 10 + 37 files changed, 2730 insertions(+), 44 deletions(-) create mode 100644 examples/ExpoChatGPTSample/components/VoiceMode/VoiceMode.tsx create mode 100644 examples/ExpoChatGPTSample/components/VoiceMode/index.ts create mode 100644 examples/ExpoChatGPTSample/components/VoiceMode/useVoiceConversation.ts create mode 100644 examples/ExpoChatGPTSample/hooks/useSendAIMessage.ts create mode 100644 package/src/ai-components/__tests__/VoiceVisualization.test.tsx create mode 100644 package/src/ai-components/__tests__/voiceDriver.test.ts create mode 100644 package/src/ai-components/__tests__/voicePalette.test.ts create mode 100644 package/src/ai-components/transcription/useDictationAudioLevel.ts create mode 100644 package/src/ai-components/voice-visualization/VoiceGlow.tsx create mode 100644 package/src/ai-components/voice-visualization/VoiceVisualizationView.tsx create mode 100644 package/src/ai-components/voice-visualization/VoiceWave.tsx create mode 100644 package/src/ai-components/voice-visualization/bounce.ts create mode 100644 package/src/ai-components/voice-visualization/driver.ts create mode 100644 package/src/ai-components/voice-visualization/glowShader.ts create mode 100644 package/src/ai-components/voice-visualization/index.ts create mode 100644 package/src/ai-components/voice-visualization/math.ts create mode 100644 package/src/ai-components/voice-visualization/palette.ts create mode 100644 package/src/ai-components/voice-visualization/simplex.ts create mode 100644 package/src/ai-components/voice-visualization/styles.ts create mode 100644 package/src/ai-components/voice-visualization/types.ts create mode 100644 package/src/ai-components/voice-visualization/useReduceMotion.ts create mode 100644 package/src/ai-components/voice-visualization/useVoiceVisualization.ts create mode 100644 package/src/ai-components/voice-visualization/waveGeometry.ts diff --git a/examples/ExpoChatGPTSample/components/VoiceMode/VoiceMode.tsx b/examples/ExpoChatGPTSample/components/VoiceMode/VoiceMode.tsx new file mode 100644 index 0000000000..44d84b85ad --- /dev/null +++ b/examples/ExpoChatGPTSample/components/VoiceMode/VoiceMode.tsx @@ -0,0 +1,105 @@ +import { useState } from 'react'; +import { Modal, Pressable, StyleSheet, Text, View } from 'react-native'; +import { useSafeAreaInsets } from 'react-native-safe-area-context'; + +import { + StreamTheme, + useDictationAudioLevel, + type VoiceAgentState, + VoiceGlow, + VoiceWave, +} from 'stream-chat-expo/ai-components'; + +import { useVoiceConversation } from './useVoiceConversation'; + +const BACKDROP = '#0b0b10'; +const COLORS = ['#005fff', '#7b5cff', '#00c2ff']; + +const LABELS: Record = { + connecting: 'Connecting…', + idle: 'Tap the glow to talk', + listening: 'Listening', + speaking: 'Speaking', + thinking: 'Thinking', +}; + +const VoiceModeOverlay = ({ onClose }: { onClose: () => void }) => { + const { top, bottom } = useSafeAreaInsets(); + const { error, listen, phase, userIsTalking } = useVoiceConversation(true); + const micLevel = useDictationAudioLevel(); + + return ( + + + + + ✕ + + + + + + + {error ? 'Couldn’t hear you. Tap the glow to try again.' : LABELS[phase]} + + + + + ); +}; + +/** Opens a full-screen voice conversation with the AI agent. */ +export const VoiceMode = () => { + const [open, setOpen] = useState(false); + + return ( + <> + + setOpen(true)} + style={styles.button} + > + + + + setOpen(false)} visible={open}> + {open ? setOpen(false)} /> : null} + + + ); +}; + +const styles = StyleSheet.create({ + button: { + alignItems: 'center', + backgroundColor: '#ffffff', + borderColor: '#ececec', + borderRadius: 24, + borderWidth: 1, + height: 48, + justifyContent: 'center', + overflow: 'hidden', + width: 48, + }, + buttonGlow: { height: 64, width: 64 }, + close: { color: '#ffffff', fontSize: 22 }, + entry: { alignItems: 'flex-end', paddingHorizontal: 16, paddingBottom: 8 }, + glow: { aspectRatio: 1, width: '100%' }, + glowContainer: { alignSelf: 'stretch', flex: 1, justifyContent: 'center' }, + header: { alignItems: 'flex-end', paddingHorizontal: 20 }, + label: { color: '#c9cbd3', fontSize: 17, marginVertical: 16, textAlign: 'center' }, + overlay: { backgroundColor: BACKDROP, flex: 1 }, + wave: { alignSelf: 'stretch', height: 140 }, +}); diff --git a/examples/ExpoChatGPTSample/components/VoiceMode/index.ts b/examples/ExpoChatGPTSample/components/VoiceMode/index.ts new file mode 100644 index 0000000000..abe97f1086 --- /dev/null +++ b/examples/ExpoChatGPTSample/components/VoiceMode/index.ts @@ -0,0 +1 @@ +export * from './VoiceMode'; diff --git a/examples/ExpoChatGPTSample/components/VoiceMode/useVoiceConversation.ts b/examples/ExpoChatGPTSample/components/VoiceMode/useVoiceConversation.ts new file mode 100644 index 0000000000..d0e01ceacd --- /dev/null +++ b/examples/ExpoChatGPTSample/components/VoiceMode/useVoiceConversation.ts @@ -0,0 +1,144 @@ +import { useEffect, useRef, useState } from 'react'; + +import * as Speech from 'expo-speech'; + +import { AIStates, useAIState, useChannelContext, useStableCallback } from 'stream-chat-expo'; + +import { + dictationStore, + useDictation, + useDictationState, + useDictationTranscript, + type VoiceAgentState, +} from 'stream-chat-expo/ai-components'; + +import { useSendAIMessage } from '../../hooks/useSendAIMessage'; + +/** A turn the agent is never seen working on ends after this long. */ +const TURN_TIMEOUT_MS = 20_000; + +const WORKING_STATES: string[] = [AIStates.Thinking, AIStates.Generating, AIStates.ExternalSources]; + +/** Markdown read aloud as plain words. */ +const toSpeech = (text: string) => + text + .replace(/```[\s\S]*?```/g, ' ') + .replace(/!\[[^\]]*]\([^)]*\)/g, ' ') + .replace(/\[([^\]]+)]\([^)]*\)/g, '$1') + .replace(/[*_#>`~|]/g, '') + .replace(/\s+/g, ' ') + .trim(); + +/** + * A hands-free conversation with the AI agent: it listens with dictation, sends what it heard, + * waits for the agent to answer, reads the answer aloud and listens again. + * + * Returns what the agent is doing, for `VoiceGlow`, and whether the user is being listened to, + * for `VoiceWave`. + */ +export const useVoiceConversation = (active: boolean) => { + const [phase, setPhase] = useState('idle'); + const { start, cancel } = useDictation(); + const { state: dictationState, error } = useDictationState(); + const { transcript } = useDictationTranscript(); + const send = useSendAIMessage(); + const { channel } = useChannelContext(); + const { aiState } = useAIState(channel); + + const turn = useRef<{ sentAt: number; sawWorking: boolean } | null>(null); + const timeout = useRef | null>(null); + + const listen = useStableCallback(() => { + if (!active) return; + turn.current = null; + // Clears what an earlier turn heard or failed with, before dictation starts again. + dictationStore.partialNext({ error: null, transcript: '' }); + setPhase('connecting'); + void start(); + }); + + const latestReply = useStableCallback(() => { + const sentAt = turn.current?.sentAt ?? 0; + const messages = channel?.state.latestMessages ?? []; + for (let i = messages.length - 1; i >= 0; i--) { + const message = messages[i]; + if (new Date(message.created_at ?? 0).getTime() < sentAt) break; + if (message.ai_generated && message.text) return message.text; + } + return null; + }); + + /** Ends the agent's turn: reads its answer aloud if there is one, then listens again. */ + const finishTurn = useStableCallback(() => { + if (timeout.current) clearTimeout(timeout.current); + timeout.current = null; + const reply = latestReply(); + turn.current = null; + if (!active) return; + if (!reply) { + listen(); + return; + } + setPhase('speaking'); + Speech.speak(toSpeech(reply), { + onDone: listen, + onError: listen, + // Its own audio session on iOS, so the reply is heard whatever dictation left the app's in. + useApplicationAudioSession: false, + }); + }); + + // Starts listening when voice mode opens, and stops everything when it closes. + useEffect(() => { + if (!active) return; + listen(); + return () => { + if (timeout.current) clearTimeout(timeout.current); + timeout.current = null; + turn.current = null; + cancel(); + void Speech.stop(); + setPhase('idle'); + }; + }, [active, cancel, listen]); + + // Follows dictation: listening while it records, and sends what it heard once it stops. + useEffect(() => { + if (!active || turn.current) return; + if (dictationState === 'listening') { + setPhase('listening'); + return; + } + if (dictationState !== 'idle' || (phase !== 'listening' && phase !== 'connecting')) return; + if (error) { + setPhase('idle'); + return; + } + if (phase === 'connecting') return; + const text = transcript.trim(); + if (!text) { + // Nothing was heard; keep listening. + listen(); + return; + } + turn.current = { sawWorking: false, sentAt: Date.now() }; + setPhase('thinking'); + void send({ text }); + timeout.current = setTimeout(() => { + if (!turn.current?.sawWorking) finishTurn(); + }, TURN_TIMEOUT_MS); + }, [active, dictationState, error, finishTurn, listen, phase, send, transcript]); + + // Follows the agent: its turn ends once it has been seen working and is done. + useEffect(() => { + const current = turn.current; + if (!active || !current) return; + if (WORKING_STATES.includes(aiState)) { + current.sawWorking = true; + } else if (current.sawWorking) { + finishTurn(); + } + }, [active, aiState, finishTurn]); + + return { error, listen, phase, userIsTalking: phase === 'listening' }; +}; diff --git a/examples/ExpoChatGPTSample/hooks/useSendAIMessage.ts b/examples/ExpoChatGPTSample/hooks/useSendAIMessage.ts new file mode 100644 index 0000000000..c81147a43b --- /dev/null +++ b/examples/ExpoChatGPTSample/hooks/useSendAIMessage.ts @@ -0,0 +1,35 @@ +import { useMessageComposer, useMessageInputContext, useStableCallback } from 'stream-chat-expo'; + +import type { ComposerViewProps } from 'stream-chat-expo/ai-components'; + +export type AIMessage = Parameters[0]; + +/** + * Sends a message to the AI channel through the channel's composer, with its attachments. Shared + * by the composer and voice mode, so both go through `preSendMessageRequest` (which starts the + * agent on the first message). + */ +export const useSendAIMessage = () => { + const messageComposer = useMessageComposer(); + const { sendMessage } = useMessageInputContext(); + + return useStableCallback(async ({ text, attachments }: AIMessage) => { + messageComposer.textComposer.setText(text); + if (attachments && attachments.length > 0) { + const localAttachments = await Promise.all( + attachments.map((asset) => + messageComposer.attachmentManager.fileToLocalUploadAttachment({ + ...asset, + // The picker can leave these out (e.g. some iOS photos have no file name). + name: asset.name ?? asset.uri.split('/').pop() ?? 'image', + size: asset.size ?? 0, + type: asset.type ?? 'image/jpeg', + }), + ), + ); + messageComposer.attachmentManager.upsertAttachments(localAttachments); + } + + await sendMessage(); + }); +}; diff --git a/examples/ExpoChatGPTSample/http/requests.ts b/examples/ExpoChatGPTSample/http/requests.ts index e8c20536b0..fce4cd3941 100644 --- a/examples/ExpoChatGPTSample/http/requests.ts +++ b/examples/ExpoChatGPTSample/http/requests.ts @@ -1,12 +1,19 @@ import { post } from './api'; +// The AI provider the sample backend uses, as in the React SDK's ai-chatbot example. +const PLATFORM = 'openai'; + export const startAI = (channelId: string) => post('https://ai-sdk-server-0f347d455e2e.herokuapp.com/start-ai-agent', { channel_id: channelId, + platform: PLATFORM, }); export const stopAI = (channelId: string) => post('https://ai-sdk-server-0f347d455e2e.herokuapp.com/stop-ai-agent', { channel_id: channelId, }); export const summarize = (text: string) => - post('https://ai-sdk-server-0f347d455e2e.herokuapp.com/summarize', { text }); + post('https://ai-sdk-server-0f347d455e2e.herokuapp.com/summarize', { + platform: PLATFORM, + text, + }); diff --git a/examples/ExpoChatGPTSample/package.json b/examples/ExpoChatGPTSample/package.json index 7b42eef92b..a1b51627e9 100644 --- a/examples/ExpoChatGPTSample/package.json +++ b/examples/ExpoChatGPTSample/package.json @@ -19,6 +19,7 @@ "expo-image-picker": "~58.0.10", "expo-linking": "~58.0.12", "expo-router": "~58.0.18", + "expo-speech": "~58.0.3", "react": "19.3.0", "react-native": "0.88.0", "react-native-gesture-handler": "~3.4.0", diff --git a/examples/ExpoChatGPTSample/screens/ChatContent.tsx b/examples/ExpoChatGPTSample/screens/ChatContent.tsx index d715ccbee1..b12acf5b3f 100644 --- a/examples/ExpoChatGPTSample/screens/ChatContent.tsx +++ b/examples/ExpoChatGPTSample/screens/ChatContent.tsx @@ -17,9 +17,7 @@ import { ThemeProvider, useAIState, useChannelContext, - useMessageComposer, useMessageContext, - useMessageInputContext, useStableCallback, useTheme, WithComponents, @@ -28,12 +26,13 @@ import { import { AITypingIndicatorView, ComposerView, - type ComposerViewProps, StreamingMessageView, } from 'stream-chat-expo/ai-components'; import { bottomSheetOptions } from '../bottomSheetOptions'; +import { VoiceMode } from '../components/VoiceMode'; import { useAppContext } from '../contexts/AppContext'; +import { useSendAIMessage } from '../hooks/useSendAIMessage'; import { startAI, summarize } from '../http/requests'; const CustomMessage = (props: MessageProps) => { @@ -93,8 +92,6 @@ const CustomStreamingMessageView = () => { }; const CustomComposerView = () => { - const messageComposer = useMessageComposer(); - const { sendMessage } = useMessageInputContext(); const { channel } = useChannelContext(); const { aiState } = useAIState(channel); @@ -114,27 +111,7 @@ const CustomComposerView = () => { [safeAreaInsets], ); - const serializeToMessage = useStableCallback( - async ({ text, attachments }: Parameters[0]) => { - messageComposer.textComposer.setText(text); - if (attachments && attachments.length > 0) { - const localAttachments = await Promise.all( - attachments.map((asset) => - messageComposer.attachmentManager.fileToLocalUploadAttachment({ - ...asset, - // The picker can leave these out (e.g. some iOS photos have no file name). - name: asset.name ?? asset.uri.split('/').pop() ?? 'image', - size: asset.size ?? 0, - type: asset.type ?? 'image/jpeg', - }), - ), - ); - messageComposer.attachmentManager.upsertAttachments(localAttachments); - } - - await sendMessage(); - }, - ); + const serializeToMessage = useSendAIMessage(); return ( { > + diff --git a/package/jest-setup.tsx b/package/jest-setup.tsx index e6b56b908c..e4519c92c7 100644 --- a/package/jest-setup.tsx +++ b/package/jest-setup.tsx @@ -65,8 +65,16 @@ jest.mock('react-native-reanimated', () => { makeMutable: (init: T) => ({ addListener: noop, removeListener: noop, value: init }), useSharedValue: (init: T) => new Proxy(RNReanimatedmock.useSharedValue(init), { - get: (target, prop) => - prop === 'addListener' || prop === 'removeListener' ? noop : Reflect.get(target, prop), + get: (target, prop) => { + if (prop === 'addListener' || prop === 'removeListener') return noop; + // The mock's shared values have no `modify`; run it in place, as on the UI thread. + if (prop === 'modify') { + return (modifier: (value: T) => T) => { + target.value = modifier(target.value); + }; + } + return Reflect.get(target, prop); + }, }), NativeEventsManager: class { attachEvents() {} @@ -150,13 +158,43 @@ jest.mock('react-native-teleport', () => { jest.mock('react-native-safe-area-context', () => mockSafeAreaContext); -// Skia and victory-native are optional peers used by the `ai-components` charts. Skia's JS needs its -// native module at import time, so both are replaced with inert stand-ins that render plain views. -jest.mock('@shopify/react-native-skia', () => ({ - LinearGradient: () => null, - matchFont: () => ({}), - vec: (x: number, y: number) => ({ x, y }), -})); +// Skia and victory-native are optional peers used by the `ai-components` charts and voice +// visualizations. Skia's JS needs its native module at import time, so both are replaced with inert +// stand-ins that render plain views. +jest.mock('@shopify/react-native-skia', () => { + const React = require('react'); + const rn = require('react-native'); + const Primitive = () => null; + const pathBuilder = () => { + const commands: Array<[string, number, number]> = []; + const builder = { + build: () => ({ commands }), + lineTo: (x: number, y: number) => { + commands.push(['L', x, y]); + return builder; + }, + moveTo: (x: number, y: number) => { + commands.push(['M', x, y]); + return builder; + }, + }; + return builder; + }; + return { + Canvas: ({ children, style }: { children?: unknown; style?: unknown }) => + React.createElement(rn.View, { style, testID: 'skia-canvas' }, children), + Fill: ({ children }: { children?: unknown }) => children ?? null, + LinearGradient: Primitive, + matchFont: () => ({}), + Path: Primitive, + Shader: Primitive, + Skia: { + PathBuilder: { Make: pathBuilder }, + RuntimeEffect: { Make: (source: string) => ({ source }) }, + }, + vec: (x: number, y: number) => ({ x, y }), + }; +}); jest.mock('victory-native', () => { const React = require('react'); diff --git a/package/shared-native/android/dictation/AIDictationController.kt b/package/shared-native/android/dictation/AIDictationController.kt index a06fae8cf9..bf5b7096f6 100644 --- a/package/shared-native/android/dictation/AIDictationController.kt +++ b/package/shared-native/android/dictation/AIDictationController.kt @@ -24,7 +24,8 @@ import java.util.concurrent.atomic.AtomicBoolean * * Shared by the `AIDictation` TurboModule of both SDK packages (each package's codegen generates its * own `NativeAIDictationSpec`), which forward their calls here. Events are delivered through - * [emitEvent] with the names in [EVENT_ON_STATE], [EVENT_ON_RESULT] and [EVENT_ON_ERROR]. + * [emitEvent] with the names in [EVENT_ON_STATE], [EVENT_ON_RESULT], [EVENT_ON_ERROR] and + * [EVENT_ON_LEVEL]. */ class AIDictationController( private val reactContext: ReactApplicationContext, @@ -41,6 +42,12 @@ class AIDictationController( const val EVENT_ON_STATE = "AIDictationOnState" const val EVENT_ON_RESULT = "AIDictationOnResult" const val EVENT_ON_ERROR = "AIDictationOnError" + const val EVENT_ON_LEVEL = "AIDictationOnLevel" + + // The recognizer's RMS reads about -2 dB in a quiet room and up to about 10 dB for speech + // close to the microphone; this span maps onto the level from 0 to 1. + private const val LEVEL_FLOOR_DB = 0f + private const val LEVEL_CEILING_DB = 10f private const val STATE_IDLE = "idle" private const val STATE_STARTING = "starting" @@ -328,7 +335,9 @@ class AIDictationController( lastSpeechTimeMs = System.currentTimeMillis() } - override fun onRmsChanged(rmsdB: Float) = Unit + override fun onRmsChanged(rmsdB: Float) { + emitLevel(rmsdB) + } override fun onBufferReceived(buffer: ByteArray?) = Unit override fun onEndOfSpeech() { @@ -471,6 +480,18 @@ class AIDictationController( emitEvent(EVENT_ON_STATE, map) } + /** + * Emits the microphone's level to JS, from 0 to 1, for the voice visualizations. The recognizer + * reports it as RMS in dB, around ten times a second while it listens. + */ + private fun emitLevel(rmsdB: Float) { + val level = ((rmsdB - LEVEL_FLOOR_DB) / (LEVEL_CEILING_DB - LEVEL_FLOOR_DB)).coerceIn(0f, 1f) + val map = Arguments.createMap().apply { + putDouble("level", level.toDouble()) + } + emitEvent(EVENT_ON_LEVEL, map) + } + /** * Emits a generic error event to JS with a code and message. */ diff --git a/package/shared-native/ios/AIDictation.mm b/package/shared-native/ios/AIDictation.mm index 5c657e9668..c3b0302ba4 100644 --- a/package/shared-native/ios/AIDictation.mm +++ b/package/shared-native/ios/AIDictation.mm @@ -17,6 +17,7 @@ static NSString *const AIDictationOnStateEventName = @"AIDictationOnState"; static NSString *const AIDictationOnResultEventName = @"AIDictationOnResult"; static NSString *const AIDictationOnErrorEventName = @"AIDictationOnError"; +static NSString *const AIDictationOnLevelEventName = @"AIDictationOnLevel"; @interface AIDictation () @end @@ -46,7 +47,12 @@ - (instancetype)init - (NSArray *)supportedEvents { - return @[ AIDictationOnStateEventName, AIDictationOnResultEventName, AIDictationOnErrorEventName ]; + return @[ + AIDictationOnStateEventName, + AIDictationOnResultEventName, + AIDictationOnErrorEventName, + AIDictationOnLevelEventName, + ]; } - (std::shared_ptr)getTurboModule: @@ -109,6 +115,11 @@ - (void)emitOnError:(NSDictionary *)value [self sendEventWithName:AIDictationOnErrorEventName body:value]; } +- (void)emitOnLevel:(NSDictionary *)value +{ + [self sendEventWithName:AIDictationOnLevelEventName body:value]; +} + @end #endif diff --git a/package/shared-native/ios/AIDictationModule.swift b/package/shared-native/ios/AIDictationModule.swift index b753af2c93..f933e8ed43 100644 --- a/package/shared-native/ios/AIDictationModule.swift +++ b/package/shared-native/ios/AIDictationModule.swift @@ -15,6 +15,7 @@ import React func emitOnState(_ value: NSDictionary) func emitOnResult(_ value: NSDictionary) func emitOnError(_ value: NSDictionary) + func emitOnLevel(_ value: NSDictionary) } @objc(AIDictationModule) @@ -67,6 +68,19 @@ public class AIDictationModule: NSObject { // Timer used for the simple silence monitor. private var silenceTimer: Timer? + // When the microphone level was last emitted, read and written only on the audio tap's thread. + private var lastLevelTime: TimeInterval = 0 + // The audio session as dictation found it, restored once dictation ends. Its `.record` + // category mutes all playback, so leaving it in place would silence the app. + private var savedAudioSession: ( + category: AVAudioSession.Category, + mode: AVAudioSession.Mode, + options: AVAudioSession.CategoryOptions + )? + // The loudness, in dBFS, that maps to a level of 0 and of 1. + private let LEVEL_FLOOR_DB: Float = -60 + private let LEVEL_CEILING_DB: Float = -35 + // Set while the permission prompts of the first `start()` are on screen. `stop()` and // `cancel()` clear it, which abandons that start. private var isAwaitingPermission: Bool = false @@ -194,6 +208,7 @@ public class AIDictationModule: NSObject { try self.startRecognition() } catch { self.isListening = false + self.stopAudioEngine() self.emitState(self.STATE_IDLE) self.emitError( self.ERROR_INTERNAL, @@ -282,15 +297,34 @@ public class AIDictationModule: NSObject { * - Uses `.record` category and `.measurement` mode for best recognition * quality. * - Ducks other audio and notifies other sessions on deactivation. + * - Remembers the session's previous configuration for `restoreAudioSession()`. * * Throws if the session cannot be configured or activated. */ private func setupAudioSession() throws { let session = AVAudioSession.sharedInstance() + if savedAudioSession == nil { + savedAudioSession = (session.category, session.mode, session.categoryOptions) + } try session.setCategory(.record, mode: .measurement, options: [.duckOthers]) try session.setActive(true, options: .notifyOthersOnDeactivation) } + /** + * Gives the audio session back as dictation found it: deactivates it, so audio that was ducked + * resumes, and restores its category, mode and options, so the app can play sound again. + * + * Failures are ignored: deactivation fails while other audio in the app is still running, and + * restoring the category is still worth trying then. + */ + private func restoreAudioSession() { + guard let saved = savedAudioSession else { return } + savedAudioSession = nil + let session = AVAudioSession.sharedInstance() + try? session.setActive(false, options: .notifyOthersOnDeactivation) + try? session.setCategory(saved.category, mode: saved.mode, options: saved.options) + } + /** * Starts a fresh `SFSpeechRecognitionTask` and connects it to `AVAudioEngine`. * @@ -324,6 +358,7 @@ public class AIDictationModule: NSObject { inputNode.installTap(onBus: 0, bufferSize: 1024, format: recordingFormat) { [weak self] (buffer, _) in self?.recognitionRequest?.append(buffer) + self?.measureLevel(buffer) } audioEngine.prepare() @@ -420,13 +455,15 @@ public class AIDictationModule: NSObject { } /** - * Stops the audio engine and removes any installed tap on the input node. + * Stops the audio engine, removes any installed tap on the input node and gives the audio + * session back. */ private func stopAudioEngine() { if audioEngine.isRunning { audioEngine.stop() audioEngine.inputNode.removeTap(onBus: 0) } + restoreAudioSession() } // ------------------------------------------------------------------------- @@ -549,6 +586,35 @@ public class AIDictationModule: NSObject { eventSink?.emitOnState(payload) } + /** + * Emits the microphone's level to JS for the voice visualizations, from 0 to 1, about 30 times + * a second at most. Called on the audio tap's thread. + * + * The level is the buffer's loudness in dBFS mapped onto 0 to 1. The session records in + * `.measurement` mode, which turns off the input's gain processing, so speech arrives quiet: + * about -52 to -44 dBFS at a normal distance on an iPhone. A quiet room (about -60 dBFS) reads + * 0 and loud speech close to the phone (about -35 dBFS) reads 1, so talking lands around the + * middle. + */ + private func measureLevel(_ buffer: AVAudioPCMBuffer) { + let now = ProcessInfo.processInfo.systemUptime + guard now - lastLevelTime >= 1.0 / 30.0, + let samples = buffer.floatChannelData?[0], + buffer.frameLength > 0 else { return } + lastLevelTime = now + let count = Int(buffer.frameLength) + var sum: Float = 0 + for i in 0..` | no | The level of whoever is talking, from 0 to 1, read once a frame on the UI thread. Feed it at 15 Hz or more for the motion to follow the words. While it is `null` or absent, speech animates on its own. | +| `colors` | `string[]` | no | One to five colors. Past five are ignored, and colors that cannot be read are skipped. Defaults to the theme's `voiceVisualization.colors`. | +| `expressivity` | `number` | no | From 0 to 2: how far and how fast it moves. Colors never change with it. Defaults to `1`. | + +```tsx +import { useDictationAudioLevel, VoiceGlow, VoiceWave } from 'stream-chat-react-native/ai-components'; + +const VoiceAgent = ({ agentState, isListening }) => { + const micLevel = useDictationAudioLevel(); + + return ( + <> + + + + ); +}; +``` + +The motion runs entirely on the UI thread with Reanimated and Skia, so the visualizations stay smooth while the JS thread is busy streaming a reply, and a changing `audioLevel` re-renders nothing. + +**Audio level.** `useDictationAudioLevel()` gives the microphone's level while `useDictation` (or the `SpeechToTextButton`) is listening, and `null` otherwise. For any other source, such as a remote participant's audio level from Stream Video, keep a shared value of your own and write to it: `const level = useSharedValue(null)`, then `level.value = participant.audioLevel`. + +**Interaction.** Tapping the shape bounces it and sends out a ripple, and a hovering mouse or pen ripples it too. Touches are only observed, never claimed, so a `Pressable` around the visualization still gets its press; touches beside the shape reach whatever is behind it. + +**Theming.** The theme's `voiceVisualization.colors` are the default palette (the accent color unless you set them). The visualizations draw for a light or a dark background following `voiceVisualization.backdropColor` when you set it, and the system color scheme otherwise. + +**Accessibility.** Each is an image named "Voice agent" with its state as the value ("Ready", "Connecting", "Listening", "Thinking", "Speaking"), as on iOS and Android. Pass `accessibilityLabel` and `accessibilityValue` to change them, for example to translate them. Under the system's Reduce Motion (animations off on Android), they barely move and ignore taps. + +**Differences from the native SDKs.** The glow has no `rendering` option: the shader runs on every Android version through Skia, so there is no Canvas fallback, and it does not switch to a cheaper look in battery saver as on Android. A visualization keeps animating while it is mounted, even when scrolled off screen; it pauses only while the app is in the background. + ### Theming Each one of the components in the SDK is fully theme-compatible. The `StreamTheme` provider takes care of this for you. diff --git a/package/src/ai-components/__tests__/VoiceVisualization.test.tsx b/package/src/ai-components/__tests__/VoiceVisualization.test.tsx new file mode 100644 index 0000000000..8ab768a807 --- /dev/null +++ b/package/src/ai-components/__tests__/VoiceVisualization.test.tsx @@ -0,0 +1,199 @@ +import React from 'react'; +import { AccessibilityInfo, Pressable } from 'react-native'; + +import { Skia } from '@shopify/react-native-skia'; +import { act, fireEvent, render, renderHook, screen } from '@testing-library/react-native'; + +import { type DeepPartial, StreamTheme, type Theme, VoiceGlow, VoiceWave } from '../index'; +import { rgbToHsv } from '../voice-visualization/palette'; +import { GLOW_STYLE, WAVE_STYLE } from '../voice-visualization/styles'; +import type { VoiceVisualizationProps } from '../voice-visualization/types'; +import { useVoiceVisualization } from '../voice-visualization/useVoiceVisualization'; +import { waveColors, wavePath } from '../voice-visualization/waveGeometry'; + +const themed = + (style?: DeepPartial) => + ({ children }: { children: React.ReactNode }) => ( + {children} + ); + +const layout = (width: number, height: number) => ({ + nativeEvent: { layout: { height, width, x: 0, y: 0 } }, +}); + +const renderVisualization = ( + props: Partial = {}, + style?: DeepPartial, + visualStyle = GLOW_STYLE, +) => + renderHook( + (hookProps: VoiceVisualizationProps) => useVoiceVisualization(hookProps, visualStyle), + { initialProps: { state: 'idle', ...props }, wrapper: themed(style) }, + ); + +const hueOf = (color: [number, number, number]) => rgbToHsv(color).hue; + +describe('VoiceGlow and VoiceWave', () => { + afterEach(() => jest.restoreAllMocks()); + + it.each([ + ['VoiceGlow', VoiceGlow], + ['VoiceWave', VoiceWave], + ])('%s names itself and its state, as on iOS and Android', (_, Component) => { + const { rerender } = render(, { wrapper: themed() }); + + const root = screen.getByLabelText('Voice agent'); + expect(root.props.accessibilityRole).toBe('image'); + expect(root.props.accessibilityValue).toEqual({ text: 'Ready' }); + + rerender(); + expect(screen.getByLabelText('Voice agent').props.accessibilityValue).toEqual({ + text: 'Speaking', + }); + }); + + it('takes a label and value of its own', () => { + render( + , + { wrapper: themed() }, + ); + + expect(screen.getByLabelText('Assistant').props.accessibilityValue).toEqual({ + text: 'Talking', + }); + }); + + it.each([ + ['VoiceGlow', VoiceGlow], + ['VoiceWave', VoiceWave], + ])('%s draws once it has a size', (_, Component) => { + render(, { wrapper: themed() }); + expect(screen.queryByTestId('skia-canvas')).toBeNull(); + + fireEvent(screen.getByLabelText('Voice agent'), 'layout', layout(300, 200)); + + expect(screen.getByTestId('skia-canvas')).toBeTruthy(); + expect(screen.getByTestId('voice-visualization-hit-area')).toBeTruthy(); + }); + + it('lets a press on the shape reach a parent', () => { + const onPress = jest.fn(); + render( + + + , + { wrapper: themed() }, + ); + fireEvent(screen.getByLabelText('Voice agent'), 'layout', layout(300, 300)); + + const hitArea = screen.getByTestId('voice-visualization-hit-area'); + fireEvent(hitArea, 'touchEnd', { nativeEvent: { locationX: 96, locationY: 96 } }); + fireEvent.press(hitArea); + + expect(onPress).toHaveBeenCalledTimes(1); + }); + + it('compiles the glow shader once for every glow', () => { + const make = jest.spyOn(Skia.RuntimeEffect, 'Make'); + render( + <> + + + , + { wrapper: themed() }, + ); + fireEvent(screen.getByTestId('first'), 'layout', layout(200, 200)); + fireEvent(screen.getByTestId('second'), 'layout', layout(200, 200)); + + expect(make.mock.calls.length).toBeLessThanOrEqual(1); + }); +}); + +describe('voice visualization motion', () => { + afterEach(() => jest.restoreAllMocks()); + + it('uses its own colors first', () => { + const { result } = renderVisualization({ colors: ['#ff0000', 'nope', '#00ff00'] }); + + expect(result.current.driver.value.colors[0].hue).toBe(hueOf([1, 0, 0])); + expect(result.current.driver.value.colors[1].hue).toBe(hueOf([0, 1, 0])); + expect(result.current.driver.value.count).toBe(2); + }); + + it('falls back to the theme, then to the default accent', () => { + const fromTheme = renderVisualization( + { colors: [] }, + { + voiceVisualization: { colors: ['#00ff00'] }, + }, + ); + expect(fromTheme.result.current.driver.value.colors[0].hue).toBe(hueOf([0, 1, 0])); + + const fallback = renderVisualization( + { colors: ['nope'] }, + { + voiceVisualization: { colors: ['also-nope'] }, + }, + ); + expect(fallback.result.current.driver.value.colors[0].hue).toBe(hueOf([0, 95 / 255, 1])); + }); + + it('draws for a dark backdrop', () => { + const light = renderVisualization({}, { voiceVisualization: { backdropColor: '#ffffff' } }); + const dark = renderVisualization({}, { voiceVisualization: { backdropColor: '#101010' } }); + + expect(light.result.current.dark.value).toBe(0); + expect(dark.result.current.dark.value).toBe(1); + }); + + it('takes the wave’s smooth level response and the glow’s steady one', () => { + const glow = renderVisualization({}, undefined, GLOW_STYLE); + const wave = renderVisualization({}, undefined, WAVE_STYLE); + + expect(glow.result.current.driver.value.levelResponse).toBe(GLOW_STYLE.levelResponse); + expect(wave.result.current.driver.value.levelResponse).toBe(WAVE_STYLE.levelResponse); + }); + + it('starts with a live level when one is given', () => { + const { result } = renderVisualization({ audioLevel: { value: 0.6 } as never }); + + expect(result.current.driver.value.voice).toBe(0); + expect(result.current.driver.value.mic).toBe(0); + }); + + it('bounces on a tap, unless motion is reduced', async () => { + const motion = renderVisualization(); + act(() => motion.result.current.tap(0.1, 0.1)); + expect(motion.result.current.driver.value.tapAge).toBe(0); + expect(motion.result.current.driver.value.tapX).toBe(0.1); + + jest.spyOn(AccessibilityInfo, 'isReduceMotionEnabled').mockResolvedValueOnce(true); + const reduced = renderVisualization(); + await act(async () => { + await Promise.resolve(); + }); + act(() => reduced.result.current.tap(0.1, 0.1)); + expect(reduced.result.current.driver.value.tapAge).toBeGreaterThan(0); + }); + + it('draws the wave as one stroke with its ends faded out', () => { + const { result } = renderVisualization({ state: 'speaking' }, undefined, WAVE_STYLE); + const d = result.current.driver.value; + + const path = wavePath(d, 300, 200) as unknown as { commands: Array<[string, number, number]> }; + expect(path.commands).toHaveLength(161); + expect(path.commands[0][0]).toBe('M'); + expect(path.commands[0][1]).toBeCloseTo(150 - 0.42 * 200, 4); + expect(path.commands[160][1]).toBeCloseTo(150 + 0.42 * 200, 4); + + const stops = waveColors(d, 0); + expect(stops).toHaveLength(13); + expect(stops[0][3]).toBe(0); + expect(stops[12][3]).toBe(0); + expect(stops[6][3]).toBeCloseTo(1, 4); + }); +}); diff --git a/package/src/ai-components/__tests__/dictation.test.tsx b/package/src/ai-components/__tests__/dictation.test.tsx index 83219584f9..0a745afeb1 100644 --- a/package/src/ai-components/__tests__/dictation.test.tsx +++ b/package/src/ai-components/__tests__/dictation.test.tsx @@ -1,7 +1,7 @@ import React from 'react'; import { DeviceEventEmitter } from 'react-native'; -import { act, fireEvent, render, screen } from '@testing-library/react-native'; +import { act, fireEvent, render, renderHook, screen } from '@testing-library/react-native'; import * as aiComponents from '../index'; import { AIDictation } from '../native/AIDictation'; @@ -162,5 +162,38 @@ describe('ai-components dictation', () => { state: 'idle', }); }); + + it('streams the microphone level while listening and clears it afterwards', () => { + mockNativeModule = createNativeModule(); + const levels: number[] = []; + const unsubscribe = AIDictation.addLevelListener(({ level }) => levels.push(level)); + const { result, unmount } = renderHook(() => aiComponents.useDictationAudioLevel()); + expect(result.current.value).toBeNull(); + + act(() => { + DeviceEventEmitter.emit('AIDictationOnState', { state: 'listening' }); + DeviceEventEmitter.emit('AIDictationOnLevel', { level: 0.4 }); + }); + expect(levels).toEqual([0.4]); + expect(result.current.value).toBe(0.4); + + act(() => { + DeviceEventEmitter.emit('AIDictationOnState', { state: 'idle' }); + }); + expect(result.current.value).toBeNull(); + + act(() => { + DeviceEventEmitter.emit('AIDictationOnLevel', { level: 0.7 }); + DeviceEventEmitter.emit('AIDictationOnError', { code: 'X', message: 'boom' }); + }); + expect(result.current.value).toBeNull(); + + unsubscribe(); + unmount(); + act(() => { + DeviceEventEmitter.emit('AIDictationOnLevel', { level: 0.9 }); + }); + expect(levels).toEqual([0.4, 0.7]); + }); }); }); diff --git a/package/src/ai-components/__tests__/voiceDriver.test.ts b/package/src/ai-components/__tests__/voiceDriver.test.ts new file mode 100644 index 0000000000..4c49a1a68d --- /dev/null +++ b/package/src/ai-components/__tests__/voiceDriver.test.ts @@ -0,0 +1,317 @@ +import { + bounceScale, + createBounce, + easeOut, + scaleAt, + startBounce, +} from '../voice-visualization/bounce'; +import { + advanceDriver, + createDriverState, + driverReact, + driverScale, + ease, + hoverDriver, + LEVEL_RESPONSE, + type LevelResponse, + NO_TAP, + tapDriver, + type VoiceDriverInput, + type VoiceDriverState, +} from '../voice-visualization/driver'; +import type { Hsv, VoiceAgentState } from '../voice-visualization/types'; + +// Ported from stream-chat-android's VoiceVisualizationDriverTest and VoiceVisualizationBounceTest, +// so the platforms keep moving alike. + +const blue: Hsv = { hue: 218 / 360, saturation: 1, value: 1 }; +const red: Hsv = { hue: 0, saturation: 1, value: 1 }; +const FRAME = 1 / 60; + +const input = ( + state: VoiceAgentState, + overrides: Partial> = {}, +): VoiceDriverInput => ({ + audioLevel: null, + colors: [blue], + expressivity: 1, + reduceMotion: false, + state, + ...overrides, +}); + +/** A driver with the input it reads each frame, as the view keeps them. */ +const createDriver = (initial: VoiceDriverInput, response?: LevelResponse) => { + const driver = { input: initial, state: createDriverState(initial, response) }; + return { + ...driver, + advance: (seconds: number) => advanceDriver(driver.state, driver.input, seconds), + run: (seconds: number) => { + for (let i = 0; i < Math.floor(seconds / FRAME); i++) { + advanceDriver(driver.state, driver.input, FRAME); + } + }, + setInput: (next: VoiceDriverInput) => { + driver.input = next; + }, + get d(): VoiceDriverState { + return driver.state; + }, + get input() { + return driver.input; + }, + }; +}; + +const expectNear = (actual: number, expected: number, tolerance: number) => + expect(Math.abs(actual - expected)).toBeLessThanOrEqual(tolerance); + +describe('voice visualization driver', () => { + it('starts settled on the state', () => { + const { d } = createDriver(input('speaking')); + + expect(driverReact(d)).toBe(1); + expect(d.load).toBe(0); + expect(d.flow).toBe(0); + expect(d.bright).toBe(1); + expect(d.time).toBe(0); + expect(d.count).toBe(1); + expect(driverScale(d)).toBe(1); + expect(d.tapAge).toBe(NO_TAP); + expect(d.colors[0]).toEqual(blue); + expect(d.colors[4]).toEqual(blue); + }); + + it('does not share its colors with the caller’s palette', () => { + const palette = [{ ...blue }]; + const driver = createDriver(input('idle', { colors: palette })); + driver.setInput(input('idle', { colors: [red] })); + + driver.run(1); + + expect(palette[0]).toEqual(blue); + }); + + it('eases toward the motion of a new state', () => { + const driver = createDriver(input('idle')); + driver.setInput(input('connecting')); + + driver.advance(FRAME); + expectNear(driver.d.load, 0.08, 1e-4); + driver.run(2); + expectNear(driver.d.load, 1, 1e-3); + expectNear(driverReact(driver.d), 0, 1e-3); + }); + + it('eases the same at any frame rate', () => { + const at60 = createDriver(input('idle')); + const at120 = createDriver(input('idle')); + at60.setInput(input('speaking')); + at120.setInput(input('speaking')); + + for (let i = 0; i < 30; i++) at60.advance(1 / 60); + for (let i = 0; i < 60; i++) at120.advance(1 / 120); + + expectNear(driverReact(at60.d), driverReact(at120.d), 1e-4); + expectNear(at60.d.time, at120.d.time, 1e-2); + }); + + it('clamps a long frame', () => { + const driver = createDriver(input('idle')); + + driver.advance(5); + + expectNear(driver.d.time, 0.05 * 0.4, 1e-5); + }); + + it('turns the flow only while thinking', () => { + const driver = createDriver(input('speaking')); + driver.run(1); + expect(driver.d.flowSpin).toBe(0); + + driver.setInput(input('thinking')); + driver.run(1); + expect(driver.d.flowSpin).toBeGreaterThan(0); + }); + + it('keeps the colors turning in every state', () => { + const driver = createDriver(input('idle')); + + driver.run(1); + + expect(driver.d.orbSpin).toBeGreaterThan(0); + }); + + it('scales amplitude and speed with expressivity', () => { + const lively = createDriver(input('speaking', { expressivity: 2 })); + const still = createDriver(input('speaking', { expressivity: 0 })); + + expectNear(driverReact(lively.d), 1.85, 1e-5); + expectNear(driverReact(still.d), 0.15, 1e-5); + lively.advance(0.05); + expectNear(lively.d.time, 0.05 * 1.25 * 1.4, 1e-5); + }); + + it('slows time and ignores the pointer under reduced motion', () => { + const driver = createDriver(input('idle', { reduceMotion: true })); + + tapDriver(driver.d, driver.input, 0.1, 0.1); + hoverDriver(driver.d, driver.input, 0.1, 0.1); + driver.advance(0.05); + + expectNear(driver.d.time, 0.05 * 0.07 * 0.4, 1e-6); + expect(driver.d.tapAge).toBeGreaterThan(NO_TAP); + expect(driverScale(driver.d)).toBe(1); + expect(driver.d.hoverAmount).toBe(0); + }); + + it('stops hovering once motion is reduced', () => { + const driver = createDriver(input('idle')); + hoverDriver(driver.d, driver.input, 0.1, 0.1); + driver.run(1); + + driver.setInput(input('idle', { reduceMotion: true })); + driver.run(1); + + expectNear(driver.d.hoverAmount, 0, 1e-3); + }); + + it('ripples from a tap and bounces', () => { + const driver = createDriver(input('idle')); + + tapDriver(driver.d, driver.input, 0.1, -0.2); + expect(driver.d.tapX).toBe(0.1); + expect(driver.d.tapY).toBe(-0.2); + expect(driver.d.tapAge).toBe(0); + + driver.advance(0.05); + expectNear(driver.d.tapAge, 0.05, 1e-6); + expect(driverScale(driver.d)).toBeLessThan(1); + driver.run(0.5); + expect(driverScale(driver.d)).toBe(1); + }); + + it('eases hovering in and out', () => { + const driver = createDriver(input('idle')); + + hoverDriver(driver.d, driver.input, 0.2, 0.1); + driver.run(1); + expectNear(driver.d.hoverAmount, 1, 1e-3); + expect(driver.d.hoverX).toBe(0.2); + + hoverDriver(driver.d, driver.input, null, null); + driver.run(1); + expectNear(driver.d.hoverAmount, 0, 1e-3); + expect(driver.d.hoverX).toBe(0.2); + }); + + it('lifts the motion to a live audio level', () => { + const driver = createDriver(input('idle', { audioLevel: 0.8 })); + + driver.run(2); + expectNear(driver.d.mic, 0.8, 1e-3); + expectNear(driver.d.voice, 1, 1e-3); + expectNear(driverReact(driver.d), 0.8, 1e-3); + + driver.setInput(input('idle')); + driver.run(2); + expectNear(driver.d.voice, 0, 1e-3); + expectNear(driverReact(driver.d), 0.12, 1e-3); + }); + + const settled = (response: LevelResponse, audioLevel: number) => { + const driver = createDriver(input('idle', { audioLevel }), response); + driver.run(1); + return driver; + }; + + it('catches up with a louder level within a word when smooth', () => { + const driver = settled(LEVEL_RESPONSE.smooth, 0); + + driver.setInput(input('idle', { audioLevel: 1 })); + driver.run(0.1); + expect(driver.d.mic).toBeGreaterThanOrEqual(0.5); + expect(driver.d.mic).toBeLessThanOrEqual(0.8); + driver.run(0.3); + expect(driver.d.mic).toBeGreaterThan(0.95); + }); + + it('lets a quieter level fall away over a pause, not a syllable, when smooth', () => { + const driver = settled(LEVEL_RESPONSE.smooth, 1); + + driver.setInput(input('idle', { audioLevel: 0 })); + driver.run(0.1); + expect(driver.d.mic).toBeGreaterThan(0.7); + driver.run(0.9); + expect(driver.d.mic).toBeLessThan(0.1); + }); + + it('follows a quieter level within a syllable when steady', () => { + const driver = settled(LEVEL_RESPONSE.steady, 1); + + driver.setInput(input('idle', { audioLevel: 0 })); + driver.run(0.1); + expect(driver.d.mic).toBeLessThan(0.1); + }); + + it('eases a hue the short way round', () => { + const driver = createDriver( + input('idle', { colors: [{ hue: 0.95, saturation: 1, value: 1 }] }), + ); + driver.setInput(input('idle', { colors: [{ hue: 0.05, saturation: 1, value: 1 }] })); + + driver.advance(FRAME); + expectNear(driver.d.colors[0].hue, 0.962, 1e-4); + driver.run(2); + expectNear(driver.d.colors[0].hue, 0.05, 1e-3); + }); + + it('repeats the first color in empty slots and crossfades the count', () => { + const driver = createDriver(input('idle', { colors: [blue, red] })); + expect(driver.d.colors[1]).toEqual(red); + expect(driver.d.colors[2]).toEqual(blue); + + driver.setInput(input('idle', { colors: [blue, red, red] })); + driver.advance(FRAME); + expectNear(driver.d.count, 2.12, 1e-4); + }); + + it('moves the per-frame share at 60 Hz', () => { + expectNear(ease(0.08, 1 / 60), 0.08, 1e-5); + expect(ease(0.08, 0)).toBe(0); + }); +}); + +describe('voice visualization bounce', () => { + it('follows the keyframes', () => { + expect(scaleAt(0)).toBe(1); + expectNear(scaleAt(0.15), 0.965, 1e-5); + expectNear(scaleAt(0.3), 0.93, 1e-5); + expectNear(scaleAt(0.62), 1.03, 1e-5); + expectNear(scaleAt(1), 1, 1e-5); + }); + + it('times them with the CSS ease-out', () => { + expectNear(easeOut(0), 0, 1e-4); + expectNear(easeOut(0.5), 0.684, 2e-3); + expectNear(easeOut(1), 1, 1e-4); + }); + + it('dips, overshoots and settles', () => { + const bounce = createBounce(); + expect(bounceScale(bounce)).toBe(1); + + startBounce(bounce); + const scales = Array.from({ length: 21 }, () => { + const scale = bounceScale(bounce); + bounce.age += 1 / 60; + return scale; + }); + + expectNear(scales[0], 1, 1e-9); + expectNear(Math.min(...scales), 0.93, 0.01); + expectNear(Math.max(...scales), 1.03, 0.01); + expect(scales.indexOf(Math.min(...scales))).toBeLessThan(scales.indexOf(Math.max(...scales))); + expect(bounceScale(bounce)).toBe(1); + }); +}); diff --git a/package/src/ai-components/__tests__/voicePalette.test.ts b/package/src/ai-components/__tests__/voicePalette.test.ts new file mode 100644 index 0000000000..fc2591b988 --- /dev/null +++ b/package/src/ai-components/__tests__/voicePalette.test.ts @@ -0,0 +1,124 @@ +import { audioLevelInput, expressivityInput, speechEnv } from '../voice-visualization/math'; +import { luminance, parseColor, rgbToHsv, toHsvPalette } from '../voice-visualization/palette'; +import { snoise } from '../voice-visualization/simplex'; +import { GLOW_STYLE, WAVE_STYLE } from '../voice-visualization/styles'; + +describe('voice visualization palette', () => { + it('keeps a color’s full hue, saturation and value', () => { + const hsv = rgbToHsv([0, 95 / 255, 1]); + + expect(hsv.hue * 360).toBeCloseTo(217.6, 1); + expect(hsv.saturation).toBeCloseTo(1, 4); + expect(hsv.value).toBeCloseTo(1, 4); + }); + + it('keeps a muted color muted', () => { + // hsv(108°, 25%, 60%) + const hsv = rgbToHsv([0.48, 0.6, 0.45]); + + expect(hsv.hue).toBeCloseTo(0.3, 2); + expect(hsv.saturation).toBeCloseTo(0.25, 2); + expect(hsv.value).toBeCloseTo(0.6, 4); + }); + + it('gives a gray no saturation', () => { + const hsv = rgbToHsv([0.5, 0.5, 0.5]); + + expect(hsv.saturation).toBe(0); + expect(hsv.value).toBe(0.5); + }); + + it.each([ + ['#005fff', [0, 95 / 255, 1]], + ['#0af', [0, 170 / 255, 1]], + ['#005fff80', [0, 95 / 255, 1]], + ['rgb(0, 95, 255)', [0, 95 / 255, 1]], + ['rgba(0, 95, 255, 0.5)', [0, 95 / 255, 1]], + ['hsl(0, 100%, 50%)', [1, 0, 0]], + ['rebeccapurple', [0.4, 0.2, 0.6]], + ['#000000', [0, 0, 0]], + ])('reads %s', (color, expected) => { + const rgb = parseColor(color); + expect(rgb).not.toBeNull(); + rgb?.forEach((channel, i) => expect(channel).toBeCloseTo(expected[i], 2)); + }); + + it('skips colors it cannot read', () => { + expect(parseColor('not-a-color')).toBeNull(); + expect(parseColor('#12345')).toBeNull(); + expect(parseColor('')).toBeNull(); + expect(toHsvPalette(['nope', '#ff0000'])).toHaveLength(1); + }); + + it('keeps only the first five colors', () => { + expect(toHsvPalette(Array.from({ length: 7 }, () => '#ff0000'))).toHaveLength(5); + }); + + it('measures luminance for the light or dark look', () => { + expect(luminance('#ffffff')).toBeCloseTo(1, 4); + expect(luminance('#000000')).toBe(0); + expect(luminance('#1c1e22')).toBeLessThan(0.5); + expect(luminance('')).toBeNull(); + }); +}); + +describe('voice visualization inputs', () => { + it('clamps expressivity to 0-2 and replaces one that is not a number', () => { + expect(expressivityInput(3)).toBe(2); + expect(expressivityInput(-1)).toBe(0); + expect(expressivityInput(0.5)).toBe(0.5); + expect(expressivityInput(Number.NaN)).toBe(1); + expect(expressivityInput(Number.POSITIVE_INFINITY)).toBe(1); + expect(expressivityInput(undefined)).toBe(1); + }); + + it('clamps the audio level to 0-1 and treats one that is not a number as no audio', () => { + expect(audioLevelInput(2)).toBe(1); + expect(audioLevelInput(-1)).toBe(0); + expect(audioLevelInput(0.4)).toBe(0.4); + expect(audioLevelInput(Number.NaN)).toBeNull(); + expect(audioLevelInput(null)).toBeNull(); + expect(audioLevelInput(undefined)).toBeNull(); + }); +}); + +describe('voice visualization noise', () => { + it('matches the shader’s simplex noise as a GPU computes it', () => { + // Read back from the shader's own snoise on a GPU (stream-chat-react's fixtures). + const gpu: Array<[number, number, number, number]> = [ + [0.3, 21, 0, 0.7696], + [1.7, 5, 0, 0.208], + [0.12, 0, 0, 0.049], + [2.5, 9, 0, 0.3869], + [10.2, 14, 0, 0.603], + [3.3, 1.1, 0.9, -0.7541], + [0.05, 0.05, 0.05, -0.5481], + ]; + for (const [x, y, z, expected] of gpu) { + expect(snoise(x, y, z)).toBeCloseTo(expected, 2); + } + }); + + it('gives speech phrases and pauses', () => { + const samples = Array.from({ length: 2000 }, (_, i) => speechEnv(i * 0.01)); + + expect(Math.min(...samples)).toBe(0); + expect(Math.max(...samples)).toBeGreaterThan(0.6); + expect(Math.max(...samples)).toBeLessThanOrEqual(1); + }); +}); + +describe('voice visualization hit areas', () => { + it('takes the globe within a third of the shorter side', () => { + expect(GLOW_STYLE.contains(0, 0)).toBe(true); + expect(GLOW_STYLE.contains(0.2, 0.2)).toBe(true); + expect(GLOW_STYLE.contains(0.3, 0.3)).toBe(false); + }); + + it('takes the string across, and only near its line', () => { + expect(WAVE_STYLE.contains(0.45, 0)).toBe(true); + expect(WAVE_STYLE.contains(0, 0.1)).toBe(true); + expect(WAVE_STYLE.contains(0, 0.2)).toBe(false); + expect(WAVE_STYLE.contains(0.55, 0)).toBe(false); + }); +}); diff --git a/package/src/ai-components/index.ts b/package/src/ai-components/index.ts index c0eec23884..f037bbfa42 100644 --- a/package/src/ai-components/index.ts +++ b/package/src/ai-components/index.ts @@ -5,6 +5,7 @@ export * from './charts'; export * from './message-composer'; export * from './StreamingMessageView'; export * from './AITypingIndicatorView'; +export * from './voice-visualization'; // components export * from './components'; @@ -15,7 +16,8 @@ export * from './contexts'; // services export * from './services'; export * from './transcription/useDictation'; -export { isDictationAvailable } from './native/AIDictation'; +export * from './transcription/useDictationAudioLevel'; +export { isDictationAvailable, type DictationLevel } from './native/AIDictation'; // state-stores export * from './store'; diff --git a/package/src/ai-components/native/AIDictation.ts b/package/src/ai-components/native/AIDictation.ts index 450abae7c8..47ad2437ef 100644 --- a/package/src/ai-components/native/AIDictation.ts +++ b/package/src/ai-components/native/AIDictation.ts @@ -11,6 +11,10 @@ import NativeAIDictation, { const ON_STATE_EVENT = 'AIDictationOnState'; const ON_RESULT_EVENT = 'AIDictationOnResult'; const ON_ERROR_EVENT = 'AIDictationOnError'; +const ON_LEVEL_EVENT = 'AIDictationOnLevel'; + +/** The microphone's level while dictation listens, from 0 to 1. */ +export type DictationLevel = { level: number }; let eventEmitter: NativeEventEmitter | undefined; const getEventEmitter = () => { @@ -74,4 +78,11 @@ export const AIDictation = { ); return sub ? () => sub.remove() : noop; }, + /** The microphone's level while dictation listens, about 10 to 30 times a second. */ + addLevelListener(listener: (payload: DictationLevel) => void) { + const sub = getEventEmitter()?.addListener(ON_LEVEL_EVENT, (event) => + listener(event as DictationLevel), + ); + return sub ? () => sub.remove() : noop; + }, }; diff --git a/package/src/ai-components/store/index.ts b/package/src/ai-components/store/index.ts index f737510a79..12ae1a9e26 100644 --- a/package/src/ai-components/store/index.ts +++ b/package/src/ai-components/store/index.ts @@ -9,3 +9,5 @@ export * from './composer/useComposerText'; // dictation export * from './dictation/store'; +export * from './dictation/useDictationState'; +export * from './dictation/useDictationTranscript'; diff --git a/package/src/ai-components/theme.ts b/package/src/ai-components/theme.ts index 8d61874359..c2047a2d7b 100644 --- a/package/src/ai-components/theme.ts +++ b/package/src/ai-components/theme.ts @@ -56,6 +56,19 @@ export type Theme = { aiTypingIndicatorView: { text: TextStyle; }; + /** + * `VoiceGlow` and `VoiceWave`. Optional, so a hand-built `Theme` without it keeps type-checking; + * the visualizations fall back to the default accent then. + */ + voiceVisualization?: { + /** One to five colors, used when a visualization is given no `colors` of its own. */ + colors: string[]; + /** + * The color the visualizations sit on, which tells them to draw for a light or a dark + * background. Defaults to the system's color scheme. + */ + backdropColor?: string; + }; }; export const defaultTheme: Theme = { @@ -135,4 +148,7 @@ export const defaultTheme: Theme = { aiTypingIndicatorView: { text: {}, }, + voiceVisualization: { + colors: [colors.accent_blue], + }, }; diff --git a/package/src/ai-components/transcription/useDictationAudioLevel.ts b/package/src/ai-components/transcription/useDictationAudioLevel.ts new file mode 100644 index 0000000000..16574389a7 --- /dev/null +++ b/package/src/ai-components/transcription/useDictationAudioLevel.ts @@ -0,0 +1,38 @@ +import { useEffect } from 'react'; + +import { type SharedValue, useSharedValue } from 'react-native-reanimated'; + +import { AIDictation } from '../native/AIDictation'; + +/** + * The microphone's live level while dictation listens, from 0 to 1, for the `audioLevel` of + * `VoiceWave` and `VoiceGlow`. It is `null` while dictation is not listening, which lets a + * visualization animate speech on its own. + * + * The level is a shared value rather than state, so a level changing many times a second + * re-renders nothing: the visualizations read it once a frame on the UI thread. + */ +export const useDictationAudioLevel = (): SharedValue => { + const level = useSharedValue(null); + + useEffect(() => { + const unsubscribeLevel = AIDictation.addLevelListener((payload) => { + level.value = typeof payload?.level === 'number' ? payload.level : null; + }); + const unsubscribeState = AIDictation.addStateListener(({ state }) => { + if (state !== 'listening' && state !== 'starting') level.value = null; + }); + const unsubscribeError = AIDictation.addErrorListener(() => { + level.value = null; + }); + + return () => { + unsubscribeLevel(); + unsubscribeState(); + unsubscribeError(); + level.value = null; + }; + }, [level]); + + return level; +}; diff --git a/package/src/ai-components/voice-visualization/VoiceGlow.tsx b/package/src/ai-components/voice-visualization/VoiceGlow.tsx new file mode 100644 index 0000000000..f521ce2444 --- /dev/null +++ b/package/src/ai-components/voice-visualization/VoiceGlow.tsx @@ -0,0 +1,79 @@ +import React from 'react'; + +import { useDerivedValue } from 'react-native-reanimated'; + +import { Canvas, Fill, Shader } from '@shopify/react-native-skia'; + +import { driverReact, driverScale } from './driver'; +import { getGlowEffect } from './glowShader'; +import { GLOW_STYLE } from './styles'; +import type { VoiceVisualizationProps } from './types'; +import type { VoiceVisualization } from './useVoiceVisualization'; +import { type VoiceCanvasSize, VoiceVisualizationView } from './VoiceVisualizationView'; + +export type VoiceGlowProps = VoiceVisualizationProps; + +const GlowCanvas = ({ + size: { height, width }, + visualization: { dark, driver }, +}: { + size: VoiceCanvasSize; + visualization: VoiceVisualization; +}) => { + const effect = getGlowEffect(); + const uniforms = useDerivedValue(() => { + const d = driver.value; + const color = (i: number) => [d.colors[i].hue, d.colors[i].saturation, d.colors[i].value, 0]; + return { + bright: d.bright, + col0: color(0), + col1: color(1), + col2: color(2), + col3: color(3), + col4: color(4), + count: d.count, + dark: dark.value, + flow: d.flow, + flowSpin: d.flowSpin, + hover: [d.hoverX, d.hoverY], + hoverAmt: d.hoverAmount, + load: d.load, + mic: d.mic, + orbSpin: d.orbSpin, + react: driverReact(d), + resolution: [width, height], + sat: d.sat, + scale: driverScale(d), + tap: [d.tapX, d.tapY], + tapTime: d.tapAge, + time: d.time, + voice: d.voice, + }; + }); + + if (!effect) return null; + + return ( + + + + + + ); +}; + +const renderGlow = (visualization: VoiceVisualization, size: VoiceCanvasSize) => ( + +); + +/** + * A voice agent as a soft, airy globe: color masses orbit inside it and swell with speech, and the + * gaps between them let the background show through. Tapping the globe bounces it and ripples it + * from the touch, and so does a hovering mouse or pen. + * + * It has no size of its own and draws in the middle of the view it is given, about half as wide + * as its shorter side, so give it a square. + */ +export const VoiceGlow = (props: VoiceGlowProps) => ( + +); diff --git a/package/src/ai-components/voice-visualization/VoiceVisualizationView.tsx b/package/src/ai-components/voice-visualization/VoiceVisualizationView.tsx new file mode 100644 index 0000000000..288120ea64 --- /dev/null +++ b/package/src/ai-components/voice-visualization/VoiceVisualizationView.tsx @@ -0,0 +1,174 @@ +import React, { type ReactNode, useCallback, useMemo, useState } from 'react'; +import { + type GestureResponderEvent, + type LayoutChangeEvent, + PixelRatio, + type PointerEvent, + StyleSheet, + View, +} from 'react-native'; + +import Animated from 'react-native-reanimated'; + +import type { VoiceVisualizationStyle } from './styles'; +import type { VoiceAgentState, VoiceVisualizationProps } from './types'; +import { useVoiceVisualization, type VoiceVisualization } from './useVoiceVisualization'; + +/** What the renderer draws in: the canvas size, in its own units, and the screen's pixels per unit. */ +export type VoiceCanvasSize = { height: number; pixelRatio: number; width: number }; + +type ViewProps = VoiceVisualizationProps & { + /** Draws the visualization into a canvas of the given size. */ + renderCanvas: (visualization: VoiceVisualization, size: VoiceCanvasSize) => ReactNode; + visualStyle: VoiceVisualizationStyle; +}; + +const STATE_DESCRIPTIONS: Record = { + connecting: 'Connecting', + idle: 'Ready', + listening: 'Listening', + speaking: 'Speaking', + thinking: 'Thinking', +}; + +/** + * What VoiceWave and VoiceGlow share: the motion, the canvas kept at the view's size and at most + * the style's density, reduced motion, taps and hover, and accessibility. + * + * It names itself "Voice agent" with its state as the value, as on iOS and Android. Touches are + * only watched, never claimed, so a press handler on the view or a parent still gets them, and + * only the shape takes them: around it they reach whatever is behind. + */ +export const VoiceVisualizationView = ({ + accessibilityLabel = 'Voice agent', + accessibilityValue, + audioLevel, + colors, + expressivity, + onLayout, + pointerEvents = 'box-none', + renderCanvas, + state, + style, + visualStyle, + ...restProps +}: ViewProps) => { + const visualization = useVoiceVisualization( + { audioLevel, colors, expressivity, state }, + visualStyle, + ); + const { hover, opacity, tap } = visualization; + const [layout, setLayout] = useState({ height: 0, width: 0 }); + + const handleLayout = useCallback( + (event: LayoutChangeEvent) => { + onLayout?.(event); + const { height, width } = event.nativeEvent.layout; + setLayout((current) => + current.width === width && current.height === height ? current : { height, width }, + ); + }, + [onLayout], + ); + + const side = Math.min(layout.width, layout.height); + const pixelRatio = PixelRatio.get(); + // Drawn at no more than the style's density, then scaled up to the view: the glow's soft + // gradients look the same at 2x with fewer pixels to fill, while the wave's line needs 3x. + const density = Math.min(1, visualStyle.maxScale / pixelRatio); + const canvasSize = useMemo( + () => ({ + height: layout.height * density, + pixelRatio: pixelRatio * density, + width: layout.width * density, + }), + [density, layout.height, layout.width, pixelRatio], + ); + const hitWidth = visualStyle.hitArea.width * side; + const hitHeight = visualStyle.hitArea.height * side; + + /** A touch on the hit area, centred on the view and normalized to its shorter side, y up. */ + const toLocalPoint = useCallback( + (locationX: number, locationY: number) => ({ + x: (locationX - hitWidth / 2) / (side || 1), + y: (hitHeight / 2 - locationY) / (side || 1), + }), + [hitHeight, hitWidth, side], + ); + + const handleTouchEnd = useCallback( + (event: GestureResponderEvent) => { + const { x, y } = toLocalPoint(event.nativeEvent.locationX, event.nativeEvent.locationY); + if (visualStyle.contains(x, y)) tap(x, y); + }, + [tap, toLocalPoint, visualStyle], + ); + + const handlePointerMove = useCallback( + (event: PointerEvent) => { + // Only a hovering mouse or pen ripples it; a finger has no hover. + const { buttons, offsetX, offsetY, pointerType } = event.nativeEvent; + if (pointerType === 'touch') return; + const { x, y } = toLocalPoint(offsetX, offsetY); + const hovering = buttons === 0 && visualStyle.contains(x, y); + hover(hovering ? x : null, hovering ? y : null); + }, + [hover, toLocalPoint, visualStyle], + ); + + const handlePointerLeave = useCallback(() => hover(null, null), [hover]); + + return ( + + {side > 0 ? ( + <> + + {renderCanvas(visualization, canvasSize)} + + + + ) : null} + + ); +}; + +const styles = StyleSheet.create({ + canvas: { position: 'absolute' }, + hitArea: { position: 'absolute' }, +}); diff --git a/package/src/ai-components/voice-visualization/VoiceWave.tsx b/package/src/ai-components/voice-visualization/VoiceWave.tsx new file mode 100644 index 0000000000..b367c3ba05 --- /dev/null +++ b/package/src/ai-components/voice-visualization/VoiceWave.tsx @@ -0,0 +1,67 @@ +import React from 'react'; + +import { useDerivedValue } from 'react-native-reanimated'; + +import { Canvas, LinearGradient, Path, vec } from '@shopify/react-native-skia'; + +import { WAVE_STYLE } from './styles'; +import type { VoiceVisualizationProps } from './types'; +import type { VoiceVisualization } from './useVoiceVisualization'; +import { type VoiceCanvasSize, VoiceVisualizationView } from './VoiceVisualizationView'; +import { + WAVE_END, + WAVE_POSITIONS, + WAVE_START, + waveColors, + wavePath, + waveStrokeWidth, +} from './waveGeometry'; + +export type VoiceWaveProps = VoiceVisualizationProps; + +const WaveCanvas = ({ + size: { height, pixelRatio, width }, + visualization: { dark, driver }, +}: { + size: VoiceCanvasSize; + visualization: VoiceVisualization; +}) => { + const path = useDerivedValue(() => wavePath(driver.value, width, height)); + const colors = useDerivedValue(() => waveColors(driver.value, dark.value)); + const m = Math.min(width, height); + + return ( + + + + + + ); +}; + +const renderWave = (visualization: VoiceVisualization, size: VoiceCanvasSize) => ( + +); + +/** + * A voice agent as a single string, flat at its ends, carrying a wave packet that erupts with + * speech while its colors flow along it. Tapping the string ripples it from the touch, and so does + * a hovering mouse or pen. + * + * It has no size of its own and draws in the middle of the view it is given, about 80% as wide as + * its shorter side, so give it a square or a wide box. + */ +export const VoiceWave = (props: VoiceWaveProps) => ( + +); diff --git a/package/src/ai-components/voice-visualization/bounce.ts b/package/src/ai-components/voice-visualization/bounce.ts new file mode 100644 index 0000000000..7e4c50a6e1 --- /dev/null +++ b/package/src/ai-components/voice-visualization/bounce.ts @@ -0,0 +1,61 @@ +/** + * The bounce of a tap: a dip to 93% and an overshoot to 103% before settling, over 340 ms. + * + * These are authored keyframes rather than a spring: the ease-out curve times the whole run and + * the keyframes are linear between their stops. + */ +export const BOUNCE_DURATION = 0.34; + +const OFFSETS = [0, 0.3, 0.62, 1]; +const SCALES = [1, 0.93, 1.03, 1]; + +/** How long ago the last bounce started, in seconds; infinite before the first. */ +export type BounceState = { age: number }; + +export const createBounce = (): BounceState => ({ age: Number.POSITIVE_INFINITY }); + +export const startBounce = (bounce: BounceState) => { + 'worklet'; + bounce.age = 0; +}; + +export const advanceBounce = (bounce: BounceState, seconds: number) => { + 'worklet'; + bounce.age += seconds; +}; + +/** CSS `ease-out`, `cubic-bezier(0, 0, 0.58, 1)`. */ +export const easeOut = (x: number) => { + 'worklet'; + const bx = 3 * 0.58; + const ax = 1 - bx; + const by = 3; + const ay = 1 - by; + let low = 0; + let high = 1; + for (let i = 0; i < 24; i++) { + const middle = (low + high) / 2; + if ((ax * middle + bx) * middle * middle < x) low = middle; + else high = middle; + } + const t = (low + high) / 2; + return (ay * t + by) * t * t; +}; + +/** The keyframed scale at `progress`, from 0 to 1. */ +export const scaleAt = (progress: number) => { + 'worklet'; + for (let i = 1; i < OFFSETS.length; i++) { + if (progress <= OFFSETS[i]) { + const fraction = (progress - OFFSETS[i - 1]) / (OFFSETS[i] - OFFSETS[i - 1]); + return SCALES[i - 1] + (SCALES[i] - SCALES[i - 1]) * fraction; + } + } + return 1; +}; + +export const bounceScale = (bounce: BounceState) => { + 'worklet'; + if (bounce.age >= BOUNCE_DURATION) return 1; + return scaleAt(easeOut(Math.max(bounce.age, 0) / BOUNCE_DURATION)); +}; diff --git a/package/src/ai-components/voice-visualization/driver.ts b/package/src/ai-components/voice-visualization/driver.ts new file mode 100644 index 0000000000..7d9c7fc070 --- /dev/null +++ b/package/src/ai-components/voice-visualization/driver.ts @@ -0,0 +1,240 @@ +import { advanceBounce, type BounceState, bounceScale, createBounce, startBounce } from './bounce'; +import { MAXIMUM_COLORS } from './palette'; +import type { Hsv, VoiceAgentState } from './types'; + +/** + * The share of the way the live level moves each frame at 60 Hz, toward a louder or a quieter + * level. + */ +export type LevelResponse = { rise: number; fall: number }; + +export const LEVEL_RESPONSE = { + /** Follows the level closely, both ways. The glow's. */ + steady: { fall: 0.35, rise: 0.35 }, + /** + * Follows the level like a VU meter: about 0.1 s up to a louder reading, about 0.3 s down from + * it, so a meter's per-syllable jumps swell with the phrase instead. The wave's. + */ + smooth: { fall: 0.05, rise: 0.16 }, +} satisfies Record; + +export type VoiceDriverInput = { + state: VoiceAgentState; + /** One to five colors. */ + colors: Hsv[]; + /** From 0 to 2. */ + expressivity: number; + /** From 0 to 1, or null when no live audio drives the motion. */ + audioLevel: number | null; + reduceMotion: boolean; +}; + +/** + * The motion of a voice visualization: where it is in time, how it moves, and its colors. A plain + * object rather than a class, so it can live in a shared value and be advanced on the UI thread. + */ +export type VoiceDriverState = { + speed: number; + bright: number; + sat: number; + load: number; + flow: number; + baseReact: number; + expressivity: number; + time: number; + flowSpin: number; + orbSpin: number; + count: number; + tapX: number; + tapY: number; + tapAge: number; + hoverX: number; + hoverY: number; + hoverAmount: number; + isHovering: boolean; + mic: number; + voice: number; + /** The colors drawn, eased in place toward the input's color slots. */ + colors: Hsv[]; + bounce: BounceState; + levelResponse: LevelResponse; +}; + +/** + * How a state moves. `load`, `flow` and `react` each select a motion pattern, one per state, so + * easing between states crossfades the patterns. + */ +type VoiceMotion = { + bright: number; + flow: number; + load: number; + react: number; + sat: number; + speed: number; +}; + +const MOTIONS: Record = { + connecting: { bright: 1, flow: 0, load: 1, react: 0, sat: 1, speed: 1 }, + idle: { bright: 1, flow: 0, load: 0, react: 0.12, sat: 1, speed: 0.4 }, + listening: { bright: 1, flow: 0, load: 0, react: 0.45, sat: 1, speed: 0.9 }, + speaking: { bright: 1, flow: 0, load: 0, react: 1, sat: 1, speed: 1.25 }, + thinking: { bright: 1, flow: 1, load: 0, react: 0, sat: 1, speed: 1.25 }, +}; + +export const NO_TAP = 100; +const MAX_FRAME = 0.05; +/** How fast time runs under reduced motion. */ +export const REDUCED = 0.07; +const DRIVERS = 0.08; +const GENTLE = 0.12; +const VOICE = 0.1; + +/** Expressivity scales amplitude and speed, each with a floor so the motion breathes at 0. */ +export const scales = (expressivity: number) => { + 'worklet'; + return { amplitude: 0.15 + 0.85 * expressivity, speed: 0.6 + 0.4 * expressivity }; +}; + +/** + * The share of the way a value moves in `dt` seconds when it moves `perFrame` of the way each + * frame at 60 Hz. + */ +export const ease = (perFrame: number, dt: number) => { + 'worklet'; + return 1 - Math.pow(1 - perFrame, dt * 60); +}; + +const rem = (x: number, m: number) => { + 'worklet'; + return ((x % m) + m) % m; +}; + +/** Eases an HSV color in place, its hue taking the shorter way round the wheel. */ +const easeHsv = (color: Hsv, target: Hsv, fraction: number) => { + 'worklet'; + const hueStep = (rem(target.hue - color.hue + 1.5, 1) - 0.5) * fraction; + color.hue = rem(color.hue + hueStep + 1, 1); + color.saturation += (target.saturation - color.saturation) * fraction; + color.value += (target.value - color.value) * fraction; +}; + +/** The five color slots the renderers read; past the palette they repeat its first color. */ +export const slots = (colors: Hsv[]): Hsv[] => { + 'worklet'; + const first = colors[0] ?? { hue: 0, saturation: 0, value: 0 }; + const result: Hsv[] = []; + for (let i = 0; i < MAXIMUM_COLORS; i++) result.push(colors[i] ?? first); + return result; +}; + +/** A driver settled on `input`, so the first frame shows it rather than easing into it. */ +export const createDriverState = ( + input: VoiceDriverInput, + levelResponse: LevelResponse = LEVEL_RESPONSE.steady, +): VoiceDriverState => { + const m = MOTIONS[input.state]; + const s = scales(input.expressivity); + return { + baseReact: m.react * s.amplitude, + bounce: createBounce(), + bright: m.bright, + // Copies, as they are eased in place and the palette belongs to the caller. + colors: slots(input.colors).map((color) => ({ ...color })), + count: input.colors.length, + expressivity: input.expressivity, + flow: m.flow * s.amplitude, + flowSpin: 0, + hoverAmount: 0, + hoverX: 0, + hoverY: 0, + isHovering: false, + levelResponse, + load: m.load * s.amplitude, + mic: 0, + orbSpin: 0, + sat: m.sat, + speed: m.speed * s.speed, + tapAge: NO_TAP, + tapX: 0, + tapY: 0, + time: 0, + voice: 0, + }; +}; + +/** Live audio lifts the motion to its level, whatever the state. */ +export const driverReact = (d: VoiceDriverState) => { + 'worklet'; + return d.baseReact + (Math.max(d.baseReact, d.mic) - d.baseReact) * d.voice; +}; + +/** The scale of a tap's bounce. */ +export const driverScale = (d: VoiceDriverState) => { + 'worklet'; + return bounceScale(d.bounce); +}; + +/** A tap at (x, y), centred and normalized to the shorter side with y up. */ +export const tapDriver = (d: VoiceDriverState, input: VoiceDriverInput, x: number, y: number) => { + 'worklet'; + if (input.reduceMotion) return; + d.tapX = x; + d.tapY = y; + d.tapAge = 0; + startBounce(d.bounce); +}; + +/** Where a pointer hovers, or null once it has left. */ +export const hoverDriver = ( + d: VoiceDriverState, + input: VoiceDriverInput, + x: number | null, + y: number | null, +) => { + 'worklet'; + if (x === null || y === null || input.reduceMotion) { + d.isHovering = false; + return; + } + d.hoverX = x; + d.hoverY = y; + d.isHovering = true; +}; + +/** + * Advances the motion by `seconds`, easing each value toward the input's motion by a fixed share + * per frame at 60 Hz, converted to time, so it moves the same at any refresh rate. + */ +export const advanceDriver = (d: VoiceDriverState, input: VoiceDriverInput, seconds: number) => { + 'worklet'; + // Clamped so a frame after a stall cannot leap the motion. + const dt = Math.min(Math.max(seconds, 0), MAX_FRAME); + if (input.reduceMotion) d.isHovering = false; + const m = MOTIONS[input.state]; + const s = scales(input.expressivity); + const drivers = ease(DRIVERS, dt); + d.speed += (m.speed * s.speed - d.speed) * drivers; + d.bright += (m.bright - d.bright) * drivers; + d.sat += (m.sat - d.sat) * drivers; + d.load += (m.load * s.amplitude - d.load) * drivers; + d.flow += (m.flow * s.amplitude - d.flow) * drivers; + d.baseReact += (m.react * s.amplitude - d.baseReact) * drivers; + d.expressivity += (input.expressivity - d.expressivity) * drivers; + // Integrated rather than computed as time times a weight, so a change of state eases the turns + // instead of jumping their angles. + const step = dt * (input.reduceMotion ? REDUCED : 1) * d.speed; + d.time += step; + d.flowSpin += step * d.flow; + d.orbSpin += step * (1 + 0.5 * d.flow + 0.1 * d.baseReact); + d.tapAge += dt; + advanceBounce(d.bounce, dt); + const gentle = ease(GENTLE, dt); + d.hoverAmount += ((d.isHovering ? 1 : 0) - d.hoverAmount) * gentle; + d.voice += ((input.audioLevel === null ? 0 : 1) - d.voice) * ease(VOICE, dt); + const level = input.audioLevel ?? 0; + const response = level > d.mic ? d.levelResponse.rise : d.levelResponse.fall; + d.mic += (level - d.mic) * ease(response, dt); + const targets = slots(input.colors); + for (let i = 0; i < d.colors.length; i++) easeHsv(d.colors[i], targets[i], gentle); + d.count += (input.colors.length - d.count) * gentle; +}; diff --git a/package/src/ai-components/voice-visualization/glowShader.ts b/package/src/ai-components/voice-visualization/glowShader.ts new file mode 100644 index 0000000000..fe2d8019d6 --- /dev/null +++ b/package/src/ai-components/voice-visualization/glowShader.ts @@ -0,0 +1,227 @@ +import { Skia, type SkRuntimeEffect } from '@shopify/react-native-skia'; + +/** + * The glow's shader, SkSL: the AGSL glow of stream-chat-android (itself ported from the iOS Metal + * glow) as is, since AGSL is SkSL, so the platforms look alike. Every pixel is computed on the + * GPU: color masses warped by simplex noise, concentric waves, ripples from a tap or a hovering + * pointer, light and a halo. Pixels past the halo return at once, so only the glow itself costs. + */ +const SOURCE = ` +uniform float2 resolution; +uniform float4 col0; +uniform float4 col1; +uniform float4 col2; +uniform float4 col3; +uniform float4 col4; +uniform float time; +uniform float count; +uniform float bright; +uniform float sat; +uniform float load; +uniform float flow; +uniform float flowSpin; +uniform float orbSpin; +uniform float react; +uniform float dark; +uniform float mic; +uniform float voice; +uniform float scale; +uniform float2 tap; +uniform float tapTime; +uniform float2 hover; +uniform float hoverAmt; + +float sq(float x) { return x * x; } +float ss(float e0, float e1, float x) { + float t = clamp((x - e0) / (e1 - e0), 0.0, 1.0); + return t * t * (3.0 - 2.0 * t); +} +float3 hsv2rgb(float3 c) { + float4 K = float4(1.0, 2.0 / 3.0, 1.0 / 3.0, 3.0); + float3 p = abs(fract(c.xxx + K.xyz) * 6.0 - K.www); + return c.z * mix(K.xxx, clamp(p - K.xxx, 0.0, 1.0), c.y); +} +float3 desat(float3 c, float s) { + float l = dot(c, float3(0.299, 0.587, 0.114)); + return mix(float3(l), c, s); +} +float3 saturate3(float3 c, float s) { + float l = dot(c, float3(0.299, 0.587, 0.114)); + return clamp(mix(float3(l), c, s), 0.0, 1.0); +} +float2 rot2(float2 p, float a) { + float s = sin(a); float c = cos(a); + return float2(c * p.x + s * p.y, -s * p.x + c * p.y); +} +float3 vivid(float4 hsv) { + return hsv2rgb(float3(hsv.x, clamp(hsv.y + dark * 0.06, 0.0, 1.0), hsv.z)); +} +// Waves spreading out from the last tap, fading within a second and a half. +float tapRipple(float2 p) { + float age = tapTime; + if (age > 1.5) return 0.0; + float d = distance(p, tap); + float front = age * 0.75; + float env = exp(-sq((d - front) / 0.14)); + float decay = exp(-age * 2.6); + return sin(d * 34.0 - age * 24.0) * env * decay; +} +// A gentle ripple following a hovering pointer. +float hoverRipple(float2 p) { + if (hoverAmt <= 0.001) return 0.0; + float d = distance(p, hover); + float env = exp(-sq(d / 0.22)); + return sin(d * 30.0 - time * 7.0) * env * hoverAmt * 0.6; +} +float4 permute(float4 x) { return mod(((x * 34.0) + 1.0) * x, 289.0); } +float4 taylorInvSqrt(float4 r) { return 1.79284291400159 - 0.85373472095314 * r; } +float snoise(float3 v) { + const float2 C = float2(1.0 / 6.0, 1.0 / 3.0); + const float4 D = float4(0.0, 0.5, 1.0, 2.0); + float3 i = floor(v + dot(v, C.yyy)); + float3 x0 = v - i + dot(i, C.xxx); + float3 g = step(x0.yzx, x0.xyz); + float3 l = 1.0 - g; + float3 i1 = min(g.xyz, l.zxy); + float3 i2 = max(g.xyz, l.zxy); + float3 x1 = x0 - i1 + 1.0 * C.xxx; + float3 x2 = x0 - i2 + 2.0 * C.xxx; + float3 x3 = x0 - 1.0 + 3.0 * C.xxx; + i = mod(i, 289.0); + float4 p = permute(permute(permute( + i.z + float4(0.0, i1.z, i2.z, 1.0)) + + i.y + float4(0.0, i1.y, i2.y, 1.0)) + + i.x + float4(0.0, i1.x, i2.x, 1.0)); + float n_ = 1.0 / 7.0; + float3 ns = n_ * D.wyz - D.xzx; + float4 j = p - 49.0 * floor(p * ns.z * ns.z); + float4 x_ = floor(j * ns.z); + float4 y_ = floor(j - 7.0 * x_); + float4 x = x_ * ns.x + ns.yyyy; + float4 y = y_ * ns.x + ns.yyyy; + float4 h = 1.0 - abs(x) - abs(y); + float4 b0 = float4(x.xy, y.xy); + float4 b1 = float4(x.zw, y.zw); + float4 s0 = floor(b0) * 2.0 + 1.0; + float4 s1 = floor(b1) * 2.0 + 1.0; + float4 sh = -step(h, float4(0.0)); + float4 a0 = b0.xzyw + s0.xzyw * sh.xxyy; + float4 a1 = b1.xzyw + s1.xzyw * sh.zzww; + float3 p0 = float3(a0.xy, h.x); + float3 p1 = float3(a0.zw, h.y); + float3 p2 = float3(a1.xy, h.z); + float3 p3 = float3(a1.zw, h.w); + float4 norm = taylorInvSqrt(float4(dot(p0, p0), dot(p1, p1), dot(p2, p2), dot(p3, p3))); + p0 *= norm.x; p1 *= norm.y; p2 *= norm.z; p3 *= norm.w; + float4 m = max(0.6 - float4(dot(x0, x0), dot(x1, x1), dot(x2, x2), dot(x3, x3)), 0.0); + m = m * m; + return 42.0 * dot(m * m, float4(dot(p0, x0), dot(p1, x1), dot(p2, x2), dot(p3, x3))); +} +float speechEnv(float t) { + float gate = ss(-0.25, 0.35, snoise(float3(t * 0.6, 21.0, 0.0))); + float syl = 0.55 + 0.45 * snoise(float3(t * 3.0, 5.0, 0.0)); + return clamp(gate * syl, 0.0, 1.0); +} +float2 coords(float2 pos) { + float m = min(resolution.x, resolution.y); + return float2(pos.x - 0.5 * resolution.x, 0.5 * resolution.y - pos.y) / m / scale; +} + +half4 main(float2 fragCoord) { + float2 q = coords(fragCoord); + float t = time; + float r = length(q); + + float speak = ss(0.6, 1.0, react); + float speech = react * mix(0.55 + 0.25 * sin(t * 1.6), speechEnv(t), speak); + speech = mix(speech, mic, voice); + float speechSmooth = mix(speech, react * (0.55 + 0.25 * sin(t * 1.6)), 0.5 * speak); + float energy = 0.12 + 0.30 * react + 0.18 * flow + 0.28 * load + 0.06 * speechSmooth; + float R = 0.24 * (1.0 + speechSmooth * 0.20 + load * 0.05 * sin(t * 2.0) + react * 0.04 * sin(t * 1.6)); + float p = r / R; + if (p > 2.2) return half4(0.0); + + float core = exp(-2.0 * p * p); + float body = ss(1.06, 0.62, p); + + float2 P = q / R; + float pl = length(P); + float nt = t * 0.15 + flowSpin * 0.05; + float nAmp = 0.20 + 0.28 * energy; + float2 sp = P + nAmp * float2(snoise(float3(P * 0.7, nt)), snoise(float3(P * 0.7 + 4.7, nt))); + float ang = atan(P.y, P.x); + float wave = (0.018 + 0.025 * energy) * sin(pl * 5.0 - t * 2.0) * ss(0.0, 0.4, pl); + sp += wave * float2(-sin(ang), cos(ang)); + sp = rot2(sp, flowSpin * 0.30); + float2 gtoP = q - (hoverAmt > 0.001 ? hover : tap); + sp += (gtoP / (length(gtoP) + 1e-4)) * (tapRipple(q) + hoverRipple(q)) * 0.5; + float bt = t; + float a0 = orbSpin * 0.42 + 2.0 * snoise(float3(bt * 0.12, 0.0, 0.0)); + float a1 = -orbSpin * 0.34 + 2.0 * snoise(float3(bt * 0.10, 5.0, 0.0)) + 2.0; + float a2 = orbSpin * 0.30 + 2.0 * snoise(float3(bt * 0.11, 9.0, 0.0)) + 1.0; + float a3 = -orbSpin * 0.26 + 2.0 * snoise(float3(bt * 0.09, 14.0, 0.0)) + 3.5; + float2 c0 = 0.40 * float2(cos(a0), sin(a0)); + float2 c1 = 0.38 * float2(cos(a1), sin(a1)); + float2 c2c = 0.36 * float2(cos(a2), sin(a2)); + float2 c3c = 0.34 * float2(cos(a3), sin(a3)); + float b0 = ss(1.15, 0.1, length(sp - c0)); + float b1 = ss(1.30, 0.1, length(sp - c1)); + float b2 = ss(1.20, 0.1, length(sp - c2c)); + float b3 = ss(1.25, 0.1, length(sp - c3c)); + + float3 k0 = saturate3(vivid(col0), 1.7); + float3 kL = saturate3(clamp(vivid(col0) * 1.18, 0.0, 1.0), 1.3); + float3 kD = saturate3(vivid(col0) * 0.85, 1.3); + float3 colB = mix(kL, saturate3(vivid(col1), 1.7), clamp(count - 1.0, 0.0, 1.0)); + float3 colC = mix(kD, saturate3(vivid(col2), 1.7), clamp(count - 2.0, 0.0, 1.0)); + float3 colD = saturate3(vivid(col3), 1.7); + float3 colE = saturate3(vivid(col4), 1.7); + float3 base = saturate3(mix(vivid(col0), float3(1.0), 0.06), 1.3); + float3 col = base; + col = mix(col, k0, b0); + col = mix(col, colB, b1); + col = mix(col, colC, b0 * 0.5); + col = mix(col, colD, b2 * clamp(count - 3.0, 0.0, 1.0)); + col = mix(col, colE, b3 * clamp(count - 4.0, 0.0, 1.0)); + col = saturate3(col, 1.15); + col = clamp(col * 1.12, 0.0, 1.0); + col = desat(col, sat); + + float z = sqrt(max(1.0 - p * p, 0.0)); + float3 nrm = normalize(float3(q / R, z + 1e-4)); + float3 Ld = normalize(float3(-0.35, 0.45, 0.85)); + float diff = clamp(dot(nrm, Ld), 0.0, 1.0); + col *= 0.80 + 0.20 * diff; + float fres = pow(1.0 - clamp(nrm.z, 0.0, 1.0), 2.5); + col += fres * 0.10 * mix(float3(1.0), k0, 0.6); + + float3 haloCol = mix(mix(vivid(col0), float3(1.0), 0.75), mix(vivid(col0), float3(1.0), 0.32), dark); + float halo = exp(-sq(max(r - R * 0.8, 0.0) / 0.09)); + float haloA = halo * (mix(0.05, 0.10, dark) + 0.05 * react); + float waves = clamp(b0 + 0.8 * b1, 0.0, 1.0); + float bodyA = clamp(body * (0.15 + 0.60 * waves + 0.06 * core) * (0.92 + 0.08 * fres), 0.0, 1.0); + float fade = mix(1.0, 0.45 + 0.35 * sin(t * 2.0), load); + float3 pm = col * bodyA + haloCol * haloA * (1.0 - bodyA); + float al = bodyA + haloA * (1.0 - bodyA); + float gain = bright * fade; + return half4(half3(pm * gain), half(al * gain)); +} +`; + +let effect: SkRuntimeEffect | null | undefined; + +/** + * The compiled shader, shared by every glow: it compiles once, the first time a glow draws. Null + * if Skia cannot compile it, in which case the glow draws nothing. + */ +export const getGlowEffect = () => { + if (effect === undefined) { + effect = Skia.RuntimeEffect.Make(SOURCE); + if (!effect && __DEV__) { + console.warn( + '[stream-chat-react-native/ai-components] VoiceGlow could not compile its shader, so it draws nothing. Please report this with your @shopify/react-native-skia version.', + ); + } + } + return effect; +}; diff --git a/package/src/ai-components/voice-visualization/index.ts b/package/src/ai-components/voice-visualization/index.ts new file mode 100644 index 0000000000..f8d09f5afb --- /dev/null +++ b/package/src/ai-components/voice-visualization/index.ts @@ -0,0 +1,3 @@ +export { VoiceGlow, type VoiceGlowProps } from './VoiceGlow'; +export { VoiceWave, type VoiceWaveProps } from './VoiceWave'; +export type { VoiceAgentState, VoiceVisualizationProps } from './types'; diff --git a/package/src/ai-components/voice-visualization/math.ts b/package/src/ai-components/voice-visualization/math.ts new file mode 100644 index 0000000000..45b2134cff --- /dev/null +++ b/package/src/ai-components/voice-visualization/math.ts @@ -0,0 +1,73 @@ +import { snoise } from './simplex'; + +/** The shader math the wave runs on the CPU, once per frame, on the UI thread. */ + +export const mix = (a: number, b: number, t: number) => { + 'worklet'; + return a + (b - a) * t; +}; + +export const fract = (x: number) => { + 'worklet'; + return x - Math.floor(x); +}; + +export const clamp = (x: number, low: number, high: number) => { + 'worklet'; + return Math.min(Math.max(x, low), high); +}; + +/** GLSL's smoothstep, evaluated the same with its edges either way round. */ +export const smoothstep = (edge0: number, edge1: number, x: number) => { + 'worklet'; + const t = clamp((x - edge0) / (edge1 - edge0), 0, 1); + return t * t * (3 - 2 * t); +}; + +/** HSV, hue a fraction of a turn, into `out` as RGB. */ +export const hsvToRgb = (hue: number, saturation: number, value: number, out: number[]) => { + 'worklet'; + for (let i = 0; i < 3; i++) { + const k = i === 0 ? 1 : i === 1 ? 2 / 3 : 1 / 3; + const p = Math.abs(fract(hue + k) * 6 - 3); + out[i] = value * mix(1, clamp(p - 1, 0, 1), saturation); + } +}; + +/** A palette color as is; dark mode lifts saturation a touch so it stays vivid on black. */ +export const vivid = ( + color: { hue: number; saturation: number; value: number }, + dark: number, + out: number[], +) => { + 'worklet'; + hsvToRgb(color.hue, clamp(color.saturation + dark * 0.06, 0, 1), color.value, out); +}; + +/** Pushes `c` away from its luminance by `amount`; below 1 desaturates. */ +export const saturate = (c: number[], amount: number) => { + 'worklet'; + const l = 0.299 * c[0] + 0.587 * c[1] + 0.114 * c[2]; + for (let i = 0; i < 3; i++) c[i] = clamp(mix(l, c[i], amount), 0, 1); +}; + +/** + * How much the agent is talking, 0 to 1: a slow phrase gate times a faster syllable wobble. The + * shader's own envelope, on the same simplex noise, so the wave and the glow speak alike. + */ +export const speechEnv = (t: number) => { + 'worklet'; + const gate = smoothstep(-0.25, 0.35, snoise(t * 0.6, 21, 0)); + const syllable = 0.55 + 0.45 * snoise(t * 3, 5, 0); + return clamp(gate * syllable, 0, 1); +}; + +/** `expressivity` from 0 to 2; one that is not a number gets the default, 1. */ +export const expressivityInput = (expressivity: number | undefined) => + typeof expressivity === 'number' && Number.isFinite(expressivity) ? clamp(expressivity, 0, 2) : 1; + +/** `level` from 0 to 1; one that is not a number counts as no live audio, so it can't stall the motion. */ +export const audioLevelInput = (level: number | null | undefined) => { + 'worklet'; + return typeof level === 'number' && Number.isFinite(level) ? clamp(level, 0, 1) : null; +}; diff --git a/package/src/ai-components/voice-visualization/palette.ts b/package/src/ai-components/voice-visualization/palette.ts new file mode 100644 index 0000000000..b23c5b8dd9 --- /dev/null +++ b/package/src/ai-components/voice-visualization/palette.ts @@ -0,0 +1,59 @@ +import { processColor } from 'react-native'; + +import type { Hsv } from './types'; + +export const MAXIMUM_COLORS = 5; + +export type Rgb = [number, number, number]; + +/** Used when neither the `colors` prop nor the theme gives a readable color. */ +export const FALLBACK_COLOR = '#005fff'; + +/** + * Reads any color React Native understands (names, `#rgb[a]`, `#rrggbb[aa]`, `rgb()`, `hsl()`, ...) + * into sRGB channels from 0 to 1, or null when it cannot be read. Platform colors have no + * channels to read and count as unreadable. + */ +export const parseColor = (color: string): Rgb | null => { + if (typeof color !== 'string' || !color.trim()) return null; + const processed = processColor(color.trim()); + if (typeof processed !== 'number') return null; + // 0xAARRGGBB, signed on Android. + const argb = processed < 0 ? processed + 2 ** 32 : processed; + const channel = (shift: number) => (Math.floor(argb / 2 ** shift) % 256) / 255; + return [channel(16), channel(8), channel(0)]; +}; + +/** RGB channels from 0 to 1 as HSV, keeping full saturation and value so muted colors render true. */ +export const rgbToHsv = ([r, g, b]: Rgb): Hsv => { + const high = Math.max(r, g, b); + const low = Math.min(r, g, b); + const delta = high - low; + let hue = 0; + if (delta !== 0) { + if (high === r) hue = ((((g - b) / delta) % 6) + 6) % 6; + else if (high === g) hue = (b - r) / delta + 2; + else hue = (r - g) / delta + 4; + } + return { + hue: (((hue / 6) % 1) + 1) % 1, + saturation: high === 0 ? 0 : Math.min(Math.max(delta / high, 0), 1), + value: Math.min(Math.max(high, 0), 1), + }; +}; + +/** Colors as the HSV the renderers read, at most five; colors that cannot be read are skipped. */ +export const toHsvPalette = (colors: readonly string[]): Hsv[] => + colors + .map(parseColor) + .filter((rgb): rgb is Rgb => rgb !== null) + .slice(0, MAXIMUM_COLORS) + .map(rgbToHsv); + +/** Relative luminance of a color, or null when it cannot be read. */ +export const luminance = (color: string) => { + const rgb = parseColor(color); + if (!rgb) return null; + const [r, g, b] = rgb.map((c) => (c <= 0.03928 ? c / 12.92 : Math.pow((c + 0.055) / 1.055, 2.4))); + return 0.2126 * r + 0.7152 * g + 0.0722 * b; +}; diff --git a/package/src/ai-components/voice-visualization/simplex.ts b/package/src/ai-components/voice-visualization/simplex.ts new file mode 100644 index 0000000000..f95594d990 --- /dev/null +++ b/package/src/ai-components/voice-visualization/simplex.ts @@ -0,0 +1,92 @@ +/** + * 3-D simplex noise, a port of the glow shader's `snoise` (Ashima Arts / Stefan Gustavson), so the + * wave drawn on the CPU moves with the same speech rhythm as the shader, as on iOS and the web. + * + * It runs in float32 like a GPU, with `mod` as a multiply by a float32 reciprocal, because the + * hash is sensitive to rounding: in exact arithmetic some inputs hash to a neighbouring gradient. + * Every function is a worklet, as the motion runs on the UI thread. + */ + +const INV_289 = Math.fround(1 / 289); +const ONE_THIRD = Math.fround(1 / 3); +const ONE_SIXTH = Math.fround(1 / 6); +const NS_X = Math.fround(2 / 7); +const NS_Y = Math.fround(Math.fround(0.5 * Math.fround(1 / 7)) - 1); +const NS_Z = Math.fround(1 / 7); + +const mod289 = (x: number) => { + 'worklet'; + const f = Math.fround; + return f(x - f(289 * Math.floor(f(x * INV_289)))); +}; + +const permute = (x: number) => { + 'worklet'; + const f = Math.fround; + return mod289(f(f(f(x * 34) + 1) * x)); +}; + +const taylorInvSqrt = (r: number) => { + 'worklet'; + const f = Math.fround; + return f(1.79284291400159 - f(0.85373472095314 * r)); +}; + +/** A corner's contribution, its gradient from its hash as the shader's x_/y_/h/sh steps compute it. */ +const corner = (p: number, x: number, y: number, z: number) => { + 'worklet'; + const f = Math.fround; + const j = f(p - f(49 * Math.floor(f(f(p * NS_Z) * NS_Z)))); + const xi = Math.floor(f(j * NS_Z)); + const yi = Math.floor(f(j - f(7 * xi))); + const gx0 = f(f(xi * NS_X) + NS_Y); + const gy0 = f(f(yi * NS_X) + NS_Y); + const gz = f(f(1 - Math.abs(gx0)) - Math.abs(gy0)); + const sh = gz <= 0 ? -1 : 0; + const gx = f(gx0 + f((Math.floor(gx0) * 2 + 1) * sh)); + const gy = f(gy0 + f((Math.floor(gy0) * 2 + 1) * sh)); + const norm = taylorInvSqrt(gx * gx + gy * gy + gz * gz); + const m = Math.max(0.6 - (x * x + y * y + z * z), 0); + return m * m * m * m * (gx * norm * x + gy * norm * y + gz * norm * z); +}; + +/** Simplex noise at (x, y, z), from about -1 to 1. */ +export const snoise = (inX: number, inY: number, inZ: number) => { + 'worklet'; + const f = Math.fround; + const vx = f(inX); + const vy = f(inY); + const vz = f(inZ); + const s = f(f(vx + vy + vz) * ONE_THIRD); + const ix = Math.floor(f(vx + s)); + const iy = Math.floor(f(vy + s)); + const iz = Math.floor(f(vz + s)); + const t = f(f(ix + iy + iz) * ONE_SIXTH); + const x0 = f(f(vx - ix) + t); + const y0 = f(f(vy - iy) + t); + const z0 = f(f(vz - iz) + t); + + const gx = x0 >= y0 ? 1 : 0; + const gy = y0 >= z0 ? 1 : 0; + const gz = z0 >= x0 ? 1 : 0; + const i1x = Math.min(gx, 1 - gz); + const i1y = Math.min(gy, 1 - gx); + const i1z = Math.min(gz, 1 - gy); + const i2x = Math.max(gx, 1 - gz); + const i2y = Math.max(gy, 1 - gx); + const i2z = Math.max(gz, 1 - gy); + + const mx = mod289(ix); + const my = mod289(iy); + const mz = mod289(iz); + const hash = (dx: number, dy: number, dz: number) => + permute(f(f(permute(f(f(permute(f(mz + dz)) + my) + dy)) + mx) + dx)); + + return ( + 42 * + (corner(hash(0, 0, 0), x0, y0, z0) + + corner(hash(i1x, i1y, i1z), x0 - i1x + 1 / 6, y0 - i1y + 1 / 6, z0 - i1z + 1 / 6) + + corner(hash(i2x, i2y, i2z), x0 - i2x + 1 / 3, y0 - i2y + 1 / 3, z0 - i2z + 1 / 3) + + corner(hash(1, 1, 1), x0 - 0.5, y0 - 0.5, z0 - 0.5)) + ); +}; diff --git a/package/src/ai-components/voice-visualization/styles.ts b/package/src/ai-components/voice-visualization/styles.ts new file mode 100644 index 0000000000..4a6a01bee5 --- /dev/null +++ b/package/src/ai-components/voice-visualization/styles.ts @@ -0,0 +1,42 @@ +import { LEVEL_RESPONSE, type LevelResponse } from './driver'; + +/** + * What tells the glow and the wave apart besides how they draw, as on iOS. + * + * - `levelResponse`: the wave's height tracks the level directly, so it follows it smoothly; + * per-syllable jumps would make it jitter. The glow follows it closely. + * - `maxScale`: the densest it renders. The glow is soft all over, so 2x is enough; the wave's + * thin line is drawn at the screen's own scale, up to 3x, to stay crisp. + * - `contains`: whether a point, centred and normalized to the shorter side with y up, is on the + * shape rather than the empty space around it. + */ +export type VoiceVisualizationStyle = { + contains: (x: number, y: number) => boolean; + /** The shape's bounds as a share of the shorter side, for the area that takes touches. */ + hitArea: { width: number; height: number }; + levelResponse: LevelResponse; + maxScale: number; +}; + +/** How far from the centre a tap still lands on the globe, as a share of the shorter side. */ +const GLOBE_REACH = 0.32; + +export const GLOW_STYLE: VoiceVisualizationStyle = { + contains: (x, y) => { + 'worklet'; + return Math.hypot(x, y) < GLOBE_REACH; + }, + hitArea: { height: 2 * GLOBE_REACH, width: 2 * GLOBE_REACH }, + levelResponse: LEVEL_RESPONSE.steady, + maxScale: 2, +}; + +export const WAVE_STYLE: VoiceVisualizationStyle = { + contains: (x, y) => { + 'worklet'; + return Math.abs(x) < 0.5 && Math.abs(y) < 0.14; + }, + hitArea: { height: 0.28, width: 1 }, + levelResponse: LEVEL_RESPONSE.smooth, + maxScale: 3, +}; diff --git a/package/src/ai-components/voice-visualization/types.ts b/package/src/ai-components/voice-visualization/types.ts new file mode 100644 index 0000000000..59593926c3 --- /dev/null +++ b/package/src/ai-components/voice-visualization/types.ts @@ -0,0 +1,38 @@ +import type { ViewProps } from 'react-native'; + +import type { SharedValue } from 'react-native-reanimated'; + +/** + * What a voice agent is doing, which a voice visualization shows as motion. + * + * The states differ only in how a visualization moves, never in its colors, and changing the + * state eases from one motion into the next. + * + * - `idle`: waiting to be spoken to, a calm, barely-there shimmer. + * - `connecting`: not ready yet, a loader sweep with a pulsing breath. + * - `listening`: hearing somebody, a gentle expand and contract. + * - `thinking`: working on an answer, a slow turn or a travelling wave. + * - `speaking`: talking, strong motion that ebbs and pauses like speech. + */ +export type VoiceAgentState = 'idle' | 'connecting' | 'listening' | 'thinking' | 'speaking'; + +export type VoiceVisualizationProps = Omit & { + /** What the agent is doing, which sets how the visualization moves. */ + state: VoiceAgentState; + /** + * The level of whoever is talking, from 0 to 1, read once a frame on the UI thread. Feed it at + * 15 Hz or more for the visualization to move with the words; `null` animates speech on its own. + * See `useDictationAudioLevel` for the level of the microphone while dictating. + */ + audioLevel?: SharedValue; + /** + * One to five colors. Past five are ignored, and colors that cannot be read are skipped. When + * omitted or empty, the theme's `voiceVisualization.colors` are used. + */ + colors?: string[]; + /** From 0 to 2: how far and how fast it moves. Colors never change with it. Defaults to 1. */ + expressivity?: number; +}; + +/** A color as hue (a fraction of a turn), saturation and value, each from 0 to 1. */ +export type Hsv = { hue: number; saturation: number; value: number }; diff --git a/package/src/ai-components/voice-visualization/useReduceMotion.ts b/package/src/ai-components/voice-visualization/useReduceMotion.ts new file mode 100644 index 0000000000..ed3188738f --- /dev/null +++ b/package/src/ai-components/voice-visualization/useReduceMotion.ts @@ -0,0 +1,34 @@ +import { useEffect, useState } from 'react'; +import { AccessibilityInfo } from 'react-native'; + +/** + * Whether the system asks for less motion: Reduce Motion on iOS, animations off on Android. It + * follows the setting while mounted. + * + * The AI entry point cannot reach the core SDK's own hook, which also only applies once its + * accessibility layer is enabled; the voice visualizations always honour the setting, as on iOS + * and Android. + */ +export const useReduceMotion = () => { + const [reduceMotion, setReduceMotion] = useState(false); + + useEffect(() => { + let mounted = true; + AccessibilityInfo.isReduceMotionEnabled?.() + .then((enabled) => { + // It starts off; only turning it on needs a render. + if (mounted && enabled) setReduceMotion(true); + }) + .catch(() => undefined); + const subscription = AccessibilityInfo.addEventListener?.( + 'reduceMotionChanged', + setReduceMotion, + ); + return () => { + mounted = false; + subscription?.remove(); + }; + }, []); + + return reduceMotion; +}; diff --git a/package/src/ai-components/voice-visualization/useVoiceVisualization.ts b/package/src/ai-components/voice-visualization/useVoiceVisualization.ts new file mode 100644 index 0000000000..0cd63af0f1 --- /dev/null +++ b/package/src/ai-components/voice-visualization/useVoiceVisualization.ts @@ -0,0 +1,154 @@ +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useColorScheme } from 'react-native'; + +import { + type FrameInfo, + runOnUI, + useFrameCallback, + useSharedValue, + withTiming, +} from 'react-native-reanimated'; + +import { + advanceDriver, + createDriverState, + hoverDriver, + tapDriver, + type VoiceDriverInput, +} from './driver'; +import { audioLevelInput, expressivityInput } from './math'; +import { FALLBACK_COLOR, luminance, toHsvPalette } from './palette'; +import type { VoiceVisualizationStyle } from './styles'; +import type { VoiceVisualizationProps } from './types'; +import { useReduceMotion } from './useReduceMotion'; + +import { useTheme } from '../contexts'; + +/** How long the visualization takes to fade in once its first frame is drawn, as on iOS. */ +const FADE_IN_MS = 300; + +/** Whether to draw for a dark background: the theme's backdrop when readable, else the scheme. */ +const useDark = (backdropColor: string | undefined) => { + const scheme = useColorScheme(); + return useMemo(() => { + const backdrop = backdropColor ? luminance(backdropColor) : null; + if (backdrop !== null) return backdrop < 0.5 ? 1 : 0; + return scheme === 'dark' ? 1 : 0; + }, [backdropColor, scheme]); +}; + +/** + * A voice visualization's motion, advanced once a frame on the UI thread. + * + * The driver lives in a shared value the renderers derive from, so a frame redraws without + * re-rendering React, and `audioLevel` is read in the frame callback, so a level changing many + * times a second re-renders nothing either. + */ +export const useVoiceVisualization = ( + { audioLevel, colors, expressivity, state }: VoiceVisualizationProps, + style: VoiceVisualizationStyle, +) => { + const { theme } = useTheme(); + const reduceMotion = useReduceMotion(); + const themeColors = theme.voiceVisualization?.colors; + const dark = useDark(theme.voiceVisualization?.backdropColor); + + const colorsKey = (colors ?? []).join('|'); + const themeKey = (themeColors ?? []).join('|'); + const palette = useMemo(() => { + const own = toHsvPalette(colors ?? []); + if (own.length) return own; + const fromTheme = toHsvPalette(themeColors ?? []); + return fromTheme.length ? fromTheme : toHsvPalette([FALLBACK_COLOR]); + // Keyed by the colors themselves, so a new array of the same colors changes nothing. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [colorsKey, themeKey]); + + const input: VoiceDriverInput = { + audioLevel: null, + colors: palette, + expressivity: expressivityInput(expressivity), + reduceMotion, + state, + }; + + // Starts settled on the first input, so the first frame shows it rather than easing into it. + const [initialDriver] = useState(() => + createDriverState( + { ...input, audioLevel: audioLevelInput(audioLevel?.value) }, + style.levelResponse, + ), + ); + const driver = useSharedValue(initialDriver); + const inputValue = useSharedValue(input); + const darkValue = useSharedValue(dark); + const opacity = useSharedValue(0); + + useEffect(() => { + inputValue.value = { + audioLevel: null, + colors: palette, + expressivity: expressivityInput(expressivity), + reduceMotion, + state, + }; + }, [expressivity, inputValue, palette, reduceMotion, state]); + + useEffect(() => { + darkValue.value = dark; + }, [dark, darkValue]); + + const onFrame = useCallback( + (frame: FrameInfo) => { + 'worklet'; + const seconds = (frame.timeSincePreviousFrame ?? 0) / 1000; + const current = inputValue.value; + const level = audioLevelInput(audioLevel ? audioLevel.value : null); + const next = level === current.audioLevel ? current : { ...current, audioLevel: level }; + driver.modify((d) => { + 'worklet'; + advanceDriver(d, next, seconds); + return d; + }); + // It fades in once it has something to show, rather than flashing empty while the first + // frame lands. + if (opacity.value === 0) opacity.value = withTiming(1, { duration: FADE_IN_MS }); + }, + [audioLevel, driver, inputValue, opacity], + ); + useFrameCallback(onFrame); + + /** A tap at a point centred and normalized to the shorter side, y up. */ + const tap = useCallback( + (x: number, y: number) => { + runOnUI((tapX: number, tapY: number) => { + 'worklet'; + driver.modify((d) => { + 'worklet'; + tapDriver(d, inputValue.value, tapX, tapY); + return d; + }); + })(x, y); + }, + [driver, inputValue], + ); + + /** Where a pointer hovers, or null once it has left. */ + const hover = useCallback( + (x: number | null, y: number | null) => { + runOnUI((hoverX: number | null, hoverY: number | null) => { + 'worklet'; + driver.modify((d) => { + 'worklet'; + hoverDriver(d, inputValue.value, hoverX, hoverY); + return d; + }); + })(x, y); + }, + [driver, inputValue], + ); + + return { dark: darkValue, driver, hover, opacity, tap }; +}; + +export type VoiceVisualization = ReturnType; diff --git a/package/src/ai-components/voice-visualization/waveGeometry.ts b/package/src/ai-components/voice-visualization/waveGeometry.ts new file mode 100644 index 0000000000..3af2a03c00 --- /dev/null +++ b/package/src/ai-components/voice-visualization/waveGeometry.ts @@ -0,0 +1,137 @@ +import { Skia, type SkPath } from '@shopify/react-native-skia'; + +import { driverReact, driverScale, type VoiceDriverState } from './driver'; +import { clamp, mix, saturate, smoothstep, speechEnv, vivid } from './math'; + +const POINTS = 160; +export const WAVE_STOPS = 13; +export const WAVE_START = -0.42; +export const WAVE_END = 0.42; + +/** The stops' positions along the gradient, evenly spaced. */ +export const WAVE_POSITIONS = Array.from({ length: WAVE_STOPS }, (_, s) => s / (WAVE_STOPS - 1)); + +const sq = (x: number) => { + 'worklet'; + return x * x; +}; + +const bandCenter = (i: number, seg: number, t: number) => { + 'worklet'; + switch (i) { + case 0: + return 0.5 * seg + 0.12 * Math.sin(t * 0.13); + case 1: + return 1.5 * seg + 0.13 * Math.sin(t * 0.11 + 2.1); + case 2: + return 2.5 * seg + 0.12 * Math.sin(t * 0.17 + 4.2); + case 3: + return 3.5 * seg + 0.12 * Math.sin(t * 0.15 + 1); + default: + return 4.5 * seg + 0.12 * Math.sin(t * 0.1 + 3.3); + } +}; + +/** The string's height above its centre line at each point, centred and normalized, y up. */ +export const waveHeights = (d: VoiceDriverState) => { + 'worklet'; + const t = d.time; + const react = driverReact(d); + const center = d.load * 0.3 * Math.sin(t * 1.3); + const freq = 24 + 26 * react + 10 * d.flow; + const speak = smoothstep(0.6, 1, react); + const drive = mix(speak * 0.26 * speechEnv(t), 0.3 * d.mic, d.voice); + const reach = 0.018 + 0.022 * react + drive + 0.14 * d.flow + 0.1 * d.load; + const tapAlive = d.tapAge < 1.5; + const hovered = d.hoverAmount > 0.001; + const xs: number[] = []; + const ys: number[] = []; + for (let i = 0; i <= POINTS; i++) { + const x = WAVE_START + ((WAVE_END - WAVE_START) * i) / POINTS; + let y = Math.exp(-sq((x - center) / 0.18)) * reach * Math.sin(x * freq - t * 5); + if (tapAlive) { + y += + Math.exp(-sq((x - d.tapX) / 0.12)) * + Math.sin((x - d.tapX) * 40 - d.tapAge * 24) * + Math.exp(-d.tapAge * 2.6) * + 0.05; + } + if (hovered) { + y += + Math.exp(-sq((x - d.hoverX) / 0.12)) * + Math.sin((x - d.hoverX) * 40 - t * 7) * + d.hoverAmount * + 0.04; + } + xs.push(x); + ys.push(y); + } + return { xs, ys }; +}; + +type LegacyPath = SkPath & { + lineTo: (x: number, y: number) => void; + moveTo: (x: number, y: number) => void; +}; + +/** + * The wave as a path in a `width` by `height` canvas. Newer Skia versions build immutable paths with + * `PathBuilder`; earlier 2.x versions mutate `SkPath` itself. + */ +export const wavePath = (d: VoiceDriverState, width: number, height: number): SkPath => { + 'worklet'; + const m = Math.min(width, height); + const scale = driverScale(d); + const { xs, ys } = waveHeights(d); + const px = (i: number) => width / 2 + xs[i] * m * scale; + const py = (i: number) => height / 2 - ys[i] * m * scale; + if (Skia.PathBuilder) { + const builder = Skia.PathBuilder.Make(); + builder.moveTo(px(0), py(0)); + for (let i = 1; i < xs.length; i++) builder.lineTo(px(i), py(i)); + return builder.build(); + } + const path = Skia.Path.Make() as LegacyPath; + path.moveTo(px(0), py(0)); + for (let i = 1; i < xs.length; i++) path.lineTo(px(i), py(i)); + return path; +}; + +/** + * The string's colors at its gradient stops, as RGBA from 0 to 1: up to five bands drifting along + * it, weighed by how near their centres are, its ends fading out. + */ +export const waveColors = (d: VoiceDriverState, dark: number) => { + 'worklet'; + const band: number[][] = []; + for (let i = 0; i < 5; i++) { + const rgb = [0, 0, 0]; + vivid(d.colors[i], dark, rgb); + band.push(rgb); + } + const seg = 1 / Math.max(d.count, 1); + const bandWidth = seg * 0.7; + const stops: Float32Array[] = []; + for (let s = 0; s < WAVE_STOPS; s++) { + const x = WAVE_START + ((WAVE_END - WAVE_START) * s) / (WAVE_STOPS - 1); + const gx = clamp(x + 0.5, 0, 1); + const color = [0, 0, 0]; + let total = 1e-4; + for (let i = 0; i < 5; i++) { + const present = i === 0 ? 1 : clamp(d.count - i, 0, 1); + const weight = Math.exp(-sq((gx - bandCenter(i, seg, d.time)) / bandWidth)) * present; + for (let j = 0; j < 3; j++) color[j] += band[i][j] * weight; + total += weight; + } + for (let j = 0; j < 3; j++) color[j] /= total; + saturate(color, 1.2); + saturate(color, d.sat); + const alpha = clamp(smoothstep(0.4, 0.24, Math.abs(x)) * d.bright, 0, 1); + stops.push(new Float32Array([color[0], color[1], color[2], alpha])); + } + return stops; +}; + +/** The line is 0.6% of the shorter side, and never under 1.5 device pixels, as on Android and iOS. */ +export const waveStrokeWidth = (width: number, height: number, pixelRatio: number) => + Math.max(0.006 * Math.min(width, height), 1.5 / pixelRatio); diff --git a/yarn.lock b/yarn.lock index 6d8e94f376..736e9fd6cc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10640,6 +10640,15 @@ __metadata: languageName: node linkType: hard +"expo-speech@npm:~58.0.3": + version: 58.0.3 + resolution: "expo-speech@npm:58.0.3" + peerDependencies: + expo: "*" + checksum: 10c0/552c3a34a96210cd0405ee278f8065500203ae87881e5b43500a9251ca2d58b6956e14e37e922872b6480669eff7a0a1ad8ff5aca500eea912cd7ae6d71bcf4d + languageName: node + linkType: hard + "expo-splash-screen@npm:~58.0.8": version: 58.0.8 resolution: "expo-splash-screen@npm:58.0.8" @@ -10747,6 +10756,7 @@ __metadata: expo-image-picker: "npm:~58.0.10" expo-linking: "npm:~58.0.12" expo-router: "npm:~58.0.18" + expo-speech: "npm:~58.0.3" react: "npm:19.3.0" react-native: "npm:0.88.0" react-native-gesture-handler: "npm:~3.4.0" From 38a224a65d0ba079be739d638f24a8c044cf4474 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 27 Oct 2026 14:36:49 +0100 Subject: [PATCH 2/2] fix(ai-components): address review findings in the voice visualizations - Dictation records who started it (`useDictation(options, { source })`, the dictation store's `source`), and `ComposerView` only fills its input with its own button's dictation, so a voice mode no longer erases or overwrites the composer's text. - Keep the shape's touch area as a real view (`collapsable={false}`) so taps and hover land. - Fade in once, with a flag, instead of re-arming while the opacity still reads 0. - Add `stateLabels` to translate the state values screen readers read. - Throttle Android's microphone level events to about 30 a second, as on iOS. - Drop a dead initial audio-level read. - ExpoChatGPTSample: find the agent's reply by message identity rather than by clock time, give voice mode its own dictation source, ignore stale dictation errors, and use a static icon for the voice-mode button instead of an always-animating glow. - Tests: the shader compiles once on a fresh module, the touch area isn't collapsable, translated state labels, and outside dictation leaving the composer's text alone. --- .../components/VoiceMode/VoiceMode.tsx | 31 +++++++++++- .../VoiceMode/useVoiceConversation.ts | 35 ++++++++++---- .../dictation/AIDictationController.kt | 13 ++++- package/src/ai-components/README.md | 3 +- .../__tests__/VoiceVisualization.test.tsx | 47 +++++++++++-------- .../__tests__/dictation.test.tsx | 24 ++++++++++ .../message-composer/ComposerView.tsx | 9 ++-- .../message-composer/SpeechToTextButton.tsx | 7 ++- .../ai-components/store/dictation/store.ts | 9 ++++ .../store/dictation/useDictationState.ts | 3 +- .../transcription/useDictation.ts | 19 ++++++-- .../VoiceVisualizationView.tsx | 9 +++- .../voice-visualization/types.ts | 6 +++ .../useVoiceVisualization.ts | 15 +++--- 14 files changed, 177 insertions(+), 53 deletions(-) diff --git a/examples/ExpoChatGPTSample/components/VoiceMode/VoiceMode.tsx b/examples/ExpoChatGPTSample/components/VoiceMode/VoiceMode.tsx index 44d84b85ad..d1a0984bff 100644 --- a/examples/ExpoChatGPTSample/components/VoiceMode/VoiceMode.tsx +++ b/examples/ExpoChatGPTSample/components/VoiceMode/VoiceMode.tsx @@ -1,6 +1,7 @@ import { useState } from 'react'; import { Modal, Pressable, StyleSheet, Text, View } from 'react-native'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; +import Svg, { Line } from 'react-native-svg'; import { StreamTheme, @@ -58,6 +59,33 @@ const VoiceModeOverlay = ({ onClose }: { onClose: () => void }) => { ); }; +/** + * A static waveform icon. The button stays still: a live `VoiceGlow` here would redraw every frame + * for as long as the chat is open. + */ +const VoiceIcon = () => ( + + {[ + [3, 8, 14], + [7, 4, 18], + [11, 2, 20], + [15, 5, 17], + [19, 9, 13], + ].map(([x, y1, y2]) => ( + + ))} + +); + /** Opens a full-screen voice conversation with the AI agent. */ export const VoiceMode = () => { const [open, setOpen] = useState(false); @@ -71,7 +99,7 @@ export const VoiceMode = () => { onPress={() => setOpen(true)} style={styles.button} > - + setOpen(false)} visible={open}> @@ -93,7 +121,6 @@ const styles = StyleSheet.create({ overflow: 'hidden', width: 48, }, - buttonGlow: { height: 64, width: 64 }, close: { color: '#ffffff', fontSize: 22 }, entry: { alignItems: 'flex-end', paddingHorizontal: 16, paddingBottom: 8 }, glow: { aspectRatio: 1, width: '100%' }, diff --git a/examples/ExpoChatGPTSample/components/VoiceMode/useVoiceConversation.ts b/examples/ExpoChatGPTSample/components/VoiceMode/useVoiceConversation.ts index d0e01ceacd..adb5e2627e 100644 --- a/examples/ExpoChatGPTSample/components/VoiceMode/useVoiceConversation.ts +++ b/examples/ExpoChatGPTSample/components/VoiceMode/useVoiceConversation.ts @@ -19,6 +19,9 @@ const TURN_TIMEOUT_MS = 20_000; const WORKING_STATES: string[] = [AIStates.Thinking, AIStates.Generating, AIStates.ExternalSources]; +// Marks voice mode's dictation as its own, so the composer underneath ignores it. +const DICTATION_SOURCE = { source: 'voice-mode' }; + /** Markdown read aloud as plain words. */ const toSpeech = (text: string) => text @@ -38,31 +41,34 @@ const toSpeech = (text: string) => */ export const useVoiceConversation = (active: boolean) => { const [phase, setPhase] = useState('idle'); - const { start, cancel } = useDictation(); + const { start, cancel } = useDictation(undefined, DICTATION_SOURCE); const { state: dictationState, error } = useDictationState(); const { transcript } = useDictationTranscript(); const send = useSendAIMessage(); const { channel } = useChannelContext(); const { aiState } = useAIState(channel); - const turn = useRef<{ sentAt: number; sawWorking: boolean } | null>(null); + // The messages already in the channel when the question was sent, so the reply is found by + // identity rather than by comparing the server's timestamps with the device's clock. + const turn = useRef<{ seenIds: Set; sawWorking: boolean } | null>(null); const timeout = useRef | null>(null); + // An error left over from an earlier turn, ignored until dictation starts again and clears it. + const staleError = useRef(dictationStore.getLatestValue().error); const listen = useStableCallback(() => { if (!active) return; turn.current = null; - // Clears what an earlier turn heard or failed with, before dictation starts again. - dictationStore.partialNext({ error: null, transcript: '' }); + staleError.current = dictationStore.getLatestValue().error; setPhase('connecting'); void start(); }); const latestReply = useStableCallback(() => { - const sentAt = turn.current?.sentAt ?? 0; + const seenIds = turn.current?.seenIds ?? new Set(); const messages = channel?.state.latestMessages ?? []; for (let i = messages.length - 1; i >= 0; i--) { const message = messages[i]; - if (new Date(message.created_at ?? 0).getTime() < sentAt) break; + if (seenIds.has(message.id)) break; if (message.ai_generated && message.text) return message.text; } return null; @@ -110,7 +116,7 @@ export const useVoiceConversation = (active: boolean) => { return; } if (dictationState !== 'idle' || (phase !== 'listening' && phase !== 'connecting')) return; - if (error) { + if (error && error !== staleError.current) { setPhase('idle'); return; } @@ -121,13 +127,16 @@ export const useVoiceConversation = (active: boolean) => { listen(); return; } - turn.current = { sawWorking: false, sentAt: Date.now() }; + turn.current = { + sawWorking: false, + seenIds: new Set((channel?.state.latestMessages ?? []).map((message) => message.id)), + }; setPhase('thinking'); void send({ text }); timeout.current = setTimeout(() => { if (!turn.current?.sawWorking) finishTurn(); }, TURN_TIMEOUT_MS); - }, [active, dictationState, error, finishTurn, listen, phase, send, transcript]); + }, [active, channel, dictationState, error, finishTurn, listen, phase, send, transcript]); // Follows the agent: its turn ends once it has been seen working and is done. useEffect(() => { @@ -140,5 +149,11 @@ export const useVoiceConversation = (active: boolean) => { } }, [active, aiState, finishTurn]); - return { error, listen, phase, userIsTalking: phase === 'listening' }; + return { + // Only an error from the current attempt, not one left over while dictation restarts. + error: phase === 'idle' ? error : null, + listen, + phase, + userIsTalking: phase === 'listening', + }; }; diff --git a/package/shared-native/android/dictation/AIDictationController.kt b/package/shared-native/android/dictation/AIDictationController.kt index bf5b7096f6..69335507fa 100644 --- a/package/shared-native/android/dictation/AIDictationController.kt +++ b/package/shared-native/android/dictation/AIDictationController.kt @@ -6,6 +6,7 @@ import android.content.pm.PackageManager import android.os.Bundle import android.os.Handler import android.os.Looper +import android.os.SystemClock import android.speech.RecognitionListener import android.speech.RecognizerIntent import android.speech.SpeechRecognizer @@ -49,6 +50,10 @@ class AIDictationController( private const val LEVEL_FLOOR_DB = 0f private const val LEVEL_CEILING_DB = 10f + // The shortest time between two level events, about 30 a second as on iOS: the recognizer + // can report RMS faster than the voice visualizations need. + private const val LEVEL_INTERVAL_MS = 33L + private const val STATE_IDLE = "idle" private const val STATE_STARTING = "starting" private const val STATE_LISTENING = "listening" @@ -70,6 +75,7 @@ class AIDictationController( private var currentLocale: Locale = Locale.getDefault() private var silenceTimeoutMs: Long = DEFAULT_SILENCE_TIMEOUT_MS private var lastSpeechTimeMs: Long = 0L + private var lastLevelAtMs: Long = 0L private var isSpeechInProgress: Boolean = false @@ -481,10 +487,13 @@ class AIDictationController( } /** - * Emits the microphone's level to JS, from 0 to 1, for the voice visualizations. The recognizer - * reports it as RMS in dB, around ten times a second while it listens. + * Emits the microphone's level to JS, from 0 to 1, for the voice visualizations, at most about + * 30 times a second. The recognizer reports it as RMS in dB while it listens. */ private fun emitLevel(rmsdB: Float) { + val now = SystemClock.elapsedRealtime() + if (now - lastLevelAtMs < LEVEL_INTERVAL_MS) return + lastLevelAtMs = now val level = ((rmsdB - LEVEL_FLOOR_DB) / (LEVEL_CEILING_DB - LEVEL_FLOOR_DB)).coerceIn(0f, 1f) val map = Arguments.createMap().apply { putDouble("level", level.toDouble()) diff --git a/package/src/ai-components/README.md b/package/src/ai-components/README.md index 7d8d612c15..131e11491a 100644 --- a/package/src/ai-components/README.md +++ b/package/src/ai-components/README.md @@ -259,6 +259,7 @@ Neither has a size of its own: each draws in the middle of the view it is given, | `audioLevel` | `SharedValue` | no | The level of whoever is talking, from 0 to 1, read once a frame on the UI thread. Feed it at 15 Hz or more for the motion to follow the words. While it is `null` or absent, speech animates on its own. | | `colors` | `string[]` | no | One to five colors. Past five are ignored, and colors that cannot be read are skipped. Defaults to the theme's `voiceVisualization.colors`. | | `expressivity` | `number` | no | From 0 to 2: how far and how fast it moves. Colors never change with it. Defaults to `1`. | +| `stateLabels` | `Partial>` | no | What screen readers read as the value for each state, for example to translate them. Defaults to "Ready", "Connecting", "Listening", "Thinking" and "Speaking". | ```tsx import { useDictationAudioLevel, VoiceGlow, VoiceWave } from 'stream-chat-react-native/ai-components'; @@ -287,7 +288,7 @@ The motion runs entirely on the UI thread with Reanimated and Skia, so the visua **Theming.** The theme's `voiceVisualization.colors` are the default palette (the accent color unless you set them). The visualizations draw for a light or a dark background following `voiceVisualization.backdropColor` when you set it, and the system color scheme otherwise. -**Accessibility.** Each is an image named "Voice agent" with its state as the value ("Ready", "Connecting", "Listening", "Thinking", "Speaking"), as on iOS and Android. Pass `accessibilityLabel` and `accessibilityValue` to change them, for example to translate them. Under the system's Reduce Motion (animations off on Android), they barely move and ignore taps. +**Accessibility.** Each is an image named "Voice agent" with its state as the value ("Ready", "Connecting", "Listening", "Thinking", "Speaking"), as on iOS and Android. Pass `stateLabels` to translate the state values, `accessibilityLabel` to change the label, or `accessibilityValue` to replace the value outright. Under the system's Reduce Motion (animations off on Android), they barely move and ignore taps. **Differences from the native SDKs.** The glow has no `rendering` option: the shader runs on every Android version through Skia, so there is no Canvas fallback, and it does not switch to a cheaper look in battery saver as on Android. A visualization keeps animating while it is mounted, even when scrolled off screen; it pauses only while the app is in the background. diff --git a/package/src/ai-components/__tests__/VoiceVisualization.test.tsx b/package/src/ai-components/__tests__/VoiceVisualization.test.tsx index 8ab768a807..4953c7872a 100644 --- a/package/src/ai-components/__tests__/VoiceVisualization.test.tsx +++ b/package/src/ai-components/__tests__/VoiceVisualization.test.tsx @@ -1,7 +1,6 @@ import React from 'react'; import { AccessibilityInfo, Pressable } from 'react-native'; -import { Skia } from '@shopify/react-native-skia'; import { act, fireEvent, render, renderHook, screen } from '@testing-library/react-native'; import { type DeepPartial, StreamTheme, type Theme, VoiceGlow, VoiceWave } from '../index'; @@ -98,18 +97,35 @@ describe('VoiceGlow and VoiceWave', () => { }); it('compiles the glow shader once for every glow', () => { - const make = jest.spyOn(Skia.RuntimeEffect, 'Make'); - render( - <> - - - , - { wrapper: themed() }, - ); - fireEvent(screen.getByTestId('first'), 'layout', layout(200, 200)); - fireEvent(screen.getByTestId('second'), 'layout', layout(200, 200)); + // A fresh copy of the module, so a shader compiled by an earlier test doesn't hide a miss. + jest.isolateModules(() => { + const { Skia: freshSkia } = require('@shopify/react-native-skia'); + const { getGlowEffect } = require('../voice-visualization/glowShader'); + const make = jest.spyOn(freshSkia.RuntimeEffect, 'Make'); + + const first = getGlowEffect(); + const second = getGlowEffect(); + + expect(make).toHaveBeenCalledTimes(1); + expect(second).toBe(first); + }); + }); + + it('keeps the touch area as a real view, so taps on the shape land', () => { + render(, { wrapper: themed() }); + fireEvent(screen.getByLabelText('Voice agent'), 'layout', layout(300, 200)); - expect(make.mock.calls.length).toBeLessThanOrEqual(1); + expect(screen.getByTestId('voice-visualization-hit-area').props.collapsable).toBe(false); + }); + + it('reads translated state values', () => { + render(, { + wrapper: themed(), + }); + + expect(screen.getByLabelText('Voice agent').props.accessibilityValue).toEqual({ + text: 'Denkt nach', + }); }); }); @@ -158,13 +174,6 @@ describe('voice visualization motion', () => { expect(wave.result.current.driver.value.levelResponse).toBe(WAVE_STYLE.levelResponse); }); - it('starts with a live level when one is given', () => { - const { result } = renderVisualization({ audioLevel: { value: 0.6 } as never }); - - expect(result.current.driver.value.voice).toBe(0); - expect(result.current.driver.value.mic).toBe(0); - }); - it('bounces on a tap, unless motion is reduced', async () => { const motion = renderVisualization(); act(() => motion.result.current.tap(0.1, 0.1)); diff --git a/package/src/ai-components/__tests__/dictation.test.tsx b/package/src/ai-components/__tests__/dictation.test.tsx index 0a745afeb1..4755ea5e5d 100644 --- a/package/src/ai-components/__tests__/dictation.test.tsx +++ b/package/src/ai-components/__tests__/dictation.test.tsx @@ -195,5 +195,29 @@ describe('ai-components dictation', () => { }); expect(levels).toEqual([0.4, 0.7]); }); + + it('leaves the composer text alone for dictation started elsewhere', async () => { + mockNativeModule = createNativeModule(); + const composer = renderComposer(); + fireEvent.changeText(composer.getByPlaceholderText('Ask anything'), 'my draft'); + + const { result } = renderHook(() => + aiComponents.useDictation(undefined, { source: 'voice-mode' }), + ); + await act(async () => { + await result.current.start(); + }); + act(() => { + DeviceEventEmitter.emit('AIDictationOnState', { state: 'listening' }); + DeviceEventEmitter.emit('AIDictationOnResult', { isFinal: true, text: 'spoken question' }); + DeviceEventEmitter.emit('AIDictationOnState', { state: 'idle' }); + }); + + expect(aiComponents.dictationStore.getLatestValue()).toMatchObject({ + source: 'voice-mode', + transcript: 'spoken question', + }); + expect(composer.getByPlaceholderText('Ask anything').props.value).toBe('my draft'); + }); }); }); diff --git a/package/src/ai-components/message-composer/ComposerView.tsx b/package/src/ai-components/message-composer/ComposerView.tsx index c2ae35564a..f214323ece 100644 --- a/package/src/ai-components/message-composer/ComposerView.tsx +++ b/package/src/ai-components/message-composer/ComposerView.tsx @@ -19,7 +19,7 @@ import type { AbstractMediaPickerService, MediaPickerState, } from '../services/media-picker-service/AbstractMediaPickerService'; -import { useComposerText } from '../store'; +import { COMPOSER_DICTATION_SOURCE, useComposerText } from '../store'; import type { ComposerState } from '../store'; import { useDictationState } from '../store/dictation/useDictationState'; import { useDictationTranscript } from '../store/dictation/useDictationTranscript'; @@ -100,7 +100,7 @@ export const AIMessageInput = () => { } = useTheme(); const styles = useStyles(); - const { error, isRecording } = useDictationState(); + const { error, isRecording, source } = useDictationState(); const { text } = useComposerText(); const { transcript } = useDictationTranscript(); const { setText } = useMessageComposerContext(); @@ -109,10 +109,13 @@ export const AIMessageInput = () => { useEffect(() => { const transcriptChanged = transcript !== previousTranscript.current; previousTranscript.current = transcript; + // Only dictation from the composer's own button fills the input; dictation started elsewhere + // (a voice mode, say) must not overwrite what the user typed. + if (source !== COMPOSER_DICTATION_SOURCE) return; if (!error && (isRecording || transcriptChanged)) { setText(transcript); } - }, [isRecording, transcript, error, setText]); + }, [isRecording, transcript, error, setText, source]); return ( ({ @@ -33,6 +33,9 @@ export const SpeechToTextButton = (props: SpeechToTextButtonProps) => { return ; }; +// Marks this button's dictation as the composer's, so `ComposerView` fills its input with it. +const DICTATION_OPTIONS = { source: COMPOSER_DICTATION_SOURCE }; + const AvailableSpeechToTextButton = ({ options }: SpeechToTextButtonProps) => { const { theme: { @@ -41,7 +44,7 @@ const AvailableSpeechToTextButton = ({ options }: SpeechToTextButtonProps) => { }, } = useTheme(); - const { cancel, toggle } = useDictation(options); + const { cancel, toggle } = useDictation(options, DICTATION_OPTIONS); const { isRecording } = useStateStore(dictationStore, selector); useEffect(() => { diff --git a/package/src/ai-components/store/dictation/store.ts b/package/src/ai-components/store/dictation/store.ts index 9c3c0514d1..41e4fbb874 100644 --- a/package/src/ai-components/store/dictation/store.ts +++ b/package/src/ai-components/store/dictation/store.ts @@ -7,6 +7,11 @@ export type DictationStoreState = { transcript: string; isRecording: boolean; error?: DictationError | null; + /** + * Who started the current dictation, as passed to `useDictation`, or `null`. Lets a consumer + * such as `ComposerView` pick up only the transcripts meant for it. + */ + source?: string | null; }; const DEFAULT_STATE: DictationStoreState = { @@ -14,8 +19,12 @@ const DEFAULT_STATE: DictationStoreState = { transcript: '', isRecording: false, error: null, + source: null, }; +/** The `source` of dictation started by the composer's own `SpeechToTextButton`. */ +export const COMPOSER_DICTATION_SOURCE = 'composer'; + export const dictationStore = new StateStore(DEFAULT_STATE); const reset = () => dictationStore.next(DEFAULT_STATE); diff --git a/package/src/ai-components/store/dictation/useDictationState.ts b/package/src/ai-components/store/dictation/useDictationState.ts index a40627005d..6a79bcb009 100644 --- a/package/src/ai-components/store/dictation/useDictationState.ts +++ b/package/src/ai-components/store/dictation/useDictationState.ts @@ -2,10 +2,11 @@ import { useStateStore } from '@stream-io/state-store/react-bindings'; import { dictationStore, type DictationStoreState } from './store'; -const selector = ({ state, isRecording, error }: DictationStoreState) => ({ +const selector = ({ state, isRecording, error, source }: DictationStoreState) => ({ state, isRecording, error, + source, }); export const useDictationState = () => useStateStore(dictationStore, selector); diff --git a/package/src/ai-components/transcription/useDictation.ts b/package/src/ai-components/transcription/useDictation.ts index eb3e3c9522..bfbf050675 100644 --- a/package/src/ai-components/transcription/useDictation.ts +++ b/package/src/ai-components/transcription/useDictation.ts @@ -27,7 +27,19 @@ const selector = ({ isRecording }: DictationStoreState) => ({ }); // TODO: add proper jsdocs -export const useDictation = (defaultOptions: DictationStartOptions = DEFAULT_OPTIONS) => { +export type UseDictationOptions = { + /** + * Who is dictating, stored as the dictation store's `source` when dictation starts. The + * `ComposerView` only fills its input with dictation whose source is the composer's own, so + * dictation started elsewhere (a voice mode, say) leaves the composer's text alone. + */ + source?: string; +}; + +export const useDictation = ( + defaultOptions: DictationStartOptions = DEFAULT_OPTIONS, + { source }: UseDictationOptions = {}, +) => { const { isRecording } = useStateStore(dictationStore, selector); useEffect(() => { @@ -65,11 +77,12 @@ export const useDictation = (defaultOptions: DictationStartOptions = DEFAULT_OPT code: 'NO_PERMISSION', message: 'Microphone permission not granted', }, + source: source ?? null, }); return; } - dictationStore.partialNext({ error: null, transcript: '' }); + dictationStore.partialNext({ error: null, source: source ?? null, transcript: '' }); const opts: DictationStartOptions = { ...defaultOptions, @@ -78,7 +91,7 @@ export const useDictation = (defaultOptions: DictationStartOptions = DEFAULT_OPT AIDictation.start(opts); }, - [defaultOptions], + [defaultOptions, source], ); const stop = useCallback(() => { diff --git a/package/src/ai-components/voice-visualization/VoiceVisualizationView.tsx b/package/src/ai-components/voice-visualization/VoiceVisualizationView.tsx index 288120ea64..8276245d84 100644 --- a/package/src/ai-components/voice-visualization/VoiceVisualizationView.tsx +++ b/package/src/ai-components/voice-visualization/VoiceVisualizationView.tsx @@ -23,7 +23,7 @@ type ViewProps = VoiceVisualizationProps & { visualStyle: VoiceVisualizationStyle; }; -const STATE_DESCRIPTIONS: Record = { +const DEFAULT_STATE_LABELS: Record = { connecting: 'Connecting', idle: 'Ready', listening: 'Listening', @@ -49,6 +49,7 @@ export const VoiceVisualizationView = ({ pointerEvents = 'box-none', renderCanvas, state, + stateLabels, style, visualStyle, ...restProps @@ -122,7 +123,9 @@ export const VoiceVisualizationView = ({ & { colors?: string[]; /** From 0 to 2: how far and how fast it moves. Colors never change with it. Defaults to 1. */ expressivity?: number; + /** + * What screen readers read as the value for each state, for example to translate them. Defaults + * to "Ready", "Connecting", "Listening", "Thinking" and "Speaking". The label, "Voice agent", is + * `accessibilityLabel`. + */ + stateLabels?: Partial>; }; /** A color as hue (a fraction of a turn), saturation and value, each from 0 to 1. */ diff --git a/package/src/ai-components/voice-visualization/useVoiceVisualization.ts b/package/src/ai-components/voice-visualization/useVoiceVisualization.ts index 0cd63af0f1..2579318396 100644 --- a/package/src/ai-components/voice-visualization/useVoiceVisualization.ts +++ b/package/src/ai-components/voice-visualization/useVoiceVisualization.ts @@ -73,16 +73,12 @@ export const useVoiceVisualization = ( }; // Starts settled on the first input, so the first frame shows it rather than easing into it. - const [initialDriver] = useState(() => - createDriverState( - { ...input, audioLevel: audioLevelInput(audioLevel?.value) }, - style.levelResponse, - ), - ); + const [initialDriver] = useState(() => createDriverState(input, style.levelResponse)); const driver = useSharedValue(initialDriver); const inputValue = useSharedValue(input); const darkValue = useSharedValue(dark); const opacity = useSharedValue(0); + const fadingIn = useSharedValue(false); useEffect(() => { inputValue.value = { @@ -112,9 +108,12 @@ export const useVoiceVisualization = ( }); // It fades in once it has something to show, rather than flashing empty while the first // frame lands. - if (opacity.value === 0) opacity.value = withTiming(1, { duration: FADE_IN_MS }); + if (!fadingIn.value) { + fadingIn.value = true; + opacity.value = withTiming(1, { duration: FADE_IN_MS }); + } }, - [audioLevel, driver, inputValue, opacity], + [audioLevel, driver, fadingIn, inputValue, opacity], ); useFrameCallback(onFrame);