feat(ai-components): add VoiceGlow and VoiceWave voice-agent visualizations

examples/ExpoChatGPTSample/components/VoiceMode/VoiceMode.tsx

@@ -0,0 +1,132 @@
+import { useState } from 'react';
+import { Modal, Pressable, StyleSheet, Text, View } from 'react-native';
+import { useSafeAreaInsets } from 'react-native-safe-area-context';
+import Svg, { Line } from 'react-native-svg';
+
+import {
+  StreamTheme,
+  useDictationAudioLevel,
+  type VoiceAgentState,
+  VoiceGlow,
+  VoiceWave,
+} from 'stream-chat-expo/ai-components';
+
+import { useVoiceConversation } from './useVoiceConversation';
+
+const BACKDROP = '#0b0b10';
+const COLORS = ['#005fff', '#7b5cff', '#00c2ff'];
+
+const LABELS: Record<VoiceAgentState, string> = {
+  connecting: 'Connecting…',
+  idle: 'Tap the glow to talk',
+  listening: 'Listening',
+  speaking: 'Speaking',
+  thinking: 'Thinking',
+};
+
+const VoiceModeOverlay = ({ onClose }: { onClose: () => void }) => {
+  const { top, bottom } = useSafeAreaInsets();
+  const { error, listen, phase, userIsTalking } = useVoiceConversation(true);
+  const micLevel = useDictationAudioLevel();
+
+  return (
+    <StreamTheme style={{ voiceVisualization: { backdropColor: BACKDROP, colors: COLORS } }}>
+      <View style={[styles.overlay, { paddingBottom: bottom + 24, paddingTop: top + 12 }]}>
+        <View style={styles.header}>
+          <Pressable accessibilityLabel='Close voice mode' hitSlop={12} onPress={onClose}>
+            <Text style={styles.close}>✕</Text>
+          </Pressable>
+        </View>
+        <Pressable
+          accessibilityHint='Starts listening'
+          disabled={phase !== 'idle'}
+          onPress={listen}
+          style={styles.glowContainer}
+        >
+          <VoiceGlow state={phase} style={styles.glow} />
+        </Pressable>
+        <Text style={styles.label}>
+          {error ? 'Couldn’t hear you. Tap the glow to try again.' : LABELS[phase]}
+        </Text>
+        <VoiceWave
+          accessibilityLabel='Your voice'
+          audioLevel={micLevel}
+          state={userIsTalking ? 'listening' : 'idle'}
+          style={styles.wave}
+        />
+      </View>
+    </StreamTheme>
+  );
+};
+
+/**
+ * A static waveform icon. The button stays still: a live `VoiceGlow` here would redraw every frame
+ * for as long as the chat is open.
+ */
+const VoiceIcon = () => (
+  <Svg height={22} viewBox='0 0 22 22' width={22}>
+    {[
+      [3, 8, 14],
+      [7, 4, 18],
+      [11, 2, 20],
+      [15, 5, 17],
+      [19, 9, 13],
+    ].map(([x, y1, y2]) => (
+      <Line
+        key={x}
+        stroke='#005fff'
+        strokeLinecap='round'
+        strokeWidth={2.4}
+        x1={x}
+        x2={x}
+        y1={y1}
+        y2={y2}
+      />
+    ))}
+  </Svg>
+);
+
+/** Opens a full-screen voice conversation with the AI agent. */
+export const VoiceMode = () => {
+  const [open, setOpen] = useState(false);
+
+  return (
+    <>
+      <View style={styles.entry}>
+        <Pressable
+          accessibilityLabel='Start voice mode'
+          accessibilityRole='button'
+          onPress={() => setOpen(true)}
+          style={styles.button}
+        >
+          <VoiceIcon />
+        </Pressable>
+      </View>
+      <Modal animationType='fade' onRequestClose={() => setOpen(false)} visible={open}>
+        {open ? <VoiceModeOverlay onClose={() => setOpen(false)} /> : null}
+      </Modal>
+    </>
+  );
+};
+
+const styles = StyleSheet.create({
+  button: {
+    alignItems: 'center',
+    backgroundColor: '#ffffff',
+    borderColor: '#ececec',
+    borderRadius: 24,
+    borderWidth: 1,
+    height: 48,
+    justifyContent: 'center',
+    overflow: 'hidden',
+    width: 48,
+  },
+  close: { color: '#ffffff', fontSize: 22 },
+  entry: { alignItems: 'flex-end', paddingHorizontal: 16, paddingBottom: 8 },
+  glow: { aspectRatio: 1, width: '100%' },
+  glowContainer: { alignSelf: 'stretch', flex: 1, justifyContent: 'center' },
+  header: { alignItems: 'flex-end', paddingHorizontal: 20 },
+  label: { color: '#c9cbd3', fontSize: 17, marginVertical: 16, textAlign: 'center' },
+  overlay: { backgroundColor: BACKDROP, flex: 1 },
+  wave: { alignSelf: 'stretch', height: 140 },
+});

examples/ExpoChatGPTSample/components/VoiceMode/index.ts

@@ -0,0 +1 @@
+export * from './VoiceMode';

examples/ExpoChatGPTSample/components/VoiceMode/useVoiceConversation.ts

@@ -0,0 +1,159 @@
+import { useEffect, useRef, useState } from 'react';
+
+import * as Speech from 'expo-speech';
+
+import { AIStates, useAIState, useChannelContext, useStableCallback } from 'stream-chat-expo';
+
+import {
+  dictationStore,
+  useDictation,
+  useDictationState,
+  useDictationTranscript,
+  type VoiceAgentState,
+} from 'stream-chat-expo/ai-components';
+
+import { useSendAIMessage } from '../../hooks/useSendAIMessage';
+
+/** A turn the agent is never seen working on ends after this long. */
+const TURN_TIMEOUT_MS = 20_000;
+
+const WORKING_STATES: string[] = [AIStates.Thinking, AIStates.Generating, AIStates.ExternalSources];
+
+// Marks voice mode's dictation as its own, so the composer underneath ignores it.
+const DICTATION_SOURCE = { source: 'voice-mode' };
+
+/** Markdown read aloud as plain words. */
+const toSpeech = (text: string) =>
+  text
+    .replace(/```[\s\S]*?```/g, ' ')
+    .replace(/!\[[^\]]*]\([^)]*\)/g, ' ')
+    .replace(/\[([^\]]+)]\([^)]*\)/g, '$1')
+    .replace(/[*_#>`~|]/g, '')
+    .replace(/\s+/g, ' ')
+    .trim();
+
+/**
+ * A hands-free conversation with the AI agent: it listens with dictation, sends what it heard,
+ * waits for the agent to answer, reads the answer aloud and listens again.
+ *
+ * Returns what the agent is doing, for `VoiceGlow`, and whether the user is being listened to,
+ * for `VoiceWave`.
+ */
+export const useVoiceConversation = (active: boolean) => {
+  const [phase, setPhase] = useState<VoiceAgentState>('idle');
+  const { start, cancel } = useDictation(undefined, DICTATION_SOURCE);
+  const { state: dictationState, error } = useDictationState();
+  const { transcript } = useDictationTranscript();
+  const send = useSendAIMessage();
+  const { channel } = useChannelContext();
+  const { aiState } = useAIState(channel);
+
+  // The messages already in the channel when the question was sent, so the reply is found by
+  // identity rather than by comparing the server's timestamps with the device's clock.
+  const turn = useRef<{ seenIds: Set<string>; sawWorking: boolean } | null>(null);
+  const timeout = useRef<ReturnType<typeof setTimeout> | null>(null);
+  // An error left over from an earlier turn, ignored until dictation starts again and clears it.
+  const staleError = useRef(dictationStore.getLatestValue().error);
+
+  const listen = useStableCallback(() => {
+    if (!active) return;
+    turn.current = null;
+    staleError.current = dictationStore.getLatestValue().error;
+    setPhase('connecting');
+    void start();
+  });
+
+  const latestReply = useStableCallback(() => {
+    const seenIds = turn.current?.seenIds ?? new Set<string>();
+    const messages = channel?.state.latestMessages ?? [];
+    for (let i = messages.length - 1; i >= 0; i--) {
+      const message = messages[i];
+      if (seenIds.has(message.id)) break;
+      if (message.ai_generated && message.text) return message.text;
+    }
+    return null;
+  });
+
+  /** Ends the agent's turn: reads its answer aloud if there is one, then listens again. */
+  const finishTurn = useStableCallback(() => {
+    if (timeout.current) clearTimeout(timeout.current);
+    timeout.current = null;
+    const reply = latestReply();
+    turn.current = null;
+    if (!active) return;
+    if (!reply) {
+      listen();
+      return;
+    }
+    setPhase('speaking');
+    Speech.speak(toSpeech(reply), {
+      onDone: listen,
+      onError: listen,
+      // Its own audio session on iOS, so the reply is heard whatever dictation left the app's in.
+      useApplicationAudioSession: false,
+    });
+  });
+
+  // Starts listening when voice mode opens, and stops everything when it closes.
+  useEffect(() => {
+    if (!active) return;
+    listen();
+    return () => {
+      if (timeout.current) clearTimeout(timeout.current);
+      timeout.current = null;
+      turn.current = null;
+      cancel();
+      void Speech.stop();
+      setPhase('idle');
+    };
+  }, [active, cancel, listen]);
+
+  // Follows dictation: listening while it records, and sends what it heard once it stops.
+  useEffect(() => {
+    if (!active || turn.current) return;
+    if (dictationState === 'listening') {
+      setPhase('listening');
+      return;
+    }
+    if (dictationState !== 'idle' || (phase !== 'listening' && phase !== 'connecting')) return;
+    if (error && error !== staleError.current) {
+      setPhase('idle');
+      return;
+    }
+    if (phase === 'connecting') return;
+    const text = transcript.trim();
+    if (!text) {
+      // Nothing was heard; keep listening.
+      listen();
+      return;
+    }
+    turn.current = {
+      sawWorking: false,
+      seenIds: new Set((channel?.state.latestMessages ?? []).map((message) => message.id)),
+    };
+    setPhase('thinking');
+    void send({ text });
+    timeout.current = setTimeout(() => {
+      if (!turn.current?.sawWorking) finishTurn();
+    }, TURN_TIMEOUT_MS);
+  }, [active, channel, dictationState, error, finishTurn, listen, phase, send, transcript]);
+
+  // Follows the agent: its turn ends once it has been seen working and is done.
+  useEffect(() => {
+    const current = turn.current;
+    if (!active || !current) return;
+    if (WORKING_STATES.includes(aiState)) {
+      current.sawWorking = true;
+    } else if (current.sawWorking) {
+      finishTurn();
+    }
+  }, [active, aiState, finishTurn]);
+
+  return {
+    // Only an error from the current attempt, not one left over while dictation restarts.
+    error: phase === 'idle' ? error : null,
+    listen,
+    phase,
+    userIsTalking: phase === 'listening',
+  };
+};

examples/ExpoChatGPTSample/hooks/useSendAIMessage.ts

@@ -0,0 +1,35 @@
+import { useMessageComposer, useMessageInputContext, useStableCallback } from 'stream-chat-expo';
+
+import type { ComposerViewProps } from 'stream-chat-expo/ai-components';
+
+export type AIMessage = Parameters<ComposerViewProps['onSendMessage']>[0];
+
+/**
+ * Sends a message to the AI channel through the channel's composer, with its attachments. Shared
+ * by the composer and voice mode, so both go through `preSendMessageRequest` (which starts the
+ * agent on the first message).
+ */
+export const useSendAIMessage = () => {
+  const messageComposer = useMessageComposer();
+  const { sendMessage } = useMessageInputContext();
+
+  return useStableCallback(async ({ text, attachments }: AIMessage) => {
+    messageComposer.textComposer.setText(text);
+    if (attachments && attachments.length > 0) {
+      const localAttachments = await Promise.all(
+        attachments.map((asset) =>
+          messageComposer.attachmentManager.fileToLocalUploadAttachment({
+            ...asset,
+            // The picker can leave these out (e.g. some iOS photos have no file name).
+            name: asset.name ?? asset.uri.split('/').pop() ?? 'image',
+            size: asset.size ?? 0,
+            type: asset.type ?? 'image/jpeg',
+          }),
+        ),
+      );
+      messageComposer.attachmentManager.upsertAttachments(localAttachments);
+    }
+
+    await sendMessage();
+  });
+};

examples/ExpoChatGPTSample/http/requests.ts

@@ -1,12 +1,19 @@
 import { post } from './api';
 
+// The AI provider the sample backend uses, as in the React SDK's ai-chatbot example.
+const PLATFORM = 'openai';
+
 export const startAI = (channelId: string) =>
   post('https://ai-sdk-server-0f347d455e2e.herokuapp.com/start-ai-agent', {
     channel_id: channelId,
+    platform: PLATFORM,
   });
 export const stopAI = (channelId: string) =>
   post('https://ai-sdk-server-0f347d455e2e.herokuapp.com/stop-ai-agent', {
     channel_id: channelId,
   });
 export const summarize = (text: string) =>
-  post('https://ai-sdk-server-0f347d455e2e.herokuapp.com/summarize', { text });
+  post('https://ai-sdk-server-0f347d455e2e.herokuapp.com/summarize', {
+    platform: PLATFORM,
+    text,
+  });

examples/ExpoChatGPTSample/package.json

@@ -19,6 +19,7 @@
     "expo-image-picker": "~58.0.10",
     "expo-linking": "~58.0.12",
     "expo-router": "~58.0.18",
+    "expo-speech": "~58.0.3",
     "react": "19.3.0",
     "react-native": "0.88.0",
     "react-native-gesture-handler": "~3.4.0",