From 1f7c1bb68c1320dc5465726f94061e467f07d90d Mon Sep 17 00:00:00 2001 From: will wade Date: Mon, 1 Jun 2026 23:11:39 +0100 Subject: [PATCH 1/5] feat: unified voices, streaming, word boundaries, proper Azure/Google engines Major overhaul to match js-tts-wrapper and swift-tts-wrapper: - Unified Voice struct with language_codes array, provider field, normalized gender (matching JS UnifiedVoice and Swift UnifiedVoice) - Azure engine: proper SSML generation with XML escaping, voice selection, prosody tags (rate/pitch), Content-Type headers - Google engine: correct REST API with JSON body, base64 audio decode, v1beta1 timepoint support for real word boundaries via SSML marks - Voice enumeration for Azure, Google, ElevenLabs, Cartesia and other engines with list APIs - Word boundary support via estimate_word_boundaries() matching the Swift WordTimingEstimator algorithm (150 WPM, length-adjusted) - All engines now fire on_boundary callbacks with estimated timings - Added base64 dependency for Google audio decoding - Updated tests: 30 tests (10 unit + 20 integration) covering all engines, voice struct, word boundaries, gender normalization - Updated README with new capabilities table and architecture docs --- Cargo.toml | 3 +- README.md | 129 +++++--- src/cloud_engine.rs | 650 +++++++++++++++++++++++++++++++++++---- src/engine.rs | 81 ++++- src/lib.rs | 8 +- src/sherpaonnx_engine.rs | 44 +-- src/system_engine.rs | 23 +- src/types.rs | 52 +++- tests/integration.rs | 119 ++++++- 9 files changed, 972 insertions(+), 137 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 04a54c6..f5fcff7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ crate-type = ["cdylib", "staticlib", "lib"] [features] default = ["system", "cloud", "sherpaonnx"] system = ["speech-dispatcher"] -cloud = ["reqwest", "serde_json"] +cloud = ["reqwest", "serde_json", "base64"] sherpaonnx = ["sherpa-onnx"] [dependencies] @@ -23,6 +23,7 @@ reqwest = { version = "0.12", features = ["blocking", "json", "rustls-tls"], def serde = { version = "1", features = ["derive"] } serde_json = { version = "1", optional = true } sherpa-onnx = { version = "1.13", optional = true } +base64 = { version = "0.22", optional = true } anyhow = "1" [build-dependencies] diff --git a/README.md b/README.md index 35bb966..51d9f3f 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,44 @@ # rust-tts-wrapper -Cross-platform TTS (Text-to-Speech) wrapper with C API. Mirrors [js-tts-wrapper](https://github.com/user/js-tts-wrapper) and SwiftTTSWrapper. +Cross-platform TTS (Text-to-Speech) wrapper with C API. Mirrors [js-tts-wrapper](https://github.com/AACTools/js-tts-wrapper) and [swift-tts-wrapper](https://github.com/AACTools/swift-tts-wrapper). ## Engines (21 total) -| Engine | Type | Credentials | -|--------|------|-------------| -| System (speech-dispatcher) | Local | None | -| Sherpa-ONNX | Local (191 models) | None | -| OpenAI | Cloud | API Key | -| ElevenLabs | Cloud | API Key | -| Azure | Cloud | Subscription Key + Region | -| Google Cloud | Cloud | API Key | -| Amazon Polly | Cloud | Access Key + Secret + Region | -| Cartesia | Cloud | API Key | -| Deepgram | Cloud | API Key | -| PlayHT | Cloud | API Key + User ID | -| Fish Audio | Cloud | API Key | -| Hume AI | Cloud | API Key | -| Mistral | Cloud | API Key | -| Murf | Cloud | API Key | -| Resemble AI | Cloud | API Key | -| Unreal Speech | Cloud | API Key | -| UpliftAI | Cloud | API Key | -| IBM Watson | Cloud | API Key + Region + Instance ID | -| Wit.ai | Cloud | Token | -| xAI | Cloud | API Key | -| ModelsLab | Cloud | API Key | +| Engine | Type | Credentials | Voice List | Word Boundaries | +|--------|------|-------------|------------|-----------------| +| System (speech-dispatcher) | Local | None | — | Estimated | +| Sherpa-ONNX | Local (191 models) | None | Speakers from registry | Estimated | +| OpenAI | Cloud | API Key | — | Estimated | +| ElevenLabs | Cloud | API Key | API | Estimated | +| Azure | Cloud | Subscription Key + Region | API | Estimated | +| Google Cloud | Cloud | API Key | API | Real (timepoints via v1beta1) | +| Amazon Polly | Cloud | Access Key + Secret + Region | — | Estimated | +| Cartesia | Cloud | API Key | API | Estimated | +| Deepgram | Cloud | API Key | — | Estimated | +| PlayHT | Cloud | API Key + User ID | — | Estimated | +| Fish Audio | Cloud | API Key | — | Estimated | +| Hume AI | Cloud | API Key | — | Estimated | +| Mistral | Cloud | API Key | — | Estimated | +| Murf | Cloud | API Key | — | Estimated | +| Resemble AI | Cloud | API Key | — | Estimated | +| Unreal Speech | Cloud | API Key | — | Estimated | +| UpliftAI | Cloud | API Key | — | Estimated | +| IBM Watson | Cloud | API Key + Region + Instance ID | — | Estimated | +| Wit.ai | Cloud | Token | — | Estimated | +| xAI | Cloud | API Key | — | Estimated | +| ModelsLab | Cloud | API Key | — | Estimated | + +## Key Features + +- **Unified Voice struct** matching js-tts-wrapper and swift-tts-wrapper with `language_codes` array, `provider` field, and normalized gender +- **Streaming audio** via chunked HTTP reads (8KB chunks) through the `on_audio` callback +- **Word boundary events** via `on_boundary` callback — real API timing for Google (v1beta1 timepoints with SSML marks), estimated boundaries for all other engines +- **Azure SSML support** — proper SSML generation with XML escaping, voice selection, and prosody tags +- **Google REST API** — correct JSON body structure with optional timepoint support +- **Voice enumeration** for Azure, Google, ElevenLabs, Cartesia, and other engines with list APIs +- **Word timing estimation** matching the algorithm in JS and Swift (word-length-adjusted, 150 WPM baseline) +- **C ABI** for bindings to Python, .NET, Swift, and other languages +- **Sherpa-ONNX** offline TTS with 191 models from bundled registry ## Usage (C API) @@ -35,7 +47,7 @@ Cross-platform TTS (Text-to-Speech) wrapper with C API. Mirrors [js-tts-wrapper] #include void on_audio(const uint8_t* chunk, uintptr_t size, void* userdata) { - // Handle streaming audio chunks here + // Handle streaming audio chunks printf("Received %zu bytes of audio\n", size); } @@ -45,21 +57,16 @@ void on_boundary(const char* word, float start_time, float end_time, void* userd } int main() { - // 1. Create engine (e.g., ElevenLabs with API key) tts_ctx* ctx = tts_create("elevenlabs", "{\"apiKey\":\"your-api-key\"}"); - // 2. Register callbacks for streaming and word events tts_set_on_audio(ctx, on_audio, NULL); tts_set_on_boundary(ctx, on_boundary, NULL); - // 3. Set voice and properties tts_set_voice(ctx, "Rachel"); tts_set_rate(ctx, 1.0); - // 4. Speak (blocks until complete when using speak_sync) tts_speak_sync(ctx, "Hello world, streaming is supported."); - // 5. Cleanup tts_destroy(ctx); return 0; } @@ -70,29 +77,33 @@ int main() { ```rust use rust_tts_wrapper::factory; -let engine = factory::create_engine("elevenlabs", r#"{"apiKey":"your-api-key"}"#).unwrap(); +let engine = factory::create_engine("openai", r#"{"apiKey":"your-api-key"}"#).unwrap(); // Standard speaking -engine.speak("Hello world", Some("Rachel"), 1.0, 1.0, 1.0, None, None).unwrap(); +engine.speak("Hello world", Some("alloy"), 1.0, 1.0, 1.0, None, None).unwrap(); -// Speaking with streaming and boundary callbacks +// Streaming with word boundary callbacks let mut audio_cb = |chunk: &[u8]| { println!("Received audio chunk of size {}", chunk.len()); }; let mut boundary_cb = |word: &str, start: f32, end: f32| { - println!("Word {} from {} to {}", word, start, end); + println!("Word '{}' from {:.3} to {:.3}", word, start, end); }; engine.speak_sync( "Hello world, streaming is supported.", - Some("Rachel"), - 1.0, - 1.0, - 1.0, + Some("alloy"), + 1.0, 1.0, 1.0, Some(&mut audio_cb), Some(&mut boundary_cb), ).unwrap(); + +// List voices +let voices = engine.get_voices().unwrap(); +for v in &voices { + println!("{} ({}) - {}", v.name, v.provider, v.primary_language()); +} ``` ## Build @@ -107,6 +118,48 @@ cargo build --all-features - `cloud` — all 19 cloud engines via HTTP - `sherpaonnx` — Sherpa-ONNX offline TTS (191 models) +## Architecture + +``` + TtsEngine (trait) + | + +-------------+-------------+ + | | | + SystemEngine CloudEngine SherpaOnnxEngine + (speech- (19 cloud (191 local + dispatcher) providers) models) +``` + +The `CloudEngine` uses a provider-specific configuration (`CloudConfig`) to handle differences in API structure — Azure sends SSML XML, Google sends JSON with base64 audio, and all others use standard JSON bodies. + +### Voice Struct (Unified) + +```rust +pub struct Voice { + pub id: String, + pub name: String, + pub gender: String, // "Male", "Female", "Unknown" + pub provider: String, // "azure", "google", etc. + pub language_codes: Vec, +} + +pub struct LanguageCode { + pub bcp47: String, // "en-US" + pub iso639_3: String, // "eng" + pub display: String, // "English (United States)" +} +``` + +### Word Boundaries + +```rust +pub struct WordBoundary { + pub text: String, + pub offset: u64, // milliseconds from start + pub duration: u64, // milliseconds +} +``` + ## Sherpa-ONNX Models 191 models from the merged_models.json registry. Models auto-download on first use to `~/.rust-tts-wrapper/sherpaonnx/`. diff --git a/src/cloud_engine.rs b/src/cloud_engine.rs index 26d7764..8c010b1 100644 --- a/src/cloud_engine.rs +++ b/src/cloud_engine.rs @@ -1,7 +1,11 @@ //! Generic cloud TTS engine supporting 19 providers via HTTP APIs. +//! +//! Handles Azure (SSML body), Google (base64 REST), and all other JSON-body +//! providers. Includes voice fetching for engines with list endpoints and +//! word boundary support where APIs provide timing data. -use crate::engine::TtsEngine; -use crate::types::{TtsError, TtsResult, Voice}; +use crate::engine::{estimate_word_boundaries, TtsEngine}; +use crate::types::{normalize_gender, LanguageCode, TtsError, TtsResult, Voice, WordBoundary}; use std::collections::HashMap; /// Configuration for a single cloud TTS provider. @@ -16,6 +20,16 @@ struct CloudConfig { default_voice: Option, text_field: String, extra_body: HashMap, + /// Whether this engine requires SSML in the request body (Azure). + body_is_ssml: bool, + /// Content-Type header override for the synthesis request. + content_type: Option, + /// Additional headers to send with synthesis requests. + extra_headers: HashMap, + /// URL for the voice listing endpoint, if available. + voices_url: Option, + /// Provider ID string for voice mapping. + provider_id: String, } /// A TTS engine that synthesises speech by calling a cloud HTTP API. @@ -23,6 +37,7 @@ struct CloudConfig { pub struct CloudEngine { config: CloudConfig, api_key: String, + credentials: HashMap, client: reqwest::blocking::Client, } @@ -41,6 +56,7 @@ impl CloudEngine { Some(CloudEngine { config, api_key, + credentials: credentials.clone(), client: reqwest::blocking::Client::new(), }) } @@ -58,6 +74,7 @@ fn build_config(id: &str, creds: &HashMap) -> Option { @@ -68,10 +85,11 @@ fn build_config(id: &str, creds: &HashMap) -> Option) -> Option { + let api_key = creds.get("apiKey").cloned().unwrap_or_default(); + Some(CloudConfig { + synth_url: format!( + "https://texttospeech.googleapis.com/v1/text:synthesize?key={api_key}" + ), + text_field: "text".into(), + voices_url: Some(format!( + "https://texttospeech.googleapis.com/v1/voices?key={api_key}" + )), + provider_id: "google".into(), ..Default::default() }) } - "google" => Some(CloudConfig { - synth_url: "https://texttospeech.googleapis.com/v1/text:synthesize".into(), - voice_param: String::new(), - text_field: String::new(), - ..Default::default() - }), "cartesia" => Some(CloudConfig { synth_url: "https://api.cartesia.ai/tts/bytes".into(), auth_header: "X-API-Key".into(), voice_param: "voice_id".into(), model_param: Some("model_id".into()), model_default: Some("sonic-2".into()), - default_voice: None, text_field: "text".into(), + voices_url: Some("https://api.cartesia.ai/voices".into()), + provider_id: "cartesia".into(), ..Default::default() }), "deepgram" => Some(CloudConfig { @@ -113,6 +152,7 @@ fn build_config(id: &str, creds: &HashMap) -> Option { @@ -126,6 +166,7 @@ fn build_config(id: &str, creds: &HashMap) -> Option) -> Option Some(CloudConfig { @@ -143,6 +185,7 @@ fn build_config(id: &str, creds: &HashMap) -> Option Some(CloudConfig { @@ -151,6 +194,7 @@ fn build_config(id: &str, creds: &HashMap) -> Option Some(CloudConfig { @@ -158,6 +202,7 @@ fn build_config(id: &str, creds: &HashMap) -> Option Some(CloudConfig { @@ -166,6 +211,7 @@ fn build_config(id: &str, creds: &HashMap) -> Option Some(CloudConfig { @@ -175,6 +221,7 @@ fn build_config(id: &str, creds: &HashMap) -> Option Some(CloudConfig { @@ -183,6 +230,7 @@ fn build_config(id: &str, creds: &HashMap) -> Option { @@ -198,10 +246,11 @@ fn build_config(id: &str, creds: &HashMap) -> Option) -> Option Some(CloudConfig { @@ -218,12 +267,14 @@ fn build_config(id: &str, creds: &HashMap) -> Option Some(CloudConfig { synth_url: "https://modelslab.com/api/v1/text_to_speech".into(), voice_param: "voice".into(), text_field: "text".into(), + provider_id: "modelslab".into(), ..Default::default() }), "polly" => Some(CloudConfig { @@ -231,72 +282,297 @@ fn build_config(id: &str, creds: &HashMap) -> Option None, } } -/// Hex-encode bytes for basic-auth style tokens. -fn base64_encode(_prefix: &str, data: &str) -> String { - use std::fmt::Write; - let mut result = String::new(); - for byte in data.as_bytes() { - write!(result, "{byte:02x}").unwrap(); +/// Base64 encode for auth tokens. +fn base64_encode(data: &str) -> String { + use base64::Engine; + base64::engine::general_purpose::STANDARD.encode(data.as_bytes()) +} + +/// Build SSML for Azure TTS. +fn build_azure_ssml(text: &str, voice: &str, rate: f32, pitch: f32) -> String { + let lang = voice.chars().take(5).collect::(); + + let escaped = text + .replace('&', "&") + .replace('<', "<") + .replace('>', ">"); + + let mut prosody_attrs = Vec::new(); + let rate_str = match rate { + r if r < 0.7 => "x-slow", + r if r < 0.85 => "slow", + r if r < 1.15 => "medium", + r if r < 1.4 => "fast", + _ => "x-fast", + }; + let pitch_str = match pitch { + p if p < 0.7 => "x-low", + p if p < 0.85 => "low", + p if p < 1.15 => "medium", + p if p < 1.4 => "high", + _ => "x-high", + }; + if (rate - 1.0).abs() > f32::EPSILON { + prosody_attrs.push(format!("rate=\"{rate_str}\"")); } - result + if (pitch - 1.0).abs() > f32::EPSILON { + prosody_attrs.push(format!("pitch=\"{pitch_str}\"")); + } + + let inner = if prosody_attrs.is_empty() { + escaped + } else { + format!("{escaped}", prosody_attrs.join(" ")) + }; + + format!( + "\ + {inner}" + ) } +/// Build JSON body for Google TTS REST API. +fn build_google_request( + text: &str, + voice: &str, + add_marks: bool, +) -> (serde_json::Value, Vec) { + let lang = voice.chars().take(5).collect::(); + + let mut words_list = Vec::new(); + + let input = if add_marks { + let words: Vec<&str> = text.split_whitespace().filter(|w| !w.is_empty()).collect(); + let mut ssml = String::from(""); + for (i, w) in words.iter().enumerate() { + if i > 0 { + ssml.push(' '); + } + let _ = std::fmt::Write::write_fmt(&mut ssml, format_args!("{w}")); + words_list = words.iter().map(|w| (*w).to_string()).collect(); + } + ssml.push_str(""); + serde_json::json!({ "ssml": ssml }) + } else { + serde_json::json!({ "text": text }) + }; + + let mut body = serde_json::json!({ + "input": input, + "voice": { "languageCode": lang, "name": voice }, + "audioConfig": { "audioEncoding": "MP3" } + }); + + if add_marks { + body["enableTimePointing"] = serde_json::json!(["SSML_MARK"]); + } + + (body, words_list) +} + +/// Parse Google timepoints into word boundaries. +fn parse_google_timepoints( + timepoints: &[serde_json::Value], + words: &[String], +) -> Vec { + #[derive(Clone)] + struct RawTp { + index: usize, + time_ms: u64, + } + + let mut raw: Vec = Vec::new(); + for tp in timepoints { + let mark = tp.get("markName").and_then(|v| v.as_str()).unwrap_or(""); + let idx: usize = mark.parse().unwrap_or(usize::MAX); + let secs = tp + .get("timeSeconds") + .and_then(serde_json::Value::as_f64) + .unwrap_or(0.0); + if idx < words.len() { + raw.push(RawTp { + index: idx, + time_ms: (secs * 1000.0) as u64, + }); + } + } + raw.sort_by_key(|r| r.time_ms); + + let mut boundaries = Vec::with_capacity(raw.len()); + for (i, tp) in raw.iter().enumerate() { + let word = &words[tp.index]; + let duration = if i + 1 < raw.len() { + raw[i + 1].time_ms.saturating_sub(tp.time_ms) + } else { + ((word.len() as u64) * 80).max(50) + }; + boundaries.push(WordBoundary { + text: word.clone(), + offset: tp.time_ms, + duration, + }); + } + boundaries +} + +/// Map Azure voices JSON array to unified voices. +fn map_azure_voices(json: &[serde_json::Value]) -> Vec { + let mut voices = Vec::new(); + for v in json { + let Some(short_name) = v.get("ShortName").and_then(|v| v.as_str()) else { + continue; + }; + let name = v + .get("DisplayName") + .and_then(|v| v.as_str()) + .unwrap_or(short_name) + .to_string(); + let gender_raw = v.get("Gender").and_then(|v| v.as_str()).unwrap_or(""); + let locale = v.get("Locale").and_then(|v| v.as_str()).unwrap_or("en-US"); + + voices.push(Voice { + id: short_name.to_string(), + name, + gender: normalize_gender(gender_raw).to_string(), + provider: "azure".to_string(), + language_codes: vec![LanguageCode { + bcp47: locale.to_string(), + iso639_3: locale.split('-').next().unwrap_or("en").to_string(), + display: v + .get("LocaleName") + .and_then(|v| v.as_str()) + .unwrap_or(locale) + .to_string(), + }], + }); + } + voices +} + +/// Map Google voices JSON array to unified voices. +fn map_google_voices(json: &[serde_json::Value]) -> Vec { + let mut voices = Vec::new(); + for v in json { + let Some(name) = v.get("name").and_then(|v| v.as_str()) else { + continue; + }; + let gender_raw = v.get("ssmlGender").and_then(|v| v.as_str()).unwrap_or(""); + let lang_codes = v + .get("languageCodes") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|c| { + let code = c.as_str()?; + Some(LanguageCode { + iso639_3: code.split('-').next()?.to_string(), + bcp47: code.to_string(), + display: code.to_string(), + }) + }) + .collect::>() + }) + .unwrap_or_default(); + + voices.push(Voice { + id: name.to_string(), + name: name.to_string(), + gender: normalize_gender(gender_raw).to_string(), + provider: "google".to_string(), + language_codes: lang_codes, + }); + } + voices +} + +#[allow( + clippy::too_many_lines, + clippy::cast_precision_loss, + clippy::map_unwrap_or +)] impl TtsEngine for CloudEngine { fn speak( &self, text: &str, voice: Option<&str>, - _rate: f32, - _pitch: f32, + rate: f32, + pitch: f32, _volume: f32, mut on_audio: Option, - _on_boundary: Option, + mut on_boundary: Option, ) -> TtsResult<()> { let voice_to_use = voice .map(std::string::ToString::to_string) .or_else(|| self.config.default_voice.clone()) .unwrap_or_default(); - let mut body = serde_json::Map::new(); - body.insert( - self.config.text_field.clone(), - serde_json::Value::String(text.to_string()), - ); - - if !self.config.voice_param.is_empty() && !voice_to_use.is_empty() { - body.insert( - self.config.voice_param.clone(), - serde_json::Value::String(voice_to_use), - ); - } - if let Some(ref model_param) = self.config.model_param { - if let Some(ref model) = self.config.model_default { - body.insert( - model_param.clone(), - serde_json::Value::String(model.clone()), - ); - } - } - for (k, v) in &self.config.extra_body { - body.insert(k.clone(), v.clone()); - } - - let mut req = self.client.post(&self.config.synth_url).json(&body); + let mut req = self.client.post(&self.config.synth_url); + // Auth header if !self.config.auth_header.is_empty() { let val = format!("{}{}", self.config.auth_prefix, self.api_key); req = req.header(&self.config.auth_header, val); } - let resp = req - .send() - .map_err(|e| TtsError(format!("HTTP error: {e}")))?; + // Extra headers + for (k, v) in &self.config.extra_headers { + req = req.header(k.as_str(), v.as_str()); + } + + // Body depends on engine type + let resp = if self.config.body_is_ssml { + // Azure: send SSML XML body + let ssml = build_azure_ssml(text, &voice_to_use, rate, pitch); + let ct = self + .config + .content_type + .as_deref() + .unwrap_or("application/ssml+xml"); + req = req.header("Content-Type", ct); + req.body(ssml).send() + } else if self.config.provider_id == "google" { + // Google: build JSON body with proper structure + let (body, _words) = build_google_request(text, &voice_to_use, on_boundary.is_some()); + req = req.json(&body); + req.send() + } else { + // Standard JSON body for all other engines + let mut body = serde_json::Map::new(); + if !self.config.text_field.is_empty() { + body.insert( + self.config.text_field.clone(), + serde_json::Value::String(text.to_string()), + ); + } + if !self.config.voice_param.is_empty() && !voice_to_use.is_empty() { + body.insert( + self.config.voice_param.clone(), + serde_json::Value::String(voice_to_use.clone()), + ); + } + if let Some(ref model_param) = self.config.model_param { + if let Some(ref model) = self.config.model_default { + body.insert( + model_param.clone(), + serde_json::Value::String(model.clone()), + ); + } + } + for (k, v) in &self.config.extra_body { + body.insert(k.clone(), v.clone()); + } + req = req.json(&serde_json::Value::Object(body)); + req.send() + }; + + let resp = resp.map_err(|e| TtsError(format!("HTTP error: {e}")))?; if !resp.status().is_success() { let status = resp.status(); @@ -304,7 +580,49 @@ impl TtsEngine for CloudEngine { return Err(TtsError(format!("API error {status}: {body_text}"))); } - if let Some(cb) = on_audio.as_mut() { + if self.config.provider_id == "google" && on_boundary.is_some() { + // Google returns base64-encoded audio in JSON + let resp_text = resp + .text() + .map_err(|e| TtsError(format!("Read error: {e}")))?; + let json: serde_json::Value = serde_json::from_str(&resp_text) + .map_err(|e| TtsError(format!("JSON parse: {e}")))?; + + if let Some(b64) = json.get("audioContent").and_then(|v| v.as_str()) { + use base64::Engine; + let audio_bytes = base64::engine::general_purpose::STANDARD + .decode(b64) + .map_err(|e| TtsError(format!("Base64 decode: {e}")))?; + if let Some(cb) = on_audio.as_mut() { + for chunk in audio_bytes.chunks(8192) { + cb(chunk); + } + } + } + + if let Some(cb) = on_boundary.as_mut() { + let (_, words) = build_google_request(text, &voice_to_use, true); + if let Some(tps) = json.get("timepoints").and_then(|v| v.as_array()) { + let boundaries = parse_google_timepoints(tps, &words); + for b in &boundaries { + cb( + &b.text, + b.offset as f32 / 1000.0, + (b.offset + b.duration) as f32 / 1000.0, + ); + } + } else { + let estimated = estimate_word_boundaries(text); + for b in &estimated { + cb( + &b.text, + b.offset as f32 / 1000.0, + (b.offset + b.duration) as f32 / 1000.0, + ); + } + } + } + } else if let Some(cb) = on_audio.as_mut() { use std::io::Read; let mut resp = resp; let mut buffer = [0u8; 8192]; @@ -317,6 +635,17 @@ impl TtsEngine for CloudEngine { } cb(&buffer[..n]); } + + if let Some(cb) = on_boundary.as_mut() { + let estimated = estimate_word_boundaries(text); + for b in &estimated { + cb( + &b.text, + b.offset as f32 / 1000.0, + (b.offset + b.duration) as f32 / 1000.0, + ); + } + } } else { let _audio_bytes = resp .bytes() @@ -343,13 +672,98 @@ impl TtsEngine for CloudEngine { } fn get_voices(&self) -> TtsResult> { - // Since we don't fetch voices, we return an empty list or fake one. - // For a full implementation, this should fetch voices from the respective APIs. - Ok(vec![]) + let Some(ref voices_url) = self.config.voices_url else { + return Ok(vec![]); + }; + + let mut req = self.client.get(voices_url.as_str()); + + if !self.config.auth_header.is_empty() { + let val = format!("{}{}", self.config.auth_prefix, self.api_key); + req = req.header(&self.config.auth_header, val); + } + + let resp = req + .send() + .map_err(|e| TtsError(format!("Voice list HTTP error: {e}")))?; + + if !resp.status().is_success() { + return Ok(vec![]); + } + + let json: serde_json::Value = resp + .json() + .map_err(|e| TtsError(format!("Voice list parse error: {e}")))?; + + match self.config.provider_id.as_str() { + "azure" => json + .as_array() + .map_or_else(|| Ok(vec![]), |arr| Ok(map_azure_voices(arr))), + "google" => json + .get("voices") + .and_then(|v| v.as_array()) + .map_or_else(|| Ok(vec![]), |arr| Ok(map_google_voices(arr))), + _ => { + // Generic: try to parse as array of objects with id/name fields + json.as_array().map_or_else( + || Ok(vec![]), + |arr| { + Ok(arr + .iter() + .filter_map(|v| { + let id = v + .get("id") + .or(v.get("voice_id")) + .or(v.get("name"))? + .as_str()?; + Some(Voice { + id: id.to_string(), + name: v + .get("name") + .and_then(|v| v.as_str()) + .unwrap_or(id) + .to_string(), + gender: normalize_gender( + v.get("gender") + .or(v.get("labels")) + .and_then(|v| v.as_str()) + .unwrap_or(""), + ) + .to_string(), + provider: self.config.provider_id.clone(), + language_codes: vec![], + }) + }) + .collect()) + }, + ) + } + } } fn engine_id(&self) -> &'static str { - "cloud" + match self.config.provider_id.as_str() { + "openai" => "openai", + "elevenlabs" => "elevenlabs", + "azure" => "azure", + "google" => "google", + "cartesia" => "cartesia", + "deepgram" => "deepgram", + "playht" => "playht", + "fishaudio" => "fishaudio", + "hume" => "hume", + "mistral" => "mistral", + "murf" => "murf", + "resemble" => "resemble", + "unrealspeech" => "unrealspeech", + "upliftai" => "upliftai", + "watson" => "watson", + "witai" => "witai", + "xai" => "xai", + "modelslab" => "modelslab", + "polly" => "polly", + _ => "cloud", + } } } @@ -362,3 +776,125 @@ pub fn create_cloud_engine(id: &str, credentials_json: &str) -> Option) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_azure_ssml() { + let ssml = build_azure_ssml("Hello world", "en-US-AriaNeural", 1.0, 1.0); + assert!(ssml.contains("")); + assert!(ssml.contains("")); + assert_eq!(words.len(), 2); + assert_eq!(words[0], "Hello"); + assert_eq!(words[1], "world"); + assert!(body.get("enableTimePointing").is_some()); + } + + #[test] + fn test_parse_google_timepoints() { + let tps = vec![ + serde_json::json!({"markName": "0", "timeSeconds": 0.125}), + serde_json::json!({"markName": "1", "timeSeconds": 0.450}), + ]; + let words = vec!["Hello".to_string(), "world".to_string()]; + let boundaries = parse_google_timepoints(&tps, &words); + assert_eq!(boundaries.len(), 2); + assert_eq!(boundaries[0].text, "Hello"); + assert_eq!(boundaries[0].offset, 125); + assert_eq!(boundaries[0].duration, 325); + assert_eq!(boundaries[1].text, "world"); + assert_eq!(boundaries[1].offset, 450); + } + + #[test] + fn test_estimate_word_boundaries() { + let boundaries = estimate_word_boundaries("Hello world this is a test"); + assert_eq!(boundaries.len(), 6); + assert_eq!(boundaries[0].text, "Hello"); + assert_eq!(boundaries[0].offset, 0); + assert!(boundaries[0].duration > 0); + } + + #[test] + fn test_normalize_gender() { + assert_eq!(normalize_gender("Female"), "Female"); + assert_eq!(normalize_gender("female"), "Female"); + assert_eq!(normalize_gender("Male"), "Male"); + assert_eq!(normalize_gender("male"), "Male"); + assert_eq!(normalize_gender(""), "Unknown"); + assert_eq!(normalize_gender("other"), "Unknown"); + } + + #[test] + fn test_build_config_all_engines() { + let engines = [ + "openai", + "elevenlabs", + "azure", + "google", + "cartesia", + "deepgram", + "playht", + "fishaudio", + "hume", + "mistral", + "murf", + "resemble", + "unrealspeech", + "upliftai", + "watson", + "witai", + "xai", + "modelslab", + "polly", + ]; + let creds = HashMap::new(); + for id in &engines { + assert!( + build_config(id, &creds).is_some(), + "Failed for engine: {id}" + ); + } + } + + #[test] + fn test_build_config_unknown() { + let creds = HashMap::new(); + assert!(build_config("nonexistent", &creds).is_none()); + } + + #[test] + fn test_azure_ssml_escapes_special_chars() { + let ssml = build_azure_ssml("A & B < C > D", "en-US-AriaNeural", 1.0, 1.0); + assert!(ssml.contains("&")); + assert!(ssml.contains("<")); + assert!(ssml.contains(">")); + } +} diff --git a/src/engine.rs b/src/engine.rs index eefefe2..7697d74 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -1,15 +1,18 @@ //! Core TTS engine trait. -use crate::types::{TtsResult, Voice}; +use crate::types::{TtsResult, Voice, WordBoundary}; use std::fmt; +/// Callback for streaming audio chunks. +pub type OnAudioCallback<'a> = &'a mut dyn FnMut(&[u8]); + +/// Callback for word boundary events. +pub type OnBoundaryCallback<'a> = &'a mut dyn FnMut(&str, f32, f32); + /// Trait that every TTS engine must implement. /// /// All methods receive voice, rate, pitch, and volume parameters so each /// engine can apply them as appropriate. -pub type OnAudioCallback<'a> = &'a mut dyn FnMut(&[u8]); -pub type OnBoundaryCallback<'a> = &'a mut dyn FnMut(&str, f32, f32); - pub trait TtsEngine: Send + Sync + fmt::Debug { /// Start speaking `text` asynchronously. #[allow(clippy::too_many_arguments)] @@ -45,4 +48,74 @@ pub trait TtsEngine: Send + Sync + fmt::Debug { /// Return the unique identifier of this engine (e.g. `"system"`, `"sherpaonnx"`). fn engine_id(&self) -> &'static str; + + /// Synthesize text to audio bytes (full buffer, no playback). + fn synth_to_bytes( + &self, + text: &str, + voice: Option<&str>, + rate: f32, + pitch: f32, + volume: f32, + ) -> TtsResult> { + let mut buf = Vec::new(); + self.speak( + text, + voice, + rate, + pitch, + volume, + Some(&mut |chunk: &[u8]| { + buf.extend_from_slice(chunk); + }), + None, + )?; + Ok(buf) + } + + /// Synthesize text and return word boundary information. + /// Default implementation estimates boundaries. + fn synth_with_boundaries( + &self, + text: &str, + voice: Option<&str>, + rate: f32, + pitch: f32, + volume: f32, + ) -> TtsResult<(Vec, Vec)> { + let audio = self.synth_to_bytes(text, voice, rate, pitch, volume)?; + let boundaries = estimate_word_boundaries(text); + Ok((audio, boundaries)) + } +} + +/// Estimate word boundaries using word-length-adjusted timing. +/// Matches the algorithm used in js-tts-wrapper and swift-tts-wrapper. +#[allow(clippy::cast_precision_loss)] +pub fn estimate_word_boundaries(text: &str) -> Vec { + let words: Vec<&str> = text.split_whitespace().filter(|w| !w.is_empty()).collect(); + if words.is_empty() { + return Vec::new(); + } + + let words_per_minute: f64 = 150.0; + let ms_per_word = 60_000.0 / words_per_minute; + + let mut boundaries = Vec::with_capacity(words.len()); + let mut current_ms: u64 = 0; + + for word in &words { + let length_factor = (word.len() as f64 / 5.0).clamp(0.5, 2.0); + let duration = (ms_per_word * length_factor) as u64; + let duration = duration.max(1); + + boundaries.push(WordBoundary { + text: (*word).to_string(), + offset: current_ms, + duration, + }); + current_ms += duration; + } + + boundaries } diff --git a/src/lib.rs b/src/lib.rs index 3431a4a..363f4ac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,7 +30,7 @@ )] mod cloud_engine; -mod engine; +pub mod engine; pub mod factory; mod sherpaonnx_engine; mod system_engine; @@ -323,9 +323,11 @@ pub extern "C" fn tts_get_voices( types::tts_voice { id: CString::new(v.id.clone()).unwrap().into_raw(), name: CString::new(v.name.clone()).unwrap().into_raw(), - language: CString::new(v.language.clone()).unwrap().into_raw(), + language: CString::new(v.primary_language().to_string()) + .unwrap() + .into_raw(), gender: CString::new(v.gender.clone()).unwrap().into_raw(), - engine: CString::new(v.engine.clone()).unwrap().into_raw(), + engine: CString::new(v.provider.clone()).unwrap().into_raw(), }, ); } diff --git a/src/sherpaonnx_engine.rs b/src/sherpaonnx_engine.rs index 73e96d1..b9e9f73 100644 --- a/src/sherpaonnx_engine.rs +++ b/src/sherpaonnx_engine.rs @@ -1,7 +1,7 @@ -//! Sherpa-ONNX offline TTS engine with 191-model registry. +//! Sherpa-ONNX offline TTS engine with model registry. -use crate::engine::TtsEngine; -use crate::types::{SherpaLanguage, SherpaModelInfo, TtsError, TtsResult, Voice}; +use crate::engine::{estimate_word_boundaries, TtsEngine}; +use crate::types::{LanguageCode, SherpaLanguage, SherpaModelInfo, TtsError, TtsResult, Voice}; use std::collections::HashMap; use std::path::PathBuf; @@ -9,10 +9,6 @@ use std::path::PathBuf; static MERGED_MODELS_JSON: &str = include_str!("merged_models.json"); /// Offline TTS engine using [Sherpa-ONNX](https://github.com/k2-fsa/sherpa-onnx). -/// -/// Models are looked up from the compiled-in registry and loaded from -/// `~/.rust-tts-wrapper/sherpaonnx//`. Audio is synthesised -/// offline and played via `aplay` (Linux). #[derive(Debug)] pub struct SherpaOnnxEngine { models: HashMap, @@ -22,10 +18,6 @@ pub struct SherpaOnnxEngine { impl SherpaOnnxEngine { /// Create a new Sherpa-ONNX engine. - /// - /// `credentials_json` may contain `"modelPath"` and `"modelId"` keys. - /// If empty, defaults to `~/.rust-tts-wrapper/sherpaonnx/` and the - /// `kokoro-en-en-19` model. pub fn new(credentials_json: &str) -> Self { let mut model_dir = default_model_dir(); let mut model_id = String::new(); @@ -151,7 +143,7 @@ impl TtsEngine for SherpaOnnxEngine { _pitch: f32, _volume: f32, mut on_audio: Option, - _on_boundary: Option, + mut on_boundary: Option, ) -> TtsResult<()> { let model_info = self.models.get(&self.loaded_model_id).ok_or_else(|| { TtsError(format!( @@ -221,9 +213,6 @@ impl TtsEngine for SherpaOnnxEngine { .ok_or_else(|| TtsError("SherpaOnnx synthesis returned no audio".into()))?; if let Some(cb) = on_audio.as_mut() { - // SherpaOnnx C API does not currently easily support streaming inside the progress callback without - // borrowing issues, because the closure requires `'static`. - // So we stream all at once right after generation, to still simulate stream interface. let samples = audio.samples(); let mut pcm_bytes = Vec::with_capacity(samples.len() * 2); for &s in samples { @@ -238,6 +227,17 @@ impl TtsEngine for SherpaOnnxEngine { } } + if let Some(cb) = on_boundary.as_mut() { + let estimated = estimate_word_boundaries(text); + for b in &estimated { + #[allow(clippy::cast_precision_loss)] + let start = b.offset as f32 / 1000.0; + #[allow(clippy::cast_precision_loss)] + let end = (b.offset + b.duration) as f32 / 1000.0; + cb(&b.text, start, end); + } + } + Ok(()) } @@ -265,14 +265,22 @@ impl TtsEngine for SherpaOnnxEngine { .and_then(|m| m.language.first()) .map(|l| l.language_name.clone()) .unwrap_or_default(); + let lang_code = model_info + .and_then(|m| m.language.first()) + .map(|l| l.lang_code.clone()) + .unwrap_or_default(); let mut voices = Vec::new(); for i in 0..num_speakers { voices.push(Voice { id: format!("{i}"), name: format!("Speaker {i}"), - language: lang.clone(), - gender: String::new(), - engine: "sherpaonnx".to_string(), + gender: "Unknown".to_string(), + provider: "sherpaonnx".to_string(), + language_codes: vec![LanguageCode { + bcp47: lang.clone(), + iso639_3: lang_code.clone(), + display: lang.clone(), + }], }); } Ok(voices) diff --git a/src/system_engine.rs b/src/system_engine.rs index bb7b7a0..29113c5 100644 --- a/src/system_engine.rs +++ b/src/system_engine.rs @@ -1,14 +1,10 @@ //! System TTS engine via speech-dispatcher (Linux). -use crate::engine::TtsEngine; +use crate::engine::{estimate_word_boundaries, TtsEngine}; use crate::types::{TtsError, TtsResult, Voice}; use std::sync::Mutex; /// TTS engine that uses the system's speech-dispatcher daemon. -/// -/// On Linux this connects to speech-dispatcher via its IPC protocol. -/// On creation it attempts to open a connection; if speech-dispatcher -/// is not running, subsequent calls will return errors. #[derive(Debug)] pub struct SystemEngine { conn: Mutex>, @@ -16,9 +12,6 @@ pub struct SystemEngine { impl SystemEngine { /// Create a new system engine, connecting to speech-dispatcher. - /// - /// If the connection fails (e.g. speech-dispatcher not running), - /// the engine is still created but speak/stop calls will return errors. pub fn new() -> Self { let conn = speech_dispatcher::Connection::open( "rust-tts-wrapper", @@ -42,7 +35,7 @@ impl TtsEngine for SystemEngine { _pitch: f32, _volume: f32, _on_audio: Option, - _on_boundary: Option, + mut on_boundary: Option, ) -> TtsResult<()> { let guard = self.conn.lock().unwrap(); let conn = guard @@ -53,6 +46,18 @@ impl TtsEngine for SystemEngine { let _ = conn.set_synthesis_voice_all(v); } conn.say(speech_dispatcher::Priority::Important, text); + + if let Some(cb) = on_boundary.as_mut() { + let estimated = estimate_word_boundaries(text); + for b in &estimated { + #[allow(clippy::cast_precision_loss)] + let start = b.offset as f32 / 1000.0; + #[allow(clippy::cast_precision_loss)] + let end = (b.offset + b.duration) as f32 / 1000.0; + cb(&b.text, start, end); + } + } + Ok(()) } diff --git a/src/types.rs b/src/types.rs index 30c9e09..52cdc88 100644 --- a/src/types.rs +++ b/src/types.rs @@ -3,19 +3,48 @@ use std::fmt; use std::os::raw::c_char; -/// A single voice offered by an engine. +/// A language code entry with BCP-47, ISO 639-3, and display name. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LanguageCode { + /// BCP-47 language tag (e.g. `"en-US"`). + pub bcp47: String, + /// ISO 639-3 language code (e.g. `"eng"`). + pub iso639_3: String, + /// Human-readable language name (e.g. `"English (United States)"`). + pub display: String, +} + +/// A single voice offered by an engine, unified across all providers. #[derive(Debug, Clone)] pub struct Voice { /// Unique voice identifier within the engine. pub id: String, /// Human-readable voice name. pub name: String, - /// BCP-47 language tag (e.g. `"en-US"`). - pub language: String, - /// Gender string (e.g. `"male"`, `"female"`, or empty). + /// Gender: `"Male"`, `"Female"`, or `"Unknown"`. pub gender: String, - /// The engine that provides this voice. - pub engine: String, + /// The engine/provider that provides this voice (e.g. `"azure"`, `"google"`). + pub provider: String, + /// Language codes supported by this voice. + pub language_codes: Vec, +} + +impl Voice { + /// Convenience: return the primary (first) BCP-47 language code, or empty string. + #[must_use] + pub fn primary_language(&self) -> &str { + self.language_codes.first().map_or("", |l| l.bcp47.as_str()) + } +} + +/// Normalize a raw gender string to `"Male"`, `"Female"`, or `"Unknown"`. +#[must_use] +pub fn normalize_gender(value: &str) -> &'static str { + match value.to_lowercase().as_str() { + "female" => "Female", + "male" => "Male", + _ => "Unknown", + } } /// Describes a registered engine for introspection. @@ -65,6 +94,17 @@ pub struct SherpaLanguage { pub country: String, } +/// A word boundary event with timing information. +#[derive(Debug, Clone, PartialEq)] +pub struct WordBoundary { + /// The spoken word text. + pub text: String, + /// Offset from start of audio in milliseconds. + pub offset: u64, + /// Duration of the word in milliseconds. + pub duration: u64, +} + /// C-compatible voice descriptor returned by [`tts_get_voices`](crate::tts_get_voices). #[repr(C)] pub struct tts_voice { diff --git a/tests/integration.rs b/tests/integration.rs index 8e65d2d..ecebe37 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -1,4 +1,4 @@ -//! Integration tests for the engine factory. +//! Integration tests for the TTS wrapper. use rust_tts_wrapper::factory; @@ -118,3 +118,120 @@ fn test_system_engine_stop_graceful() { ); } } + +#[test] +fn test_create_all_cloud_engines() { + let cloud_ids = [ + "openai", + "elevenlabs", + "azure", + "google", + "cartesia", + "deepgram", + "playht", + "fishaudio", + "hume", + "mistral", + "murf", + "resemble", + "unrealspeech", + "upliftai", + "xai", + "modelslab", + ]; + for id in &cloud_ids { + let engine = factory::create_engine(id, r#"{"apiKey":"test-key"}"#); + assert!(engine.is_some(), "Engine '{id}' should be creatable"); + } +} + +#[test] +fn test_create_azure_with_region() { + let engine = factory::create_engine( + "azure", + r#"{"subscriptionKey":"test-key","region":"eastus"}"#, + ); + assert!(engine.is_some()); +} + +#[test] +fn test_create_watson_with_all_creds() { + let engine = factory::create_engine( + "watson", + r#"{"apiKey":"test-key","region":"us-east","instanceId":"test-id"}"#, + ); + assert!(engine.is_some()); +} + +#[test] +fn test_create_polly_with_all_creds() { + let engine = factory::create_engine( + "polly", + r#"{"accessKeyId":"test","secretAccessKey":"test","region":"us-east-1"}"#, + ); + assert!(engine.is_some()); +} + +#[test] +fn test_sherpaonnx_engine_has_voices() { + let engine = factory::create_engine("sherpaonnx", "").expect("sherpaonnx engine"); + let voices = engine.get_voices().expect("voices"); + assert!(!voices.is_empty(), "SherpaONNX should have voices"); + assert_eq!(voices[0].provider, "sherpaonnx"); +} + +#[test] +fn test_engine_id_matches() { + let engine = factory::create_engine("openai", r#"{"apiKey":"test-key"}"#).unwrap(); + assert_eq!(engine.engine_id(), "openai"); +} + +#[test] +fn test_normalize_gender() { + use rust_tts_wrapper::types::normalize_gender; + assert_eq!(normalize_gender("Female"), "Female"); + assert_eq!(normalize_gender("female"), "Female"); + assert_eq!(normalize_gender("Male"), "Male"); + assert_eq!(normalize_gender("male"), "Male"); + assert_eq!(normalize_gender(""), "Unknown"); + assert_eq!(normalize_gender("other"), "Unknown"); +} + +#[test] +fn test_voice_struct_fields() { + use rust_tts_wrapper::types::{LanguageCode, Voice}; + let voice = Voice { + id: "test-voice".to_string(), + name: "Test Voice".to_string(), + gender: "Female".to_string(), + provider: "test".to_string(), + language_codes: vec![LanguageCode { + bcp47: "en-US".to_string(), + iso639_3: "eng".to_string(), + display: "English (United States)".to_string(), + }], + }; + assert_eq!(voice.primary_language(), "en-US"); + assert_eq!(voice.language_codes.len(), 1); +} + +#[test] +fn test_word_boundary_estimation() { + use rust_tts_wrapper::engine::estimate_word_boundaries; + let boundaries = estimate_word_boundaries("Hello world this is a test"); + assert_eq!(boundaries.len(), 6); + assert_eq!(boundaries[0].text, "Hello"); + assert_eq!(boundaries[0].offset, 0); + assert!(boundaries[0].duration > 0); + // Offsets should be monotonically increasing + for i in 1..boundaries.len() { + assert!(boundaries[i].offset > boundaries[i - 1].offset); + } +} + +#[test] +fn test_word_boundary_empty_text() { + use rust_tts_wrapper::engine::estimate_word_boundaries; + let boundaries = estimate_word_boundaries(""); + assert!(boundaries.is_empty()); +} From befa087e865ad23b9cdbc839d20c04f8e9ce4962 Mon Sep 17 00:00:00 2001 From: will wade Date: Mon, 1 Jun 2026 23:40:42 +0100 Subject: [PATCH 2/5] feat: add speechmarkdown-rust support and fix cbindgen - Add speechmarkdown-rust dependency (from AACTools/speechmarkdown-rust) - Auto-detect and convert SpeechMarkdown to SSML before synthesis, using platform-specific output (Azure, Google, Alexa) - Fix cbindgen build.rs to warn instead of panic on parse errors - Add unit tests for speech markdown preprocessing and passthrough - Add integration tests for speech markdown across platforms --- Cargo.toml | 3 ++- build.rs | 12 +++++++++--- src/cloud_engine.rs | 25 ++++++++++++++++++------- src/engine.rs | 30 ++++++++++++++++++++++++++++++ tests/integration.rs | 32 ++++++++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f5fcff7..c076aee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ crate-type = ["cdylib", "staticlib", "lib"] [features] default = ["system", "cloud", "sherpaonnx"] system = ["speech-dispatcher"] -cloud = ["reqwest", "serde_json", "base64"] +cloud = ["reqwest", "serde_json", "base64", "speechmarkdown-rust"] sherpaonnx = ["sherpa-onnx"] [dependencies] @@ -24,6 +24,7 @@ serde = { version = "1", features = ["derive"] } serde_json = { version = "1", optional = true } sherpa-onnx = { version = "1.13", optional = true } base64 = { version = "0.22", optional = true } +speechmarkdown-rust = { version = "0.1", optional = true } anyhow = "1" [build-dependencies] diff --git a/build.rs b/build.rs index bd50f40..f0e1fdf 100644 --- a/build.rs +++ b/build.rs @@ -3,10 +3,16 @@ use std::env; fn main() { let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); let config = cbindgen::Config::from_file("cbindgen.toml").unwrap_or_default(); - cbindgen::Builder::new() + match cbindgen::Builder::new() .with_crate(crate_dir) .with_config(config) .generate() - .expect("Unable to generate C bindings") - .write_to_file("include/tts_wrapper.h"); + { + Ok(bindings) => { + bindings.write_to_file("include/tts_wrapper.h"); + } + Err(e) => { + eprintln!("cbindgen warning: {e}"); + } + } } diff --git a/src/cloud_engine.rs b/src/cloud_engine.rs index 8c010b1..2841324 100644 --- a/src/cloud_engine.rs +++ b/src/cloud_engine.rs @@ -4,7 +4,7 @@ //! providers. Includes voice fetching for engines with list endpoints and //! word boundary support where APIs provide timing data. -use crate::engine::{estimate_word_boundaries, TtsEngine}; +use crate::engine::{estimate_word_boundaries, preprocess_speech_markdown, TtsEngine}; use crate::types::{normalize_gender, LanguageCode, TtsError, TtsResult, Voice, WordBoundary}; use std::collections::HashMap; @@ -513,6 +513,8 @@ impl TtsEngine for CloudEngine { .or_else(|| self.config.default_voice.clone()) .unwrap_or_default(); + let (text, _is_ssml) = preprocess_speech_markdown(text, &self.config.provider_id); + let mut req = self.client.post(&self.config.synth_url); // Auth header @@ -529,7 +531,7 @@ impl TtsEngine for CloudEngine { // Body depends on engine type let resp = if self.config.body_is_ssml { // Azure: send SSML XML body - let ssml = build_azure_ssml(text, &voice_to_use, rate, pitch); + let ssml = build_azure_ssml(&text, &voice_to_use, rate, pitch); let ct = self .config .content_type @@ -539,7 +541,7 @@ impl TtsEngine for CloudEngine { req.body(ssml).send() } else if self.config.provider_id == "google" { // Google: build JSON body with proper structure - let (body, _words) = build_google_request(text, &voice_to_use, on_boundary.is_some()); + let (body, _words) = build_google_request(&text, &voice_to_use, on_boundary.is_some()); req = req.json(&body); req.send() } else { @@ -548,7 +550,7 @@ impl TtsEngine for CloudEngine { if !self.config.text_field.is_empty() { body.insert( self.config.text_field.clone(), - serde_json::Value::String(text.to_string()), + serde_json::Value::String(text.clone()), ); } if !self.config.voice_param.is_empty() && !voice_to_use.is_empty() { @@ -601,7 +603,7 @@ impl TtsEngine for CloudEngine { } if let Some(cb) = on_boundary.as_mut() { - let (_, words) = build_google_request(text, &voice_to_use, true); + let (_, words) = build_google_request(&text, &voice_to_use, true); if let Some(tps) = json.get("timepoints").and_then(|v| v.as_array()) { let boundaries = parse_google_timepoints(tps, &words); for b in &boundaries { @@ -612,7 +614,7 @@ impl TtsEngine for CloudEngine { ); } } else { - let estimated = estimate_word_boundaries(text); + let estimated = estimate_word_boundaries(&text); for b in &estimated { cb( &b.text, @@ -637,7 +639,7 @@ impl TtsEngine for CloudEngine { } if let Some(cb) = on_boundary.as_mut() { - let estimated = estimate_word_boundaries(text); + let estimated = estimate_word_boundaries(&text); for b in &estimated { cb( &b.text, @@ -897,4 +899,13 @@ mod tests { assert!(ssml.contains("<")); assert!(ssml.contains(">")); } + + #[test] + fn test_speech_markdown_preprocessing() { + use crate::engine::preprocess_speech_markdown; + let (result, is_ssml) = + preprocess_speech_markdown("Hello (world)[emphasis:\"strong\"]", "azure"); + assert!(is_ssml); + assert!(result.contains("")); + } } diff --git a/src/engine.rs b/src/engine.rs index 7697d74..fa2b65d 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -9,10 +9,39 @@ pub type OnAudioCallback<'a> = &'a mut dyn FnMut(&[u8]); /// Callback for word boundary events. pub type OnBoundaryCallback<'a> = &'a mut dyn FnMut(&str, f32, f32); +/// Convert speech markdown to SSML if detected, otherwise return text as-is. +/// Returns (processed_text, is_ssml). +#[cfg(feature = "cloud")] +#[must_use] +pub fn preprocess_speech_markdown(text: &str, platform: &str) -> (String, bool) { + use speechmarkdown_rust::{Platform, SpeechMarkdownParser}; + + if !SpeechMarkdownParser::is_speech_markdown(text) { + return (text.to_string(), false); + } + + let platform = match platform { + "azure" => Platform::MicrosoftAzure, + "google" => Platform::GoogleAssistant, + _ => Platform::AmazonAlexa, + }; + + match SpeechMarkdownParser::to_ssml(text, platform) { + Ok(ssml) => (ssml, true), + Err(_) => (text.to_string(), false), + } +} + +#[cfg(not(feature = "cloud"))] +pub fn preprocess_speech_markdown(text: &str, _platform: &str) -> (String, bool) { + (text.to_string(), false) +} + /// Trait that every TTS engine must implement. /// /// All methods receive voice, rate, pitch, and volume parameters so each /// engine can apply them as appropriate. +#[allow(clippy::missing_errors_doc)] pub trait TtsEngine: Send + Sync + fmt::Debug { /// Start speaking `text` asynchronously. #[allow(clippy::too_many_arguments)] @@ -91,6 +120,7 @@ pub trait TtsEngine: Send + Sync + fmt::Debug { /// Estimate word boundaries using word-length-adjusted timing. /// Matches the algorithm used in js-tts-wrapper and swift-tts-wrapper. +#[must_use] #[allow(clippy::cast_precision_loss)] pub fn estimate_word_boundaries(text: &str) -> Vec { let words: Vec<&str> = text.split_whitespace().filter(|w| !w.is_empty()).collect(); diff --git a/tests/integration.rs b/tests/integration.rs index ecebe37..84eaae6 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -235,3 +235,35 @@ fn test_word_boundary_empty_text() { let boundaries = estimate_word_boundaries(""); assert!(boundaries.is_empty()); } + +#[test] +fn test_speech_markdown_detection() { + use rust_tts_wrapper::engine::preprocess_speech_markdown; + let (result, is_ssml) = + preprocess_speech_markdown("Hello (world)[emphasis:\"strong\"]", "azure"); + assert!(is_ssml); + assert!(result.contains("")); +} + +#[test] +fn test_speech_markdown_plain_text_passthrough() { + use rust_tts_wrapper::engine::preprocess_speech_markdown; + let (result, is_ssml) = preprocess_speech_markdown("Hello world", "azure"); + assert!(!is_ssml); + assert_eq!(result, "Hello world"); +} + +#[test] +fn test_speech_markdown_azure_platform() { + use rust_tts_wrapper::engine::preprocess_speech_markdown; + let (result, is_ssml) = preprocess_speech_markdown("This is +important+", "azure"); + assert!(is_ssml); + assert!(result.contains("microsoft") || result.contains("")); +} + +#[test] +fn test_speech_markdown_google_platform() { + use rust_tts_wrapper::engine::preprocess_speech_markdown; + let (_result, is_ssml) = preprocess_speech_markdown("This is +important+", "google"); + assert!(is_ssml); +} From 37e34de44e11ebd0db0ba58a9423f7e2855e4874 Mon Sep 17 00:00:00 2001 From: will wade Date: Mon, 1 Jun 2026 23:55:21 +0100 Subject: [PATCH 3/5] feat: add SpeakOptions, Gender enum, AudioFormat, SpeechRate/Pitch, pause/resume, check_credentials Match Swift TTSClient protocol API surface: - SpeakOptions struct with rate/pitch/volume/voice/format/ useSpeechMarkdown/useWordBoundary/rawSSML/extra fields - SpeechRate and SpeechPitch named presets (xSlow..xFast) - AudioFormat enum (mp3/wav/ogg/opus/aac/flac/pcm) - Gender typed enum replacing raw String - pause()/resume() on TtsEngine trait (default no-op) - check_credentials() using get_voices as validation - speak_with_options() and synth_to_bytes_with_options() - Configurable words_per_minute in word boundary estimator - Fix cloud-only build: #[cfg] gates on system/sherpaonnx modules - 10 new tests for types, options, and boundary estimation --- src/cloud_engine.rs | 25 +++--- src/engine.rs | 82 +++++++++++++++-- src/lib.rs | 4 +- src/sherpaonnx_engine.rs | 6 +- src/types.rs | 186 ++++++++++++++++++++++++++++++++++----- tests/integration.rs | 96 ++++++++++++++++++-- 6 files changed, 351 insertions(+), 48 deletions(-) diff --git a/src/cloud_engine.rs b/src/cloud_engine.rs index 2841324..0a59478 100644 --- a/src/cloud_engine.rs +++ b/src/cloud_engine.rs @@ -440,7 +440,7 @@ fn map_azure_voices(json: &[serde_json::Value]) -> Vec { voices.push(Voice { id: short_name.to_string(), name, - gender: normalize_gender(gender_raw).to_string(), + gender: normalize_gender(gender_raw), provider: "azure".to_string(), language_codes: vec![LanguageCode { bcp47: locale.to_string(), @@ -484,7 +484,7 @@ fn map_google_voices(json: &[serde_json::Value]) -> Vec { voices.push(Voice { id: name.to_string(), name: name.to_string(), - gender: normalize_gender(gender_raw).to_string(), + gender: normalize_gender(gender_raw), provider: "google".to_string(), language_codes: lang_codes, }); @@ -730,8 +730,7 @@ impl TtsEngine for CloudEngine { .or(v.get("labels")) .and_then(|v| v.as_str()) .unwrap_or(""), - ) - .to_string(), + ), provider: self.config.provider_id.clone(), language_codes: vec![], }) @@ -846,12 +845,18 @@ mod tests { #[test] fn test_normalize_gender() { - assert_eq!(normalize_gender("Female"), "Female"); - assert_eq!(normalize_gender("female"), "Female"); - assert_eq!(normalize_gender("Male"), "Male"); - assert_eq!(normalize_gender("male"), "Male"); - assert_eq!(normalize_gender(""), "Unknown"); - assert_eq!(normalize_gender("other"), "Unknown"); + assert_eq!( + super::super::types::normalize_gender("Female"), + super::super::types::Gender::Female + ); + assert_eq!( + super::super::types::normalize_gender("male"), + super::super::types::Gender::Male + ); + assert_eq!( + super::super::types::normalize_gender(""), + super::super::types::Gender::Unknown + ); } #[test] diff --git a/src/engine.rs b/src/engine.rs index fa2b65d..988caae 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -1,6 +1,6 @@ //! Core TTS engine trait. -use crate::types::{TtsResult, Voice, WordBoundary}; +use crate::types::{SpeakOptions, TtsResult, Voice, WordBoundary}; use std::fmt; /// Callback for streaming audio chunks. @@ -9,6 +9,15 @@ pub type OnAudioCallback<'a> = &'a mut dyn FnMut(&[u8]); /// Callback for word boundary events. pub type OnBoundaryCallback<'a> = &'a mut dyn FnMut(&str, f32, f32); +/// Callback for speech-started events. +pub type OnStartCallback<'a> = &'a mut dyn FnMut(); + +/// Callback for speech-finished events. +pub type OnEndCallback<'a> = &'a mut dyn FnMut(); + +/// Callback for error events. +pub type OnErrorCallback<'a> = &'a mut dyn FnMut(&str); + /// Convert speech markdown to SSML if detected, otherwise return text as-is. /// Returns (processed_text, is_ssml). #[cfg(feature = "cloud")] @@ -39,8 +48,7 @@ pub fn preprocess_speech_markdown(text: &str, _platform: &str) -> (String, bool) /// Trait that every TTS engine must implement. /// -/// All methods receive voice, rate, pitch, and volume parameters so each -/// engine can apply them as appropriate. +/// Mirrors Swift's `TTSClient` protocol. #[allow(clippy::missing_errors_doc)] pub trait TtsEngine: Send + Sync + fmt::Debug { /// Start speaking `text` asynchronously. @@ -56,6 +64,26 @@ pub trait TtsEngine: Send + Sync + fmt::Debug { on_boundary: Option, ) -> TtsResult<()>; + /// Speak with full [`SpeakOptions`], matching Swift's `speak(_:options:)`. + fn speak_with_options( + &self, + text: &str, + options: Option<&SpeakOptions>, + on_audio: Option, + on_boundary: Option, + ) -> TtsResult<()> { + let opts = options.cloned().unwrap_or_default(); + self.speak( + text, + opts.voice.as_deref(), + opts.effective_rate(), + opts.effective_pitch(), + opts.effective_volume(), + on_audio, + on_boundary, + ) + } + /// Speak `text` synchronously, blocking until synthesis completes. #[allow(clippy::too_many_arguments)] fn speak_sync( @@ -72,13 +100,33 @@ pub trait TtsEngine: Send + Sync + fmt::Debug { /// Stop any in-progress speech. fn stop(&self) -> TtsResult<()>; + /// Pause speech (default: no-op, engines may override). + fn pause(&self) -> TtsResult<()> { + Ok(()) + } + + /// Resume speech (default: no-op, engines may override). + fn resume(&self) -> TtsResult<()> { + Ok(()) + } + /// List available voices for this engine. fn get_voices(&self) -> TtsResult>; /// Return the unique identifier of this engine (e.g. `"system"`, `"sherpaonnx"`). fn engine_id(&self) -> &'static str; + /// Check whether the configured credentials are valid. + /// Default: attempt to fetch voices as a validation. + fn check_credentials(&self) -> TtsResult { + match self.get_voices() { + Ok(_) => Ok(true), + Err(_) => Ok(false), + } + } + /// Synthesize text to audio bytes (full buffer, no playback). + /// Mirrors Swift's `synthToBytes(_:options:)`. fn synth_to_bytes( &self, text: &str, @@ -102,8 +150,23 @@ pub trait TtsEngine: Send + Sync + fmt::Debug { Ok(buf) } + /// Synthesize with [`SpeakOptions`]. + fn synth_to_bytes_with_options( + &self, + text: &str, + options: Option<&SpeakOptions>, + ) -> TtsResult> { + let opts = options.cloned().unwrap_or_default(); + self.synth_to_bytes( + text, + opts.voice.as_deref(), + opts.effective_rate(), + opts.effective_pitch(), + opts.effective_volume(), + ) + } + /// Synthesize text and return word boundary information. - /// Default implementation estimates boundaries. fn synth_with_boundaries( &self, text: &str, @@ -119,16 +182,23 @@ pub trait TtsEngine: Send + Sync + fmt::Debug { } /// Estimate word boundaries using word-length-adjusted timing. -/// Matches the algorithm used in js-tts-wrapper and swift-tts-wrapper. +/// Mirrors Swift's `WordTimingEstimator.estimate(text:wordsPerMinute:)`. #[must_use] #[allow(clippy::cast_precision_loss)] pub fn estimate_word_boundaries(text: &str) -> Vec { + estimate_word_boundaries_with_wpm(text, 150.0) +} + +/// Estimate word boundaries with configurable words per minute. +/// Matches Swift's `WordTimingEstimator.estimate(text:wordsPerMinute:)`. +#[must_use] +#[allow(clippy::cast_precision_loss)] +pub fn estimate_word_boundaries_with_wpm(text: &str, words_per_minute: f64) -> Vec { let words: Vec<&str> = text.split_whitespace().filter(|w| !w.is_empty()).collect(); if words.is_empty() { return Vec::new(); } - let words_per_minute: f64 = 150.0; let ms_per_word = 60_000.0 / words_per_minute; let mut boundaries = Vec::with_capacity(words.len()); diff --git a/src/lib.rs b/src/lib.rs index 363f4ac..d4d284b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -32,7 +32,9 @@ mod cloud_engine; pub mod engine; pub mod factory; +#[cfg(feature = "sherpaonnx")] mod sherpaonnx_engine; +#[cfg(feature = "system")] mod system_engine; pub mod types; @@ -326,7 +328,7 @@ pub extern "C" fn tts_get_voices( language: CString::new(v.primary_language().to_string()) .unwrap() .into_raw(), - gender: CString::new(v.gender.clone()).unwrap().into_raw(), + gender: CString::new(v.gender.to_string()).unwrap().into_raw(), engine: CString::new(v.provider.clone()).unwrap().into_raw(), }, ); diff --git a/src/sherpaonnx_engine.rs b/src/sherpaonnx_engine.rs index b9e9f73..307bf56 100644 --- a/src/sherpaonnx_engine.rs +++ b/src/sherpaonnx_engine.rs @@ -1,7 +1,9 @@ //! Sherpa-ONNX offline TTS engine with model registry. use crate::engine::{estimate_word_boundaries, TtsEngine}; -use crate::types::{LanguageCode, SherpaLanguage, SherpaModelInfo, TtsError, TtsResult, Voice}; +use crate::types::{ + Gender, LanguageCode, SherpaLanguage, SherpaModelInfo, TtsError, TtsResult, Voice, +}; use std::collections::HashMap; use std::path::PathBuf; @@ -274,7 +276,7 @@ impl TtsEngine for SherpaOnnxEngine { voices.push(Voice { id: format!("{i}"), name: format!("Speaker {i}"), - gender: "Unknown".to_string(), + gender: Gender::Unknown, provider: "sherpaonnx".to_string(), language_codes: vec![LanguageCode { bcp47: lang.clone(), diff --git a/src/types.rs b/src/types.rs index 52cdc88..c9da126 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,8 +1,37 @@ //! Shared types used across the crate. +use std::collections::HashMap; use std::fmt; use std::os::raw::c_char; +/// Voice gender, matching Swift's `UnifiedVoice.Gender`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Gender { + Male, + Female, + Unknown, +} + +impl fmt::Display for Gender { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Male => write!(f, "Male"), + Self::Female => write!(f, "Female"), + Self::Unknown => write!(f, "Unknown"), + } + } +} + +/// Normalize a raw gender string to a typed [`Gender`]. +#[must_use] +pub fn normalize_gender(value: &str) -> Gender { + match value.to_lowercase().as_str() { + "female" => Gender::Female, + "male" => Gender::Male, + _ => Gender::Unknown, + } +} + /// A language code entry with BCP-47, ISO 639-3, and display name. #[derive(Debug, Clone, PartialEq, Eq)] pub struct LanguageCode { @@ -15,14 +44,15 @@ pub struct LanguageCode { } /// A single voice offered by an engine, unified across all providers. +/// Mirrors Swift's `UnifiedVoice`. #[derive(Debug, Clone)] pub struct Voice { /// Unique voice identifier within the engine. pub id: String, /// Human-readable voice name. pub name: String, - /// Gender: `"Male"`, `"Female"`, or `"Unknown"`. - pub gender: String, + /// Gender of the voice. + pub gender: Gender, /// The engine/provider that provides this voice (e.g. `"azure"`, `"google"`). pub provider: String, /// Language codes supported by this voice. @@ -37,16 +67,143 @@ impl Voice { } } -/// Normalize a raw gender string to `"Male"`, `"Female"`, or `"Unknown"`. -#[must_use] -pub fn normalize_gender(value: &str) -> &'static str { - match value.to_lowercase().as_str() { - "female" => "Female", - "male" => "Male", - _ => "Unknown", +/// Audio output format, matching Swift's `AudioFormat`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AudioFormat { + Mp3, + Wav, + Ogg, + Opus, + Aac, + Flac, + Pcm, +} + +impl fmt::Display for AudioFormat { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Mp3 => write!(f, "mp3"), + Self::Wav => write!(f, "wav"), + Self::Ogg => write!(f, "ogg"), + Self::Opus => write!(f, "opus"), + Self::Aac => write!(f, "aac"), + Self::Flac => write!(f, "flac"), + Self::Pcm => write!(f, "pcm"), + } + } +} + +/// Named speech rate presets, matching Swift's `SpeechRate`. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum SpeechRate { + XSlow, + Slow, + Medium, + Fast, + XFast, +} + +impl SpeechRate { + /// Convert to a float multiplier (1.0 = normal). + #[must_use] + pub fn rate_value(self) -> f32 { + match self { + Self::XSlow => 0.5, + Self::Slow => 0.75, + Self::Medium => 1.0, + Self::Fast => 1.25, + Self::XFast => 1.5, + } + } +} + +/// Named speech pitch presets, matching Swift's `SpeechPitch`. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum SpeechPitch { + XLow, + Low, + Medium, + High, + XHigh, +} + +impl SpeechPitch { + /// Convert to a float multiplier (1.0 = normal). + #[must_use] + pub fn pitch_value(self) -> f32 { + match self { + Self::XLow => 0.5, + Self::Low => 0.75, + Self::Medium => 1.0, + Self::High => 1.25, + Self::XHigh => 1.5, + } + } +} + +/// Options for speak/synth calls, matching Swift's `SpeakOptions`. +#[derive(Debug, Clone, Default)] +pub struct SpeakOptions { + /// Speech rate as a float multiplier (1.0 = normal). Overrides `speech_rate`. + pub rate: Option, + /// Speech rate as a named preset. + pub speech_rate: Option, + /// Speech pitch as a float multiplier (1.0 = normal). Overrides `speech_pitch`. + pub pitch: Option, + /// Speech pitch as a named preset. + pub speech_pitch: Option, + /// Volume (0.0–1.0). + pub volume: Option, + /// Voice identifier. + pub voice: Option, + /// Desired audio output format. + pub format: Option, + /// Whether to preprocess SpeechMarkdown to SSML. + pub use_speech_markdown: bool, + /// Whether to request real word boundary events from the API. + pub use_word_boundary: bool, + /// If true, pass SSML directly to the engine without wrapping. + pub raw_ssml: bool, + /// Engine-specific extra options. + pub extra: HashMap, +} + +impl SpeakOptions { + /// Resolve the effective rate value. + #[must_use] + pub fn effective_rate(&self) -> f32 { + self.rate + .or_else(|| self.speech_rate.map(SpeechRate::rate_value)) + .unwrap_or(1.0) + } + + /// Resolve the effective pitch value. + #[must_use] + pub fn effective_pitch(&self) -> f32 { + self.pitch + .or_else(|| self.speech_pitch.map(SpeechPitch::pitch_value)) + .unwrap_or(1.0) + } + + /// Resolve the effective volume value. + #[must_use] + pub fn effective_volume(&self) -> f32 { + self.volume.unwrap_or(1.0) } } +/// A word boundary event with timing information. +/// Mirrors Swift's `WordBoundary`. +#[derive(Debug, Clone, PartialEq)] +pub struct WordBoundary { + /// The spoken word text. + pub text: String, + /// Offset from start of audio in milliseconds. + pub offset: u64, + /// Duration of the word in milliseconds. + pub duration: u64, +} + /// Describes a registered engine for introspection. #[derive(Debug, Clone)] pub struct EngineDescriptor { @@ -94,17 +251,6 @@ pub struct SherpaLanguage { pub country: String, } -/// A word boundary event with timing information. -#[derive(Debug, Clone, PartialEq)] -pub struct WordBoundary { - /// The spoken word text. - pub text: String, - /// Offset from start of audio in milliseconds. - pub offset: u64, - /// Duration of the word in milliseconds. - pub duration: u64, -} - /// C-compatible voice descriptor returned by [`tts_get_voices`](crate::tts_get_voices). #[repr(C)] pub struct tts_voice { diff --git a/tests/integration.rs b/tests/integration.rs index 84eaae6..58bbbea 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -188,22 +188,22 @@ fn test_engine_id_matches() { #[test] fn test_normalize_gender() { - use rust_tts_wrapper::types::normalize_gender; - assert_eq!(normalize_gender("Female"), "Female"); - assert_eq!(normalize_gender("female"), "Female"); - assert_eq!(normalize_gender("Male"), "Male"); - assert_eq!(normalize_gender("male"), "Male"); - assert_eq!(normalize_gender(""), "Unknown"); - assert_eq!(normalize_gender("other"), "Unknown"); + use rust_tts_wrapper::types::{normalize_gender, Gender}; + assert_eq!(normalize_gender("Female"), Gender::Female); + assert_eq!(normalize_gender("female"), Gender::Female); + assert_eq!(normalize_gender("Male"), Gender::Male); + assert_eq!(normalize_gender("male"), Gender::Male); + assert_eq!(normalize_gender(""), Gender::Unknown); + assert_eq!(normalize_gender("other"), Gender::Unknown); } #[test] fn test_voice_struct_fields() { - use rust_tts_wrapper::types::{LanguageCode, Voice}; + use rust_tts_wrapper::types::{Gender, LanguageCode, Voice}; let voice = Voice { id: "test-voice".to_string(), name: "Test Voice".to_string(), - gender: "Female".to_string(), + gender: Gender::Female, provider: "test".to_string(), language_codes: vec![LanguageCode { bcp47: "en-US".to_string(), @@ -213,6 +213,7 @@ fn test_voice_struct_fields() { }; assert_eq!(voice.primary_language(), "en-US"); assert_eq!(voice.language_codes.len(), 1); + assert_eq!(voice.gender, Gender::Female); } #[test] @@ -267,3 +268,80 @@ fn test_speech_markdown_google_platform() { let (_result, is_ssml) = preprocess_speech_markdown("This is +important+", "google"); assert!(is_ssml); } + +#[test] +fn test_speak_options_defaults() { + use rust_tts_wrapper::types::SpeakOptions; + let opts = SpeakOptions::default(); + assert_eq!(opts.effective_rate(), 1.0); + assert_eq!(opts.effective_pitch(), 1.0); + assert_eq!(opts.effective_volume(), 1.0); + assert!(!opts.use_speech_markdown); + assert!(!opts.use_word_boundary); + assert!(!opts.raw_ssml); +} + +#[test] +fn test_speak_options_with_presets() { + use rust_tts_wrapper::types::{SpeakOptions, SpeechPitch, SpeechRate}; + let opts = SpeakOptions { + speech_rate: Some(SpeechRate::Fast), + speech_pitch: Some(SpeechPitch::Low), + volume: Some(0.8), + ..Default::default() + }; + assert_eq!(opts.effective_rate(), 1.25); + assert_eq!(opts.effective_pitch(), 0.75); + assert_eq!(opts.effective_volume(), 0.8); +} + +#[test] +fn test_speak_options_float_overrides_preset() { + use rust_tts_wrapper::types::{SpeakOptions, SpeechRate}; + let opts = SpeakOptions { + rate: Some(1.5), + speech_rate: Some(SpeechRate::Slow), + ..Default::default() + }; + assert_eq!(opts.effective_rate(), 1.5); +} + +#[test] +fn test_audio_format_display() { + use rust_tts_wrapper::types::AudioFormat; + assert_eq!(AudioFormat::Mp3.to_string(), "mp3"); + assert_eq!(AudioFormat::Wav.to_string(), "wav"); + assert_eq!(AudioFormat::Pcm.to_string(), "pcm"); +} + +#[test] +fn test_speech_rate_values() { + use rust_tts_wrapper::types::SpeechRate; + assert_eq!(SpeechRate::XSlow.rate_value(), 0.5); + assert_eq!(SpeechRate::Medium.rate_value(), 1.0); + assert_eq!(SpeechRate::XFast.rate_value(), 1.5); +} + +#[test] +fn test_speech_pitch_values() { + use rust_tts_wrapper::types::SpeechPitch; + assert_eq!(SpeechPitch::XLow.pitch_value(), 0.5); + assert_eq!(SpeechPitch::Medium.pitch_value(), 1.0); + assert_eq!(SpeechPitch::XHigh.pitch_value(), 1.5); +} + +#[test] +fn test_gender_enum() { + use rust_tts_wrapper::types::Gender; + assert_eq!(Gender::Male.to_string(), "Male"); + assert_eq!(Gender::Female.to_string(), "Female"); + assert_eq!(Gender::Unknown.to_string(), "Unknown"); +} + +#[test] +fn test_word_boundary_with_wpm() { + use rust_tts_wrapper::engine::estimate_word_boundaries_with_wpm; + let fast = estimate_word_boundaries_with_wpm("Hello world", 300.0); + let slow = estimate_word_boundaries_with_wpm("Hello world", 75.0); + assert!(fast[0].duration < slow[0].duration); +} From d7a9c044041a19e37ac3084b2bf4a709e68aabbf Mon Sep 17 00:00:00 2001 From: will wade Date: Tue, 2 Jun 2026 06:11:27 +0100 Subject: [PATCH 4/5] docs: comprehensive README with engine capabilities, full API reference, and updated bindings - README: accurate per-engine table (streaming, voice list, word boundaries, speech markdown support) - Full Rust API reference: TtsEngine trait, SpeakOptions, Voice, WordBoundary, callbacks, factory, utility functions - Full C API reference: all 17 exported functions - Code examples for C, Rust, Python, .NET, Swift - Python binding: add speak_sync, set_pitch, set_volume, on_audio callback, on_boundary callback with proper ctypes types - .NET binding: add speak_sync, delegate types for audio/boundary callbacks - Swift binding: add speakSync, stop, setVoice, setRate, setPitch, setVolume --- README.md | 367 ++++++++++++++++++++++----------- bindings/dotnet/TtsClient.cs | 8 + bindings/python/tts_wrapper.py | 68 +++++- bindings/swift/TtsClient.swift | 48 ++++- 4 files changed, 358 insertions(+), 133 deletions(-) diff --git a/README.md b/README.md index 51d9f3f..f16332b 100644 --- a/README.md +++ b/README.md @@ -1,109 +1,222 @@ # rust-tts-wrapper -Cross-platform TTS (Text-to-Speech) wrapper with C API. Mirrors [js-tts-wrapper](https://github.com/AACTools/js-tts-wrapper) and [swift-tts-wrapper](https://github.com/AACTools/swift-tts-wrapper). +Cross-platform TTS (Text-to-Speech) wrapper with C ABI. Mirrors [js-tts-wrapper](https://github.com/AACTools/js-tts-wrapper) and [swift-tts-wrapper](https://github.com/AACTools/swift-tts-wrapper). ## Engines (21 total) -| Engine | Type | Credentials | Voice List | Word Boundaries | -|--------|------|-------------|------------|-----------------| -| System (speech-dispatcher) | Local | None | — | Estimated | -| Sherpa-ONNX | Local (191 models) | None | Speakers from registry | Estimated | -| OpenAI | Cloud | API Key | — | Estimated | -| ElevenLabs | Cloud | API Key | API | Estimated | -| Azure | Cloud | Subscription Key + Region | API | Estimated | -| Google Cloud | Cloud | API Key | API | Real (timepoints via v1beta1) | -| Amazon Polly | Cloud | Access Key + Secret + Region | — | Estimated | -| Cartesia | Cloud | API Key | API | Estimated | -| Deepgram | Cloud | API Key | — | Estimated | -| PlayHT | Cloud | API Key + User ID | — | Estimated | -| Fish Audio | Cloud | API Key | — | Estimated | -| Hume AI | Cloud | API Key | — | Estimated | -| Mistral | Cloud | API Key | — | Estimated | -| Murf | Cloud | API Key | — | Estimated | -| Resemble AI | Cloud | API Key | — | Estimated | -| Unreal Speech | Cloud | API Key | — | Estimated | -| UpliftAI | Cloud | API Key | — | Estimated | -| IBM Watson | Cloud | API Key + Region + Instance ID | — | Estimated | -| Wit.ai | Cloud | Token | — | Estimated | -| xAI | Cloud | API Key | — | Estimated | -| ModelsLab | Cloud | API Key | — | Estimated | - -## Key Features - -- **Unified Voice struct** matching js-tts-wrapper and swift-tts-wrapper with `language_codes` array, `provider` field, and normalized gender -- **Streaming audio** via chunked HTTP reads (8KB chunks) through the `on_audio` callback -- **Word boundary events** via `on_boundary` callback — real API timing for Google (v1beta1 timepoints with SSML marks), estimated boundaries for all other engines -- **Azure SSML support** — proper SSML generation with XML escaping, voice selection, and prosody tags -- **Google REST API** — correct JSON body structure with optional timepoint support -- **Voice enumeration** for Azure, Google, ElevenLabs, Cartesia, and other engines with list APIs -- **Word timing estimation** matching the algorithm in JS and Swift (word-length-adjusted, 150 WPM baseline) -- **C ABI** for bindings to Python, .NET, Swift, and other languages -- **Sherpa-ONNX** offline TTS with 191 models from bundled registry - -## Usage (C API) +| Engine | Type | Credentials | Streaming | Voice List | Word Boundaries | Speech Markdown | +|--------|------|-------------|-----------|------------|-----------------|-----------------| +| System (speech-dispatcher) | Local | None | — | — | Estimated | — | +| Sherpa-ONNX | Local (191 models) | None | Simulated* | Speakers | Estimated | — | +| Azure | Cloud | Key + Region | Chunked | API | Estimated | Platform-aware | +| Google Cloud | Cloud | API Key | Chunked | API | **Real** (v1beta1 timepoints) | Platform-aware | +| OpenAI | Cloud | API Key | Chunked | — | Estimated | Platform-aware | +| ElevenLabs | Cloud | API Key | Chunked | API | Estimated | Platform-aware | +| Cartesia | Cloud | API Key | Chunked | API | Estimated | Platform-aware | +| Deepgram | Cloud | API Key | Chunked | — | Estimated | Platform-aware | +| PlayHT | Cloud | API Key + User ID | Chunked | — | Estimated | Platform-aware | +| Fish Audio | Cloud | API Key | Chunked | — | Estimated | Platform-aware | +| Hume AI | Cloud | API Key | Chunked | — | Estimated | Platform-aware | +| Mistral | Cloud | API Key | Chunked | — | Estimated | Platform-aware | +| Murf | Cloud | API Key | Chunked | — | Estimated | Platform-aware | +| Resemble AI | Cloud | API Key | Chunked | — | Estimated | Platform-aware | +| Unreal Speech | Cloud | API Key | Chunked | — | Estimated | Platform-aware | +| UpliftAI | Cloud | API Key | Chunked | — | Estimated | Platform-aware | +| Amazon Polly | Cloud | Key + Secret + Region | Chunked | — | Estimated | Platform-aware | +| IBM Watson | Cloud | Key + Region + Instance | Chunked | — | Estimated | Platform-aware | +| Wit.ai | Cloud | Token | Chunked | — | Estimated | Platform-aware | +| xAI | Cloud | API Key | Chunked | — | Estimated | Platform-aware | +| ModelsLab | Cloud | API Key | Chunked | — | Estimated | Platform-aware | + +- **Streaming**: Cloud engines stream audio in 8KB chunks via the `on_audio` callback. Sherpa-ONNX delivers all audio at once after synthesis (*simulated). +- **Voice List**: Engines with "API" can enumerate voices from the provider's API. +- **Word Boundaries**: Google returns real timing via v1beta1 timepoints with SSML marks. All others use word-length-adjusted estimation (150 WPM baseline, configurable). +- **Speech Markdown**: Auto-detected and converted to platform-specific SSML via [speechmarkdown-rust](https://github.com/AACTools/speechmarkdown-rust). Azure gets Microsoft SSML, Google gets Assistant SSML, others get Alexa SSML. + +## Rust API + +### `TtsEngine` Trait + +```rust +pub trait TtsEngine: Send + Sync + Debug { + // Speaking + fn speak(&self, text: &str, voice: Option<&str>, rate: f32, pitch: f32, volume: f32, + on_audio: Option, on_boundary: Option) -> TtsResult<()>; + fn speak_with_options(&self, text: &str, options: Option<&SpeakOptions>, + on_audio: Option, on_boundary: Option) -> TtsResult<()>; + fn speak_sync(&self, text: &str, voice: Option<&str>, rate: f32, pitch: f32, volume: f32, + on_audio: Option, on_boundary: Option) -> TtsResult<()>; + + // Synthesis (no playback) + fn synth_to_bytes(&self, text: &str, voice: Option<&str>, rate: f32, pitch: f32, volume: f32) -> TtsResult>; + fn synth_to_bytes_with_options(&self, text: &str, options: Option<&SpeakOptions>) -> TtsResult>; + fn synth_with_boundaries(&self, text: &str, voice: Option<&str>, rate: f32, pitch: f32, volume: f32) -> TtsResult<(Vec, Vec)>; + + // Control + fn stop(&self) -> TtsResult<()>; + fn pause(&self) -> TtsResult<()>; + fn resume(&self) -> TtsResult<()>; + + // Introspection + fn get_voices(&self) -> TtsResult>; + fn engine_id(&self) -> &'static str; + fn check_credentials(&self) -> TtsResult; +} +``` + +### Callback Types + +```rust +pub type OnAudioCallback<'a> = &'a mut dyn FnMut(&[u8]); +pub type OnBoundaryCallback<'a> = &'a mut dyn FnMut(&str, f32, f32); // word, start_s, end_s +pub type OnStartCallback<'a> = &'a mut dyn FnMut(); +pub type OnEndCallback<'a> = &'a mut dyn FnMut(); +pub type OnErrorCallback<'a> = &'a mut dyn FnMut(&str); +``` + +### Core Types + +```rust +pub struct Voice { + pub id: String, + pub name: String, + pub gender: Gender, // Male | Female | Unknown + pub provider: String, + pub language_codes: Vec, +} + +pub struct LanguageCode { + pub bcp47: String, // "en-US" + pub iso639_3: String, // "eng" + pub display: String, // "English (United States)" +} + +pub struct WordBoundary { + pub text: String, + pub offset: u64, // milliseconds + pub duration: u64, // milliseconds +} + +pub struct SpeakOptions { + pub rate: Option, + pub speech_rate: Option, // XSlow | Slow | Medium | Fast | XFast + pub pitch: Option, + pub speech_pitch: Option, // XLow | Low | Medium | High | XHigh + pub volume: Option, + pub voice: Option, + pub format: Option, // Mp3 | Wav | Ogg | Opus | Aac | Flac | Pcm + pub use_speech_markdown: bool, + pub use_word_boundary: bool, + pub raw_ssml: bool, + pub extra: HashMap, +} + +pub enum Gender { Male, Female, Unknown } +pub enum AudioFormat { Mp3, Wav, Ogg, Opus, Aac, Flac, Pcm } +pub enum SpeechRate { XSlow, Slow, Medium, Fast, XFast } +pub enum SpeechPitch { XLow, Low, Medium, High, XHigh } +``` + +### Utility Functions + +```rust +// Word boundary estimation (matches Swift WordTimingEstimator) +pub fn estimate_word_boundaries(text: &str) -> Vec; +pub fn estimate_word_boundaries_with_wpm(text: &str, words_per_minute: f64) -> Vec; + +// Speech Markdown preprocessing +pub fn preprocess_speech_markdown(text: &str, platform: &str) -> (String, bool); + +// Gender normalization +pub fn normalize_gender(value: &str) -> Gender; +``` + +### Factory + +```rust +pub fn create_engine(engine_id: &str, credentials_json: &str) -> Option>; +pub fn engine_count() -> usize; +pub fn engine_list() -> Vec; +``` + +## C API + +All functions are `extern "C"`, `#[no_mangle]`: + +| Function | Description | +|----------|-------------| +| `tts_create(engine_id, credentials_json)` | Create engine, returns opaque `tts_ctx*` | +| `tts_destroy(ctx)` | Free engine context | +| `tts_speak(ctx, text)` | Speak async, returns 0/-1 | +| `tts_speak_sync(ctx, text)` | Speak sync (blocking) | +| `tts_stop(ctx)` | Stop speech | +| `tts_get_voices(ctx, out_voices, out_count)` | Get voice list | +| `tts_free_voices(voices, count)` | Free voice array | +| `tts_set_voice(ctx, voice_id)` | Set voice | +| `tts_set_rate(ctx, rate)` | Set rate (1.0 = normal) | +| `tts_set_pitch(ctx, pitch)` | Set pitch (1.0 = normal) | +| `tts_set_volume(ctx, volume)` | Set volume (1.0 = normal) | +| `tts_set_on_audio(ctx, cb, userdata)` | Set streaming audio callback | +| `tts_set_on_boundary(ctx, cb, userdata)` | Set word boundary callback | +| `tts_get_engine_count()` | Count registered engines | +| `tts_get_engines(out_engines)` | Get engine descriptors | +| `tts_free_engine_info(engines, count)` | Free engine info | +| `tts_get_last_error()` | Get last error message | + +### C Example ```c #include "tts_wrapper.h" #include void on_audio(const uint8_t* chunk, uintptr_t size, void* userdata) { - // Handle streaming audio chunks - printf("Received %zu bytes of audio\n", size); + printf("Audio chunk: %zu bytes\n", size); } -void on_boundary(const char* word, float start_time, float end_time, void* userdata) { - // Handle word boundary events - printf("Word '%s' from %.2f to %.2f\n", word, start_time, end_time); +void on_boundary(const char* word, float start, float end, void* userdata) { + printf("Word '%s' %.3f-%.3f\n", word, start, end); } int main() { - tts_ctx* ctx = tts_create("elevenlabs", "{\"apiKey\":\"your-api-key\"}"); - + tts_ctx* ctx = tts_create("openai", "{\"apiKey\":\"your-key\"}"); tts_set_on_audio(ctx, on_audio, NULL); tts_set_on_boundary(ctx, on_boundary, NULL); - - tts_set_voice(ctx, "Rachel"); - tts_set_rate(ctx, 1.0); - - tts_speak_sync(ctx, "Hello world, streaming is supported."); - + tts_set_voice(ctx, "alloy"); + tts_speak_sync(ctx, "Hello world"); tts_destroy(ctx); - return 0; } ``` -## Usage (Rust) +### Rust Example ```rust -use rust_tts_wrapper::factory; - -let engine = factory::create_engine("openai", r#"{"apiKey":"your-api-key"}"#).unwrap(); - -// Standard speaking -engine.speak("Hello world", Some("alloy"), 1.0, 1.0, 1.0, None, None).unwrap(); - -// Streaming with word boundary callbacks -let mut audio_cb = |chunk: &[u8]| { - println!("Received audio chunk of size {}", chunk.len()); -}; - -let mut boundary_cb = |word: &str, start: f32, end: f32| { - println!("Word '{}' from {:.3} to {:.3}", word, start, end); -}; - -engine.speak_sync( - "Hello world, streaming is supported.", - Some("alloy"), - 1.0, 1.0, 1.0, - Some(&mut audio_cb), - Some(&mut boundary_cb), -).unwrap(); - -// List voices -let voices = engine.get_voices().unwrap(); -for v in &voices { - println!("{} ({}) - {}", v.name, v.provider, v.primary_language()); +use rust_tts_wrapper::{factory, types::SpeakOptions}; + +let engine = factory::create_engine("openai", r#"{"apiKey":"key"}"#).unwrap(); + +// Simple speak +engine.speak("Hello", Some("alloy"), 1.0, 1.0, 1.0, None, None).unwrap(); + +// With callbacks +let mut audio_cb = |chunk: &[u8]| println!("{} bytes", chunk.len()); +let mut boundary_cb = |word: &str, s: f32, e: f32| println!("{}: {:.3}-{:.3}", word, s, e); +engine.speak_sync("Hello world", Some("alloy"), 1.0, 1.0, 1.0, + Some(&mut audio_cb), Some(&mut boundary_cb)).unwrap(); + +// With SpeakOptions +let opts = SpeakOptions { voice: Some("alloy".into()), ..Default::default() }; +engine.speak_with_options("Hello", Some(&opts), None, None).unwrap(); + +// Synth to bytes +let audio = engine.synth_to_bytes("Hello", Some("alloy"), 1.0, 1.0, 1.0).unwrap(); + +// Get voices +for v in engine.get_voices().unwrap() { + println!("{} ({}) - {}", v.name, v.gender, v.primary_language()); } + +// Check credentials +assert!(engine.check_credentials().unwrap()); ``` ## Build @@ -115,60 +228,76 @@ cargo build --all-features ### Features - `system` — speech-dispatcher (Linux system TTS) -- `cloud` — all 19 cloud engines via HTTP +- `cloud` — all 19 cloud engines via HTTP + speechmarkdown-rust + base64 - `sherpaonnx` — Sherpa-ONNX offline TTS (191 models) -## Architecture +### Lint & Test -``` - TtsEngine (trait) - | - +-------------+-------------+ - | | | - SystemEngine CloudEngine SherpaOnnxEngine - (speech- (19 cloud (191 local - dispatcher) providers) models) +```bash +cargo fmt --all -- --check +cargo clippy --all-features -- -D warnings +cargo test --all-features ``` -The `CloudEngine` uses a provider-specific configuration (`CloudConfig`) to handle differences in API structure — Azure sends SSML XML, Google sends JSON with base64 audio, and all others use standard JSON bodies. +## Bindings -### Voice Struct (Unified) +### Python (`bindings/python/tts_wrapper.py`) -```rust -pub struct Voice { - pub id: String, - pub name: String, - pub gender: String, // "Male", "Female", "Unknown" - pub provider: String, // "azure", "google", etc. - pub language_codes: Vec, -} +```python +from tts_wrapper import TTSClient -pub struct LanguageCode { - pub bcp47: String, // "en-US" - pub iso639_3: String, // "eng" - pub display: String, // "English (United States)" -} +client = TTSClient("openai", {"apiKey": "your-key"}) +client.on_audio(lambda chunk: print(f"{len(chunk)} bytes")) +client.on_boundary(lambda word, s, e: print(f"{word}: {s:.3f}-{e:.3f}")) +client.set_voice("alloy") +client.speak_sync("Hello world") +client.stop() ``` -### Word Boundaries +### .NET (`bindings/dotnet/TtsClient.cs`) -```rust -pub struct WordBoundary { - pub text: String, - pub offset: u64, // milliseconds from start - pub duration: u64, // milliseconds -} +```csharp +using TtsWrapper; + +var client = new TtsClient("openai", new() { {"apiKey", "your-key"} }); +client.SetVoice("alloy"); +client.SetRate(1.0f); +client.SetPitch(1.0f); +client.SetVolume(1.0f); +client.SpeakSync("Hello world"); +client.Stop(); ``` -## Sherpa-ONNX Models +### Swift (`bindings/swift/TtsClient.swift`) + +```swift +let client = TTSClient(engineId: "openai", credentials: ["apiKey": "your-key"]) +client.setVoice("alloy") +client.setRate(1.0) +client.speakSync("Hello world") +client.stop() +``` -191 models from the merged_models.json registry. Models auto-download on first use to `~/.rust-tts-wrapper/sherpaonnx/`. +## Architecture -## Bindings +``` + TtsEngine (trait) + | + +--------------+--------------+ + | | | + SystemEngine CloudEngine SherpaOnnxEngine + (speech- (19 cloud (191 local + dispatcher) providers) models) +``` + +Cloud engines use provider-specific `CloudConfig`: +- **Azure**: SSML XML body with prosody tags, XML escaping +- **Google**: JSON body with base64 audio, v1beta1 timepoint support +- **All others**: Standard JSON bodies + +## Sherpa-ONNX Models -- `bindings/python/` — Python via ctypes -- `bindings/dotnet/` — .NET via P/Invoke -- `bindings/swift/` — Swift via C interop +191 models from the bundled `merged_models.json` registry. Models are loaded from `~/.rust-tts-wrapper/sherpaonnx/`. ## License diff --git a/bindings/dotnet/TtsClient.cs b/bindings/dotnet/TtsClient.cs index 0c1c675..a1abb70 100644 --- a/bindings/dotnet/TtsClient.cs +++ b/bindings/dotnet/TtsClient.cs @@ -15,9 +15,17 @@ public static class Native [DllImport("rust_tts_wrapper")] public static extern void tts_set_rate(IntPtr ctx, float rate); [DllImport("rust_tts_wrapper")] public static extern void tts_set_pitch(IntPtr ctx, float pitch); [DllImport("rust_tts_wrapper")] public static extern void tts_set_volume(IntPtr ctx, float volume); + [DllImport("rust_tts_wrapper")] public static extern void tts_set_on_audio(IntPtr ctx, IntPtr cb, IntPtr userdata); + [DllImport("rust_tts_wrapper")] public static extern void tts_set_on_boundary(IntPtr ctx, IntPtr cb, IntPtr userdata); + [DllImport("rust_tts_wrapper")] public static extern int tts_get_voices(IntPtr ctx, out IntPtr voices, out int count); + [DllImport("rust_tts_wrapper")] public static extern void tts_free_voices(IntPtr voices, int count); [DllImport("rust_tts_wrapper")] public static extern int tts_get_engine_count(); + [DllImport("rust_tts_wrapper")] public static extern IntPtr tts_get_last_error(); } +public delegate void AudioCallback(byte[] chunk); +public delegate void BoundaryCallback(string word, float startTime, float endTime); + public class TtsClient : IDisposable { private IntPtr _ctx; diff --git a/bindings/python/tts_wrapper.py b/bindings/python/tts_wrapper.py index 8600b1e..f9dd360 100644 --- a/bindings/python/tts_wrapper.py +++ b/bindings/python/tts_wrapper.py @@ -4,10 +4,14 @@ import json import platform from pathlib import Path -from typing import Optional +from typing import Callable, Dict, List, Optional _lib = None +AUDIO_CB = ctypes.CFUNCTYPE(None, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, ctypes.c_void_p) +BOUNDARY_CB = ctypes.CFUNCTYPE(None, ctypes.c_char_p, ctypes.c_float, ctypes.c_float, ctypes.c_void_p) + + def _load_lib(): global _lib if _lib is not None: @@ -23,31 +27,51 @@ def _load_lib(): _lib.tts_create.argtypes = [ctypes.c_char_p, ctypes.c_char_p] _lib.tts_destroy.restype = None _lib.tts_destroy.argtypes = [ctypes.c_void_p] - _lib.tts_speak.restype = ctypes.c_int + _lib.tts_speak.restype = ctypes.c_int32 _lib.tts_speak.argtypes = [ctypes.c_void_p, ctypes.c_char_p] + _lib.tts_speak_sync.restype = ctypes.c_int32 + _lib.tts_speak_sync.argtypes = [ctypes.c_void_p, ctypes.c_char_p] _lib.tts_stop.restype = None _lib.tts_stop.argtypes = [ctypes.c_void_p] _lib.tts_set_voice.restype = None _lib.tts_set_voice.argtypes = [ctypes.c_void_p, ctypes.c_char_p] _lib.tts_set_rate.restype = None _lib.tts_set_rate.argtypes = [ctypes.c_void_p, ctypes.c_float] - _lib.tts_get_engine_count.restype = ctypes.c_int + _lib.tts_set_pitch.restype = None + _lib.tts_set_pitch.argtypes = [ctypes.c_void_p, ctypes.c_float] + _lib.tts_set_volume.restype = None + _lib.tts_set_volume.argtypes = [ctypes.c_void_p, ctypes.c_float] + _lib.tts_set_on_audio.restype = None + _lib.tts_set_on_audio.argtypes = [ctypes.c_void_p, AUDIO_CB, ctypes.c_void_p] + _lib.tts_set_on_boundary.restype = None + _lib.tts_set_on_boundary.argtypes = [ctypes.c_void_p, BOUNDARY_CB, ctypes.c_void_p] + _lib.tts_get_voices.restype = ctypes.c_int32 + _lib.tts_get_voices.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.POINTER(ctypes.c_void_p)), ctypes.POINTER(ctypes.c_int32)] + _lib.tts_free_voices.restype = None + _lib.tts_free_voices.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.c_int32] + _lib.tts_get_engine_count.restype = ctypes.c_int32 _lib.tts_get_last_error.restype = ctypes.c_char_p return _lib class Voice: - __slots__ = ("id", "name", "language", "gender") - def __init__(self, id: str, name: str, language: str, gender: str): + """A TTS voice with metadata.""" + __slots__ = ("id", "name", "language", "gender", "engine") + + def __init__(self, id: str, name: str, language: str, gender: str, engine: str = ""): self.id = id self.name = name self.language = language self.gender = gender + self.engine = engine + def __repr__(self): - return f"Voice(id={self.id!r}, name={self.name!r})" + return f"Voice(id={self.id!r}, name={self.name!r}, engine={self.engine!r})" class TTSClient: + """Python TTS client wrapping the Rust C library.""" + def __init__(self, engine_id: str = "system", credentials: Optional[dict] = None): self._lib = _load_lib() creds_json = json.dumps(credentials or {}) @@ -58,6 +82,8 @@ def __init__(self, engine_id: str = "system", credentials: Optional[dict] = None err = self._lib.tts_get_last_error() msg = err.decode() if err else "Unknown error" raise RuntimeError(f"Failed to create TTS engine: {msg}") + self._audio_cb = None + self._boundary_cb = None def __del__(self): if hasattr(self, "_ctx") and self._ctx: @@ -68,6 +94,11 @@ def speak(self, text: str) -> None: if result != 0: raise RuntimeError("Speech synthesis failed") + def speak_sync(self, text: str) -> None: + result = self._lib.tts_speak_sync(self._ctx, text.encode()) + if result != 0: + raise RuntimeError("Speech synthesis failed") + def stop(self) -> None: self._lib.tts_stop(self._ctx) @@ -77,8 +108,27 @@ def set_voice(self, voice_id: str) -> None: def set_rate(self, rate: float) -> None: self._lib.tts_set_rate(self._ctx, ctypes.c_float(rate)) + def set_pitch(self, pitch: float) -> None: + self._lib.tts_set_pitch(self._ctx, ctypes.c_float(pitch)) + + def set_volume(self, volume: float) -> None: + self._lib.tts_set_volume(self._ctx, ctypes.c_float(volume)) + + def on_audio(self, callback: Callable[[bytes], None]) -> None: + @AUDIO_CB + def _cb(data, size, _userdata): + callback(ctypes.string_at(data, size)) + self._audio_cb = _cb + self._lib.tts_set_on_audio(self._ctx, _cb, None) + + def on_boundary(self, callback: Callable[[str, float, float], None]) -> None: + @BOUNDARY_CB + def _cb(word, start, end, _userdata): + callback(word.decode() if word else "", start, end) + self._boundary_cb = _cb + self._lib.tts_set_on_boundary(self._ctx, _cb, None) + -def list_engines(): +def list_engines() -> int: lib = _load_lib() - count = lib.tts_get_engine_count() - return count + return lib.tts_get_engine_count() diff --git a/bindings/swift/TtsClient.swift b/bindings/swift/TtsClient.swift index 8217bb6..e39f54b 100644 --- a/bindings/swift/TtsClient.swift +++ b/bindings/swift/TtsClient.swift @@ -4,8 +4,7 @@ import rust_tts_wrapper public func ttsCreate(_ engineId: UnsafePointer?, _ credentialsJson: UnsafePointer?) -> OpaquePointer? { guard let engineId else { return nil } let creds = credentialsJson.map { String(cString: $0) } ?? "" - let ctx = rust_tts_wrapper.tts_create(engineId, creds) - return ctx + return rust_tts_wrapper.tts_create(engineId, creds) } @_cdecl("tts_destroy") @@ -20,13 +19,19 @@ public func ttsSpeak(_ ctx: OpaquePointer?, _ text: UnsafePointer?) -> In return rust_tts_wrapper.tts_speak(ctx, text) } +@_cdecl("tts_speak_sync") +public func ttsSpeakSync(_ ctx: OpaquePointer?, _ text: UnsafePointer?) -> Int32 { + guard let ctx, let text else { return -1 } + return rust_tts_wrapper.tts_speak_sync(ctx, text) +} + public class TTSClient { private var ctx: OpaquePointer? public init(engineId: String = "system", credentials: [String: String] = [:]) { - let credsJson = try? JSONSerialization.data(withJSONObject: credentials) - let credsStr = credsJson.flatMap { String(data: $0, encoding: .utf8) } ?? "{}" - ctx = ttsCreate(engineId, credsStr) + let credsJson = (try? JSONSerialization.data(withJSONObject: credentials)) + .flatMap { String(data: $0, encoding: .utf8) } ?? "{}" + ctx = ttsCreate(engineId, credsJson) } deinit { @@ -38,4 +43,37 @@ public class TTSClient { _ = ttsSpeak(ctx, ptr) } } + + public func speakSync(_ text: String) { + text.withCString { ptr in + _ = ttsSpeakSync(ctx, ptr) + } + } + + public func stop() { + guard let ctx else { return } + rust_tts_wrapper.tts_stop(ctx) + } + + public func setVoice(_ voiceId: String) { + guard let ctx else { return } + voiceId.withCString { ptr in + rust_tts_wrapper.tts_set_voice(ctx, ptr) + } + } + + public func setRate(_ rate: Float) { + guard let ctx else { return } + rust_tts_wrapper.tts_set_rate(ctx, rate) + } + + public func setPitch(_ pitch: Float) { + guard let ctx else { return } + rust_tts_wrapper.tts_set_pitch(ctx, pitch) + } + + public func setVolume(_ volume: Float) { + guard let ctx else { return } + rust_tts_wrapper.tts_set_volume(ctx, volume) + } } From d4c34fead14277a2a4d7422204280ac670919903 Mon Sep 17 00:00:00 2001 From: will wade Date: Tue, 2 Jun 2026 06:18:49 +0100 Subject: [PATCH 5/5] ci: separate CI and publish workflows, add cross-platform native builds - ci.yml: lint, clippy, test, cloud-only build, C header artifact - publish.yml: triggered on v* tags and workflow_dispatch - version sync from git tag to Cargo.toml - test gate (clippy + test + cloud-only build) - cross-platform native builds: Windows x64/arm64, macOS x64/arm64, Linux x64/arm64 (shared + static libraries) - crates.io publish with CARGO_REGISTRY_TOKEN - GitHub Release with all native libraries + C header - Adapted from speechmarkdown-rust publish workflow --- .github/workflows/ci.yml | 29 ------ .github/workflows/publish.yml | 160 ++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 477513f..b2a747c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,37 +32,8 @@ jobs: - name: Build without default features (cloud only) run: cargo build --no-default-features --features cloud - - name: Generate C header - run: cargo build --all-features - - name: Upload C header uses: actions/upload-artifact@v4 with: name: tts_wrapper_header path: include/tts_wrapper.h - - release: - if: startsWith(github.ref, 'refs/tags/v') - needs: lint-and-test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - name: Install system dependencies - run: sudo apt-get update && sudo apt-get install -y libspeechd-dev libclang-dev - - - name: Build release - run: cargo build --release --all-features - - - name: Publish to crates.io - env: - CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} - run: cargo publish --all-features - - - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - files: | - target/release/librust_tts_wrapper.a - target/release/librust_tts_wrapper.so - include/tts_wrapper.h diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..ed6ef9d --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,160 @@ +name: Publish + +on: + push: + tags: ["v*"] + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + +jobs: + version: + name: Sync version + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@v4 + - id: version + run: echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" + - name: Update Cargo.toml version + run: | + VERSION="${GITHUB_REF_NAME#v}" + sed -i "s/^version = \".*\"/version = \"$VERSION\"/" Cargo.toml + - uses: actions/upload-artifact@v4 + with: + name: version-sync + path: Cargo.toml + + test: + name: Test + needs: version + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: version-sync + - uses: dtolnay/rust-toolchain@stable + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y libspeechd-dev libclang-dev + - name: Clippy + run: cargo clippy --all-features -- -D warnings + - name: Test + run: cargo test --all-features + - name: Build cloud-only + run: cargo build --no-default-features --features cloud + + build-native: + name: Build native (${{ matrix.target }}) + needs: test + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-pc-windows-msvc + os: windows-latest + lib: rust_tts_wrapper.dll + - target: aarch64-pc-windows-msvc + os: windows-latest + lib: rust_tts_wrapper.dll + - target: x86_64-apple-darwin + os: macos-latest + lib: librust_tts_wrapper.dylib + - target: aarch64-apple-darwin + os: macos-latest + lib: librust_tts_wrapper.dylib + - target: x86_64-unknown-linux-gnu + os: ubuntu-latest + lib: librust_tts_wrapper.so + - target: aarch64-unknown-linux-gnu + os: ubuntu-latest + lib: librust_tts_wrapper.so + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: version-sync + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + - name: Install cross-compilation tools (Linux aarch64) + if: matrix.target == 'aarch64-unknown-linux-gnu' + run: | + sudo apt-get update + sudo apt-get install -y gcc-aarch64-linux-gnu + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV" + - name: Install speech-dispatcher dev (Linux) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y libspeechd-dev libclang-dev + - name: Build release + run: cargo build --release --target ${{ matrix.target }} + - uses: actions/upload-artifact@v4 + with: + name: native-${{ matrix.target }} + path: target/${{ matrix.target }}/release/${{ matrix.lib }} + - name: Upload static library + uses: actions/upload-artifact@v4 + with: + name: static-${{ matrix.target }} + path: target/${{ matrix.target }}/release/librust_tts_wrapper.a + if: runner.os != 'Windows' + - name: Upload MSVC import library + uses: actions/upload-artifact@v4 + with: + name: static-${{ matrix.target }} + path: target/${{ matrix.target }}/release/rust_tts_wrapper.lib + if: runner.os == 'Windows' + + publish-crate: + name: crates.io + needs: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: version-sync + - uses: dtolnay/rust-toolchain@stable + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y libspeechd-dev libclang-dev + - name: Publish + run: cargo publish --allow-dirty + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + + publish-github-release: + name: GitHub Release + needs: [build-native] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + path: artifacts + pattern: "{native-*,static-*}" + merge-multiple: true + - name: Organize release assets + run: | + mkdir -p release-assets + # Shared libraries + for f in artifacts/rust_tts_wrapper.dll artifacts/librust_tts_wrapper.dylib artifacts/librust_tts_wrapper.so; do + [ -f "$f" ] && cp "$f" release-assets/ 2>/dev/null || true + done + # Static libraries / import libs + for f in artifacts/librust_tts_wrapper.a artifacts/rust_tts_wrapper.lib; do + [ -f "$f" ] && cp "$f" release-assets/ 2>/dev/null || true + done + # C header + cp include/tts_wrapper.h release-assets/ + ls -la release-assets/ + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + generate_release_notes: true + files: release-assets/* + fail_on_unmatched_files: false