diff --git a/docs/models/irodori_tts.md b/docs/models/irodori_tts.md index 981cbf2c4..095d8e662 100644 --- a/docs/models/irodori_tts.md +++ b/docs/models/irodori_tts.md @@ -65,6 +65,26 @@ audiocpp_cli --task clon --family irodori_tts \ --out out.wav ``` +v4 with a Speaker Inversion embedding: + +```bash +audiocpp_cli --task tts --family irodori_tts \ + --model models/Irodori-TTS-v4-Small-GGUF/irodori-tts-v4-small-q8_0.gguf \ + --backend cuda --language ja \ + --text "こんにちは、これは学習した話者埋め込みを使った推論です。" \ + --request-option speaker_embedding_path=path/to/name.speaker.safetensors \ + --out out.wav +``` + +## Speaker Inversion Embeddings + +[Irodori-TTS](https://github.com/Aratako/Irodori-TTS)'s Speaker Inversion training learns a few speaker tokens for one voice while the base model stays frozen, and saves them as `*.speaker.safetensors`. audio.cpp uses such a file as the speaker condition in place of reference audio, like `--ref-embed` in the Python `infer.py`: the tokens go to the model as they are, without the speaker encoder. + +- **By path:** request option `speaker_embedding_path` (CLI `--request-option`, or `"options"` in a server request). +- **By name:** put the file in an `embeddings` directory next to the model weights as `.safetensors` and pass `` as the voice id (CLI `--voice-id`, server `"voice"`). With the model path set to that directory, the server's `GET /v1/audio/voices` lists the names. A voice id with no such file is ignored, as before. + +The file must hold a `speaker_embedding` tensor of shape `[tokens, speaker_dim]` (F32, F16 or BF16). An embedding cannot be combined with reference audio or `no_ref=true`. Use it with the base checkpoint it was trained on: an embedding from another checkpoint with the same `speaker_dim` loads, but the speaker match is not guaranteed. + ## Request Options (use with `--request-option`) v4 uses the normalized schema-v1 option names directly. New requests should use these names: @@ -74,6 +94,7 @@ v4 uses the normalized schema-v1 option names directly. New requests should use | `language` | `ja` | `ja` | Text language code; Irodori-TTS is Japanese-only. | | `instruction` | text | empty | Voice-design instruction; only useful on caption-conditioned checkpoints. Legacy `caption` is accepted as an alias. | | `no_ref` | bool | `true` unless a reference is provided | Use no-reference generation. Set `false` with `--voice-ref` for reference conditioning. | +| `speaker_embedding_path` | path | unset | Speaker Inversion embedding (`*.speaker.safetensors`) used instead of reference audio. See [Speaker Inversion Embeddings](#speaker-inversion-embeddings). | | `num_inference_steps` | integer | `40` | RF diffusion steps. | | `duration_sec` | seconds | unset | Explicit output duration; omitted uses predicted duration. | | `duration_scale` | float | `1.0` | Multiplier for predicted duration. | diff --git a/include/engine/models/irodori_tts/assets.h b/include/engine/models/irodori_tts/assets.h index 6f84f884e..5609a1d8e 100644 --- a/include/engine/models/irodori_tts/assets.h +++ b/include/engine/models/irodori_tts/assets.h @@ -87,6 +87,10 @@ struct IrodoriCodecConfig { }; struct IrodoriTTSAssets { + // The model directory as given (or the directory of the given file). Unlike + // resources.model_root(), it is not replaced by the extracted sidecars of a + // standalone GGUF, so it is where user files such as embeddings/ live. + std::filesystem::path model_dir; assets::ResourceBundle resources; IrodoriModelConfig config; IrodoriCodecConfig codec; diff --git a/include/engine/models/irodori_tts/types.h b/include/engine/models/irodori_tts/types.h index 38fd9265f..565cddc53 100644 --- a/include/engine/models/irodori_tts/types.h +++ b/include/engine/models/irodori_tts/types.h @@ -3,6 +3,8 @@ #include "engine/framework/runtime/session.h" #include +#include +#include #include namespace engine::models::irodori_tts { @@ -35,6 +37,9 @@ struct IrodoriRequest { bool no_ref = true; runtime::AudioBuffer reference_audio; bool has_reference_audio = false; + // Speaker Inversion embedding used as the speaker condition instead of + // reference audio. + std::optional speaker_embedding_path; IrodoriGenerationOptions generation; }; diff --git a/model_specs/irodori_tts.json b/model_specs/irodori_tts.json index ba8581fbe..8e56b5196 100644 --- a/model_specs/irodori_tts.json +++ b/model_specs/irodori_tts.json @@ -46,6 +46,12 @@ "required": false, "default": true }, + { + "name": "speaker_embedding_path", + "type": "path", + "description": "Speaker Inversion embedding (*.speaker.safetensors written by Irodori-TTS training) used as the speaker condition instead of reference audio; use it with the base checkpoint it was trained on. Cannot be combined with reference audio or no_ref=true.", + "required": false + }, { "name": "num_inference_steps", "type": "int", diff --git a/src/models/irodori_tts/assets.cpp b/src/models/irodori_tts/assets.cpp index 7549be232..fdb1777cb 100644 --- a/src/models/irodori_tts/assets.cpp +++ b/src/models/irodori_tts/assets.cpp @@ -293,6 +293,7 @@ float IrodoriModelConfig::caption_mlp_ratio_resolved() const noexcept { std::shared_ptr load_irodori_tts_assets(const std::filesystem::path & model_path) { auto assets = std::make_shared(); + assets->model_dir = std::filesystem::is_directory(model_path) ? model_path : model_path.parent_path(); assets->resources = engine::model_spec::load_resource_bundle( model_path, engine::model_spec::default_spec_path("irodori_tts")); diff --git a/src/models/irodori_tts/session.cpp b/src/models/irodori_tts/session.cpp index 5d12d5854..1047d3373 100644 --- a/src/models/irodori_tts/session.cpp +++ b/src/models/irodori_tts/session.cpp @@ -1,6 +1,8 @@ #include "engine/models/irodori_tts/session.h" #include "engine/framework/debug/profiler.h" +#include "engine/framework/io/binary.h" +#include "engine/framework/io/safetensors.h" #include "engine/framework/runtime/options.h" #include "engine/framework/runtime/spec_backed_model.h" #include "engine/framework/text/chunking.h" @@ -8,6 +10,8 @@ #include "engine/models/irodori_tts/condition_encoder.h" #include "engine/models/irodori_tts/rf_dit.h" +#include + #include #include #include @@ -268,6 +272,108 @@ no_reference_speaker_condition(const IrodoriModelConfig &config) { return out; } +// Speaker Inversion embeddings are written by Irodori-TTS training as +// `*.speaker.safetensors` holding one tensor, `speaker_embedding`, of shape +// [tokens, speaker_dim] (or [1, tokens, speaker_dim]). Like the Python +// runtime, the tokens become the speaker condition as they are: no speaker +// encoder, no normalization and no prepended mean token. +IrodoriSpeakerCondition +load_speaker_embedding(const std::filesystem::path &path, + const IrodoriModelConfig &config) { + constexpr const char *kTensorName = "speaker_embedding"; + const auto index = io::load_safetensors_index(path); + const auto found = index.tensors.find(kTensorName); + if (found == index.tensors.end()) { + throw std::runtime_error( + "Irodori-TTS speaker embedding has no speaker_embedding tensor: " + + path.string()); + } + const auto &info = found->second; + auto shape = info.shape; + if (shape.size() == 3 && shape.front() == 1) { + shape.erase(shape.begin()); + } + if (shape.size() != 2 || shape[0] <= 0 || shape[1] != config.speaker_dim) { + throw std::runtime_error( + "Irodori-TTS speaker embedding must have shape [tokens, " + + std::to_string(config.speaker_dim) + "]: " + path.string()); + } + size_t element_bytes = 0; + if (info.dtype == "F32") { + element_bytes = sizeof(float); + } else if (info.dtype == "F16" || info.dtype == "BF16") { + element_bytes = sizeof(uint16_t); + } else { + throw std::runtime_error( + "Irodori-TTS speaker embedding dtype must be F32, F16 or BF16, got " + + info.dtype + ": " + path.string()); + } + // Compare by division so that a huge token count in the header cannot + // overflow into a matching byte size. + const size_t byte_size = info.data_end - info.data_begin; + const size_t row_bytes = static_cast(shape[1]) * element_bytes; + const size_t count = byte_size / element_bytes; + const auto bytes = io::read_binary_file(path); + if (info.data_end < info.data_begin || byte_size % row_bytes != 0 || + byte_size / row_bytes != static_cast(shape[0]) || + index.header_bytes > bytes.size() || + info.data_end > bytes.size() - index.header_bytes) { + throw std::runtime_error( + "Irodori-TTS speaker embedding data does not match its header: " + + path.string()); + } + const std::byte *data = bytes.data() + index.header_bytes + info.data_begin; + + IrodoriSpeakerCondition out; + out.state.resize(count); + if (info.dtype == "F32") { + std::memcpy(out.state.data(), data, byte_size); + } else if (info.dtype == "F16") { + ggml_fp16_to_fp32_row(reinterpret_cast(data), + out.state.data(), static_cast(count)); + } else { + ggml_bf16_to_fp32_row(reinterpret_cast(data), + out.state.data(), static_cast(count)); + } + out.tokens = shape[0]; + out.mask.assign(static_cast(out.tokens), 1); + out.has_speaker = true; + return out; +} + +// The request's Speaker Inversion embedding, if any: the speaker_embedding_path +// option, or a cached voice id naming /embeddings/.safetensors +// (the layout GET /v1/audio/voices lists). A voice id without such a file is +// ignored, as before, so clients that always send an OpenAI voice name keep +// working. +std::optional +resolve_speaker_embedding_path(const runtime::TaskRequest &request, + const std::filesystem::path &model_dir) { + if (const auto value = + runtime::find_option(request.options, {"speaker_embedding_path"})) { + const std::filesystem::path path(*value); + if (!std::filesystem::is_regular_file(path)) { + throw std::runtime_error( + "Irodori-TTS speaker_embedding_path does not exist: " + *value); + } + return path; + } + if (!request.voice.has_value() || !request.voice->speaker.has_value() || + !request.voice->speaker->cached_voice_id.has_value()) { + return std::nullopt; + } + const std::string &voice_id = *request.voice->speaker->cached_voice_id; + if (voice_id.empty() || voice_id == "." || voice_id == ".." || + voice_id.find_first_of("/\\:") != std::string::npos) { + return std::nullopt; + } + auto path = model_dir / "embeddings" / (voice_id + ".safetensors"); + if (!std::filesystem::is_regular_file(path)) { + return std::nullopt; + } + return path; +} + std::string trim_ascii(std::string text) { while (!text.empty() && (text.front() == ' ' || text.front() == '\n' || text.front() == '\r' || text.front() == '\t')) { @@ -444,6 +550,11 @@ IrodoriTTSSession::run(const runtime::TaskRequest &request) { contract_->request_option_keys.end()) { validation_options.erase("instruction"); } + // Likewise for packages whose embedded contract predates speaker embeddings. + if (contract_->request_option_keys.find("speaker_embedding_path") == + contract_->request_option_keys.end()) { + validation_options.erase("speaker_embedding_path"); + } runtime::validate_spec_backed_request_options( validation_options, *contract_, "Irodori-TTS"); const auto wall_start = Clock::now(); @@ -463,7 +574,12 @@ IrodoriTTSSession::run(const runtime::TaskRequest &request) { IrodoriSpeakerCondition speaker = no_reference_speaker_condition(assets_->config); bool reference_cache_hit = false; - if (!first_request.no_ref) { + if (first_request.speaker_embedding_path.has_value()) { + speaker = load_speaker_embedding(*first_request.speaker_embedding_path, + assets_->config); + debug::trace_log_scalar("irodori_tts.speaker_embedding.tokens", + speaker.tokens); + } else if (!first_request.no_ref) { if (!first_request.has_reference_audio) { throw std::runtime_error( "Irodori-TTS reference mode requires reference audio"); @@ -660,6 +776,26 @@ IrodoriTTSSession::make_request(const runtime::TaskRequest &request) const { out.has_reference_audio = true; out.no_ref = false; } + if (auto embedding = resolve_speaker_embedding_path( + request, assets_->model_dir)) { + // One speaker source per request, as in the Python runtime (--ref-embed). + if (out.has_reference_audio) { + throw std::runtime_error( + "Irodori-TTS speaker embedding cannot be combined with reference audio"); + } + if (runtime::find_option(request.options, {"no_ref"}).has_value() && + out.no_ref) { + throw std::runtime_error( + "Irodori-TTS speaker embedding cannot be combined with no_ref=true"); + } + if (!assets_->config.use_speaker_condition) { + throw std::runtime_error( + "Irodori-TTS checkpoint does not use speaker conditioning, so it " + "cannot take a speaker embedding"); + } + out.speaker_embedding_path = std::move(embedding); + out.no_ref = false; + } out.generation = generation_options_from_request(request); if (out.generation.duration_scale <= 0.0F) { throw std::runtime_error("Irodori-TTS duration_scale must be positive");