irodori_tts: accept Speaker Inversion embeddings as the speaker condition

docs/models/irodori_tts.md

@@ -65,6 +65,26 @@ audiocpp_cli --task clon --family irodori_tts \
   --out out.wav
 ```
 
+v4 with a Speaker Inversion embedding:
+
+```bash
+audiocpp_cli --task tts --family irodori_tts \
+  --model models/Irodori-TTS-v4-Small-GGUF/irodori-tts-v4-small-q8_0.gguf \
+  --backend cuda --language ja \
+  --text "こんにちは、これは学習した話者埋め込みを使った推論です。" \
+  --request-option speaker_embedding_path=path/to/name.speaker.safetensors \
+  --out out.wav
+```
+
+## Speaker Inversion Embeddings
+
+[Irodori-TTS](https://github.com/Aratako/Irodori-TTS)'s Speaker Inversion training learns a few speaker tokens for one voice while the base model stays frozen, and saves them as `*.speaker.safetensors`. audio.cpp uses such a file as the speaker condition in place of reference audio, like `--ref-embed` in the Python `infer.py`: the tokens go to the model as they are, without the speaker encoder.
+
+- **By path:** request option `speaker_embedding_path` (CLI `--request-option`, or `"options"` in a server request).
+- **By name:** put the file in an `embeddings` directory next to the model weights as `<name>.safetensors` and pass `<name>` as the voice id (CLI `--voice-id`, server `"voice"`). With the model path set to that directory, the server's `GET /v1/audio/voices` lists the names. A voice id with no such file is ignored, as before.
+
+The file must hold a `speaker_embedding` tensor of shape `[tokens, speaker_dim]` (F32, F16 or BF16). An embedding cannot be combined with reference audio or `no_ref=true`. Use it with the base checkpoint it was trained on: an embedding from another checkpoint with the same `speaker_dim` loads, but the speaker match is not guaranteed.
+
 ## Request Options (use with `--request-option`)
 
 v4 uses the normalized schema-v1 option names directly. New requests should use these names:
@@ -74,6 +94,7 @@ v4 uses the normalized schema-v1 option names directly. New requests should use
 | `language` | `ja` | `ja` | Text language code; Irodori-TTS is Japanese-only. |
 | `instruction` | text | empty | Voice-design instruction; only useful on caption-conditioned checkpoints. Legacy `caption` is accepted as an alias. |
 | `no_ref` | bool | `true` unless a reference is provided | Use no-reference generation. Set `false` with `--voice-ref` for reference conditioning. |
+| `speaker_embedding_path` | path | unset | Speaker Inversion embedding (`*.speaker.safetensors`) used instead of reference audio. See [Speaker Inversion Embeddings](#speaker-inversion-embeddings). |
 | `num_inference_steps` | integer | `40` | RF diffusion steps. |
 | `duration_sec` | seconds | unset | Explicit output duration; omitted uses predicted duration. |
 | `duration_scale` | float | `1.0` | Multiplier for predicted duration. |

include/engine/models/irodori_tts/assets.h

@@ -87,6 +87,10 @@ struct IrodoriCodecConfig {
 };
 
 struct IrodoriTTSAssets {
+    // The model directory as given (or the directory of the given file). Unlike
+    // resources.model_root(), it is not replaced by the extracted sidecars of a
+    // standalone GGUF, so it is where user files such as embeddings/ live.
+    std::filesystem::path model_dir;
     assets::ResourceBundle resources;
     IrodoriModelConfig config;
     IrodoriCodecConfig codec;

include/engine/models/irodori_tts/types.h

@@ -3,6 +3,8 @@
 #include "engine/framework/runtime/session.h"
 
 #include <cstdint>
+#include <filesystem>
+#include <optional>
 #include <string>
 
 namespace engine::models::irodori_tts {
@@ -35,6 +37,9 @@ struct IrodoriRequest {
   bool no_ref = true;
   runtime::AudioBuffer reference_audio;
   bool has_reference_audio = false;
+  // Speaker Inversion embedding used as the speaker condition instead of
+  // reference audio.
+  std::optional<std::filesystem::path> speaker_embedding_path;
   IrodoriGenerationOptions generation;
 };
 

model_specs/irodori_tts.json

@@ -46,6 +46,12 @@
         "required": false,
         "default": true
       },
+      {
+        "name": "speaker_embedding_path",
+        "type": "path",
+        "description": "Speaker Inversion embedding (*.speaker.safetensors written by Irodori-TTS training) used as the speaker condition instead of reference audio; use it with the base checkpoint it was trained on. Cannot be combined with reference audio or no_ref=true.",
+        "required": false
+      },
       {
         "name": "num_inference_steps",
         "type": "int",

src/models/irodori_tts/assets.cpp

@@ -293,6 +293,7 @@ float IrodoriModelConfig::caption_mlp_ratio_resolved() const noexcept {
 
 std::shared_ptr<const IrodoriTTSAssets> load_irodori_tts_assets(const std::filesystem::path & model_path) {
     auto assets = std::make_shared<IrodoriTTSAssets>();
+    assets->model_dir = std::filesystem::is_directory(model_path) ? model_path : model_path.parent_path();
     assets->resources = engine::model_spec::load_resource_bundle(
         model_path,
         engine::model_spec::default_spec_path("irodori_tts"));

src/models/irodori_tts/session.cpp

@@ -1,13 +1,17 @@
 #include "engine/models/irodori_tts/session.h"
 
 #include "engine/framework/debug/profiler.h"
+#include "engine/framework/io/binary.h"
+#include "engine/framework/io/safetensors.h"
 #include "engine/framework/runtime/options.h"
 #include "engine/framework/runtime/spec_backed_model.h"
 #include "engine/framework/text/chunking.h"
 #include "engine/models/irodori_tts/codec.h"
 #include "engine/models/irodori_tts/condition_encoder.h"
 #include "engine/models/irodori_tts/rf_dit.h"
 
+#include <ggml.h>
+
 #include <algorithm>
 #include <cctype>
 #include <chrono>
@@ -268,6 +272,108 @@ no_reference_speaker_condition(const IrodoriModelConfig &config) {
   return out;
 }
 
+// Speaker Inversion embeddings are written by Irodori-TTS training as
+// `*.speaker.safetensors` holding one tensor, `speaker_embedding`, of shape
+// [tokens, speaker_dim] (or [1, tokens, speaker_dim]). Like the Python
+// runtime, the tokens become the speaker condition as they are: no speaker
+// encoder, no normalization and no prepended mean token.
+IrodoriSpeakerCondition
+load_speaker_embedding(const std::filesystem::path &path,
+                       const IrodoriModelConfig &config) {
+  constexpr const char *kTensorName = "speaker_embedding";
+  const auto index = io::load_safetensors_index(path);
+  const auto found = index.tensors.find(kTensorName);
+  if (found == index.tensors.end()) {
+    throw std::runtime_error(
+        "Irodori-TTS speaker embedding has no speaker_embedding tensor: " +
+        path.string());
+  }
+  const auto &info = found->second;
+  auto shape = info.shape;
+  if (shape.size() == 3 && shape.front() == 1) {
+    shape.erase(shape.begin());
+  }
+  if (shape.size() != 2 || shape[0] <= 0 || shape[1] != config.speaker_dim) {
+    throw std::runtime_error(
+        "Irodori-TTS speaker embedding must have shape [tokens, " +
+        std::to_string(config.speaker_dim) + "]: " + path.string());
+  }
+  size_t element_bytes = 0;
+  if (info.dtype == "F32") {
+    element_bytes = sizeof(float);
+  } else if (info.dtype == "F16" || info.dtype == "BF16") {
+    element_bytes = sizeof(uint16_t);
+  } else {
+    throw std::runtime_error(
+        "Irodori-TTS speaker embedding dtype must be F32, F16 or BF16, got " +
+        info.dtype + ": " + path.string());
+  }
+  // Compare by division so that a huge token count in the header cannot
+  // overflow into a matching byte size.
+  const size_t byte_size = info.data_end - info.data_begin;
+  const size_t row_bytes = static_cast<size_t>(shape[1]) * element_bytes;
+  const size_t count = byte_size / element_bytes;
+  const auto bytes = io::read_binary_file(path);
+  if (info.data_end < info.data_begin || byte_size % row_bytes != 0 ||
+      byte_size / row_bytes != static_cast<size_t>(shape[0]) ||
+      index.header_bytes > bytes.size() ||
+      info.data_end > bytes.size() - index.header_bytes) {
+    throw std::runtime_error(
+        "Irodori-TTS speaker embedding data does not match its header: " +
+        path.string());
+  }
+  const std::byte *data = bytes.data() + index.header_bytes + info.data_begin;
+
+  IrodoriSpeakerCondition out;
+  out.state.resize(count);
+  if (info.dtype == "F32") {
+    std::memcpy(out.state.data(), data, byte_size);
+  } else if (info.dtype == "F16") {
+    ggml_fp16_to_fp32_row(reinterpret_cast<const ggml_fp16_t *>(data),
+                          out.state.data(), static_cast<int64_t>(count));
+  } else {
+    ggml_bf16_to_fp32_row(reinterpret_cast<const ggml_bf16_t *>(data),
+                          out.state.data(), static_cast<int64_t>(count));
+  }
+  out.tokens = shape[0];
+  out.mask.assign(static_cast<size_t>(out.tokens), 1);
+  out.has_speaker = true;
+  return out;
+}
+
+// The request's Speaker Inversion embedding, if any: the speaker_embedding_path
+// option, or a cached voice id naming <model dir>/embeddings/<id>.safetensors
+// (the layout GET /v1/audio/voices lists). A voice id without such a file is
+// ignored, as before, so clients that always send an OpenAI voice name keep
+// working.
+std::optional<std::filesystem::path>
+resolve_speaker_embedding_path(const runtime::TaskRequest &request,
+                               const std::filesystem::path &model_dir) {
+  if (const auto value =
+          runtime::find_option(request.options, {"speaker_embedding_path"})) {
+    const std::filesystem::path path(*value);
+    if (!std::filesystem::is_regular_file(path)) {
+      throw std::runtime_error(
+          "Irodori-TTS speaker_embedding_path does not exist: " + *value);
+    }
+    return path;
+  }
+  if (!request.voice.has_value() || !request.voice->speaker.has_value() ||
+      !request.voice->speaker->cached_voice_id.has_value()) {
+    return std::nullopt;
+  }
+  const std::string &voice_id = *request.voice->speaker->cached_voice_id;
+  if (voice_id.empty() || voice_id == "." || voice_id == ".." ||
+      voice_id.find_first_of("/\\:") != std::string::npos) {
+    return std::nullopt;
+  }
+  auto path = model_dir / "embeddings" / (voice_id + ".safetensors");
+  if (!std::filesystem::is_regular_file(path)) {
+    return std::nullopt;
+  }
+  return path;
+}
+
 std::string trim_ascii(std::string text) {
   while (!text.empty() && (text.front() == ' ' || text.front() == '\n' ||
                            text.front() == '\r' || text.front() == '\t')) {
@@ -444,6 +550,11 @@ IrodoriTTSSession::run(const runtime::TaskRequest &request) {
           contract_->request_option_keys.end()) {
     validation_options.erase("instruction");
   }
+  // Likewise for packages whose embedded contract predates speaker embeddings.
+  if (contract_->request_option_keys.find("speaker_embedding_path") ==
+      contract_->request_option_keys.end()) {
+    validation_options.erase("speaker_embedding_path");
+  }
   runtime::validate_spec_backed_request_options(
       validation_options, *contract_, "Irodori-TTS");
   const auto wall_start = Clock::now();
@@ -463,7 +574,12 @@ IrodoriTTSSession::run(const runtime::TaskRequest &request) {
   IrodoriSpeakerCondition speaker =
       no_reference_speaker_condition(assets_->config);
   bool reference_cache_hit = false;
-  if (!first_request.no_ref) {
+  if (first_request.speaker_embedding_path.has_value()) {
+    speaker = load_speaker_embedding(*first_request.speaker_embedding_path,
+                                     assets_->config);
+    debug::trace_log_scalar("irodori_tts.speaker_embedding.tokens",
+                            speaker.tokens);
+  } else if (!first_request.no_ref) {
     if (!first_request.has_reference_audio) {
       throw std::runtime_error(
           "Irodori-TTS reference mode requires reference audio");
@@ -660,6 +776,26 @@ IrodoriTTSSession::make_request(const runtime::TaskRequest &request) const {
     out.has_reference_audio = true;
     out.no_ref = false;
   }
+  if (auto embedding = resolve_speaker_embedding_path(
+          request, assets_->model_dir)) {
+    // One speaker source per request, as in the Python runtime (--ref-embed).
+    if (out.has_reference_audio) {
+      throw std::runtime_error(
+          "Irodori-TTS speaker embedding cannot be combined with reference audio");
+    }
+    if (runtime::find_option(request.options, {"no_ref"}).has_value() &&
+        out.no_ref) {
+      throw std::runtime_error(
+          "Irodori-TTS speaker embedding cannot be combined with no_ref=true");
+    }
+    if (!assets_->config.use_speaker_condition) {
+      throw std::runtime_error(
+          "Irodori-TTS checkpoint does not use speaker conditioning, so it "
+          "cannot take a speaker embedding");
+    }
+    out.speaker_embedding_path = std::move(embedding);
+    out.no_ref = false;
+  }
   out.generation = generation_options_from_request(request);
   if (out.generation.duration_scale <= 0.0F) {
     throw std::runtime_error("Irodori-TTS duration_scale must be positive");