feat: input artifacts in server and CLI task requests

CMakeLists.txt

@@ -2611,6 +2611,7 @@ function(audiocpp_use_utf8_codepage target)
 endfunction()
 
 add_executable(audiocpp_cli
+    app/common/base64.cpp
     app/common/build_info.cpp
     app/cli/main.cpp
     app/cli/args.cpp
@@ -2832,9 +2833,9 @@ configure_file(
 )
 
 add_executable(audiocpp_server
+    app/common/base64.cpp
     app/common/build_info.cpp
     app/server/main.cpp
-    app/server/base64.cpp
     app/server/config.cpp
     app/server/frontend.cpp
     app/server/http.cpp
@@ -2890,6 +2891,7 @@ add_executable(model_perf
     tests/perf/model_perf.cpp
     app/cli/args.cpp
     app/cli/request.cpp
+    app/common/base64.cpp
 )
 target_link_libraries(model_perf PRIVATE engine_runtime ggml)
 audiocpp_use_utf8_codepage(model_perf)
@@ -3135,6 +3137,11 @@ if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TEST
         add_test(NAME nemotron_3_diar_streaming_test COMMAND nemotron_3_diar_streaming_test)
     endif()
 
+    if(yue2 IN_LIST AUDIOCPP_LINKED_MODELS)
+        add_engine_unittest(yue2_request_test tests/yue2/request_test.cpp)
+        add_test(NAME yue2_request_test COMMAND yue2_request_test)
+    endif()
+
     # Registered here rather than in the extended block below so it runs in the
     # linux/mac build workflows (they only set ENGINE_BUILD_TESTS). It is
     # platform-safe: the Metal timing section skips itself when Metal is absent.
@@ -3526,6 +3533,7 @@ if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TEST
             tests/unittests/test_cli_request_options.cpp
             app/cli/args.cpp
             app/cli/request.cpp
+            app/common/base64.cpp
         )
         target_link_libraries(cli_request_options_test PRIVATE engine_runtime ggml)
         if (ENGINE_ENABLE_OPENMP)
@@ -3544,12 +3552,11 @@ if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TEST
         target_include_directories(server_multipart_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/app/server)
         add_test(NAME server_multipart_test COMMAND server_multipart_test)
 
-        add_executable(server_base64_test
-            tests/unittests/test_server_base64.cpp
-            app/server/base64.cpp
+        add_executable(base64_test
+            tests/unittests/test_base64.cpp
+            app/common/base64.cpp
         )
-        target_include_directories(server_base64_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/app/server)
-        add_test(NAME server_base64_test COMMAND server_base64_test)
+        add_test(NAME base64_test COMMAND base64_test)
 
         add_executable(server_busy_guard_test
             tests/unittests/test_server_busy_guard.cpp
@@ -3567,6 +3574,7 @@ if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TEST
             app/server/model_memory.cpp
             app/cli/args.cpp
             app/cli/request.cpp
+            app/common/base64.cpp
         )
         target_include_directories(server_config_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/app/server)
         target_link_libraries(server_config_test PRIVATE engine_runtime ggml)

app/cli/request.cpp

@@ -2,16 +2,26 @@
 
 #include "args.h"
 
+#include "../common/base64.h"
+
 #include "engine/framework/audio/wav_reader.h"
 
 #include <cmath>
+#include <cstddef>
 #include <filesystem>
+#include <fstream>
+#include <initializer_list>
 #include <iomanip>
+#include <iterator>
 #include <sstream>
 #include <string>
+#include <system_error>
 #include <utility>
 
 namespace minitts::cli {
+
+using engine::runtime::InvalidRequestError;
+
 namespace {
 
 std::string path_arg_string(const std::filesystem::path & path) {
@@ -36,6 +46,157 @@ void set_option_from_json_field(
     }
 }
 
+struct ArtifactKindName {
+    const char * name;
+    engine::runtime::ArtifactKind kind;
+};
+
+// The names the server writes a result artifact's kind with.
+constexpr ArtifactKindName kArtifactKinds[] = {
+    {"speaker_embedding", engine::runtime::ArtifactKind::SpeakerEmbedding},
+    {"style_embedding", engine::runtime::ArtifactKind::StyleEmbedding},
+    {"prompt_embedding", engine::runtime::ArtifactKind::PromptEmbedding},
+    {"acoustic_tokens", engine::runtime::ArtifactKind::AcousticTokens},
+    {"midi", engine::runtime::ArtifactKind::Midi},
+    {"transcript_alignment", engine::runtime::ArtifactKind::TranscriptAlignment},
+    {"diarization_state", engine::runtime::ArtifactKind::DiarizationState},
+    {"vad_state", engine::runtime::ArtifactKind::VadState},
+    {"custom", engine::runtime::ArtifactKind::Custom},
+};
+
+constexpr bool artifact_kinds_in_enum_order() {
+    for (size_t index = 0; index < std::size(kArtifactKinds); ++index) {
+        if (kArtifactKinds[index].kind != static_cast<engine::runtime::ArtifactKind>(index)) {
+            return false;
+        }
+    }
+    return true;
+}
+
+// Custom is the last ArtifactKind, so a kind added to the enum without a name
+// here fails to build instead of being turned away as unknown.
+static_assert(
+    std::size(kArtifactKinds) == static_cast<size_t>(engine::runtime::ArtifactKind::Custom) + 1 &&
+        artifact_kinds_in_enum_order(),
+    "kArtifactKinds must name every ArtifactKind, in enum order");
+
+bool is_artifact_field(const std::string & key) {
+    for (const char * field : {"id", "kind", "payload", "path", "meta"}) {
+        if (key == field) {
+            return true;
+        }
+    }
+    return false;
+}
+
+engine::runtime::ArtifactKind parse_artifact_kind(const std::string & name, const std::string & where) {
+    for (const auto & entry : kArtifactKinds) {
+        if (name == entry.name) {
+            return entry.kind;
+        }
+    }
+    std::string names;
+    for (const auto & entry : kArtifactKinds) {
+        names += (names.empty() ? "" : ", ") + std::string(entry.name);
+    }
+    throw InvalidRequestError(where + ": unknown kind '" + name + "', expected one of " + names);
+}
+
+// A field that is absent or null reads as not given.
+const engine::io::json::Value * artifact_field(
+    const engine::io::json::Value & entry,
+    const std::string & key) {
+    const auto * value = entry.find(key);
+    return value == nullptr || value->is_null() ? nullptr : value;
+}
+
+std::string artifact_string(
+    const engine::io::json::Value & entry,
+    const std::string & key,
+    const std::string & where) {
+    const auto * value = artifact_field(entry, key);
+    if (value == nullptr || !value->is_string() || value->as_string().empty()) {
+        throw InvalidRequestError(where + ": " + key + " must be a non-empty string");
+    }
+    return value->as_string();
+}
+
+[[noreturn]] void throw_artifact_limit(const std::string & where) {
+    throw InvalidRequestError(
+        where + ": the request's artifact payloads total more than " +
+        std::to_string(kMaxRequestArtifactBytes) + " bytes");
+}
+
+std::vector<std::byte> decode_artifact_payload(
+    const engine::io::json::Value & payload,
+    uint64_t budget,
+    const std::string & where) {
+    if (!payload.is_string()) {
+        throw InvalidRequestError(where + ": payload must be a base64 string");
+    }
+    // An inline payload is already in memory as part of the request, so the
+    // server's request body limit bounds this decode.
+    std::vector<std::byte> bytes;
+    try {
+        bytes = minitts::app::base64_decode_bytes(payload.as_string());
+    } catch (const std::runtime_error & error) {
+        throw InvalidRequestError(where + ": payload is not valid base64 (" + error.what() + ")");
+    }
+    if (bytes.size() > budget) {
+        throw_artifact_limit(where);
+    }
+    return bytes;
+}
+
+std::vector<std::byte> read_artifact_file(
+    const std::filesystem::path & path,
+    uint64_t budget,
+    const std::string & where) {
+    // Regular files only: a FIFO or a device such as /dev/zero has no size to
+    // check and may never end.
+    std::error_code error;
+    if (!std::filesystem::is_regular_file(path, error)) {
+        throw InvalidRequestError(where + ": path is not a regular file: " + path.string());
+    }
+    const auto size = std::filesystem::file_size(path, error);
+    if (error) {
+        throw InvalidRequestError(where + ": cannot read the size of " + path.string() + ": " + error.message());
+    }
+    if (static_cast<uint64_t>(size) > budget) {
+        throw_artifact_limit(where);
+    }
+    std::ifstream input(path, std::ios::binary);
+    if (!input) {
+        throw InvalidRequestError(where + ": failed to open " + path.string());
+    }
+    std::vector<std::byte> bytes(static_cast<size_t>(size));
+    if (!bytes.empty() &&
+        !input.read(reinterpret_cast<char *>(bytes.data()), static_cast<std::streamsize>(bytes.size()))) {
+        throw InvalidRequestError(where + ": failed to read " + path.string());
+    }
+    return bytes;
+}
+
+std::unordered_map<std::string, std::string> artifact_meta(
+    const engine::io::json::Value & entry,
+    const std::string & where) {
+    std::unordered_map<std::string, std::string> meta;
+    const auto * value = artifact_field(entry, "meta");
+    if (value == nullptr) {
+        return meta;
+    }
+    if (!value->is_object()) {
+        throw InvalidRequestError(where + ": meta must be an object");
+    }
+    for (const auto & [key, child] : value->as_object()) {
+        if (!child.is_string() && !child.is_number() && !child.is_bool()) {
+            throw InvalidRequestError(where + ": meta." + key + " must be a string, number or boolean");
+        }
+        meta[key] = json_option_string(child);
+    }
+    return meta;
+}
+
 }  // namespace
 
 bool is_stdin_audio_source(std::string_view audio_arg) {
@@ -120,6 +281,55 @@ std::optional<float> json_optional_float(
     return value->as_f32();
 }
 
+std::vector<engine::runtime::VoiceArtifact> json_request_artifacts(
+    const engine::io::json::Value * value,
+    const std::filesystem::path & base_dir) {
+    std::vector<engine::runtime::VoiceArtifact> artifacts;
+    if (value == nullptr || value->is_null()) {
+        return artifacts;
+    }
+    if (!value->is_array()) {
+        throw InvalidRequestError("artifacts must be an array of artifact objects");
+    }
+    uint64_t total_bytes = 0;
+    const auto & entries = value->as_array();
+    for (size_t index = 0; index < entries.size(); ++index) {
+        const auto & entry = entries[index];
+        std::string where = "artifacts[" + std::to_string(index) + "]";
+        if (!entry.is_object()) {
+            throw InvalidRequestError(where + " must be an object with id, kind, and payload or path");
+        }
+        for (const auto & field : entry.as_object()) {
+            if (!is_artifact_field(field.first)) {
+                throw InvalidRequestError(
+                    where + ": unknown field '" + field.first + "', an artifact has id, kind, payload or path, and meta");
+            }
+        }
+        engine::runtime::VoiceArtifact artifact;
+        artifact.id = artifact_string(entry, "id", where);
+        where += " (" + artifact.id + ")";
+        const auto * kind = artifact_field(entry, "kind");
+        if (kind == nullptr || !kind->is_string()) {
+            throw InvalidRequestError(where + ": kind must be a string");
+        }
+        artifact.kind = parse_artifact_kind(kind->as_string(), where);
+        const auto * payload = artifact_field(entry, "payload");
+        const bool has_path = artifact_field(entry, "path") != nullptr;
+        if ((payload != nullptr) == has_path) {
+            throw InvalidRequestError(
+                where + ": give exactly one of payload (base64) and path (a file holding the payload)");
+        }
+        const uint64_t budget = kMaxRequestArtifactBytes - total_bytes;
+        artifact.payload = payload != nullptr
+            ? decode_artifact_payload(*payload, budget, where)
+            : read_artifact_file(resolve_case_path(base_dir, artifact_string(entry, "path", where)), budget, where);
+        total_bytes += artifact.payload.size();
+        artifact.meta = artifact_meta(entry, where);
+        artifacts.push_back(std::move(artifact));
+    }
+    return artifacts;
+}
+
 engine::runtime::TaskRequest build_request_from_json(
     const engine::io::json::Value & value,
     const std::filesystem::path & base_dir) {
@@ -180,6 +390,7 @@ engine::runtime::TaskRequest build_request_from_json(
     if (has_voice) {
         request.voice = std::move(voice);
     }
+    request.input_artifacts = json_request_artifacts(value.find("artifacts"), base_dir);
 
     request.options = json_options_map(value.find("options"));
     if (!language.empty()) {

app/cli/request.h

@@ -1,16 +1,23 @@
 #pragma once
 
 #include "engine/framework/io/json.h"
+#include "engine/framework/runtime/errors.h"
 #include "engine/framework/runtime/session.h"
 
+#include <cstdint>
 #include <filesystem>
 #include <optional>
 #include <string>
 #include <string_view>
 #include <unordered_map>
+#include <vector>
 
 namespace minitts::cli {
 
+// Most artifact payload bytes one request may carry, inline and from files
+// together: the server's default max_request_body_bytes (2 GiB).
+inline constexpr uint64_t kMaxRequestArtifactBytes = 2ull * 1024ull * 1024ull * 1024ull;
+
 // True when --audio selects raw PCM on stdin ("-") rather than a file path.
 bool is_stdin_audio_source(std::string_view audio_arg);
 engine::runtime::AudioBuffer read_audio_buffer(const std::filesystem::path & path);
@@ -24,6 +31,15 @@ std::optional<std::string> json_optional_string(
 std::optional<float> json_optional_float(
     const engine::io::json::Value & object,
     const std::string & key);
+// The request's `artifacts` array, in order, as TaskRequest::input_artifacts.
+// Each entry has the shape the server writes a result's artifacts in: `id`,
+// `kind`, `payload` (base64, or a data URI) and optional `meta` (string,
+// number or boolean values). `path` may replace `payload`: the payload is then
+// that file's bytes, resolved against base_dir. Throws
+// engine::runtime::InvalidRequestError, which the server answers with 400.
+std::vector<engine::runtime::VoiceArtifact> json_request_artifacts(
+    const engine::io::json::Value * value,
+    const std::filesystem::path & base_dir);
 engine::runtime::TaskRequest build_request_from_json(
     const engine::io::json::Value & value,
     const std::filesystem::path & base_dir);