From e3d214419f8fbc29e5a115b6c618537e7c736730 Mon Sep 17 00:00:00 2001 From: kossum Date: Sun, 13 Sep 2026 22:33:05 +0900 Subject: [PATCH 1/2] server: support input_image in Responses tool outputs Convert Responses API input_image content in function_call_output to Chat Completions image_url content. This allows multimodal tool results, such as Codex view_image output, to pass through the existing multimodal pipeline. Add regression tests for text/image tool outputs and invalid image content. --- tests/test-chat.cpp | 135 +++++++++++++++++++++++++++++++++++ tools/server/server-chat.cpp | 18 ++++- 2 files changed, 150 insertions(+), 3 deletions(-) diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 30a7237e314b..8a03a5d9067a 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -1999,6 +1999,141 @@ static void test_convert_responses_to_chatcmpl() { assert_equals(false, result.contains("tools")); } + + // Test function_call_output with array output containing input_text + { + json input = json::parse(R"({ + "input": [ + { + "type": "function_call", + "name": "view_image", + "arguments": "{}", + "call_id": "call_1" + }, + { + "type": "function_call_output", + "call_id": "call_1", + "output": [ + { + "type": "input_text", + "text": "here is the image" + } + ] + } + ], + "model": "test-model" + })"); + + json result = server_chat_convert_responses_to_chatcmpl(input); + assert_equals((size_t)2, result.at("messages").size()); + + const auto & tool_msg = result.at("messages")[1]; + assert_equals(std::string("tool"), tool_msg.at("role").get()); + assert_equals(std::string("call_1"), tool_msg.at("tool_call_id").get()); + assert_equals(true, tool_msg.at("content").is_array()); + assert_equals(std::string("text"), tool_msg.at("content")[0].at("type").get()); + assert_equals(std::string("here is the image"), tool_msg.at("content")[0].at("text").get()); + } + + // Test function_call_output with input_image (e.g. Codex view_image tool output) + { + json input = json::parse(R"({ + "input": [ + { + "type": "function_call", + "name": "view_image", + "arguments": "{}", + "call_id": "call_1" + }, + { + "type": "function_call_output", + "call_id": "call_1", + "output": [ + { + "type": "input_text", + "text": "here you go" + }, + { + "type": "input_image", + "image_url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB" + } + ] + } + ], + "model": "test-model" + })"); + + json result = server_chat_convert_responses_to_chatcmpl(input); + assert_equals((size_t)2, result.at("messages").size()); + + const auto & tool_msg = result.at("messages")[1]; + assert_equals(std::string("tool"), tool_msg.at("role").get()); + assert_equals(std::string("call_1"), tool_msg.at("tool_call_id").get()); + assert_equals((size_t)2, tool_msg.at("content").size()); + + assert_equals(std::string("text"), tool_msg.at("content")[0].at("type").get()); + assert_equals(std::string("here you go"), tool_msg.at("content")[0].at("text").get()); + + assert_equals(std::string("image_url"), tool_msg.at("content")[1].at("type").get()); + assert_equals( + std::string("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB"), + tool_msg.at("content")[1].at("image_url").at("url").get() + ); + } + + // Test function_call_output input_image without image_url + { + json input = json::parse(R"({ + "input": [ + { + "type": "function_call_output", + "call_id": "call_1", + "output": [ + { + "type": "input_image" + } + ] + } + ], + "model": "test-model" + })"); + + try { + server_chat_convert_responses_to_chatcmpl(input); + throw std::runtime_error("Expected exception"); + } catch (const std::exception & e) { + if (std::string(e.what()).find("image_url") == std::string::npos) { + throw std::runtime_error("Expected exception about missing 'image_url'"); + } + } + } + + // Test function_call_output with unsupported type + { + json input = json::parse(R"({ + "input": [ + { + "type": "function_call_output", + "call_id": "call_1", + "output": [ + { + "type": "input_file" + } + ] + } + ], + "model": "test-model" + })"); + + try { + server_chat_convert_responses_to_chatcmpl(input); + throw std::runtime_error("Expected exception"); + } catch (const std::exception & e) { + if (std::string(e.what()).find("must be one of") == std::string::npos) { + throw std::runtime_error("Expected exception about 'type'"); + } + } + } } // Shared LFM2 parser cases - all variants use one output format and parser diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp index a6fe3c6ba619..f0b909797339 100644 --- a/tools/server/server-chat.cpp +++ b/tools/server/server-chat.cpp @@ -203,10 +203,22 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) { } else { json chatcmpl_outputs = item.at("output"); for (json & chatcmpl_output : chatcmpl_outputs) { - if (!chatcmpl_output.contains("type") || chatcmpl_output.at("type") != "input_text") { - throw std::invalid_argument("Output of tool call should be 'Input text'"); + const std::string out_type = json_value(chatcmpl_output, "type", std::string()); + if (out_type == "input_text") { + chatcmpl_output["type"] = "text"; + } else if (out_type == "input_image") { + if (!chatcmpl_output.contains("image_url")) { + throw std::invalid_argument("'image_url' is required for 'input_image' tool output"); + } + chatcmpl_output = json { + {"image_url", json { + {"url", chatcmpl_output.at("image_url")} + }}, + {"type", "image_url"}, + }; + } else { + throw std::invalid_argument("'type' must be one of 'input_text' or 'input_image'"); } - chatcmpl_output["type"] = "text"; } chatcmpl_messages.push_back(json { {"content", chatcmpl_outputs}, From bd1d4105879a1787da480552f4a2e162aa0c6787 Mon Sep 17 00:00:00 2001 From: kossum Date: Mon, 14 Sep 2026 10:19:59 +0900 Subject: [PATCH 2/2] server : validate image_url and text in Responses tool output The function_call_output branches now check field types with exists_and_is_string instead of a bare contains(): - input_image: image_url must be a string, so a non-string value (e.g. a Chat-Completions-style object) fails fast with a clear message here instead of surfacing later in the media pipeline - input_text: text must be present and a string, consistent with the user-message path --- tests/test-chat.cpp | 57 ++++++++++++++++++++++++++++++++++++ tools/server/server-chat.cpp | 7 +++-- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 8a03a5d9067a..ac8099fc92b4 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -2134,6 +2134,63 @@ static void test_convert_responses_to_chatcmpl() { } } } + + // Test function_call_output input_image where image_url is an object (not a string) + { + json input = json::parse(R"({ + "input": [ + { + "type": "function_call_output", + "call_id": "call_1", + "output": [ + { + "type": "input_image", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB" + } + } + ] + } + ], + "model": "test-model" + })"); + + try { + server_chat_convert_responses_to_chatcmpl(input); + throw std::runtime_error("Expected exception"); + } catch (const std::exception & e) { + if (std::string(e.what()).find("image_url") == std::string::npos) { + throw std::runtime_error("Expected exception about 'image_url'"); + } + } + } + + // Test function_call_output input_text without text + { + json input = json::parse(R"({ + "input": [ + { + "type": "function_call_output", + "call_id": "call_1", + "output": [ + { + "type": "input_text" + } + ] + } + ], + "model": "test-model" + })"); + + try { + server_chat_convert_responses_to_chatcmpl(input); + throw std::runtime_error("Expected exception"); + } catch (const std::exception & e) { + if (std::string(e.what()).find("text") == std::string::npos) { + throw std::runtime_error("Expected exception about 'text'"); + } + } + } } // Shared LFM2 parser cases - all variants use one output format and parser diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp index f0b909797339..865771721419 100644 --- a/tools/server/server-chat.cpp +++ b/tools/server/server-chat.cpp @@ -205,10 +205,13 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) { for (json & chatcmpl_output : chatcmpl_outputs) { const std::string out_type = json_value(chatcmpl_output, "type", std::string()); if (out_type == "input_text") { + if (!exists_and_is_string(chatcmpl_output, "text")) { + throw std::invalid_argument("'text' is required for 'input_text' tool output"); + } chatcmpl_output["type"] = "text"; } else if (out_type == "input_image") { - if (!chatcmpl_output.contains("image_url")) { - throw std::invalid_argument("'image_url' is required for 'input_image' tool output"); + if (!exists_and_is_string(chatcmpl_output, "image_url")) { + throw std::invalid_argument("'image_url' is required for 'input_image' tool output and must be a string"); } chatcmpl_output = json { {"image_url", json {