Skip to content

server: support input_image in Responses tool outputs #28847

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
kossum wants to merge 2 commits into from
Closed

server: support input_image in Responses tool outputs #28847

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e3d2144
server: support input_image in Responses tool outputs
kossum Sep 13, 2026
bd1d410
server : validate image_url and text in Responses tool output
kossum Sep 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
192 changes: 192 additions & 0 deletions tests/test-chat.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1999,6 +1999,198 @@ static void test_convert_responses_to_chatcmpl() {

assert_equals(false, result.contains("tools"));
}

// Test function_call_output with array output containing input_text
{
json input = json::parse(R"({
"input": [
{
"type": "function_call",
"name": "view_image",
"arguments": "{}",
"call_id": "call_1"
},
{
"type": "function_call_output",
"call_id": "call_1",
"output": [
{
"type": "input_text",
"text": "here is the image"
}
]
}
],
"model": "test-model"
})");

json result = server_chat_convert_responses_to_chatcmpl(input);
assert_equals((size_t)2, result.at("messages").size());

const auto & tool_msg = result.at("messages")[1];
assert_equals(std::string("tool"), tool_msg.at("role").get<std::string>());
assert_equals(std::string("call_1"), tool_msg.at("tool_call_id").get<std::string>());
assert_equals(true, tool_msg.at("content").is_array());
assert_equals(std::string("text"), tool_msg.at("content")[0].at("type").get<std::string>());
assert_equals(std::string("here is the image"), tool_msg.at("content")[0].at("text").get<std::string>());
}

// Test function_call_output with input_image (e.g. Codex view_image tool output)
{
json input = json::parse(R"({
"input": [
{
"type": "function_call",
"name": "view_image",
"arguments": "{}",
"call_id": "call_1"
},
{
"type": "function_call_output",
"call_id": "call_1",
"output": [
{
"type": "input_text",
"text": "here you go"
},
{
"type": "input_image",
"image_url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB"
}
]
}
],
"model": "test-model"
})");

json result = server_chat_convert_responses_to_chatcmpl(input);
assert_equals((size_t)2, result.at("messages").size());

const auto & tool_msg = result.at("messages")[1];
assert_equals(std::string("tool"), tool_msg.at("role").get<std::string>());
assert_equals(std::string("call_1"), tool_msg.at("tool_call_id").get<std::string>());
assert_equals((size_t)2, tool_msg.at("content").size());

assert_equals(std::string("text"), tool_msg.at("content")[0].at("type").get<std::string>());
assert_equals(std::string("here you go"), tool_msg.at("content")[0].at("text").get<std::string>());

assert_equals(std::string("image_url"), tool_msg.at("content")[1].at("type").get<std::string>());
assert_equals(
std::string("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB"),
tool_msg.at("content")[1].at("image_url").at("url").get<std::string>()
);
}

// Test function_call_output input_image without image_url
{
json input = json::parse(R"({
"input": [
{
"type": "function_call_output",
"call_id": "call_1",
"output": [
{
"type": "input_image"
}
]
}
],
"model": "test-model"
})");

try {
server_chat_convert_responses_to_chatcmpl(input);
throw std::runtime_error("Expected exception");
} catch (const std::exception & e) {
if (std::string(e.what()).find("image_url") == std::string::npos) {
throw std::runtime_error("Expected exception about missing 'image_url'");
}
}
}

// Test function_call_output with unsupported type
{
json input = json::parse(R"({
"input": [
{
"type": "function_call_output",
"call_id": "call_1",
"output": [
{
"type": "input_file"
}
]
}
],
"model": "test-model"
})");

try {
server_chat_convert_responses_to_chatcmpl(input);
throw std::runtime_error("Expected exception");
} catch (const std::exception & e) {
if (std::string(e.what()).find("must be one of") == std::string::npos) {
throw std::runtime_error("Expected exception about 'type'");
}
}
}

// Test function_call_output input_image where image_url is an object (not a string)
{
json input = json::parse(R"({
"input": [
{
"type": "function_call_output",
"call_id": "call_1",
"output": [
{
"type": "input_image",
"image_url": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB"
}
}
]
}
],
"model": "test-model"
})");

try {
server_chat_convert_responses_to_chatcmpl(input);
throw std::runtime_error("Expected exception");
} catch (const std::exception & e) {
if (std::string(e.what()).find("image_url") == std::string::npos) {
throw std::runtime_error("Expected exception about 'image_url'");
}
}
}

// Test function_call_output input_text without text
{
json input = json::parse(R"({
"input": [
{
"type": "function_call_output",
"call_id": "call_1",
"output": [
{
"type": "input_text"
}
]
}
],
"model": "test-model"
})");

try {
server_chat_convert_responses_to_chatcmpl(input);
throw std::runtime_error("Expected exception");
} catch (const std::exception & e) {
if (std::string(e.what()).find("text") == std::string::npos) {
throw std::runtime_error("Expected exception about 'text'");
}
}
}
}

// Shared LFM2 parser cases - all variants use one output format and parser
Expand Down
21 changes: 18 additions & 3 deletions tools/server/server-chat.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,10 +203,25 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) {
} else {
json chatcmpl_outputs = item.at("output");
for (json & chatcmpl_output : chatcmpl_outputs) {
if (!chatcmpl_output.contains("type") || chatcmpl_output.at("type") != "input_text") {
throw std::invalid_argument("Output of tool call should be 'Input text'");
const std::string out_type = json_value(chatcmpl_output, "type", std::string());
if (out_type == "input_text") {
if (!exists_and_is_string(chatcmpl_output, "text")) {
throw std::invalid_argument("'text' is required for 'input_text' tool output");
}
chatcmpl_output["type"] = "text";
} else if (out_type == "input_image") {
if (!exists_and_is_string(chatcmpl_output, "image_url")) {
throw std::invalid_argument("'image_url' is required for 'input_image' tool output and must be a string");
}
chatcmpl_output = json {
{"image_url", json {
{"url", chatcmpl_output.at("image_url")}
}},
{"type", "image_url"},
};
} else {
throw std::invalid_argument("'type' must be one of 'input_text' or 'input_image'");
}
chatcmpl_output["type"] = "text";
}
chatcmpl_messages.push_back(json {
{"content", chatcmpl_outputs},
Expand Down