Skip to content

Commit 63cef2a

Browse files
LeonardW-slLeonardW-sl
committed
feat(bridge): support image input (multimodal)
- ConvertIncomingMessage extracts image parts and builds Qoder contents with {type:image_url, image_url:{url}} before the text part - image-only user messages are no longer silently dropped - ExtractMessageImages handles OpenAI image_url, Responses input_image and Claude image blocks (base64 + url sources), with a contents[] fallback - reject non data:/http(s) image URLs - add unit tests and an end-to-end test asserting the encoded upstream body carries the image part
1 parent ae3d42f commit 63cef2a

3 files changed

Lines changed: 299 additions & 8 deletions

File tree

‎internal/bridge/image_integration_test.go‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
package bridge
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io"
7+
"net/http"
8+
"net/http/httptest"
9+
"strings"
10+
"testing"
11+
12+
"qoder2api/internal/cosy"
13+
)
14+
15+
// TestCallQoderSendsImageParts 验证图片 part 从客户端请求一路传到上游请求体
16+
// （端到端覆盖 BuildQoderMessages -> CallQoderWithOpts -> cosy.Encode）。
17+
func TestCallQoderSendsImageParts(t *testing.T) {
18+
var rawBody []byte
19+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
20+
rawBody, _ = io.ReadAll(r.Body)
21+
w.Header().Set("Content-Type", "text/event-stream")
22+
fmt.Fprint(w, sseContent("ok"))
23+
fmt.Fprint(w, "data: [DONE]\n\n")
24+
}))
25+
defer srv.Close()
26+
27+
msgs := BuildQoderMessages(nil, []interface{}{
28+
map[string]interface{}{
29+
"role": "user",
30+
"content": []interface{}{
31+
map[string]interface{}{"type": "text", "text": "看图"},
32+
map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "data:image/png;base64,ZZZZ"}},
33+
},
34+
},
35+
}, "看图", false)
36+
37+
err := envelopeBridge(t, srv.URL).CallQoderWithOpts(
38+
context.Background(), "codex", msgs, "auto", nil, CallOpts{},
39+
func(Delta) {})
40+
if err != nil {
41+
t.Fatalf("CallQoderWithOpts: %v", err)
42+
}
43+
44+
plain, err := cosy.Decode(string(rawBody))
45+
if err != nil {
46+
t.Fatalf("decode upstream body: %v", err)
47+
}
48+
body := string(plain)
49+
if !strings.Contains(body, `"image_url"`) {
50+
t.Fatalf("upstream body missing image part: %s", body)
51+
}
52+
if !strings.Contains(body, "data:image/png;base64,ZZZZ") {
53+
t.Fatalf("upstream body missing image url: %s", body)
54+
}
55+
if !strings.Contains(body, `"type":"text"`) {
56+
t.Fatalf("upstream body missing text part: %s", body)
57+
}
58+
}

‎internal/bridge/image_test.go‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
package bridge
2+
3+
import (
4+
"encoding/json"
5+
"strings"
6+
"testing"
7+
)
8+
9+
func TestBuildUserMessageWithImages(t *testing.T) {
10+
msg := BuildUserMessageWithImages("看图", []string{"data:image/png;base64,AAAA"})
11+
data, _ := json.Marshal(msg)
12+
s := string(data)
13+
if !strings.Contains(s, `"type":"image_url"`) {
14+
t.Fatalf("missing image part: %s", s)
15+
}
16+
if !strings.Contains(s, `"url":"data:image/png;base64,AAAA"`) {
17+
t.Fatalf("missing image url: %s", s)
18+
}
19+
if !strings.Contains(s, `"type":"text"`) {
20+
t.Fatalf("missing text part: %s", s)
21+
}
22+
}
23+
24+
func TestBuildUserMessageTextOnlyUnchanged(t *testing.T) {
25+
msg := BuildUserMessage("hi")
26+
contents, _ := msg["contents"].([]interface{})
27+
if len(contents) != 1 {
28+
t.Fatalf("text-only should have exactly 1 part, got %d", len(contents))
29+
}
30+
first, _ := contents[0].(map[string]interface{})
31+
if first["type"] != "text" || first["text"] != "hi" {
32+
t.Fatalf("unexpected text part: %v", first)
33+
}
34+
}
35+
36+
func TestConvertIncomingMessageImageOnly(t *testing.T) {
37+
msg := map[string]interface{}{
38+
"role": "user",
39+
"content": []interface{}{
40+
map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "data:image/jpeg;base64,BBBB"}},
41+
},
42+
}
43+
out := ConvertIncomingMessage(msg, false)
44+
if out == nil {
45+
t.Fatal("image-only user message was dropped")
46+
}
47+
contents, _ := out["contents"].([]interface{})
48+
if len(contents) != 1 {
49+
t.Fatalf("want 1 part (image), got %d: %v", len(contents), contents)
50+
}
51+
first, _ := contents[0].(map[string]interface{})
52+
if first["type"] != "image_url" {
53+
t.Fatalf("want image part, got %v", first)
54+
}
55+
}
56+
57+
func TestConvertIncomingMessageTextAndImage(t *testing.T) {
58+
msg := map[string]interface{}{
59+
"role": "user",
60+
"content": []interface{}{
61+
map[string]interface{}{"type": "text", "text": "这是什么"},
62+
map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "https://example.com/a.png"}},
63+
},
64+
}
65+
out := ConvertIncomingMessage(msg, false)
66+
contents, _ := out["contents"].([]interface{})
67+
if len(contents) != 2 {
68+
t.Fatalf("want 2 parts, got %d: %v", len(contents), contents)
69+
}
70+
first, _ := contents[0].(map[string]interface{})
71+
if first["type"] != "image_url" {
72+
t.Fatalf("image should come first, got %v", first)
73+
}
74+
last, _ := contents[1].(map[string]interface{})
75+
if last["type"] != "text" || last["text"] != "这是什么" {
76+
t.Fatalf("text should follow image, got %v", last)
77+
}
78+
}
79+
80+
func TestConvertIncomingMessageClaudeImage(t *testing.T) {
81+
msg := map[string]interface{}{
82+
"role": "user",
83+
"content": []interface{}{
84+
map[string]interface{}{
85+
"type": "image",
86+
"source": map[string]interface{}{
87+
"type": "base64", "media_type": "image/png", "data": "CCCC",
88+
},
89+
},
90+
map[string]interface{}{"type": "text", "text": "hi"},
91+
},
92+
}
93+
out := ConvertIncomingMessage(msg, false)
94+
contents, _ := out["contents"].([]interface{})
95+
if len(contents) != 2 {
96+
t.Fatalf("want 2 parts, got %d: %v", len(contents), contents)
97+
}
98+
first, _ := contents[0].(map[string]interface{})
99+
if first["type"] != "image_url" {
100+
t.Fatalf("claude image not converted: %v", first)
101+
}
102+
iu, _ := first["image_url"].(map[string]interface{})
103+
if iu["url"] != "data:image/png;base64,CCCC" {
104+
t.Fatalf("bad data url: %v", iu)
105+
}
106+
}
107+
108+
func TestConvertIncomingMessageRejectsJunkImageURL(t *testing.T) {
109+
msg := map[string]interface{}{
110+
"role": "user",
111+
"content": []interface{}{
112+
map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "not-a-url"}},
113+
},
114+
}
115+
if out := ConvertIncomingMessage(msg, false); out != nil {
116+
t.Fatalf("junk image should be dropped, got %v", out)
117+
}
118+
}
119+
120+
func TestExtractMessageImagesInputImageString(t *testing.T) {
121+
msg := map[string]interface{}{
122+
"content": []interface{}{
123+
map[string]interface{}{"type": "input_image", "image_url": "data:image/webp;base64,DDDD"},
124+
},
125+
}
126+
imgs := ExtractMessageImages(msg)
127+
if len(imgs) != 1 || imgs[0] != "data:image/webp;base64,DDDD" {
128+
t.Fatalf("want 1 data url, got %v", imgs)
129+
}
130+
}

‎internal/bridge/messages.go‎

Lines changed: 111 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package bridge
22

3-
import "encoding/json"
3+
import (
4+
"encoding/json"
5+
"strings"
6+
)
47

58
func BuildQoderMessages(templateMsgs []interface{}, incoming []interface{}, prompt string, toolsEnabled bool) []interface{} {
69
var rebuilt []interface{}
@@ -174,6 +177,7 @@ func ConvertIncomingMessage(msg map[string]interface{}, toolsEnabled bool) map[s
174177
}
175178

176179
text := NormalizeMessageContent(msg)
180+
images := ExtractMessageImages(msg)
177181

178182
if role == "assistant" && toolsEnabled {
179183
if tc, ok := msg["tool_calls"]; ok {
@@ -194,12 +198,12 @@ func ConvertIncomingMessage(msg map[string]interface{}, toolsEnabled bool) map[s
194198
return out
195199
}
196200

197-
if text == "" {
201+
if text == "" && len(images) == 0 {
198202
return nil
199203
}
200204

201205
if role == "user" {
202-
return BuildUserMessage(text)
206+
return BuildUserMessageWithImages(text, images)
203207
}
204208
return BuildStructuredMessage(role, text)
205209
}
@@ -215,18 +219,117 @@ func HasBlockType(blocks []interface{}, blockType string) bool {
215219
return false
216220
}
217221

222+
// BuildUserMessage 构造纯文本 user 消息（保留原有签名，内部转发到多模态版本）。
218223
func BuildUserMessage(text string) map[string]interface{} {
224+
return BuildUserMessageWithImages(text, nil)
225+
}
226+
227+
// BuildUserMessageWithImages 构造 Qoder 上游的 user 消息。
228+
//
229+
// Qoder 的 contents 数组接受 OpenAI 风格的 content parts：图片在前
230+
// ({type:image_url, image_url:{url}})，文本在后 ({type:text, text})。
231+
// 图片 URL 支持 data: base64 与 http(s) 两种形态，与官方客户端一致。
232+
func BuildUserMessageWithImages(text string, images []string) map[string]interface{} {
233+
contents := make([]interface{}, 0, len(images)+1)
234+
for _, url := range images {
235+
contents = append(contents, map[string]interface{}{
236+
"type": "image_url",
237+
"image_url": map[string]interface{}{"url": url},
238+
})
239+
}
240+
if text != "" || len(contents) == 0 {
241+
contents = append(contents, map[string]interface{}{"type": "text", "text": text})
242+
}
219243
return map[string]interface{}{
220-
"role": "user",
221-
"content": "",
222-
"contents": []interface{}{
223-
map[string]interface{}{"type": "text", "text": text},
224-
},
244+
"role": "user",
245+
"content": "",
246+
"contents": contents,
225247
"response_meta": BlankResponseMeta(),
226248
"reasoning_content_signature": "",
227249
}
228250
}
229251

252+
// ExtractMessageImages 从消息的 content 数组中提取图片 URL。
253+
// 支持三种形态：
254+
// - OpenAI Chat Completions: {type:image_url, image_url:{url}}
255+
// - OpenAI Responses API: {type:input_image, image_url:"..."}
256+
// - Claude Messages API: {type:image, source:{type:base64|url, media_type, data|url}}
257+
//
258+
// 空 URL 与非图片 URL（既不是 data: 也不是 http(s)）会被丢弃。
259+
func ExtractMessageImages(msg map[string]interface{}) []string {
260+
arr, ok := msg["content"].([]interface{})
261+
if !ok {
262+
// 兼容 Qoder 风格：图片可能直接放在 contents 数组里（参考 EchoPing07）
263+
arr, ok = msg["contents"].([]interface{})
264+
if !ok {
265+
return nil
266+
}
267+
}
268+
var images []string
269+
for _, block := range arr {
270+
b, ok := block.(map[string]interface{})
271+
if !ok {
272+
continue
273+
}
274+
t, _ := b["type"].(string)
275+
switch t {
276+
case "image_url", "input_image":
277+
if url := imageURLFromPart(b); url != "" && looksLikeImageURL(url) {
278+
images = append(images, url)
279+
}
280+
case "image":
281+
if url := claudeImageToURL(b); url != "" {
282+
images = append(images, url)
283+
}
284+
}
285+
}
286+
return images
287+
}
288+
289+
// imageURLFromPart 兼容 {image_url:{url}}、{image_url:"..."} 与 {url:"..."} 三种写法。
290+
func imageURLFromPart(part map[string]interface{}) string {
291+
switch v := part["image_url"].(type) {
292+
case string:
293+
return v
294+
case map[string]interface{}:
295+
if s, ok := v["url"].(string); ok {
296+
return s
297+
}
298+
}
299+
if s, ok := part["url"].(string); ok {
300+
return s
301+
}
302+
return ""
303+
}
304+
305+
// claudeImageToURL 把 Claude 的 image block 转成 data:/http URL。
306+
func claudeImageToURL(block map[string]interface{}) string {
307+
src, ok := block["source"].(map[string]interface{})
308+
if !ok {
309+
return ""
310+
}
311+
switch st, _ := src["type"].(string); st {
312+
case "base64":
313+
mt, _ := src["media_type"].(string)
314+
data, _ := src["data"].(string)
315+
if mt != "" && data != "" {
316+
return "data:" + mt + ";base64," + data
317+
}
318+
case "url":
319+
if u, ok := src["url"].(string); ok && looksLikeImageURL(u) {
320+
return u
321+
}
322+
}
323+
return ""
324+
}
325+
326+
// looksLikeImageURL 只放行 data: 与 http(s) 两类，避免把垃圾塞给上游。
327+
func looksLikeImageURL(url string) bool {
328+
return strings.HasPrefix(url, "data:") ||
329+
strings.HasPrefix(url, "http://") ||
330+
strings.HasPrefix(url, "https://")
331+
}
332+
230333
func BuildStructuredMessage(role, text string) map[string]interface{} {
231334
return map[string]interface{}{
232335
"role": role,

0 commit comments

Comments
 (0)