From 01c98784c54c338de084c3321a66e4f59c07cf98 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Mon, 10 Aug 2026 16:06:12 +0800 Subject: [PATCH 1/4] fix(web): render local-path media in both dashboard chains Drop the legacy Gradio convention from `messages_to_markdown` and make the Web dashboard resolve local media paths in Markdown as well. `messages_to_markdown` emitted `![image](gradio_api/file=)`, a convention from the Gradio-era dashboard (#1019). `gradio` is no longer a dependency, `evalscope app` is only a deprecated alias of `evalscope service`, and nothing in the codebase interprets the prefix, so local-path images were unusable in the compare view and printed as noise by `Sample.pretty_print`. Dropping the prefix alone is not enough: `MarkdownRenderer` passed the image `src` straight to `ImageLightbox`, while only the structured `Messages` chain proxied media through `/api/v1/reports/media/file?path=`. `resolveMediaSrc` is therefore extracted into `utils/media.ts` and shared by both chains. The structured chain had a related gap: `ContentImage` values may be relative paths (e.g. the custom VQA data used by `general_vqa`), which the client cannot tell apart from a base64 payload, producing an invalid `data:` URI. Local media paths are now absolutised when serialising prediction messages, matching what `messages_to_markdown` already does, so both chains expose media identically. Verified against a real `general_vqa` run (DashScope qwen-vl-plus / qwen-vl-max on the custom VQA sample data): images now render in both the compare view and the single-report chat view, requesting the media proxy endpoint. --- evalscope/api/messages/chat_message.py | 5 +- evalscope/utils/data_utils.py | 29 ++++++++++- .../common/MarkdownRenderer.test.tsx | 23 +++++++++ .../components/common/MarkdownRenderer.tsx | 5 +- .../components/single/chat/MediaBlocks.tsx | 16 +----- evalscope/web/src/utils/media.ts | 18 +++++++ tests/api/test_messages_to_markdown.py | 45 +++++++++++++++++ tests/report/test_prediction_media_paths.py | 49 +++++++++++++++++++ 8 files changed, 171 insertions(+), 19 deletions(-) create mode 100644 evalscope/web/src/utils/media.ts create mode 100644 tests/api/test_messages_to_markdown.py create mode 100644 tests/report/test_prediction_media_paths.py diff --git a/evalscope/api/messages/chat_message.py b/evalscope/api/messages/chat_message.py index 665dfe7c6..4bde85905 100644 --- a/evalscope/api/messages/chat_message.py +++ b/evalscope/api/messages/chat_message.py @@ -225,9 +225,10 @@ def messages_to_markdown(messages: List[ChatMessage], max_length: Optional[int] # Use markdown image syntax image_base64_or_url = content_item.image if os.path.isfile(image_base64_or_url): + # Local file: emit the absolute path so that renderers can resolve it, + # e.g. the Web dashboard proxies it through its media file endpoint. image_base64_or_url = os.path.abspath(image_base64_or_url) - # If it's a file, convert to a markdown image with a gradio-compatible path - content_parts.append(f'![image](gradio_api/file={image_base64_or_url})') + content_parts.append(f'![image]({image_base64_or_url})') else: # If it's not a file, assume it's a base64 string if max_length and len(image_base64_or_url) > max_length: diff --git a/evalscope/utils/data_utils.py b/evalscope/utils/data_utils.py index dd582a942..c45c2cb76 100644 --- a/evalscope/utils/data_utils.py +++ b/evalscope/utils/data_utils.py @@ -171,6 +171,33 @@ def _load_perf_map(cache_manager: CacheManager, dataset_name: str, subset_name: return perf_map +# Serialised ContentBlock types that may carry a server-side media path. +_MEDIA_BLOCK_FIELDS = {'image': 'image', 'audio': 'audio', 'video': 'video'} + + +def _absolutize_media_path(block: Dict[str, Any]) -> Dict[str, Any]: + """Rewrite a local media path in a serialised ContentBlock to an absolute path. + + A renderer can only resolve a server-side file when it is given an absolute + path: a relative path is indistinguishable from a base64 payload on the + client side. This mirrors what :func:`messages_to_markdown` does for the + markdown chain, so both chains expose local media the same way. + + Args: + block (Dict[str, Any]): A serialised ContentBlock, mutated in place. + + Returns: + Dict[str, Any]: The same block, for convenient chaining. + """ + field = _MEDIA_BLOCK_FIELDS.get(block.get('type')) + if not field: + return block + value = block.get(field) + if isinstance(value, str) and value and not value.startswith('data:') and os.path.isfile(value): + block[field] = os.path.abspath(value) + return block + + def _serialize_messages(review_result: ReviewResult) -> List[Dict[str, Any]]: """Serialize a ReviewResult's message list into frontend-compatible dicts. @@ -200,7 +227,7 @@ def _serialize_messages(review_result: ReviewResult) -> List[Dict[str, Any]]: # reasoning, text, …). model_dump() mirrors the Python # ContentBase subclasses to plain dicts that match the # TypeScript ContentBlock interface in types.ts. - serialised_content = [c.model_dump() for c in m.content] + serialised_content = [_absolutize_media_path(c.model_dump()) for c in m.content] else: # Text-only / legacy path – content is already a str. serialised_content = m.content diff --git a/evalscope/web/src/components/common/MarkdownRenderer.test.tsx b/evalscope/web/src/components/common/MarkdownRenderer.test.tsx index 28d9c4207..da6afafc7 100644 --- a/evalscope/web/src/components/common/MarkdownRenderer.test.tsx +++ b/evalscope/web/src/components/common/MarkdownRenderer.test.tsx @@ -33,3 +33,26 @@ describe('MarkdownRenderer heavy-content gating', () => { expect(screen.queryByRole('status')).not.toBeInTheDocument() }) }) + +describe('MarkdownRenderer image sources', () => { + it('proxies a local absolute image path through the media file endpoint', () => { + // The markdown chain (e.g. the `Input` column built by `messages_to_markdown`) + // must resolve server-side paths the same way structured messages do. + const { container } = renderMarkdown('![shot](/tmp/miniwob/step-000.png)') + + expect(container.querySelector('img')).toHaveAttribute( + 'src', + '/api/v1/reports/media/file?path=%2Ftmp%2Fminiwob%2Fstep-000.png', + ) + }) + + it('keeps http and data URI image sources untouched', () => { + const { container } = renderMarkdown( + '![remote](https://example.com/a.png)\n\n![inline](data:image/png;base64,aGVsbG8=)', + ) + + const images = container.querySelectorAll('img') + expect(images[0]).toHaveAttribute('src', 'https://example.com/a.png') + expect(images[1]).toHaveAttribute('src', 'data:image/png;base64,aGVsbG8=') + }) +}) diff --git a/evalscope/web/src/components/common/MarkdownRenderer.tsx b/evalscope/web/src/components/common/MarkdownRenderer.tsx index 8c1e9962f..187f4390b 100644 --- a/evalscope/web/src/components/common/MarkdownRenderer.tsx +++ b/evalscope/web/src/components/common/MarkdownRenderer.tsx @@ -4,6 +4,7 @@ import remarkGfm from 'remark-gfm' import type { Components, Options } from 'react-markdown' import { useTheme } from '@/contexts/ThemeContext' import { useLocale } from '@/contexts/LocaleContext' +import { resolveMediaSrc } from '@/utils/media' import ImageLightbox from './ImageLightbox' import LazyCodeBlock from './LazyCodeBlock' @@ -115,7 +116,9 @@ export default function MarkdownRenderer({ content, collapsed = false }: Props) const { plugins: mathPlugins, state: mathState } = useMathPlugins(content, collapsed) const markdownComponents = useMemo(() => ({ - img: ({ src, alt }) => , + img: ({ src, alt }) => ( + + ), code: ({ className, children }) => { const match = /language-(\w+)/.exec(className || '') if (match) { diff --git a/evalscope/web/src/components/single/chat/MediaBlocks.tsx b/evalscope/web/src/components/single/chat/MediaBlocks.tsx index ffeef2f62..1682c80b4 100644 --- a/evalscope/web/src/components/single/chat/MediaBlocks.tsx +++ b/evalscope/web/src/components/single/chat/MediaBlocks.tsx @@ -5,21 +5,7 @@ import { useLocale } from '@/contexts/LocaleContext' import MarkdownRenderer from '@/components/common/MarkdownRenderer' import ImageLightbox from '@/components/common/ImageLightbox' import ChatBubble, { bubbleAccent } from '@/components/ui/ChatBubble' - -/** - * Resolve a server-side path / URL / base64 payload to a browser-loadable src. - * - * - http(s) and data: URIs are returned as-is. - * - Absolute POSIX/Windows paths are proxied through the media file endpoint. - * - Anything else is treated as base64 and wrapped in a data: URI with `mimeType`. - */ -function resolveMediaSrc(src: string, mimeType: string): string { - if (src.startsWith('http://') || src.startsWith('https://') || src.startsWith('data:')) return src - if (src.startsWith('/') || /^[A-Za-z]:[/\\]/.test(src)) { - return `/api/v1/reports/media/file?path=${encodeURIComponent(src)}` - } - return `data:${mimeType};base64,${src}` -} +import { resolveMediaSrc } from '@/utils/media' const IMG_INLINE_CLASS = 'cursor-zoom-in hover:scale-[1.02] transition-transform max-w-full max-h-[360px] rounded-lg border border-[var(--border)] block' diff --git a/evalscope/web/src/utils/media.ts b/evalscope/web/src/utils/media.ts new file mode 100644 index 000000000..b9d99f691 --- /dev/null +++ b/evalscope/web/src/utils/media.ts @@ -0,0 +1,18 @@ +/** + * Resolve a server-side path / URL / base64 payload to a browser-loadable src. + * + * - http(s) and data: URIs are returned as-is. + * - Absolute POSIX/Windows paths are proxied through the media file endpoint. + * - Anything else is treated as base64 and wrapped in a data: URI with `mimeType`. + * + * Shared by both rendering chains so that a local media path shows up the same + * way in structured `ContentBlock` messages and in Markdown text (e.g. the + * `Input` column produced by `messages_to_markdown`). + */ +export function resolveMediaSrc(src: string, mimeType: string): string { + if (src.startsWith('http://') || src.startsWith('https://') || src.startsWith('data:')) return src + if (src.startsWith('/') || /^[A-Za-z]:[/\\]/.test(src)) { + return `/api/v1/reports/media/file?path=${encodeURIComponent(src)}` + } + return `data:${mimeType};base64,${src}` +} diff --git a/tests/api/test_messages_to_markdown.py b/tests/api/test_messages_to_markdown.py new file mode 100644 index 000000000..2aeb52283 --- /dev/null +++ b/tests/api/test_messages_to_markdown.py @@ -0,0 +1,45 @@ +import pytest +from pathlib import Path + +from evalscope.api.messages import ChatMessageUser, ContentImage, ContentText, messages_to_markdown + + +def test_local_image_path_is_emitted_as_plain_absolute_path(tmp_path: Path) -> None: + # Regression test: the legacy `gradio_api/file=` prefix is no longer + # understood by any renderer, so a local file must be emitted as-is. + image_path = tmp_path / 'screenshot.png' + image_path.write_bytes(b'fake-png') + messages = [ChatMessageUser(content=[ContentText(text='Look:'), ContentImage(image=str(image_path))])] + + markdown = messages_to_markdown(messages) + + assert f'![image]({image_path})' in markdown + assert 'gradio_api' not in markdown + + +def test_relative_image_path_is_absolutised(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + image_path = tmp_path / 'frame.jpg' + image_path.write_bytes(b'fake-jpg') + monkeypatch.chdir(tmp_path) + messages = [ChatMessageUser(content=[ContentImage(image='frame.jpg')])] + + markdown = messages_to_markdown(messages) + + assert f'![image]({image_path.resolve()})' in markdown + + +def test_data_uri_image_is_passed_through() -> None: + data_uri = 'data:image/png;base64,aGVsbG8=' + messages = [ChatMessageUser(content=[ContentImage(image=data_uri)])] + + markdown = messages_to_markdown(messages) + + assert f'![image]({data_uri})' in markdown + + +def test_base64_image_is_truncated_by_max_length() -> None: + messages = [ChatMessageUser(content=[ContentImage(image='a' * 100)])] + + markdown = messages_to_markdown(messages, max_length=10) + + assert f'![image]({"a" * 10})' in markdown diff --git a/tests/report/test_prediction_media_paths.py b/tests/report/test_prediction_media_paths.py new file mode 100644 index 000000000..591f309bb --- /dev/null +++ b/tests/report/test_prediction_media_paths.py @@ -0,0 +1,49 @@ +import pytest +from pathlib import Path + +from evalscope.api.evaluator import ReviewResult +from evalscope.api.messages import ChatMessageUser, ContentImage, ContentText +from evalscope.api.metric import SampleScore, Score +from evalscope.utils.data_utils import _serialize_messages + + +def _review_result(image: str) -> ReviewResult: + return ReviewResult( + index=0, + target='Dog', + messages=[ChatMessageUser(content=[ContentText(text='What animal is this?'), ContentImage(image=image)])], + sample_score=SampleScore(score=Score(value={'acc': 1.0}), sample_id='0'), + ) + + +def _image_values(review_result: ReviewResult) -> list: + values = [] + for message in _serialize_messages(review_result): + content = message['content'] + if isinstance(content, list): + values.extend(block['image'] for block in content if block['type'] == 'image') + return values + + +def test_local_image_path_is_absolutised_for_the_frontend(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + # A relative path cannot be told apart from a base64 payload by the client, + # so it must be absolutised before it reaches the dashboard. + image_path = tmp_path / 'dog.jpg' + image_path.write_bytes(b'fake-jpg') + monkeypatch.chdir(tmp_path) + + assert _image_values(_review_result('dog.jpg')) == [str(image_path.resolve())] + + +def test_absolute_image_path_is_preserved(tmp_path: Path) -> None: + image_path = tmp_path / 'dog.jpg' + image_path.write_bytes(b'fake-jpg') + + assert _image_values(_review_result(str(image_path))) == [str(image_path)] + + +def test_data_uri_and_base64_images_are_left_alone() -> None: + data_uri = 'data:image/png;base64,aGVsbG8=' + + assert _image_values(_review_result(data_uri)) == [data_uri] + assert _image_values(_review_result('aGVsbG8=')) == ['aGVsbG8='] From f4b9a773341fecaa97af721456443f1f99e65b53 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Mon, 10 Aug 2026 16:42:37 +0800 Subject: [PATCH 2/4] fix(web): keep escaped local paths resolvable in markdown media MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local paths were emitted as bare markdown destinations, which breaks in two ways found while addressing review feedback: - A destination containing a space or an unbalanced parenthesis does not parse as an image at all, so the whole `![image](...)` degrades to literal text. - A markdown parser percent-encodes the destination, so a path with non-ASCII characters (common in the OCR datasets) or spaces reached the frontend as `/tmp/%E5%9B%BE.jpg`. `encodeURIComponent` then escaped the `%` again and the media proxy received a literal `%E5%9B%BE` path and answered 404. The destination is now wrapped in `<...>`, and the markdown image renderer decodes the src before resolving it via `resolveMarkdownMediaSrc`. Remote URLs and data: URIs are still passed through untouched so an intentionally escaped remote URL is never rewritten. Also simplify `_MEDIA_BLOCK_TYPES` (the previous mapping was an identity dict) and tighten a test helper's return type. Verified end to end on a path containing both CJK characters and spaces (`.../图片 目录/狗 图片.jpg`): the compare view and the single-report chat view both render the image, the src carries single-encoded UTF-8 with no `%25`, and the media endpoint answers 200/304. --- evalscope/api/messages/chat_message.py | 4 +++- evalscope/utils/data_utils.py | 13 +++++----- .../common/MarkdownRenderer.test.tsx | 24 +++++++++++++++++-- .../components/common/MarkdownRenderer.tsx | 4 ++-- evalscope/web/src/utils/media.ts | 22 +++++++++++++++++ tests/api/test_messages_to_markdown.py | 22 +++++++++++++---- tests/report/test_prediction_media_paths.py | 3 ++- 7 files changed, 76 insertions(+), 16 deletions(-) diff --git a/evalscope/api/messages/chat_message.py b/evalscope/api/messages/chat_message.py index 4bde85905..2ce2412c7 100644 --- a/evalscope/api/messages/chat_message.py +++ b/evalscope/api/messages/chat_message.py @@ -227,8 +227,10 @@ def messages_to_markdown(messages: List[ChatMessage], max_length: Optional[int] if os.path.isfile(image_base64_or_url): # Local file: emit the absolute path so that renderers can resolve it, # e.g. the Web dashboard proxies it through its media file endpoint. + # The destination is wrapped in <> so that paths containing spaces or + # unbalanced parentheses still parse as a markdown image. image_base64_or_url = os.path.abspath(image_base64_or_url) - content_parts.append(f'![image]({image_base64_or_url})') + content_parts.append(f'![image](<{image_base64_or_url}>)') else: # If it's not a file, assume it's a base64 string if max_length and len(image_base64_or_url) > max_length: diff --git a/evalscope/utils/data_utils.py b/evalscope/utils/data_utils.py index c45c2cb76..cf7ee22c4 100644 --- a/evalscope/utils/data_utils.py +++ b/evalscope/utils/data_utils.py @@ -171,8 +171,9 @@ def _load_perf_map(cache_manager: CacheManager, dataset_name: str, subset_name: return perf_map -# Serialised ContentBlock types that may carry a server-side media path. -_MEDIA_BLOCK_FIELDS = {'image': 'image', 'audio': 'audio', 'video': 'video'} +# Serialised ContentBlock types that may carry a server-side media path. The +# payload field of each of these blocks is named after the block type. +_MEDIA_BLOCK_TYPES = frozenset({'image', 'audio', 'video'}) def _absolutize_media_path(block: Dict[str, Any]) -> Dict[str, Any]: @@ -189,12 +190,12 @@ def _absolutize_media_path(block: Dict[str, Any]) -> Dict[str, Any]: Returns: Dict[str, Any]: The same block, for convenient chaining. """ - field = _MEDIA_BLOCK_FIELDS.get(block.get('type')) - if not field: + block_type = block.get('type') + if block_type not in _MEDIA_BLOCK_TYPES: return block - value = block.get(field) + value = block.get(block_type) if isinstance(value, str) and value and not value.startswith('data:') and os.path.isfile(value): - block[field] = os.path.abspath(value) + block[block_type] = os.path.abspath(value) return block diff --git a/evalscope/web/src/components/common/MarkdownRenderer.test.tsx b/evalscope/web/src/components/common/MarkdownRenderer.test.tsx index da6afafc7..dc7d6dabc 100644 --- a/evalscope/web/src/components/common/MarkdownRenderer.test.tsx +++ b/evalscope/web/src/components/common/MarkdownRenderer.test.tsx @@ -46,13 +46,33 @@ describe('MarkdownRenderer image sources', () => { ) }) + it('decodes a percent-encoded non-ascii path back to the real filesystem path', () => { + // The markdown parser encodes non-ascii destinations, so without decoding + // the proxy would receive the literal escape sequences and return 404. + const { container } = renderMarkdown('![shot]()') + + expect(container.querySelector('img')).toHaveAttribute( + 'src', + `/api/v1/reports/media/file?path=${encodeURIComponent('/tmp/\u6570\u636e\u96c6/a.png')}`, + ) + }) + + it('renders a local path containing spaces via the angle-bracket destination', () => { + const { container } = renderMarkdown('![shot]()') + + expect(container.querySelector('img')).toHaveAttribute( + 'src', + `/api/v1/reports/media/file?path=${encodeURIComponent('/tmp/my dir/a.png')}`, + ) + }) + it('keeps http and data URI image sources untouched', () => { const { container } = renderMarkdown( - '![remote](https://example.com/a.png)\n\n![inline](data:image/png;base64,aGVsbG8=)', + '![remote](https://example.com/a%20b.png)\n\n![inline](data:image/png;base64,aGVsbG8=)', ) const images = container.querySelectorAll('img') - expect(images[0]).toHaveAttribute('src', 'https://example.com/a.png') + expect(images[0]).toHaveAttribute('src', 'https://example.com/a%20b.png') expect(images[1]).toHaveAttribute('src', 'data:image/png;base64,aGVsbG8=') }) }) diff --git a/evalscope/web/src/components/common/MarkdownRenderer.tsx b/evalscope/web/src/components/common/MarkdownRenderer.tsx index 187f4390b..ade438402 100644 --- a/evalscope/web/src/components/common/MarkdownRenderer.tsx +++ b/evalscope/web/src/components/common/MarkdownRenderer.tsx @@ -4,7 +4,7 @@ import remarkGfm from 'remark-gfm' import type { Components, Options } from 'react-markdown' import { useTheme } from '@/contexts/ThemeContext' import { useLocale } from '@/contexts/LocaleContext' -import { resolveMediaSrc } from '@/utils/media' +import { resolveMarkdownMediaSrc } from '@/utils/media' import ImageLightbox from './ImageLightbox' import LazyCodeBlock from './LazyCodeBlock' @@ -117,7 +117,7 @@ export default function MarkdownRenderer({ content, collapsed = false }: Props) const markdownComponents = useMemo(() => ({ img: ({ src, alt }) => ( - + ), code: ({ className, children }) => { const match = /language-(\w+)/.exec(className || '') diff --git a/evalscope/web/src/utils/media.ts b/evalscope/web/src/utils/media.ts index b9d99f691..5ddf79906 100644 --- a/evalscope/web/src/utils/media.ts +++ b/evalscope/web/src/utils/media.ts @@ -16,3 +16,25 @@ export function resolveMediaSrc(src: string, mimeType: string): string { } return `data:${mimeType};base64,${src}` } + +/** + * Resolve an image src taken from rendered Markdown. + * + * A Markdown parser percent-encodes the link destination, so a local path + * containing spaces, non-ASCII characters or backslashes arrives here already + * encoded (`/tmp/my img.jpg` -> `/tmp/my%20img.jpg`). It has to be decoded back + * to the real filesystem path before the media endpoint can look it up, + * otherwise the proxy receives the literal escape sequences. + * + * Remote URLs and data: URIs are returned untouched, so an intentionally + * escaped remote URL is never rewritten. + */ +export function resolveMarkdownMediaSrc(src: string, mimeType: string): string { + if (src.startsWith('http://') || src.startsWith('https://') || src.startsWith('data:')) return src + try { + return resolveMediaSrc(decodeURI(src), mimeType) + } catch { + // Malformed escape sequence – fall back to the raw value. + return resolveMediaSrc(src, mimeType) + } +} diff --git a/tests/api/test_messages_to_markdown.py b/tests/api/test_messages_to_markdown.py index 2aeb52283..df08feb47 100644 --- a/tests/api/test_messages_to_markdown.py +++ b/tests/api/test_messages_to_markdown.py @@ -4,19 +4,33 @@ from evalscope.api.messages import ChatMessageUser, ContentImage, ContentText, messages_to_markdown -def test_local_image_path_is_emitted_as_plain_absolute_path(tmp_path: Path) -> None: +def test_local_image_path_is_emitted_as_an_absolute_path(tmp_path: Path) -> None: # Regression test: the legacy `gradio_api/file=` prefix is no longer - # understood by any renderer, so a local file must be emitted as-is. + # understood by any renderer, so a local file must be emitted as a path. image_path = tmp_path / 'screenshot.png' image_path.write_bytes(b'fake-png') messages = [ChatMessageUser(content=[ContentText(text='Look:'), ContentImage(image=str(image_path))])] markdown = messages_to_markdown(messages) - assert f'![image]({image_path})' in markdown + assert f'![image](<{image_path}>)' in markdown assert 'gradio_api' not in markdown +def test_local_image_path_with_spaces_stays_a_single_destination(tmp_path: Path) -> None: + # An unwrapped destination containing a space does not parse as a markdown + # image at all, so the <> form is required here. + image_dir = tmp_path / 'my images' + image_dir.mkdir() + image_path = image_dir / 'a shot.png' + image_path.write_bytes(b'fake-png') + messages = [ChatMessageUser(content=[ContentImage(image=str(image_path))])] + + markdown = messages_to_markdown(messages) + + assert f'![image](<{image_path}>)' in markdown + + def test_relative_image_path_is_absolutised(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: image_path = tmp_path / 'frame.jpg' image_path.write_bytes(b'fake-jpg') @@ -25,7 +39,7 @@ def test_relative_image_path_is_absolutised(tmp_path: Path, monkeypatch: pytest. markdown = messages_to_markdown(messages) - assert f'![image]({image_path.resolve()})' in markdown + assert f'![image](<{image_path.resolve()}>)' in markdown def test_data_uri_image_is_passed_through() -> None: diff --git a/tests/report/test_prediction_media_paths.py b/tests/report/test_prediction_media_paths.py index 591f309bb..07d4af196 100644 --- a/tests/report/test_prediction_media_paths.py +++ b/tests/report/test_prediction_media_paths.py @@ -1,5 +1,6 @@ import pytest from pathlib import Path +from typing import List from evalscope.api.evaluator import ReviewResult from evalscope.api.messages import ChatMessageUser, ContentImage, ContentText @@ -16,7 +17,7 @@ def _review_result(image: str) -> ReviewResult: ) -def _image_values(review_result: ReviewResult) -> list: +def _image_values(review_result: ReviewResult) -> List[str]: values = [] for message in _serialize_messages(review_result): content = message['content'] From ab86ad81bc4a2438a46142a2685b341e21e20395 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Mon, 10 Aug 2026 17:05:28 +0800 Subject: [PATCH 3/4] fix(web): keep valid remote media sources out of the local-path branch Follow-up from review: the resolver classified sources with a case-sensitive prefix check, so `HTTPS://host/a.png` and `blob:` URIs fell through to the base64 branch, and `//cdn/a.png` was proxied as if it were a local path. In the markdown chain these all rendered fine before this PR, so it was a regression. Scheme matching is now case-insensitive and covers `blob:`. Protocol-relative URLs are only treated as remote in the markdown resolver: markdown is authored text, whereas a structured `ContentBlock` carries a server-side media path where a `//server/share/...` value should still go through the proxy. Adds `utils/media.test.ts` covering both resolvers, including the `decodeURI` fallback for a malformed escape sequence. --- .../common/MarkdownRenderer.test.tsx | 10 +++++ evalscope/web/src/utils/media.test.ts | 43 +++++++++++++++++++ evalscope/web/src/utils/media.ts | 14 +++--- 3 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 evalscope/web/src/utils/media.test.ts diff --git a/evalscope/web/src/components/common/MarkdownRenderer.test.tsx b/evalscope/web/src/components/common/MarkdownRenderer.test.tsx index dc7d6dabc..66cef336a 100644 --- a/evalscope/web/src/components/common/MarkdownRenderer.test.tsx +++ b/evalscope/web/src/components/common/MarkdownRenderer.test.tsx @@ -75,4 +75,14 @@ describe('MarkdownRenderer image sources', () => { expect(images[0]).toHaveAttribute('src', 'https://example.com/a%20b.png') expect(images[1]).toHaveAttribute('src', 'data:image/png;base64,aGVsbG8=') }) + + it('keeps an uppercase scheme and a protocol-relative url untouched', () => { + // These are valid remote sources; treating them as local paths or base64 + // payloads would break images that rendered fine before. + const { container } = renderMarkdown('![up](HTTPS://example.com/a.png)\n\n![rel](//cdn.example.com/a.png)') + + const images = container.querySelectorAll('img') + expect(images[0]).toHaveAttribute('src', 'HTTPS://example.com/a.png') + expect(images[1]).toHaveAttribute('src', '//cdn.example.com/a.png') + }) }) diff --git a/evalscope/web/src/utils/media.test.ts b/evalscope/web/src/utils/media.test.ts new file mode 100644 index 000000000..1c270826a --- /dev/null +++ b/evalscope/web/src/utils/media.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest' + +import { resolveMarkdownMediaSrc, resolveMediaSrc } from './media' + +const PROXY = '/api/v1/reports/media/file?path=' + +describe('resolveMediaSrc', () => { + it('proxies absolute posix and windows paths', () => { + expect(resolveMediaSrc('/tmp/a.png', 'image/jpeg')).toBe(`${PROXY}${encodeURIComponent('/tmp/a.png')}`) + expect(resolveMediaSrc('C:\\tmp\\a.png', 'image/jpeg')).toBe(`${PROXY}${encodeURIComponent('C:\\tmp\\a.png')}`) + }) + + it('passes remote sources through regardless of scheme case', () => { + for (const src of ['http://e.com/a.png', 'HTTPS://e.com/a.png', 'data:image/png;base64,x', 'blob:abc-123']) { + expect(resolveMediaSrc(src, 'image/jpeg')).toBe(src) + } + }) + + it('treats anything else as a base64 payload', () => { + expect(resolveMediaSrc('aGVsbG8=', 'image/png')).toBe('data:image/png;base64,aGVsbG8=') + }) +}) + +describe('resolveMarkdownMediaSrc', () => { + it('decodes a percent-encoded destination before proxying it', () => { + const raw = '/tmp/my dir/\u56fe.png' + expect(resolveMarkdownMediaSrc(encodeURI(raw), 'image/jpeg')).toBe(`${PROXY}${encodeURIComponent(raw)}`) + }) + + it('round-trips a path whose name contains a percent sign', () => { + const raw = '/tmp/100%.png' + expect(resolveMarkdownMediaSrc(encodeURI(raw), 'image/jpeg')).toBe(`${PROXY}${encodeURIComponent(raw)}`) + }) + + it('leaves a protocol-relative url alone instead of proxying it as a path', () => { + expect(resolveMarkdownMediaSrc('//cdn.example.com/a.png', 'image/jpeg')).toBe('//cdn.example.com/a.png') + }) + + it('falls back to the raw value when the escape sequence is malformed', () => { + // A lone '%' makes decodeURI throw; the src must still be resolved. + expect(resolveMarkdownMediaSrc('/tmp/a%zz.png', 'image/jpeg')).toBe(`${PROXY}${encodeURIComponent('/tmp/a%zz.png')}`) + }) +}) diff --git a/evalscope/web/src/utils/media.ts b/evalscope/web/src/utils/media.ts index 5ddf79906..401aa112d 100644 --- a/evalscope/web/src/utils/media.ts +++ b/evalscope/web/src/utils/media.ts @@ -1,7 +1,10 @@ +/** Schemes that already point at something the browser can load directly. */ +const REMOTE_SCHEME_RE = /^(?:https?|data|blob):/i + /** * Resolve a server-side path / URL / base64 payload to a browser-loadable src. * - * - http(s) and data: URIs are returned as-is. + * - http(s), data: and blob: URIs are returned as-is (scheme match is case-insensitive). * - Absolute POSIX/Windows paths are proxied through the media file endpoint. * - Anything else is treated as base64 and wrapped in a data: URI with `mimeType`. * @@ -10,7 +13,7 @@ * `Input` column produced by `messages_to_markdown`). */ export function resolveMediaSrc(src: string, mimeType: string): string { - if (src.startsWith('http://') || src.startsWith('https://') || src.startsWith('data:')) return src + if (REMOTE_SCHEME_RE.test(src)) return src if (src.startsWith('/') || /^[A-Za-z]:[/\\]/.test(src)) { return `/api/v1/reports/media/file?path=${encodeURIComponent(src)}` } @@ -26,11 +29,12 @@ export function resolveMediaSrc(src: string, mimeType: string): string { * to the real filesystem path before the media endpoint can look it up, * otherwise the proxy receives the literal escape sequences. * - * Remote URLs and data: URIs are returned untouched, so an intentionally - * escaped remote URL is never rewritten. + * Markdown content is authored text rather than a media field, so anything that + * looks like a URL is passed through untouched - including protocol-relative + * URLs and intentionally escaped remote URLs. */ export function resolveMarkdownMediaSrc(src: string, mimeType: string): string { - if (src.startsWith('http://') || src.startsWith('https://') || src.startsWith('data:')) return src + if (REMOTE_SCHEME_RE.test(src) || src.startsWith('//')) return src try { return resolveMediaSrc(decodeURI(src), mimeType) } catch { From 25b94d69f5a2dacc61eb13ccb38540803beefeef Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Mon, 10 Aug 2026 17:48:31 +0800 Subject: [PATCH 4/4] refactor(web): drop the speculative blob: scheme from the media resolver A blob URL is created client-side by URL.createObjectURL and is scoped to a browser session, so it can never appear in a server-side prediction record. Matching it only served a hypothetical case. --- evalscope/web/src/utils/media.test.ts | 2 +- evalscope/web/src/utils/media.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/evalscope/web/src/utils/media.test.ts b/evalscope/web/src/utils/media.test.ts index 1c270826a..1d378fd6b 100644 --- a/evalscope/web/src/utils/media.test.ts +++ b/evalscope/web/src/utils/media.test.ts @@ -11,7 +11,7 @@ describe('resolveMediaSrc', () => { }) it('passes remote sources through regardless of scheme case', () => { - for (const src of ['http://e.com/a.png', 'HTTPS://e.com/a.png', 'data:image/png;base64,x', 'blob:abc-123']) { + for (const src of ['http://e.com/a.png', 'HTTPS://e.com/a.png', 'data:image/png;base64,x']) { expect(resolveMediaSrc(src, 'image/jpeg')).toBe(src) } }) diff --git a/evalscope/web/src/utils/media.ts b/evalscope/web/src/utils/media.ts index 401aa112d..6f36cbe9a 100644 --- a/evalscope/web/src/utils/media.ts +++ b/evalscope/web/src/utils/media.ts @@ -1,10 +1,10 @@ /** Schemes that already point at something the browser can load directly. */ -const REMOTE_SCHEME_RE = /^(?:https?|data|blob):/i +const REMOTE_SCHEME_RE = /^(?:https?|data):/i /** * Resolve a server-side path / URL / base64 payload to a browser-loadable src. * - * - http(s), data: and blob: URIs are returned as-is (scheme match is case-insensitive). + * - http(s) and data: URIs are returned as-is (the scheme match is case-insensitive). * - Absolute POSIX/Windows paths are proxied through the media file endpoint. * - Anything else is treated as base64 and wrapped in a data: URI with `mimeType`. *