diff --git a/agentlightning/verl/rollout_adapter.py b/agentlightning/verl/rollout_adapter.py index 18f2e1cca..a8dda25fe 100644 --- a/agentlightning/verl/rollout_adapter.py +++ b/agentlightning/verl/rollout_adapter.py @@ -9,6 +9,7 @@ import json import zipfile from typing import Any, cast +from urllib.request import url2pathname import numpy as np import torch @@ -247,7 +248,7 @@ def _load_pil_image(url: str) -> Any: _, _, payload = url.partition(",") return Image.open(io.BytesIO(base64.b64decode(payload))).convert("RGB") if url.startswith("file://"): - return Image.open(url[len("file://") :]).convert("RGB") + return Image.open(url2pathname(url.removeprefix("file:"))).convert("RGB") if url.startswith(("http://", "https://")): # [multimodal-patch] Remote images are fetched from the trainer process with # content-type and size validation. diff --git a/docs/80-example-multimodal-qa.md b/docs/80-example-multimodal-qa.md index 6cd6e9e5c..eb6ecb970 100644 --- a/docs/80-example-multimodal-qa.md +++ b/docs/80-example-multimodal-qa.md @@ -16,6 +16,8 @@ The dataset is generated with PIL at startup β€” there is nothing to download: - The agent sends the image (as a base64 `image_url` content part) plus the question "How many red circles are in the image?" to the proxied VLM. - A rule-based reward scores the answer: 1.0 if the first integer in the reply equals the true circle count, 0.0 otherwise. +For images already stored on the trainer host, construct each `image_url` with `Path("images/frame 1.png").resolve().as_uri()` (from `pathlib`). Local file URIs preserve spaces, Unicode, and literal percent characters in filenames. + Any mrope VLM supported by verl works; pass `--model` to change the default model. ## Training diff --git a/tests/verl/test_rollout_adapter.py b/tests/verl/test_rollout_adapter.py index 8d0843926..dc289ae9a 100644 --- a/tests/verl/test_rollout_adapter.py +++ b/tests/verl/test_rollout_adapter.py @@ -7,6 +7,7 @@ import json import sys import zipfile +from pathlib import Path from types import SimpleNamespace from typing import ClassVar @@ -594,6 +595,38 @@ def test_load_pil_image_decodes_data_url() -> None: assert image.mode == "RGB" +@pytest.mark.parametrize("filename", ["plain.png", "frame one.png", "图像.png", "frame%20.png"]) +def test_load_pil_image_decodes_file_uri(tmp_path: Path, filename: str) -> None: + from PIL import Image + + image_path = tmp_path / filename + Image.new("RGB", (2, 2), (255, 0, 0)).save(image_path) + # A second unquote would load the wrong image for the literal-percent case. + if filename == "frame%20.png": + Image.new("RGB", (2, 2), (0, 0, 255)).save(tmp_path / "frame .png") + + image = rollout_adapter_module._load_pil_image(image_path.as_uri()) + + assert image.size == (2, 2) + assert image.mode == "RGB" + assert image.getpixel((0, 0)) == (255, 0, 0) + + +def test_file_uri_image_row_is_kept_for_training(tmp_path: Path) -> None: + from PIL import Image + + image_path = tmp_path / "frame 50%.png" + Image.new("RGB", (2, 2), (255, 0, 0)).save(image_path) + rollout = _transition_rollout([_image_triplet([1, 2], [3], image_urls=[image_path.as_uri()])]) + + batch, _ = _transition_adapter(FakeMropeProcessor()).get_train_data_batch([rollout]) + + assert batch.batch["is_drop_mask"].tolist() == [False] + assert batch.non_tensor_batch["multi_modal_inputs"][0] is not None + keep = (~batch.batch["is_drop_mask"].bool()).nonzero(as_tuple=True)[0].tolist() + assert batch[keep].non_tensor_batch["rollout_id_list"].tolist() == [rollout.rollout_id] + + def test_load_pil_image_fetches_remote_url_by_default(monkeypatch: pytest.MonkeyPatch) -> None: pytest.importorskip("PIL") httpx = pytest.importorskip("httpx")