From 8dbe563301f000ed568194a82910446a7e620f34 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 2 Nov 2026 21:26:12 -0800 Subject: [PATCH 01/22] feat(browsecomp_advanced_harness): serve typed resources sessions An Environment Server seeds with ResourcesSeedSessionRequest and closes with /close_session; the harness accepted neither, and its per-rollout state (the per_session page workspace and the provider-call metrics) was released only by /verify, and the metrics not at all. - /seed_session takes the typed request, keyed by resources_session_id, and still takes an agent /run's flat row under the cookie session. A typed seed is refused with 422 when task_data lacks question or ground_truth, or when a workspace ID is not a plain directory name; a seed for another episode or after the close gets 409. - /close_session removes the workspace and the metrics, idempotently, through ResourcesSessionFence. A failed removal raises and keeps the session for a retried close. - A tool call for a closed session gets 404 (session_lost) instead of recreating its workspace. - /verify drops the session's metrics entry after reporting it, unless dump_session_id_to_metrics_on_exit keeps it for the shutdown dump. Refs GYMINT-161. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- .../browsecomp_advanced_harness/README.md | 9 + .../browsecomp_advanced_harness/app.py | 132 +++++++++-- .../tests/test_resources_sessions.py | 214 ++++++++++++++++++ 3 files changed, 342 insertions(+), 13 deletions(-) create mode 100644 resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py diff --git a/resources_servers/browsecomp_advanced_harness/README.md b/resources_servers/browsecomp_advanced_harness/README.md index 8b2917f32e9..f35ed8c8509 100644 --- a/resources_servers/browsecomp_advanced_harness/README.md +++ b/resources_servers/browsecomp_advanced_harness/README.md @@ -25,6 +25,15 @@ judge_model_api_key: "" judge_model_name: Qwen/Qwen3-235B-A22B-Instruct-2507 ``` +## Sessions + +Each rollout has its own session: the provider-call metrics that `/verify` reports and, with `workspace: per_session`, the disk workspace that `search` and `browse` write pages to and `bash_command` reads. + +- **Environment Server.** `/seed_session` takes a `ResourcesSeedSessionRequest` and keys the session by its `resources_session_id`. The seed is refused with 422 when `task_data` lacks the `question` and `ground_truth` that `/verify` needs, or, with a workspace, when the ID is not a plain directory name. `/close_session` removes the workspace and the metrics, and is idempotent. A close that cannot remove the workspace fails and keeps the session, so the caller can retry it. After a close, a seed for the same ID is refused with 409, and a tool call for it gets 404 (`session_lost`) instead of recreating the workspace. `/verify` of a closed session still grades the answer: the reward depends only on the response and the task. +- **An agent's `/run`.** `/seed_session` takes the flat row and keys the session by the cookie. `/verify` removes the workspace, since this path sends no close. + +`/verify` drops the session's metrics after reporting them, unless `dump_session_id_to_metrics_on_exit` keeps them for the dump at shutdown. + ## Running ```bash diff --git a/resources_servers/browsecomp_advanced_harness/app.py b/resources_servers/browsecomp_advanced_harness/app.py index 14f237d340b..a5e2f80cf1c 100644 --- a/resources_servers/browsecomp_advanced_harness/app.py +++ b/resources_servers/browsecomp_advanced_harness/app.py @@ -31,19 +31,33 @@ from fastapi import FastAPI, Request from httpx import AsyncClient from pocketshell import run as pocketshell_run -from pydantic import BaseModel, Field, PrivateAttr, model_validator +from pydantic import BaseModel, Field, PrivateAttr, ValidationError, model_validator from tavily import AsyncTavilyClient from tavily.errors import BadRequestError from nemo_gym.client import RATE_LIMIT_ERROR_CODES, RETRY_ERROR_CODES, SESSION_ID_KEY, raise_for_status, request from nemo_gym.config import ModelServerRef -from nemo_gym.core import NeMoGymEasyInputMessage, NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming +from nemo_gym.core import ( + EpisodeId, + NeMoGymEasyInputMessage, + NeMoGymResponse, + NeMoGymResponseCreateParamsNonStreaming, + TaskId, +) from nemo_gym.server_runtime import ( BaseResourcesServerConfig, BaseRunRequest, BaseSeedSessionRequest, BaseSeedSessionResponse, BaseVerifyRequest, + ResourcesCloseSessionRequest, + ResourcesCloseSessionResponse, + ResourcesSeedRejectedError, + ResourcesSeedSessionRequest, + ResourcesSeedSessionResponse, + ResourcesSessionConflictError, + ResourcesSessionFence, + ResourcesSessionLostError, SimpleResourcesServer, ) from nemo_gym.verification import JudgeError, call_judge @@ -218,6 +232,13 @@ class TavilySearchVerifyRequest(TavilySearchRunRequest, BaseVerifyRequest): pass +class BrowsecompTaskData(BaseModel): + """The task fields /verify reads, checked when an Environment Server seeds the session.""" + + question: str + ground_truth: str + + class JudgeEvaluation(BaseModel): judge_response_create_params: Optional[NeMoGymResponseCreateParamsNonStreaming] = None reasoning: str @@ -879,6 +900,20 @@ def write_browse_page(self, url, title, content) -> str: return f"pages/{fname}" +# A typed seed's resources_session_id names its workspace directory, so it must be one plain path component. +_WORKSPACE_SESSION_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") + + +def _remove_workspace(workspace: Path) -> None: + """Remove a session workspace, raising on any failure other than the directory already being gone.""" + + def ignore_missing(function, path, error: BaseException) -> None: + if not isinstance(error, FileNotFoundError): + raise error + + shutil.rmtree(workspace, onexc=ignore_missing) + + # Judge retry backoff cap. The 2026-07-15 inference-api 529 outage outlasted the old # 10-attempt/30s-cap schedule (~2.9 min total) and falsely zeroed 25 samples in one # full-400 run; 15 attempts capped at 120s gives ~18 min of cumulative window. @@ -916,6 +951,9 @@ class TavilySearchResourcesServer(SimpleResourcesServer): _num_requests: int = 0 _session_id_to_metrics: Optional[Dict[str, TavilySearchMetrics]] = PrivateAttr(default=None) _session_workspaces: Dict[str, "_PageWriter"] = PrivateAttr(default_factory=dict) + # Environment Server sessions by resources_session_id, with the episode and task each was seeded for. + _typed_sessions: Dict[str, tuple[EpisodeId, TaskId]] = PrivateAttr(default_factory=dict) + _fence: Optional[ResourcesSessionFence] = PrivateAttr(default=None) _bash_semaphore: Optional[asyncio.Semaphore] = PrivateAttr(default=None) _workspace_root: Optional[str] = PrivateAttr(default=None) @@ -957,6 +995,7 @@ def model_post_init(self, __context) -> None: print(f"Search provider: exa ({len(self._exa_clients)} key(s))") self._session_id_to_metrics = defaultdict(TavilySearchMetrics) + self._fence = ResourcesSessionFence(self.config.session_close_retry_window_seconds) self._exclude_domains = self._parse_exclude_domains() self._page_cache: dict[str, str] = {} @@ -998,11 +1037,74 @@ async def lifespan_wrapper(app): return app - async def seed_session(self, request: Request, body: BaseSeedSessionRequest) -> BaseSeedSessionResponse: - if self.config.workspace == "per_session": - sid = request.session[SESSION_ID_KEY] - self._session_workspaces[sid] = _PageWriter(Path(self._workspace_root) / sid) - return BaseSeedSessionResponse() + async def seed_session( + self, request: Request, body: ResourcesSeedSessionRequest | BaseSeedSessionRequest + ) -> ResourcesSeedSessionResponse | BaseSeedSessionResponse: + if not isinstance(body, ResourcesSeedSessionRequest): + # An agent's /run seeds with its row; the session is the cookie's, and /verify cleans it up. + if self.config.workspace == "per_session": + sid = request.session[SESSION_ID_KEY] + self._session_workspaces[sid] = _PageWriter(Path(self._workspace_root) / sid) + return BaseSeedSessionResponse() + + sid = body.resources_session_id + async with self._fence.locked(sid): + self._fence.raise_if_closed(sid) + identity = self._typed_sessions.get(sid) + if identity is None: + try: + BrowsecompTaskData.model_validate(body.task_data) + except ValidationError as error: + raise ResourcesSeedRejectedError(f"task_data is not a BrowseComp task: {error}") from error + if self.config.workspace == "per_session": + if not _WORKSPACE_SESSION_ID_RE.fullmatch(sid): + raise ResourcesSeedRejectedError( + f"resources_session_id {sid!r} cannot name a workspace directory" + ) + self._session_workspaces[sid] = _PageWriter(Path(self._workspace_root) / sid) + self._typed_sessions[sid] = (body.episode_id, body.task_id) + elif identity != (body.episode_id, body.task_id): + raise ResourcesSessionConflictError("resources_session_id is already bound to another episode or task") + # Tool calls and /verify find the session by the cookie. A served app's session router already names it + # resources_session_id; an app without one (a test's) does not. + request.session[SESSION_ID_KEY] = sid + return ResourcesSeedSessionResponse(resources_session_id=sid) + + async def close_resources_session(self, body: ResourcesCloseSessionRequest) -> ResourcesCloseSessionResponse: + # Sessions are keyed by resources_session_id, so a close without the seed's cookie releases the same one. + sid = body.resources_session_id + async with self._fence.locked(sid): + closed_episode = self._fence.closed_episode(sid) + if closed_episode is None: + identity = self._typed_sessions.get(sid) + if identity is not None and identity[0] != body.episode_id: + raise ResourcesSessionConflictError("episode_id does not match the seeded resources session") + await self._release_session(sid) + self._typed_sessions.pop(sid, None) + self._fence.mark_closed(sid, body.episode_id) + elif closed_episode != body.episode_id: + raise ResourcesSessionConflictError("episode_id does not match the closed resources session") + return ResourcesCloseSessionResponse(resources_session_id=sid) + + async def _release_session(self, sid: str) -> None: + """Remove the session's workspace and metrics. A failed removal raises and keeps the workspace for a retry.""" + page_writer = self._session_workspaces.get(sid) + if page_writer is not None: + await asyncio.to_thread(_remove_workspace, page_writer.workspace) + self._session_workspaces.pop(sid, None) + self._release_metrics(sid) + + def _release_metrics(self, sid: str) -> None: + # With the dump on, the entries are kept for it; otherwise one entry per rollout would accumulate. + if not self.config.dump_session_id_to_metrics_on_exit: + self._session_id_to_metrics.pop(sid, None) + + def _open_session_id(self, request: Request) -> str: + """The request's session, refusing a closed one so a late tool call cannot recreate its workspace.""" + sid = request.session[SESSION_ID_KEY] + if self._fence.closed_episode(sid) is not None: + raise ResourcesSessionLostError(f"Resources session {sid} is closed") + return sid def _get_page_writer(self, sid: str) -> Optional["_PageWriter"]: if self.config.workspace != "per_session": @@ -1019,7 +1121,7 @@ def _cleanup_workspace(self, sid: str) -> None: shutil.rmtree(pw.workspace, ignore_errors=True) async def bash_command(self, request: Request, body: BashCommandRequest) -> BashCommandResponse: - sid = request.session[SESSION_ID_KEY] + sid = self._open_session_id(request) page_writer = self._get_page_writer(sid) if page_writer is None: return BashCommandResponse( @@ -1320,7 +1422,7 @@ async def _search_one_to_disk( return "\n".join(blocks) async def search(self, request: Request, body: TavilySearchRequest) -> TavilySearchResponse: - sid = request.session[SESSION_ID_KEY] + sid = self._open_session_id(request) metrics = self._session_id_to_metrics[sid] if self.config.debug: @@ -1359,7 +1461,8 @@ async def search(self, request: Request, body: TavilySearchRequest) -> TavilySea return TavilySearchResponse(results_string="\n\n".join(results)) async def browse(self, request: Request, body: BrowseRequest) -> BrowseResponse: - metrics = self._session_id_to_metrics[request.session[SESSION_ID_KEY]] + sid = self._open_session_id(request) + metrics = self._session_id_to_metrics[sid] if self.config.debug: print("\n\n browse urls: ", body.urls) @@ -1423,7 +1526,7 @@ async def browse(self, request: Request, body: BrowseRequest) -> BrowseResponse: if not result_list: return BrowseResponse(results_string="No content extracted.") - page_writer = self._get_page_writer(request.session[SESSION_ID_KEY]) + page_writer = self._get_page_writer(sid) if page_writer is not None: # terminal mode: write each page to disk, return metadata + preview. # Mirrors the reference harness tavily_extract (disk mode) formatting exactly. @@ -1472,7 +1575,8 @@ async def verify(self, request: Request, body: TavilySearchVerifyRequest) -> Tav agent_num_tool_calls = getattr(body.response, "num_tool_calls", None) if agent_num_tool_calls is None: agent_num_tool_calls = sum(o.type == "function_call" for o in body.response.output) - session_metrics = self._session_id_to_metrics[request.session[SESSION_ID_KEY]] + sid = request.session[SESSION_ID_KEY] + session_metrics = self._session_id_to_metrics[sid] num_provider_429s, num_provider_other_retries = _sum_provider_retry_counts(session_metrics) verify_response = TavilySearchVerifyResponse( **body.model_dump(), @@ -1484,9 +1588,11 @@ async def verify(self, request: Request, body: TavilySearchVerifyRequest) -> Tav num_provider_other_retries=num_provider_other_retries, ) + # The verify response carries the session's metrics, and an agent's /run sends no close, so release here. + self._release_metrics(sid) # terminal mode: clean up this session's disk workspace if self.config.workspace == "per_session": - self._cleanup_workspace(request.session[SESSION_ID_KEY]) + self._cleanup_workspace(sid) if judge_error is not None: raise JudgeError(judge_error) diff --git a/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py b/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py new file mode 100644 index 00000000000..fd917566a73 --- /dev/null +++ b/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py @@ -0,0 +1,214 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The typed resources sessions an Environment Server seeds and closes, next to the legacy /run seed.""" + +import os +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from fastapi.testclient import TestClient + +from nemo_gym.client import ServerClient +from nemo_gym.core import ( + SESSION_LOST, + EpisodeId, + NeMoGymResponse, + NeMoGymResponseOutputMessage, + NeMoGymResponseOutputText, + TaskId, +) +from nemo_gym.server_runtime import ResourcesCloseSessionRequest, ResourcesSeedSessionRequest +from nemo_gym.testing.session_conformance import check_resources_session_contract +from resources_servers.browsecomp_advanced_harness import app as harness_app +from resources_servers.browsecomp_advanced_harness.app import ( + TavilySearchMetrics, + TavilySearchResourcesServer, + TavilySearchResourcesServerConfig, +) + + +_EXCLUDE_DOMAINS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "dummy_exclude_domains_file.json") +_EPISODE = EpisodeId(rollout_id="rollout", attempt=0) +_TASK_DATA = {"question": "What is the capital of France?", "ground_truth": "Paris"} + + +def _server(tmp_path: Path, **overrides) -> TavilySearchResourcesServer: + config = TavilySearchResourcesServerConfig( + host="0.0.0.0", + port=8080, + entrypoint="", + name="", + tavily_api_key="test_api_key", # pragma: allowlist secret + exclude_domains_file_path=_EXCLUDE_DOMAINS_FILE, + use_judge=False, + workspace_root=str(tmp_path), + **overrides, + ) + return TavilySearchResourcesServer(config=config, server_client=MagicMock(spec=ServerClient)) + + +def _seed(resources_session_id: str = "resources-session", task_data: dict | None = None) -> dict: + return ResourcesSeedSessionRequest( + resources_session_id=resources_session_id, + episode_id=_EPISODE, + task_id=TaskId(taskset="browsecomp", task_id="0"), + task_data=_TASK_DATA if task_data is None else task_data, + ).model_dump(mode="json") + + +def _close(resources_session_id: str = "resources-session") -> dict: + return ResourcesCloseSessionRequest(resources_session_id=resources_session_id, episode_id=_EPISODE).model_dump( + mode="json" + ) + + +def _verify_body(answer: str) -> dict: + response = NeMoGymResponse( + id="resp", + created_at=0.0, + model="policy", + object="response", + output=[ + NeMoGymResponseOutputMessage( + id="msg", + content=[NeMoGymResponseOutputText(annotations=[], text=answer, type="output_text")], + role="assistant", + status="completed", + type="message", + ) + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ) + return {**_TASK_DATA, "responses_create_params": {"input": []}, "response": response.model_dump(mode="json")} + + +@pytest.mark.parametrize("workspace", ["per_session", "none"]) +def test_typed_sessions_follow_the_session_contract(tmp_path: Path, workspace: str) -> None: + server = _server(tmp_path, workspace=workspace) + check_resources_session_contract( + server.setup_webserver(), + ResourcesSeedSessionRequest.model_validate(_seed("contract-session")), + keeps_state=True, + holds_state=lambda resources_session_id: ( + resources_session_id in server._typed_sessions or resources_session_id in server._session_workspaces + ), + ) + # Both closed sessions' workspaces are gone from disk, not only from the server's map. + assert list(tmp_path.iterdir()) == [] + + +def test_close_removes_the_workspace_and_metrics_and_refuses_later_tool_calls(tmp_path: Path) -> None: + server = _server(tmp_path, workspace="per_session") + client = TestClient(server.setup_webserver(), raise_server_exceptions=False) + assert client.post("/seed_session", json=_seed()).status_code == 200 + workspace = tmp_path / "resources-session" + assert (workspace / "manifest.tsv").is_file() + server._session_id_to_metrics["resources-session"] = TavilySearchMetrics() + + assert client.post("/close_session", json=_close()).status_code == 200 + + assert not workspace.exists() + assert "resources-session" not in server._session_id_to_metrics + # A late tool call names the closed session by its cookie: it is refused and recreates nothing. + for path, body in (("/bash_command", {"keystrokes": "ls"}), ("/search", {"queries": ["q"]})): + response = client.post(path, json=body) + assert response.status_code == 404, response.text + assert response.json()["detail"]["failure_kind"] == SESSION_LOST + assert not workspace.exists() + assert "resources-session" not in server._session_id_to_metrics + + +def test_a_failed_workspace_removal_fails_the_close_and_a_retry_releases_it( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + server = _server(tmp_path, workspace="per_session") + client = TestClient(server.setup_webserver(), raise_server_exceptions=False) + assert client.post("/seed_session", json=_seed()).status_code == 200 + workspace = tmp_path / "resources-session" + remove_workspace = harness_app._remove_workspace + + def fail(path: Path) -> None: + raise PermissionError(f"cannot remove {path}") + + monkeypatch.setattr(harness_app, "_remove_workspace", fail) + assert client.post("/close_session", json=_close()).status_code == 500 + # Not recorded as closed, so the workspace is still held for the caller's retry. + assert workspace.exists() + assert "resources-session" in server._session_workspaces + assert server._fence.closed_episode("resources-session") is None + + monkeypatch.setattr(harness_app, "_remove_workspace", remove_workspace) + assert client.post("/close_session", json=_close()).status_code == 200 + assert not workspace.exists() + assert server._fence.closed_episode("resources-session") == _EPISODE + + +def test_a_seed_without_the_verify_fields_is_refused_as_final(tmp_path: Path) -> None: + client = TestClient(_server(tmp_path).setup_webserver(), raise_server_exceptions=False) + response = client.post("/seed_session", json=_seed(task_data={"question": "q"})) + assert response.status_code == 422 + assert "ground_truth" in response.json()["detail"] + + +def test_a_workspace_seed_refuses_an_id_that_is_not_a_directory_name(tmp_path: Path) -> None: + server = _server(tmp_path / "root", workspace="per_session") + client = TestClient(server.setup_webserver(), raise_server_exceptions=False) + response = client.post("/seed_session", json=_seed("../escape")) + assert response.status_code == 422 + assert not (tmp_path / "escape").exists() + assert server._typed_sessions == {} + + +def test_the_legacy_run_seed_and_verify_still_own_the_cookie_session(tmp_path: Path) -> None: + """An agent's /run seeds with its flat row and never closes; its /verify releases the session.""" + server = _server(tmp_path, workspace="per_session") + client = TestClient(server.setup_webserver(), raise_server_exceptions=False) + row = {**_TASK_DATA, "responses_create_params": {"input": []}} + seed = client.post("/seed_session", json=row) + assert seed.status_code == 200 + assert seed.json() == {} + assert server._typed_sessions == {} + [sid] = server._session_workspaces + assert (tmp_path / sid / "pages").is_dir() + + listing = client.post("/bash_command", json={"keystrokes": "ls"}) + assert listing.status_code == 200 + assert "manifest.tsv" in listing.json()["results_string"] + server._session_id_to_metrics[sid] = TavilySearchMetrics() + + verify = client.post("/verify", json=_verify_body("Answer: Paris Confidence: 90%")) + assert verify.status_code == 200, verify.text + assert verify.json()["reward"] == 1.0 + assert server._session_workspaces == {} + assert not (tmp_path / sid).exists() + assert sid not in server._session_id_to_metrics + + +def test_verify_keeps_metrics_for_the_dump_at_shutdown(tmp_path: Path) -> None: + server = _server(tmp_path, dump_session_id_to_metrics_on_exit=True) + client = TestClient(server.setup_webserver(), raise_server_exceptions=False) + assert client.post("/seed_session", json=_seed()).status_code == 200 + server._session_id_to_metrics["resources-session"] = TavilySearchMetrics() + + assert client.post("/verify", json=_verify_body("Answer: London Confidence: 90%")).json()["reward"] == 0.0 + assert client.post("/close_session", json=_close()).status_code == 200 + + assert "resources-session" in server._session_id_to_metrics + + +def test_verify_after_close_still_grades_the_answer(tmp_path: Path) -> None: + """The reward depends only on the response and the task, not on the session's workspace, so no mask.""" + server = _server(tmp_path, workspace="per_session") + client = TestClient(server.setup_webserver(), raise_server_exceptions=False) + assert client.post("/seed_session", json=_seed()).status_code == 200 + assert client.post("/close_session", json=_close()).status_code == 200 + + verify = client.post("/verify", json=_verify_body("Answer: Paris Confidence: 90%")) + + assert verify.status_code == 200, verify.text + assert verify.json()["reward"] == 1.0 + assert "mask_sample" not in verify.json() + assert not (tmp_path / "resources-session").exists() From 65c7c1c5cdd9d7d54b5d9481dd83fcb9e26e78f2 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 2 Nov 2026 21:43:59 -0800 Subject: [PATCH 02/22] feat(browsecomp_agent): serve agent sessions for single_agent_turn browsecomp_agent ran only through its own /run behind the legacy_agent relay. It now serves agent sessions, so an Environment Server can drive it against browsecomp_advanced_harness in steps. - The seed must grant exactly one direct HTTP tool access and no sandbox or required MCP access; anything else is refused with a final 422. - In a session, tool calls go to the grant's URL with its headers and cookies, and the close returns the resources cookies for /verify. The context compaction, resets, progress board and deadline are the same loop as on the /run path. - max_run_retries retries an empty final answer inside the one activation, keeping the harness session, as /run does. - A model call that fails with an HTTP error returns a failed response with model_request_error and its status, for the Environment Server's outcome policy; other errors fail the activation with a 500. - The harness's /verify scores a timed-out rollout 0 without the judge, as /run did by not verifying it, and every row reports timed_out, which only /run added before. - benchmarks/browsecomp/config_single_agent_turn.yaml runs the benchmark through single_agent_turn_legacy; config.yaml keeps the relay. Refs GYMINT-211. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- benchmarks/browsecomp/README.md | 22 ++ .../browsecomp/config_single_agent_turn.yaml | 21 ++ .../browsecomp_advanced_harness/app.py | 13 +- .../tests/test_app.py | 20 ++ .../browsecomp_agent/README.md | 12 + responses_api_agents/browsecomp_agent/app.py | 172 +++++++++++- .../tests/test_agent_sessions.py | 251 ++++++++++++++++++ 7 files changed, 498 insertions(+), 13 deletions(-) create mode 100644 benchmarks/browsecomp/config_single_agent_turn.yaml create mode 100644 responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py diff --git a/benchmarks/browsecomp/README.md b/benchmarks/browsecomp/README.md index 7d7d2ef9dd3..04842cd2064 100644 --- a/benchmarks/browsecomp/README.md +++ b/benchmarks/browsecomp/README.md @@ -35,3 +35,25 @@ ng_e2e_collect_rollouts \ ++policy_api_key=??? \ ++policy_model_name=??? ``` + +## Run through the Environment Server + +`config_single_agent_turn.yaml` runs the same benchmark through the `single_agent_turn_legacy` Environment Server instead of the `legacy_agent` relay. A crash then repeats only the current step (seed, agent turn, agent close, verify, close) rather than the whole episode. Use it in place of `config.yaml` in step 3: + +```bash +config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ +benchmarks/browsecomp/config_single_agent_turn.yaml" +``` + +Correct answers are judged the same way on both paths. Errors are handled differently, so the counts can differ: + +| Case | `config.yaml` (`/run`) | `config_single_agent_turn.yaml` | +| --- | --- | --- | +| Rollout reaches `rollout_timeout_s` | Reward 0 without a judge call, `timed_out: 1` | The same | +| Model call returns 400, 413 or 422 | Reward 0, with `agent_error` | Verified with an empty answer, which scores 0 | +| Model call returns 5xx after the client's retries, or a connection drops or times out | Set aside unscored (`kill_shaped`) for resume | A non-terminal failure row (`_ng_failure_*`), unscored, which `gym eval run --resume` retries | +| Model call returns 408, 425 or 429 after the client's retries | Reward 0, with `agent_error` | A non-terminal failure row, which a resume retries | +| Model call returns any other 4xx | Reward 0, with `agent_error` | A terminal failure row, not scored and not retried | +| Any other error in the agent turn | Reward 0, with `agent_error` | A non-terminal failure row, which a resume retries | + +So an error that `config.yaml` scored 0 now leaves the scored set instead of counting as a wrong answer. Check the failure rows and the coverage file alongside the accuracy. diff --git a/benchmarks/browsecomp/config_single_agent_turn.yaml b/benchmarks/browsecomp/config_single_agent_turn.yaml new file mode 100644 index 00000000000..dfb7bd2d42b --- /dev/null +++ b/benchmarks/browsecomp/config_single_agent_turn.yaml @@ -0,0 +1,21 @@ +# BrowseComp through the single_agent_turn_legacy Environment Server instead of the legacy_agent relay. +# The servers and dataset are config.yaml's. The Environment Server seeds the harness session and the agent session, +# grants the agent direct HTTP access to the harness tools, runs the agent turn, closes the agent, verifies, and +# closes the harness session, so a crash repeats only the current step. +config_paths: +- benchmarks/browsecomp/config.yaml + +browsecomp_benchmark_environment_server: + environment_servers: + _delete_key: legacy_agent + single_agent_turn_legacy: + entrypoint: app.py + resources_server: + type: resources_servers + name: browsecomp_benchmark_resources_server + agent_server: + type: responses_api_agents + name: browsecomp_benchmark_agent + # The agent's search, browse and bash_command calls reach its harness session through this grant. + resources_tool_transports: + - direct_http diff --git a/resources_servers/browsecomp_advanced_harness/app.py b/resources_servers/browsecomp_advanced_harness/app.py index a5e2f80cf1c..60f10b7146e 100644 --- a/resources_servers/browsecomp_advanced_harness/app.py +++ b/resources_servers/browsecomp_advanced_harness/app.py @@ -286,6 +286,8 @@ class TavilySearchVerifyResponse(TavilySearchVerifyRequest, JudgeEvaluation): # reward in _metrics.json (nested metrics.* records are not recursed). num_provider_429s: int = 0 num_provider_other_retries: int = 0 + # 1 when the agent stopped at its rollout_timeout_s, so the mean is the share of samples it stopped. + timed_out: int = 0 # Task-local accumulator for provider retry counts. The client retry loops @@ -1560,9 +1562,17 @@ async def verify(self, request: Request, body: TavilySearchVerifyRequest) -> Tav question = body.question ground_truth = body.ground_truth last_assistant_response = _last_assistant_text(body.response) + # browsecomp_agent marks a rollout it stopped at rollout_timeout_s. + timed_out = bool(getattr(body.response, "timed_out", False)) judge_error = None - if self.config.use_judge: + if timed_out: + # Wrong by construction: the agent's /run scores it 0 without a verify, and an Environment Server's + # verify does the same without calling the judge. + judge_evaluation = JudgeEvaluation( + reasoning="The rollout reached its time limit.", extracted_final_answer="", reward=0.0 + ) + elif self.config.use_judge: judge_evaluation, judge_error = await self._verify_answer_with_judge( question, ground_truth, last_assistant_response ) @@ -1586,6 +1596,7 @@ async def verify(self, request: Request, body: TavilySearchVerifyRequest) -> Tav metrics=session_metrics, num_provider_429s=num_provider_429s, num_provider_other_retries=num_provider_other_retries, + timed_out=int(timed_out), ) # The verify response carries the session's metrics, and an agent's /run sends no close, so release here. diff --git a/resources_servers/browsecomp_advanced_harness/tests/test_app.py b/resources_servers/browsecomp_advanced_harness/tests/test_app.py index 2a257ca0ea6..a4d768bb101 100644 --- a/resources_servers/browsecomp_advanced_harness/tests/test_app.py +++ b/resources_servers/browsecomp_advanced_harness/tests/test_app.py @@ -250,6 +250,26 @@ async def test_verify_correct_answer(self, config: TavilySearchResourcesServerCo assert res.reward == approx(1.0) assert res.extracted_final_answer == "Paris" + assert res.timed_out == 0 + + async def test_verify_of_a_timed_out_rollout_scores_zero_without_the_judge( + self, config: TavilySearchResourcesServerConfig + ) -> None: + """An Environment Server verifies every rollout, including one the agent stopped at rollout_timeout_s, + which the agent's /run scored 0 without a verify.""" + server_client = MagicMock(spec=ServerClient) + server_client.post = AsyncMock() + server = TavilySearchResourcesServer(config=config, server_client=server_client) + response = self._create_model_response("Exact Answer: Paris").model_dump(mode="json") | {"timed_out": True} + + req = TavilySearchVerifyRequest.model_validate( + {"responses_create_params": {"input": []}, "response": response, "ground_truth": "Paris", "question": "?"} + ) + res = await server.verify(self._create_dummy_request(), req) + + assert res.reward == 0.0 + assert res.timed_out == 1 + server_client.post.assert_not_awaited() async def test_verify_incorrect_answer(self, config: TavilySearchResourcesServerConfig) -> None: server_client = MagicMock(spec=ServerClient) diff --git a/responses_api_agents/browsecomp_agent/README.md b/responses_api_agents/browsecomp_agent/README.md index 873d4f17537..9ca2a19db2d 100644 --- a/responses_api_agents/browsecomp_agent/README.md +++ b/responses_api_agents/browsecomp_agent/README.md @@ -1,5 +1,17 @@ # Description +The BrowseComp research agent, paired with the [`browsecomp_advanced_harness`](../../resources_servers/browsecomp_advanced_harness) resources server. Its loop calls the harness's `search`, `browse` and `bash_command` tools. It compacts old tool results (`keep_rounds`) and resets the context past a token threshold (`context_reset_tokens`). With `progress: true` it keeps a progress board that survives the resets. With `rollout_timeout_s` set, it stops at a wall-clock limit. + +It runs an episode in one of two ways: + +- **Through an Environment Server** (`benchmarks/browsecomp/config_single_agent_turn.yaml`). `single_agent_turn_legacy` seeds the harness session and an agent session, calls `/v1/responses` once, closes the agent session, then verifies and closes the harness session. + - The seed must grant exactly one direct HTTP tool access to the harness (`resources_tool_transports: [direct_http]`), and no sandbox or required MCP access. Any other seed is refused with a final 422. + - Tool calls go to the grant's URL with its headers and cookies. The close returns the harness cookies, which the Environment Server sends to `/verify`. + - `max_run_retries` retries an empty final answer (after `` stripping) inside the one activation, keeping the harness session, as `/run` does. + - A model call that fails with an HTTP error returns a `failed` response that names the status (`model_request_error`). The Environment Server's outcome policy then verifies 400, 413 and 422, records 408, 425, 429 and 5xx as non-terminal failures that a resume retries, and fails the episode on any other status. Other errors, such as a dropped connection, fail the activation with a 500, which is also recorded as a non-terminal failure. +- **Through its own `/run`** (`benchmarks/browsecomp/config.yaml`, behind the `legacy_agent` relay). `/run` seeds the harness, retries an empty answer, and verifies. A rollout stopped at `rollout_timeout_s` scores 0 without a verify. An error scores 0 with `agent_error`, except an infrastructure failure (5xx, a dropped connection, a timeout), which is set aside unscored for resume. + +On both paths, a rollout stopped at `rollout_timeout_s` scores 0 without a judge call and reports `timed_out: 1`. # Licensing information Code: Apache 2.0 diff --git a/responses_api_agents/browsecomp_agent/app.py b/responses_api_agents/browsecomp_agent/app.py index 8d8732a3178..ee4d6393d39 100644 --- a/responses_api_agents/browsecomp_agent/app.py +++ b/responses_api_agents/browsecomp_agent/app.py @@ -18,20 +18,29 @@ import re import time import traceback +from dataclasses import dataclass from pathlib import Path -from typing import List, Optional +from typing import Any, List, Optional +from uuid import uuid4 import aiohttp from fastapi import Request, Response from pydantic import ConfigDict, Field, ValidationError from nemo_gym.agents import ( + AGENT_OUTCOME_METADATA_KEY, + AgentCloseSessionResponse, + AgentOutcome, + AgentSeedRejectedError, + AgentSeedSessionRequest, + AgentSessionState, BaseResponsesAPIAgentConfig, Body, SimpleResponsesAPIAgent, ) from nemo_gym.artifacts import NG_FAILURE_CLASS_KEY, NG_NO_PERSIST_KEY from nemo_gym.client import NeMoGymAsyncOpenAI, get_response_json, raise_for_status +from nemo_gym.client import request as http_request from nemo_gym.config import ( AggregateMetrics, AggregateMetricsRequest, @@ -41,6 +50,7 @@ get_global_config_dict, ) from nemo_gym.core import ( + MODEL_REQUEST_ERROR, NeMoGymEasyInputMessage, NeMoGymFunctionCallOutput, NeMoGymResponse, @@ -50,7 +60,13 @@ NeMoGymResponseOutputText, accumulate_response_usage, ) -from nemo_gym.server_runtime import BaseRunRequest, BaseVerifyRequest, BaseVerifyResponse +from nemo_gym.server_runtime import ( + BaseRunRequest, + BaseVerifyRequest, + BaseVerifyResponse, + DirectHTTPToolAccess, + MCPToolAccess, +) from responses_api_models.vllm_model.app import VLLMConverter @@ -228,6 +244,19 @@ def _is_infrastructure_failure(exc: BaseException) -> bool: return isinstance(exc, (aiohttp.ClientConnectionError, TimeoutError)) +@dataclass +class BrowsecompAgentSessionState(AgentSessionState): + """An Environment Server episode: its grant to the resources server and the cookie jar its tool calls carry.""" + + tool_access: DirectHTTPToolAccess + resources_cookies: dict[str, str] + + +def _cookie_values(cookies: Any) -> dict[str, str]: + """Flatten a plain dict or an aiohttp cookie jar of morsels into name-to-value pairs.""" + return {str(name): str(getattr(value, "value", value)) for name, value in (cookies or {}).items()} + + class BrowsecompAgent(SimpleResponsesAPIAgent): config: BrowsecompAgentConfig _policy_model_openai_client: Optional[NeMoGymAsyncOpenAI] = None @@ -287,17 +316,115 @@ async def responses( # RE-RAISE unchanged so run()'s containment behaves exactly as before. # Grep the next run for [browsecomp][responses_exc]. try: - return await self._responses_impl(request, response, body) + agent_session_id = self._agent_session_id_from_request(request) + if agent_session_id is None: + return await self._responses_impl(request, response, body) + state = self._require_agent_session(agent_session_id) + if not isinstance(state, BrowsecompAgentSessionState): + raise TypeError("Expected BrowseComp agent session state") + return await self._session_responses(request, response, body, state) except Exception as e: print(f"[browsecomp][responses_exc] error_type={type(e).__name__} error={str(e)[:300]}", flush=True) traceback.print_exc() raise + async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> BrowsecompAgentSessionState: + if body.sandbox_access is not None: + raise AgentSeedRejectedError("BrowseComp agent does not support sandbox access") + accesses = self.effective_tool_accesses(body) + unsupported = [access.name for access in accesses if isinstance(access, MCPToolAccess) and access.required] + if unsupported: + raise AgentSeedRejectedError( + f"BrowseComp agent does not support required MCP tool access: {', '.join(unsupported)}" + ) + direct_accesses = [access for access in accesses if isinstance(access, DirectHTTPToolAccess)] + if len(direct_accesses) != 1: + raise AgentSeedRejectedError( + "BrowseComp agent needs exactly one direct HTTP tool access to its resources server, got " + f"{len(direct_accesses)}; set resources_tool_transports: [direct_http] on the Environment Server" + ) + [tool_access] = direct_accesses + return BrowsecompAgentSessionState( + request=body, tool_access=tool_access, resources_cookies=dict(tool_access.cookies) + ) + + async def _close_agent_session_state(self, state: AgentSessionState) -> AgentCloseSessionResponse: + if not isinstance(state, BrowsecompAgentSessionState): + raise TypeError("Expected BrowseComp agent session state") + return AgentCloseSessionResponse( + agent_session_id=state.request.agent_session_id, resources_cookies=state.resources_cookies + ) + + async def _session_responses( + self, + request: Request, + response: Response, + body: NeMoGymResponseCreateParamsNonStreaming, + state: BrowsecompAgentSessionState, + ) -> NeMoGymResponse: + """Run an Environment Server's one activation, with the empty-answer retries run() makes around it. + + Like run(), a retry keeps the session's resources state; it does not seed again. A model call that fails with + an HTTP error returns a failed response that names its status, for the Environment Server's outcome policy. + """ + attempts = max(1, self.config.max_run_retries) + for attempt in range(attempts): + attempt_body = body + if self.config.snap_dir: + # The keys run() sets: the rollout's capture key is unique across tasks, repeats and attempts. + metadata = dict(body.metadata or {}) + metadata.update(task_index=state.request.episode_id.capture_key, attempt=str(attempt)) + attempt_body = body.model_copy(update={"metadata": metadata}) + try: + model_response = await self._responses_impl(request, response, attempt_body, session=state) + except aiohttp.ClientResponseError as error: + return self._model_request_failure(body, error) + answer = re.sub(r".*?", "", self._last_message_text(model_response), flags=re.DOTALL) + if answer.strip() or getattr(model_response, "timed_out", False) or attempt == attempts - 1: + return model_response + print( + f"[browsecomp][retry][{state.request.episode_id.capture_key}] attempt={attempt + 1}/{attempts} " + f"reason=empty_output_after_think_strip", + flush=True, + ) + + def _model_request_failure( + self, body: NeMoGymResponseCreateParamsNonStreaming, error: aiohttp.ClientResponseError + ) -> NeMoGymResponse: + """A failed response for a model call that answered with an HTTP error. + + run() scored a 4xx 0 and set a 5xx aside as an infrastructure failure. The Environment Server's default + outcome policy verifies 400, 413 and 422, retries 408, 425, 429 and 5xx, and fails the episode on the rest. + The output is empty, as run() scored no partial answer. + """ + content = getattr(error, "response_content", b"") + detail = content.decode(errors="replace") if isinstance(content, bytes) else str(content) + return NeMoGymResponse( + id=f"resp_{uuid4().hex}", + created_at=time.time(), + model=body.model or self.config.model_server.name, + object="response", + output=[], + status="failed", + error={"code": "server_error", "message": f"Model HTTP {error.status}: {detail or error.message}"}, + tools=body.tools, + tool_choice=body.tool_choice, + parallel_tool_calls=body.parallel_tool_calls, + metadata={ + **(body.metadata or {}), + AGENT_OUTCOME_METADATA_KEY: AgentOutcome( + failure_kind=MODEL_REQUEST_ERROR, status_code=error.status + ).model_dump_json(), + }, + ) + async def _responses_impl( self, request: Request, response: Response, body: NeMoGymResponseCreateParamsNonStreaming, + *, + session: Optional[BrowsecompAgentSessionState] = None, ) -> NeMoGymResponse: body = body.model_copy(deep=True) @@ -311,7 +438,8 @@ async def _responses_impl( usage = None step = 0 model_server_cookies = None # update the cookies on every model response - resources_server_cookies = request.cookies # update the cookies on every resources server response + # update the cookies on every resources server response + resources_server_cookies = request.cookies if session is None else dict(session.resources_cookies) reset_threshold = self._reset_threshold(self.config) loop_started = time.monotonic() @@ -606,12 +734,23 @@ async def _model_call(): else: async def _tool_call(): - response = await self.server_client.post( - server_name=self.config.resources_server.name, - url_path=f"/{output_function_call.name}", - json=tool_args, - cookies=resources_server_cookies, - ) + if session is None: + response = await self.server_client.post( + server_name=self.config.resources_server.name, + url_path=f"/{output_function_call.name}", + json=tool_args, + cookies=resources_server_cookies, + ) + else: + # An Environment Server episode reaches its resources session only through the grant. + response = await http_request( + method="POST", + url=f"{str(session.tool_access.base_url).rstrip('/')}/{output_function_call.name}", + json=tool_args, + cookies=resources_server_cookies, + headers=dict(session.tool_access.headers), + _internal=True, + ) return response, (await response.content.read()).decode() try: @@ -621,7 +760,10 @@ async def _tool_call(): timed_out = True break # We don't raise for status here since it's a valid return for the API to error e.g. if the model outputs an invalid call or something. - resources_server_cookies = api_response.cookies + if session is None: + resources_server_cookies = api_response.cookies + else: + resources_server_cookies = resources_server_cookies | _cookie_values(api_response.cookies) # reference-harness parity: the resources server wraps every tool result in a one-field # JSON envelope ({"results_string": "..."}), which JSON-escapes newlines — the # model then sees a single escaped line instead of raw multi-line text. This was @@ -796,7 +938,13 @@ async def _tool_call(): ) # Propogate any extra cookies necessary for downstream verification - for k, v in (*resources_server_cookies.items(), *(model_server_cookies or {}).items()): + if session is None: + downstream_cookies = (*resources_server_cookies.items(), *(model_server_cookies or {}).items()) + else: + # The Environment Server takes the resources cookies from the session's close, not from this reply. + session.resources_cookies = resources_server_cookies + downstream_cookies = (model_server_cookies or {}).items() + for k, v in downstream_cookies: response.set_cookie(k, v) if model_response is None: diff --git a/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py b/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py new file mode 100644 index 00000000000..fce211f7799 --- /dev/null +++ b/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py @@ -0,0 +1,251 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The agent sessions an Environment Server (single_agent_turn) seeds, activates once, and closes.""" + +import json as jsonlib +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from aiohttp import ClientResponseError +from fastapi.testclient import TestClient + +from nemo_gym.agents import ( + AgentCloseSessionRequest, + AgentSeedSessionRequest, + OutcomeAction, + agent_outcome_policy, +) +from nemo_gym.client import ServerClient +from nemo_gym.config import ModelServerRef, ResourcesServerRef +from nemo_gym.core import ( + EpisodeId, + NeMoGymResponse, + NeMoGymResponseFunctionToolCall, + NeMoGymResponseOutputMessage, + NeMoGymResponseOutputText, + TaskId, +) +from nemo_gym.sandbox import DirectSandboxConnection, SandboxAccess +from nemo_gym.server_runtime import DirectHTTPToolAccess, MCPStreamableHTTPConnection, MCPToolAccess +from nemo_gym.testing.agent_session_conformance import check_agent_seed_rejected +from responses_api_agents.browsecomp_agent.app import BrowsecompAgent, BrowsecompAgentConfig + + +EPISODE = EpisodeId(rollout_id="7-0", attempt=0) +GRANT = DirectHTTPToolAccess( + name="harness.direct_http", + required=True, + base_url="http://harness:9000", + cookies={"harness": "seed-cookie"}, + headers={"x-grant": "1"}, +) +QUESTION = {"input": [{"role": "user", "content": "What is the capital of France?"}]} + + +def _seed(**overrides) -> AgentSeedSessionRequest: + return AgentSeedSessionRequest( + agent_session_id="agent-session", + episode_id=EPISODE, + task_id=TaskId(taskset="browsecomp", task_id="7"), + **({"tool_accesses": [GRANT]} | overrides), + ) + + +def _message(text: str) -> NeMoGymResponseOutputMessage: + return NeMoGymResponseOutputMessage( + id="msg", + content=[NeMoGymResponseOutputText(annotations=[], text=text, type="output_text")], + role="assistant", + status="completed", + type="message", + ) + + +def _tool_call(keystrokes: str) -> NeMoGymResponseFunctionToolCall: + return NeMoGymResponseFunctionToolCall( + id="fc", call_id="call", name="bash_command", arguments=jsonlib.dumps({"keystrokes": keystrokes}) + ) + + +def _model_response(*output) -> dict: + return NeMoGymResponse( + id="resp", + created_at=0.0, + model="policy", + object="response", + output=list(output), + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ).model_dump(mode="json") + + +class _Harness: + """Stands in for the resources server behind the grant; each call sets the next cookie value.""" + + def __init__(self) -> None: + self.calls: list[dict] = [] + + async def __call__(self, **kwargs) -> MagicMock: + self.calls.append(kwargs) + reply = MagicMock(status=200, cookies={"harness": SimpleNamespace(value=f"after-call-{len(self.calls)}")}) + reply.content.read = AsyncMock(return_value=b'{"results_string": "manifest.tsv\\npages"}') + return reply + + +def _agent(model_replies: list, monkeypatch: pytest.MonkeyPatch, **config) -> tuple[TestClient, MagicMock, _Harness]: + """An agent app whose model returns ``model_replies`` in order (a dict, or an int HTTP error status).""" + + async def post(server_name=None, url_path=None, json=None, cookies=None): + reply = model_replies.pop(0) + if isinstance(reply, int): + response = MagicMock(ok=False, status=reply, cookies={}) + request_info = SimpleNamespace(url="http://model", method="POST", headers={}, real_url="http://model") + response.raise_for_status = MagicMock( + side_effect=ClientResponseError(request_info, (), status=reply, message="refused") + ) + response.content.read = AsyncMock(return_value=b"model refused the request") + return response + response = MagicMock(ok=True, status=200, cookies={}) + response.read = AsyncMock(return_value=jsonlib.dumps(reply).encode()) + return response + + server_client = MagicMock(spec=ServerClient) + server_client.global_config_dict = {} + server_client.post = AsyncMock(side_effect=post) + harness = _Harness() + monkeypatch.setattr("responses_api_agents.browsecomp_agent.app.http_request", harness) + agent = BrowsecompAgent( + config=BrowsecompAgentConfig( + host="0.0.0.0", + port=8080, + entrypoint="", + name="browsecomp_agent", + resources_server=ResourcesServerRef(type="resources_servers", name="configured_harness"), + model_server=ModelServerRef(type="responses_api_models", name="policy_model"), + nudge_steps=False, + **config, + ), + server_client=server_client, + ) + return TestClient(agent.setup_webserver()), server_client, harness + + +@pytest.mark.parametrize( + ("unsupported", "reason"), + [ + ({"tool_accesses": []}, "needs exactly one direct HTTP tool access to its resources server, got 0"), + ( + {"tool_accesses": [GRANT, GRANT.model_copy(update={"name": "other.direct_http"})]}, + "needs exactly one direct HTTP tool access to its resources server, got 2", + ), + ( + { + "tool_accesses": [ + GRANT, + MCPToolAccess( + name="resources", + required=True, + connection=MCPStreamableHTTPConnection(url="http://harness:9000/mcp"), + ), + ] + }, + "does not support required MCP tool access: resources", + ), + ( + { + "sandbox_access": SandboxAccess( + connection=DirectSandboxConnection(provider_config_ref="runtime", descriptor={}), + workdir="/work", + ) + }, + "does not support sandbox access", + ), + ], +) +def test_a_seed_it_cannot_serve_is_refused_as_final( + unsupported: dict, reason: str, monkeypatch: pytest.MonkeyPatch +) -> None: + client, _, _ = _agent([], monkeypatch) + assert reason in check_agent_seed_rejected(client.app, _seed(**unsupported)) + + +def test_tool_calls_use_the_grant_and_close_returns_the_resources_cookies(monkeypatch: pytest.MonkeyPatch) -> None: + client, server_client, harness = _agent( + [_model_response(_tool_call("ls")), _model_response(_message("Exact Answer: Paris"))], monkeypatch + ) + assert client.post("/v1/agent_sessions", json=_seed().model_dump(mode="json")).status_code == 200 + + reply = client.post("/v1/responses", json=QUESTION) + + assert reply.status_code == 200, reply.text + assert [call.kwargs["server_name"] for call in server_client.post.await_args_list] == ["policy_model"] * 2 + [call] = harness.calls + assert call["url"] == "http://harness:9000/bash_command" + assert call["json"] == {"keystrokes": "ls"} + assert call["cookies"] == {"harness": "seed-cookie"} + assert call["headers"] == {"x-grant": "1"} + # The tool output reaches the model unwrapped, as on the /run path. + assert reply.json()["output"][1]["output"] == "manifest.tsv\npages" + # The resources cookie returns through the close, not on the reply, where it would join the agent's cookies. + assert "harness" not in reply.cookies + close = client.post( + "/v1/agent_sessions/close", + json=AgentCloseSessionRequest(agent_session_id="agent-session", episode_id=EPISODE).model_dump(mode="json"), + ) + assert close.status_code == 200 + assert close.json()["resources_cookies"] == {"harness": "after-call-1"} + + +def test_an_empty_answer_is_retried_inside_the_activation(monkeypatch: pytest.MonkeyPatch) -> None: + client, server_client, harness = _agent( + [ + _model_response(_tool_call("ls")), + _model_response(_message("still thinking")), + _model_response(_tool_call("cat manifest.tsv")), + _model_response(_message("Exact Answer: Paris")), + ], + monkeypatch, + max_run_retries=2, + ) + assert client.post("/v1/agent_sessions", json=_seed().model_dump(mode="json")).status_code == 200 + + reply = client.post("/v1/responses", json=QUESTION) + + assert reply.status_code == 200, reply.text + assert reply.json()["output"][-1]["content"][0]["text"] == "Exact Answer: Paris" + assert server_client.post.await_count == 4 + # Like run(), the retry keeps the session's resources state: its tool call carries the first attempt's cookie. + assert [call["cookies"] for call in harness.calls] == [{"harness": "seed-cookie"}, {"harness": "after-call-1"}] + + +def test_the_last_attempt_returns_its_empty_answer(monkeypatch: pytest.MonkeyPatch) -> None: + client, server_client, _ = _agent([_model_response(_message("no answer"))], monkeypatch) + assert client.post("/v1/agent_sessions", json=_seed().model_dump(mode="json")).status_code == 200 + + reply = client.post("/v1/responses", json=QUESTION) + + assert reply.status_code == 200, reply.text + assert reply.json()["output"][-1]["content"][0]["text"] == "no answer" + assert server_client.post.await_count == 1 + + +@pytest.mark.parametrize( + ("status", "action"), + [(400, OutcomeAction.VERIFY), (503, OutcomeAction.RETRY), (429, OutcomeAction.RETRY), (404, OutcomeAction.FAIL)], +) +def test_a_model_http_error_is_a_failed_response_for_the_outcome_policy( + status: int, action: OutcomeAction, monkeypatch: pytest.MonkeyPatch +) -> None: + client, _, _ = _agent([_model_response(_tool_call("ls")), status], monkeypatch) + assert client.post("/v1/agent_sessions", json=_seed().model_dump(mode="json")).status_code == 200 + + reply = client.post("/v1/responses", json=QUESTION) + + assert reply.status_code == 200, reply.text + response = NeMoGymResponse.model_validate(reply.json()) + assert response.status == "failed" + assert response.output == [] + assert f"Model HTTP {status}: model refused the request" in response.error.message + assert agent_outcome_policy(response) == action From c78b6f4253ecb8c326cbea672d5d9f6a91dfa4de Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 3 Nov 2026 09:35:09 -0800 Subject: [PATCH 03/22] fix(browsecomp_advanced_harness): mask a verify after its session closed Every server under the resources session contract answers a verify for a closed session with lost_session_verification, including those whose reward reads only the response, so a closed session reads the same everywhere. single_agent_turn verifies before it closes, so the mask only catches an out-of-order verify. TavilySearchVerifyResponse now inherits BaseVerifyResponse so it can carry mask_sample and failure_kind. A session the fence never recorded as closed, such as an agent's /run or a caller that never seeds, is graded as before. Refs GYMINT-161. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- .../browsecomp_advanced_harness/README.md | 2 +- .../browsecomp_advanced_harness/app.py | 35 ++++++++++++++----- .../tests/test_resources_sessions.py | 24 ++++++++++--- 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/resources_servers/browsecomp_advanced_harness/README.md b/resources_servers/browsecomp_advanced_harness/README.md index f35ed8c8509..bde35df5c30 100644 --- a/resources_servers/browsecomp_advanced_harness/README.md +++ b/resources_servers/browsecomp_advanced_harness/README.md @@ -29,7 +29,7 @@ judge_model_name: Qwen/Qwen3-235B-A22B-Instruct-2507 Each rollout has its own session: the provider-call metrics that `/verify` reports and, with `workspace: per_session`, the disk workspace that `search` and `browse` write pages to and `bash_command` reads. -- **Environment Server.** `/seed_session` takes a `ResourcesSeedSessionRequest` and keys the session by its `resources_session_id`. The seed is refused with 422 when `task_data` lacks the `question` and `ground_truth` that `/verify` needs, or, with a workspace, when the ID is not a plain directory name. `/close_session` removes the workspace and the metrics, and is idempotent. A close that cannot remove the workspace fails and keeps the session, so the caller can retry it. After a close, a seed for the same ID is refused with 409, and a tool call for it gets 404 (`session_lost`) instead of recreating the workspace. `/verify` of a closed session still grades the answer: the reward depends only on the response and the task. +- **Environment Server.** `/seed_session` takes a `ResourcesSeedSessionRequest` and keys the session by its `resources_session_id`. The seed is refused with 422 when `task_data` lacks the `question` and `ground_truth` that `/verify` needs, or, with a workspace, when the ID is not a plain directory name. `/close_session` removes the workspace and the metrics, and is idempotent. A close that cannot remove the workspace fails and keeps the session, so the caller can retry it. After a close, a seed for the same ID is refused with 409, and a tool call for it gets 404 (`session_lost`) instead of recreating the workspace. `/verify` of a closed session returns a masked sample (`reward: 0`, `mask_sample: true`, `failure_kind: session_lost`). The reward depends only on the response and the task, so the mask is kept on purpose: a closed session reads the same on every server under the resources session contract. `single_agent_turn` verifies before it closes, so only an out-of-order verify gets the mask. - **An agent's `/run`.** `/seed_session` takes the flat row and keys the session by the cookie. `/verify` removes the workspace, since this path sends no close. `/verify` drops the session's metrics after reporting them, unless `dump_session_id_to_metrics_on_exit` keeps them for the dump at shutdown. diff --git a/resources_servers/browsecomp_advanced_harness/app.py b/resources_servers/browsecomp_advanced_harness/app.py index a5e2f80cf1c..60f749d1555 100644 --- a/resources_servers/browsecomp_advanced_harness/app.py +++ b/resources_servers/browsecomp_advanced_harness/app.py @@ -50,6 +50,7 @@ BaseSeedSessionRequest, BaseSeedSessionResponse, BaseVerifyRequest, + BaseVerifyResponse, ResourcesCloseSessionRequest, ResourcesCloseSessionResponse, ResourcesSeedRejectedError, @@ -59,6 +60,7 @@ ResourcesSessionFence, ResourcesSessionLostError, SimpleResourcesServer, + lost_session_verification, ) from nemo_gym.verification import JudgeError, call_judge from resources_servers.browsecomp_advanced_harness.judge_prompt import JUDGE_PROMPT_TEMPLATE @@ -278,7 +280,7 @@ class TavilySearchMetrics(BaseModel): async_tavily_calls: List[TavilySearchSingleAsyncTavilyMetrics] = Field(default_factory=list) -class TavilySearchVerifyResponse(TavilySearchVerifyRequest, JudgeEvaluation): +class TavilySearchVerifyResponse(TavilySearchVerifyRequest, BaseVerifyResponse, JudgeEvaluation): num_tool_calls: int reset_count: int = 0 metrics: TavilySearchMetrics @@ -1557,6 +1559,28 @@ async def browse(self, request: Request, body: BrowseRequest) -> BrowseResponse: return BrowseResponse(results_string=results_string) async def verify(self, request: Request, body: TavilySearchVerifyRequest) -> TavilySearchVerifyResponse: + # num_tool_calls now comes from the agent loop (counts ALL tool calls, + # including those made before context resets). Fall back to the old + # final-output count only if the agent didn't surface it. + agent_num_tool_calls = getattr(body.response, "num_tool_calls", None) + if agent_num_tool_calls is None: + agent_num_tool_calls = sum(o.type == "function_call" for o in body.response.output) + reset_count = getattr(body.response, "reset_count", 0) or 0 + sid = request.session[SESSION_ID_KEY] + if self._fence.closed_episode(sid) is not None: + # Masked although the score reads only the response, so every server under the resources session contract + # treats a closed session alike. single_agent_turn verifies before it closes, so only an out-of-order verify + # lands here. A session the fence never closed, such as an agent's /run, is graded as usual. + lost = lost_session_verification("This browsecomp_advanced_harness session is already closed") + return TavilySearchVerifyResponse( + **(body.model_dump() | lost), + reasoning="", + extracted_final_answer="", + num_tool_calls=agent_num_tool_calls, + reset_count=reset_count, + metrics=TavilySearchMetrics(), + ) + question = body.question ground_truth = body.ground_truth last_assistant_response = _last_assistant_text(body.response) @@ -1569,20 +1593,13 @@ async def verify(self, request: Request, body: TavilySearchVerifyRequest) -> Tav else: judge_evaluation = self._verify_answer_with_regex(ground_truth, last_assistant_response) - # num_tool_calls now comes from the agent loop (counts ALL tool calls, - # including those made before context resets). Fall back to the old - # final-output count only if the agent didn't surface it. - agent_num_tool_calls = getattr(body.response, "num_tool_calls", None) - if agent_num_tool_calls is None: - agent_num_tool_calls = sum(o.type == "function_call" for o in body.response.output) - sid = request.session[SESSION_ID_KEY] session_metrics = self._session_id_to_metrics[sid] num_provider_429s, num_provider_other_retries = _sum_provider_retry_counts(session_metrics) verify_response = TavilySearchVerifyResponse( **body.model_dump(), **judge_evaluation.model_dump(), num_tool_calls=agent_num_tool_calls, - reset_count=getattr(body.response, "reset_count", 0) or 0, + reset_count=reset_count, metrics=session_metrics, num_provider_429s=num_provider_429s, num_provider_other_retries=num_provider_other_retries, diff --git a/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py b/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py index fd917566a73..b25db0a54ba 100644 --- a/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py +++ b/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py @@ -95,6 +95,7 @@ def test_typed_sessions_follow_the_session_contract(tmp_path: Path, workspace: s holds_state=lambda resources_session_id: ( resources_session_id in server._typed_sessions or resources_session_id in server._session_workspaces ), + verify_body=_verify_body("Answer: Paris Confidence: 90%"), ) # Both closed sessions' workspaces are gone from disk, not only from the server's map. assert list(tmp_path.iterdir()) == [] @@ -181,7 +182,7 @@ def test_the_legacy_run_seed_and_verify_still_own_the_cookie_session(tmp_path: P verify = client.post("/verify", json=_verify_body("Answer: Paris Confidence: 90%")) assert verify.status_code == 200, verify.text - assert verify.json()["reward"] == 1.0 + assert (verify.json()["reward"], verify.json()["mask_sample"]) == (1.0, False) assert server._session_workspaces == {} assert not (tmp_path / sid).exists() assert sid not in server._session_id_to_metrics @@ -199,8 +200,8 @@ def test_verify_keeps_metrics_for_the_dump_at_shutdown(tmp_path: Path) -> None: assert "resources-session" in server._session_id_to_metrics -def test_verify_after_close_still_grades_the_answer(tmp_path: Path) -> None: - """The reward depends only on the response and the task, not on the session's workspace, so no mask.""" +def test_verify_after_close_is_masked(tmp_path: Path) -> None: + """Masked like every server under the contract, although the answer alone would score 1.""" server = _server(tmp_path, workspace="per_session") client = TestClient(server.setup_webserver(), raise_server_exceptions=False) assert client.post("/seed_session", json=_seed()).status_code == 200 @@ -209,6 +210,19 @@ def test_verify_after_close_still_grades_the_answer(tmp_path: Path) -> None: verify = client.post("/verify", json=_verify_body("Answer: Paris Confidence: 90%")) assert verify.status_code == 200, verify.text - assert verify.json()["reward"] == 1.0 - assert "mask_sample" not in verify.json() + verified = verify.json() + assert (verified["reward"], verified["mask_sample"], verified["failure_kind"]) == (0.0, True, SESSION_LOST) + assert verified["metrics"] == {"async_tavily_calls": []} + # The masked verify recreates neither the workspace nor a metrics entry. assert not (tmp_path / "resources-session").exists() + assert "resources-session" not in server._session_id_to_metrics + + +def test_a_caller_that_never_seeds_is_still_graded(tmp_path: Path) -> None: + """A verify whose session the fence never closed, here one nobody seeded, is graded from the response.""" + client = TestClient(_server(tmp_path, workspace="per_session").setup_webserver(), raise_server_exceptions=False) + + verify = client.post("/verify", json=_verify_body("Answer: Paris Confidence: 90%")) + + assert verify.status_code == 200, verify.text + assert (verify.json()["reward"], verify.json()["mask_sample"]) == (1.0, False) From 615a506700a2bdeb54c7dbf1931bfcf27d586c76 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 3 Nov 2026 09:37:02 -0800 Subject: [PATCH 04/22] fix(browsecomp_advanced_harness): report timed_out on a masked verify The verify of a closed session now returns a masked sample before the judge runs, so read timed_out first and put it on that row too. Every row keeps reporting whether the agent stopped at rollout_timeout_s. Refs GYMINT-211. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- resources_servers/browsecomp_advanced_harness/app.py | 5 +++-- .../tests/test_resources_sessions.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/resources_servers/browsecomp_advanced_harness/app.py b/resources_servers/browsecomp_advanced_harness/app.py index 99053d0666e..4598c89576c 100644 --- a/resources_servers/browsecomp_advanced_harness/app.py +++ b/resources_servers/browsecomp_advanced_harness/app.py @@ -1568,6 +1568,8 @@ async def verify(self, request: Request, body: TavilySearchVerifyRequest) -> Tav if agent_num_tool_calls is None: agent_num_tool_calls = sum(o.type == "function_call" for o in body.response.output) reset_count = getattr(body.response, "reset_count", 0) or 0 + # browsecomp_agent marks a rollout it stopped at rollout_timeout_s. + timed_out = bool(getattr(body.response, "timed_out", False)) sid = request.session[SESSION_ID_KEY] if self._fence.closed_episode(sid) is not None: # Masked although the score reads only the response, so every server under the resources session contract @@ -1581,13 +1583,12 @@ async def verify(self, request: Request, body: TavilySearchVerifyRequest) -> Tav num_tool_calls=agent_num_tool_calls, reset_count=reset_count, metrics=TavilySearchMetrics(), + timed_out=int(timed_out), ) question = body.question ground_truth = body.ground_truth last_assistant_response = _last_assistant_text(body.response) - # browsecomp_agent marks a rollout it stopped at rollout_timeout_s. - timed_out = bool(getattr(body.response, "timed_out", False)) judge_error = None if timed_out: diff --git a/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py b/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py index b25db0a54ba..512390b7fbf 100644 --- a/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py +++ b/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py @@ -218,6 +218,18 @@ def test_verify_after_close_is_masked(tmp_path: Path) -> None: assert "resources-session" not in server._session_id_to_metrics +def test_a_masked_verify_still_reports_a_timed_out_rollout(tmp_path: Path) -> None: + client = TestClient(_server(tmp_path).setup_webserver(), raise_server_exceptions=False) + assert client.post("/seed_session", json=_seed()).status_code == 200 + assert client.post("/close_session", json=_close()).status_code == 200 + body = _verify_body("Answer: Paris Confidence: 90%") + body["response"]["timed_out"] = True + + verified = client.post("/verify", json=body).json() + + assert (verified["mask_sample"], verified["timed_out"]) == (True, 1) + + def test_a_caller_that_never_seeds_is_still_graded(tmp_path: Path) -> None: """A verify whose session the fence never closed, here one nobody seeded, is graded from the response.""" client = TestClient(_server(tmp_path, workspace="per_session").setup_webserver(), raise_server_exceptions=False) From 6c6fcd1b8ae3e37b58ebf63016f868c8e6945101 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 3 Nov 2026 19:47:36 -0800 Subject: [PATCH 05/22] fix(config): list browsecomp_agent as an agent-session agent type browsecomp_agent now seeds agent sessions, so the registry test requires it, and an --agent-type swap to it keeps the session environment server instead of relaying. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- nemo_gym/config/global_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nemo_gym/config/global_config.py b/nemo_gym/config/global_config.py index 9625d27480d..4a3913ee1cf 100644 --- a/nemo_gym/config/global_config.py +++ b/nemo_gym/config/global_config.py @@ -211,6 +211,7 @@ def _installed_version(distribution: str) -> Optional[str]: # so this list is the source of truth; a unit test keeps it in step with the agent apps. AGENT_SESSION_AGENT_TYPES = frozenset( { + "browsecomp_agent", "claude_code_agent", "codex_agent", "harbor_oracle_agent", From d7a83fbb13fd0e11cd5b37c443e2be2483e76da1 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 3 Nov 2026 20:01:21 -0800 Subject: [PATCH 06/22] test(browsecomp_advanced_harness): import the session contract check from nemo_gym.server_runtime main removed nemo_gym.testing.session_conformance; the check now lives in the server_runtime facade. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- .../tests/test_resources_sessions.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py b/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py index b25db0a54ba..8f6c24b74f8 100644 --- a/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py +++ b/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py @@ -18,8 +18,11 @@ NeMoGymResponseOutputText, TaskId, ) -from nemo_gym.server_runtime import ResourcesCloseSessionRequest, ResourcesSeedSessionRequest -from nemo_gym.testing.session_conformance import check_resources_session_contract +from nemo_gym.server_runtime import ( + ResourcesCloseSessionRequest, + ResourcesSeedSessionRequest, + check_resources_session_contract, +) from resources_servers.browsecomp_advanced_harness import app as harness_app from resources_servers.browsecomp_advanced_harness.app import ( TavilySearchMetrics, From 61b7ebcd871da3c5eed8398575ff5806016cfd50 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 4 Nov 2026 09:48:21 -0800 Subject: [PATCH 07/22] fix(browsecomp): isolate workspace tools in owned sandboxes Signed-off-by: yaoyu-33 --- .../browsecomp_advanced_harness/README.md | 47 +- .../browsecomp_advanced_harness/app.py | 244 ++++--- .../pyproject.toml | 2 +- .../browsecomp_advanced_harness/sandbox.py | 98 +++ .../shell_output.py | 29 + .../shell_runner.py | 25 + .../tests/test_bash_guard.py | 1 + .../tests/test_pocketshell_wiring.py | 1 + .../tests/test_resources_sessions.py | 2 +- .../tests/test_rollout_isolation.py | 1 + .../tests/test_sandbox_shell.py | 273 ++++++++ .../browsecomp_advanced_harness/uv.lock | 595 +++++++++++++++++- 12 files changed, 1239 insertions(+), 79 deletions(-) create mode 100644 resources_servers/browsecomp_advanced_harness/sandbox.py create mode 100644 resources_servers/browsecomp_advanced_harness/shell_output.py create mode 100644 resources_servers/browsecomp_advanced_harness/shell_runner.py create mode 100644 resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py diff --git a/resources_servers/browsecomp_advanced_harness/README.md b/resources_servers/browsecomp_advanced_harness/README.md index bde35df5c30..18cf57af5ae 100644 --- a/resources_servers/browsecomp_advanced_harness/README.md +++ b/resources_servers/browsecomp_advanced_harness/README.md @@ -5,10 +5,12 @@ requiring multi-hop investigation through search and page-level browsing. ## Overview -This resources server provides two browsing tools backed by [Tavily](https://tavily.com/): +This resources server provides browsing tools backed by [Tavily](https://tavily.com/) or Exa: - **`search`** — parallel web search returning top results with content snippets - **`browse`** — full-page content extraction from given URLs +- **`bash_command`** — the existing read-only pocketshell interpreter over retrieved pages, executed inside a + resources-owned sandbox when `workspace: per_session` is enabled Reward is computed via an LLM judge that compares the model's final answer against the ground truth. @@ -25,14 +27,53 @@ judge_model_api_key: "" judge_model_name: Qwen/Qwen3-235B-A22B-Instruct-2507 ``` +## Workspace sandbox + +Workspace tools require a configured sandbox provider. Add these fields under the resources server's +`resources_servers` block and compose the corresponding provider configuration: + +```yaml +workspace: per_session +sandbox_provider: sandbox +sandbox_config: + image: python:3.13-slim + workdir: /workspace + ttl_s: 7200 +sandbox_install_timeout_seconds: 900 +sandbox_close_timeout_seconds: 60 +``` + +For example, composing `nemo_gym/sandbox/providers/docker/configs/docker.yaml` defines the named `sandbox` provider. +The shared sandbox API also supports other providers; Enroot needs an image available on its cluster. Provider +`default_metadata` is merged with `sandbox_config.metadata`, then the resource server and session labels. + +Each resources session owns one tool sandbox. An agent that runs its loop in a sandbox owns a separate sandbox; +neither component borrows the other's container. This works with providers such as Enroot that do not support +serializing and reconnecting to a sandbox. Search and browse retain their existing responses and write immutable +retrieved pages. Only those pages and their manifest are copied into the tool workspace. Search/judge credentials +and verifier inputs are not copied into the sandbox. + +The shell uses the checked-out pocketshell implementation and `SandboxPythonRuntime` with the installed `regex` +version pinned into its isolated Python runtime. The first seed builds that runtime if needed; subsequent sessions +reuse its host-side artifact cache. Setup failures surface to the caller and retain ownership when cleanup fails, +so a later close can retry. Commands are supervised inside the sandbox; their guard, supported commands, output +truncation, and exit-code envelope are unchanged. There is no fallback to host execution after a sandbox failure. + +`allow_host_execution: true` explicitly enables the former host interpreter for development when no provider is +configured, or permits the non-isolating `local` provider. It defaults to `false`; production and benchmark configs +must provide an isolating provider. Configurations with `workspace: none` do not need a tool sandbox. + ## Sessions Each rollout has its own session: the provider-call metrics that `/verify` reports and, with `workspace: per_session`, the disk workspace that `search` and `browse` write pages to and `bash_command` reads. - **Environment Server.** `/seed_session` takes a `ResourcesSeedSessionRequest` and keys the session by its `resources_session_id`. The seed is refused with 422 when `task_data` lacks the `question` and `ground_truth` that `/verify` needs, or, with a workspace, when the ID is not a plain directory name. `/close_session` removes the workspace and the metrics, and is idempotent. A close that cannot remove the workspace fails and keeps the session, so the caller can retry it. After a close, a seed for the same ID is refused with 409, and a tool call for it gets 404 (`session_lost`) instead of recreating the workspace. `/verify` of a closed session returns a masked sample (`reward: 0`, `mask_sample: true`, `failure_kind: session_lost`). The reward depends only on the response and the task, so the mask is kept on purpose: a closed session reads the same on every server under the resources session contract. `single_agent_turn` verifies before it closes, so only an out-of-order verify gets the mask. -- **An agent's `/run`.** `/seed_session` takes the flat row and keys the session by the cookie. `/verify` removes the workspace, since this path sends no close. +- **An agent's `/run`.** `/seed_session` takes the flat row and keys the session by the cookie. `/verify` releases the sandbox and removes the workspace, since this path sends no close. -`/verify` drops the session's metrics after reporting them, unless `dump_session_id_to_metrics_on_exit` keeps them for the dump at shutdown. +`/verify` and `/close_session` stop the owned sandbox before deleting its workspace and metrics. A failed stop leaves +the session available for a close retry. Shutdown also stops any remaining tool sandboxes. Tool calls, verification, +and close are serialized per session, so cleanup cannot delete files while a tool is running. `/verify` drops the +session's metrics after reporting them, unless `dump_session_id_to_metrics_on_exit` keeps them for the dump at shutdown. ## Running diff --git a/resources_servers/browsecomp_advanced_harness/app.py b/resources_servers/browsecomp_advanced_harness/app.py index f0c9223d719..ff8325b2a15 100644 --- a/resources_servers/browsecomp_advanced_harness/app.py +++ b/resources_servers/browsecomp_advanced_harness/app.py @@ -15,6 +15,7 @@ import asyncio import functools import json +import logging import os import re import shutil @@ -23,20 +24,22 @@ from collections import defaultdict from contextlib import asynccontextmanager from contextvars import ContextVar -from pathlib import Path +from pathlib import Path, PurePosixPath from time import time from typing import Any, ClassVar, Dict, List, Optional from urllib.parse import urlparse +from uuid import uuid4 from fastapi import FastAPI, Request from httpx import AsyncClient from pocketshell import run as pocketshell_run -from pydantic import BaseModel, Field, PrivateAttr, ValidationError, model_validator +from pydantic import BaseModel, Field, JsonValue, PrivateAttr, ValidationError, model_validator from tavily import AsyncTavilyClient from tavily.errors import BadRequestError +from nemo_gym.agents import SandboxSession from nemo_gym.client import RATE_LIMIT_ERROR_CODES, RETRY_ERROR_CODES, SESSION_ID_KEY, raise_for_status, request -from nemo_gym.config import ModelServerRef +from nemo_gym.config import ModelServerRef, get_global_config_dict from nemo_gym.core import ( EpisodeId, NeMoGymEasyInputMessage, @@ -44,6 +47,13 @@ NeMoGymResponseCreateParamsNonStreaming, TaskId, ) +from nemo_gym.sandbox import ( + AsyncSandbox, + SandboxSpec, + create_provider, + resolve_provider_config, + resolve_provider_metadata, +) from nemo_gym.server_runtime import ( BaseResourcesServerConfig, BaseRunRequest, @@ -64,6 +74,11 @@ ) from nemo_gym.verification import JUDGE_FAILURES, JudgeError, call_judge, is_judge_failure from resources_servers.browsecomp_advanced_harness.judge_prompt import JUDGE_PROMPT_TEMPLATE +from resources_servers.browsecomp_advanced_harness.sandbox import BrowsecompShellSandbox, shell_source_files +from resources_servers.browsecomp_advanced_harness.shell_output import format_shell_output + + +LOG = logging.getLogger(__name__) # Exa /search "type" values. The deep variants run Exa's multi-step research path: @@ -99,6 +114,14 @@ class TavilySearchResourcesServerConfig(BaseResourcesServerConfig): workspace_root: Optional[str] = None # default: $BROWSECOMP_WS_ROOT or /tmp/browsecomp_ws bash_timeout_s: float = 60.0 # match the reference harness's _BASH_MAX_DURATION_S bash_max_concurrency: int = 64 + sandbox_provider: str | None = None + sandbox_config: dict[str, JsonValue] = Field( + default_factory=lambda: {"image": "python:3.13-slim", "workdir": "/workspace", "ttl_s": 7200} + ) + sandbox_install_timeout_seconds: float = Field(default=900, gt=0) + sandbox_close_timeout_seconds: float = Field(default=60, gt=0) + # Development-only opt-in. Never selected after a sandbox setup/execution failure. + allow_host_execution: bool = False max_page_bytes: int = 2_000_000 # cap per-page bytes written to disk # Results returned per search query (both providers). The reference Exa # reference uses 10; its Tavily path (and this harness historically) uses 5. @@ -593,9 +616,6 @@ async def get_contents(self, urls: List[str], max_characters: int) -> Dict[str, _SLUG_RE = re.compile(r"[^a-z0-9]+") _URL_RE = re.compile(r"https?://([^/]+)(/.*)?") -_BASH_HEAD_BYTES = 8192 -_BASH_TAIL_BYTES = 4096 - def _slugify(text: str, max_len: int = 40) -> str: s = (text or "").lower() @@ -613,17 +633,6 @@ def _slug_from_url(url: str) -> str: return _slugify(f"{host} {tail}") -def _bash_truncate(s: str) -> str: - if len(s) <= _BASH_HEAD_BYTES + _BASH_TAIL_BYTES + 64: - return s - omitted = len(s) - _BASH_HEAD_BYTES - _BASH_TAIL_BYTES - return ( - f"{s[:_BASH_HEAD_BYTES]}\n" - f"[...truncated {omitted} bytes; head shown above, tail follows...]\n" - f"{s[-_BASH_TAIL_BYTES:]}" - ) - - # --- bash_command guards (ported byte-for-byte from an internal reference # harness: _bash_denylisted + _bash_allowlisted). Layered deny-first, # then default-deny allow-list. NOT a security boundary (the ulimit -f 0 in @@ -958,6 +967,7 @@ class TavilySearchResourcesServer(SimpleResourcesServer): _fence: Optional[ResourcesSessionFence] = PrivateAttr(default=None) _bash_semaphore: Optional[asyncio.Semaphore] = PrivateAttr(default=None) _workspace_root: Optional[str] = PrivateAttr(default=None) + _shell_sessions: dict[str, BrowsecompShellSandbox] = PrivateAttr(default_factory=dict) JUDGE_PROMPT_TEMPLATE: ClassVar[str] = JUDGE_PROMPT_TEMPLATE JUDGE_MAX_ATTEMPTS: ClassVar[int] = 15 @@ -1024,16 +1034,26 @@ def setup_webserver(self) -> FastAPI: @asynccontextmanager async def lifespan_wrapper(app): - async with main_app_lifespan(app) as maybe_state: - yield maybe_state - - if self.config.dump_session_id_to_metrics_on_exit: - out_file = Path(__file__).parent / "session_id_metrics.json" - print(f"Dumping session_id metrics to {out_file}") + async def close_shell(sid: str) -> None: + try: + async with asyncio.timeout(self.config.sandbox_close_timeout_seconds * 5): + async with self._fence.locked(sid): + await self._release_session(sid) + except Exception: + LOG.exception("Could not release BrowseComp shell session %s at shutdown", sid) - to_dump = {k: v.model_dump(mode="json") for k, v in self._session_id_to_metrics.items()} - with out_file.open("w") as f: - json.dump(to_dump, f) + try: + async with main_app_lifespan(app) as maybe_state: + yield maybe_state + finally: + await asyncio.gather(*(close_shell(sid) for sid in list(self._shell_sessions))) + if self.config.dump_session_id_to_metrics_on_exit: + out_file = Path(__file__).parent / "session_id_metrics.json" + print(f"Dumping session_id metrics to {out_file}") + + to_dump = {k: v.model_dump(mode="json") for k, v in self._session_id_to_metrics.items()} + with out_file.open("w") as f: + json.dump(to_dump, f) app.router.lifespan_context = lifespan_wrapper @@ -1042,11 +1062,16 @@ async def lifespan_wrapper(app): async def seed_session( self, request: Request, body: ResourcesSeedSessionRequest | BaseSeedSessionRequest ) -> ResourcesSeedSessionResponse | BaseSeedSessionResponse: + if self.config.workspace == "per_session": + self._validate_shell_config() if not isinstance(body, ResourcesSeedSessionRequest): - # An agent's /run seeds with its row; the session is the cookie's, and /verify cleans it up. - if self.config.workspace == "per_session": - sid = request.session[SESSION_ID_KEY] - self._session_workspaces[sid] = _PageWriter(Path(self._workspace_root) / sid) + # An agent's /run seeds with its row; /verify owns its eventual cleanup. + sid = request.session[SESSION_ID_KEY] + async with self._fence.locked(sid): + if self.config.workspace == "per_session": + await self._release_session(sid) + self._session_workspaces[sid] = _PageWriter(Path(self._workspace_root) / sid) + await self._ensure_shell(sid) return BaseSeedSessionResponse() sid = body.resources_session_id @@ -1067,6 +1092,8 @@ async def seed_session( self._typed_sessions[sid] = (body.episode_id, body.task_id) elif identity != (body.episode_id, body.task_id): raise ResourcesSessionConflictError("resources_session_id is already bound to another episode or task") + if self.config.workspace == "per_session": + await self._ensure_shell(sid) # Tool calls and /verify find the session by the cookie. A served app's session router already names it # resources_session_id; an app without one (a test's) does not. request.session[SESSION_ID_KEY] = sid @@ -1090,12 +1117,76 @@ async def close_resources_session(self, body: ResourcesCloseSessionRequest) -> R async def _release_session(self, sid: str) -> None: """Remove the session's workspace and metrics. A failed removal raises and keeps the workspace for a retry.""" - page_writer = self._session_workspaces.get(sid) - if page_writer is not None: - await asyncio.to_thread(_remove_workspace, page_writer.workspace) - self._session_workspaces.pop(sid, None) + shell = self._shell_sessions.get(sid) + if shell is not None: + await shell.close(timeout=self.config.sandbox_close_timeout_seconds) + self._shell_sessions.pop(sid, None) + if self.config.workspace == "per_session": + await asyncio.to_thread(self._cleanup_workspace, sid) self._release_metrics(sid) + def _validate_shell_config(self) -> None: + """Refuse a missing isolation boundary before creating a workspace or provider.""" + if self.config.sandbox_provider is None: + if not self.config.allow_host_execution: + raise ResourcesSeedRejectedError( + "BrowseComp workspace tools require sandbox_provider; " + "allow_host_execution=true is only for development" + ) + return + provider_config = resolve_provider_config(self.config.sandbox_provider, get_global_config_dict()) + if "local" in provider_config and not self.config.allow_host_execution: + raise ResourcesSeedRejectedError("The local provider does not isolate BrowseComp workspace tools") + workdir = self.config.sandbox_config.get("workdir", "/workspace") + if not isinstance(workdir, str) or not PurePosixPath(workdir).is_absolute() or workdir == "/": + raise ResourcesSeedRejectedError( + "BrowseComp sandbox_config.workdir must be an absolute non-root directory" + ) + + async def _ensure_shell(self, sid: str) -> BrowsecompShellSandbox | None: + """Create the Resources-owned tool sandbox once; retain failed cleanup for close.""" + self._validate_shell_config() + if self.config.sandbox_provider is None: + return None + shell = self._shell_sessions.get(sid) + if shell is not None: + if not shell.ready or shell.session.closing: + raise RuntimeError("BrowseComp shell setup/close is incomplete; close this resources session") + return shell + global_config = get_global_config_dict() + provider = create_provider(resolve_provider_config(self.config.sandbox_provider, global_config)) + sandbox = AsyncSandbox(provider) + config = {"image": "python:3.13-slim", "workdir": "/workspace", "ttl_s": 7200, **self.config.sandbox_config} + config["metadata"] = ( + resolve_provider_metadata(self.config.sandbox_provider, global_config) + | (config.get("metadata") or {}) + | {"nemo_gym_resources": self.config.name[:63], "resources_session_id": sid[:63]} + ) + config["files"] = (config.get("files") or {}) | shell_source_files() + shell = BrowsecompShellSandbox( + session=SandboxSession( + sandbox=sandbox, + session_dir=f"/tmp/nemo-gym-browsecomp-shell/{uuid4().hex}", + workdir=config["workdir"], + harness="BrowseComp read-only shell", + owns_sandbox=True, + ) + ) + # Store ownership before the first await, including a partially completed start. + self._shell_sessions[sid] = shell + try: + await sandbox.start(SandboxSpec(**config)) + await shell.install(timeout=self.config.sandbox_install_timeout_seconds) + except BaseException: + try: + await shell.close(timeout=self.config.sandbox_close_timeout_seconds) + except BaseException: + LOG.exception("BrowseComp shell setup cleanup failed for %s; retaining it for close", sid) + else: + self._shell_sessions.pop(sid, None) + raise + return shell + def _release_metrics(self, sid: str) -> None: # With the dump on, the entries are kept for it; otherwise one entry per rollout would accumulate. if not self.config.dump_session_id_to_metrics_on_exit: @@ -1118,37 +1209,45 @@ def _get_page_writer(self, sid: str) -> Optional["_PageWriter"]: return pw def _cleanup_workspace(self, sid: str) -> None: - pw = self._session_workspaces.pop(sid, None) + pw = self._session_workspaces.get(sid) if pw is not None: - shutil.rmtree(pw.workspace, ignore_errors=True) + _remove_workspace(pw.workspace) + self._session_workspaces.pop(sid, None) async def bash_command(self, request: Request, body: BashCommandRequest) -> BashCommandResponse: sid = self._open_session_id(request) - page_writer = self._get_page_writer(sid) - if page_writer is None: - return BashCommandResponse( - results_string="[bash error: terminal workspace is not enabled for this server]" - ) - duration = max(1.0, min(self.config.bash_timeout_s, float(body.duration or 10))) - out = await self._run_bash_readonly(body.keystrokes or "", duration, str(page_writer.workspace)) - return BashCommandResponse(results_string=out) + async with self._fence.locked(sid): + self._open_session_id(request) + page_writer = self._get_page_writer(sid) + if page_writer is None: + return BashCommandResponse( + results_string="[bash error: terminal workspace is not enabled for this server]" + ) + shell = await self._ensure_shell(sid) + command = body.keystrokes or "" + blocked = _bash_denylisted(command) or _bash_allowlisted(command) + if blocked is not None: + return BashCommandResponse(results_string=f"[blocked: {blocked}]\n[exit_code=-3]") + duration = max(1.0, min(self.config.bash_timeout_s, float(body.duration or 10))) + if shell is None: + out = await self._run_bash_readonly(command, duration, str(page_writer.workspace)) + else: + async with self._bash_semaphore: + out = await shell.execute( + command=command, + duration=duration, + workspace=page_writer.workspace, + close_timeout=self.config.sandbox_close_timeout_seconds, + ) + return BashCommandResponse(results_string=out) async def _run_bash_readonly(self, keystrokes: str, duration: float, cwd: str) -> str: - """Run the command IN-PROCESS via pocketshell -- no subprocess, no sandbox. - - Replaces `asyncio.create_subprocess_exec("bash", "-c", ...)`. The old - design relied on a command-NAME deny/allow list plus `ulimit -f 0`, which - never inspected path arguments (so `cat /etc/passwd` passed the guard) and - which its own comment described as "NOT a security boundary". pocketshell - cannot execute anything it does not implement, and confines every path to - the sample workspace, so the protection is structural. - - The legacy guard is deliberately kept IN FRONT so that anything it used to - reject still returns the identical `[blocked: ...]` string -- existing - synthetic-data corpora, and models trained on them, saw those exact responses. + """Development-only pocketshell execution; HTTP routes require explicit host opt-in. - Output shape (stdout / --- stderr --- / [exit_code=N]) is unchanged. + Production routes use a Resources-owned sandbox, with the same interpreter and output format. """ + if not self.config.allow_host_execution: + raise RuntimeError("BrowseComp host shell execution requires allow_host_execution=true") blocked = _bash_denylisted(keystrokes) or _bash_allowlisted(keystrokes) if blocked is not None: print(f"[browsecomp][bash][blocked] {blocked}: {(keystrokes or '')[:160]!r}", flush=True) @@ -1174,17 +1273,7 @@ async def _run_bash_readonly(self, keystrokes: str, duration: float, cwd: str) - print(f"🚨 [browsecomp][bash] pocketshell error: {e!r}", flush=True) return f"[bash error: {e}]\n[exit_code=-2]" - stdout = _bash_truncate(result.stdout) - stderr = _bash_truncate(result.stderr) - truncated = (len(result.stdout) + len(result.stderr)) > (_BASH_HEAD_BYTES + _BASH_TAIL_BYTES) * 2 - parts = [] - if stdout: - parts.append(stdout) - if stderr: - parts.append("--- stderr ---") - parts.append(stderr) - parts.append(f"[exit_code={result.exit_code}{' truncated' if truncated else ''}]") - return "\n".join(parts) + return format_shell_output(result.stdout, result.stderr, result.exit_code) def _select_tavily_client(self) -> AsyncTavilyClient: client = self._async_tavily_clients[self._num_requests % len(self._async_tavily_clients)] @@ -1424,6 +1513,11 @@ async def _search_one_to_disk( return "\n".join(blocks) async def search(self, request: Request, body: TavilySearchRequest) -> TavilySearchResponse: + async with self._fence.locked(request.session[SESSION_ID_KEY]): + self._open_session_id(request) + return await self._search(request, body) + + async def _search(self, request: Request, body: TavilySearchRequest) -> TavilySearchResponse: sid = self._open_session_id(request) metrics = self._session_id_to_metrics[sid] @@ -1463,6 +1557,11 @@ async def search(self, request: Request, body: TavilySearchRequest) -> TavilySea return TavilySearchResponse(results_string="\n\n".join(results)) async def browse(self, request: Request, body: BrowseRequest) -> BrowseResponse: + async with self._fence.locked(request.session[SESSION_ID_KEY]): + self._open_session_id(request) + return await self._browse(request, body) + + async def _browse(self, request: Request, body: BrowseRequest) -> BrowseResponse: sid = self._open_session_id(request) metrics = self._session_id_to_metrics[sid] @@ -1559,6 +1658,10 @@ async def browse(self, request: Request, body: BrowseRequest) -> BrowseResponse: return BrowseResponse(results_string=results_string) async def verify(self, request: Request, body: TavilySearchVerifyRequest) -> TavilySearchVerifyResponse: + async with self._fence.locked(request.session[SESSION_ID_KEY]): + return await self._verify(request, body) + + async def _verify(self, request: Request, body: TavilySearchVerifyRequest) -> TavilySearchVerifyResponse: # num_tool_calls now comes from the agent loop (counts ALL tool calls, # including those made before context resets). Fall back to the old # final-output count only if the agent didn't surface it. @@ -1606,10 +1709,7 @@ async def verify(self, request: Request, body: TavilySearchVerifyRequest) -> Tav ) # The verify response carries the session's metrics, and an agent's /run sends no close, so release here. - self._release_metrics(sid) - # terminal mode: clean up this session's disk workspace - if self.config.workspace == "per_session": - self._cleanup_workspace(sid) + await self._release_session(sid) if judge_error is not None: raise judge_error diff --git a/resources_servers/browsecomp_advanced_harness/pyproject.toml b/resources_servers/browsecomp_advanced_harness/pyproject.toml index 111b582aafd..cb043b42fc3 100644 --- a/resources_servers/browsecomp_advanced_harness/pyproject.toml +++ b/resources_servers/browsecomp_advanced_harness/pyproject.toml @@ -6,7 +6,7 @@ name = "gym-resources-servers-browsecomp-advanced-harness" version = "0.0.0" requires-python = ">=3.13.14" dependencies = [ - "nemo-gym[dev]", + "nemo-gym[dev,sandbox]", "tavily-python==0.7.21", "pocketshell[pcre]", ] diff --git a/resources_servers/browsecomp_advanced_harness/sandbox.py b/resources_servers/browsecomp_advanced_harness/sandbox.py new file mode 100644 index 00000000000..2cad0339d26 --- /dev/null +++ b/resources_servers/browsecomp_advanced_harness/sandbox.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Resources-owned, supervised execution of the existing read-only shell.""" + +import json +from dataclasses import dataclass, field +from pathlib import Path +from shlex import quote +from uuid import uuid4 + +from nemo_gym.agents import SandboxCommand, SandboxPythonRuntime, SandboxSession +from nemo_gym.sandbox import read_text, upload_text + + +_RUNTIME = SandboxPythonRuntime(harness="browsecomp-shell", requirements=("regex",), import_check="import regex") +_RUNNER = f"{_RUNTIME.directory}/shell_runner.py" + + +def shell_source_files() -> dict[str, str]: + """Deliver the checked-out shell implementation through SandboxSpec's cleanup-aware startup.""" + source = Path(__file__).parent + files = { + f"{_RUNTIME.directory}/pocketshell/{path.name}": path.read_text(encoding="utf-8") + for path in sorted((source / "pocketshell/src/pocketshell").glob("*.py")) + } + for name in ("shell_runner.py", "shell_output.py"): + files[f"{_RUNTIME.directory}/{name}"] = (source / name).read_text(encoding="utf-8") + return files + + +@dataclass +class BrowsecompShellSandbox: + """One Resources session owns its shell sandbox and immutable page copies.""" + + session: SandboxSession[str] + ready: bool = False + uploaded_pages: set[str] = field(default_factory=set) + + async def install(self, *, timeout: float) -> None: + """Install the pinned Python/regex runtime and the checked-out pocketshell source.""" + sandbox = self.session.sandbox + if not await _RUNTIME.is_installed(sandbox): + await _RUNTIME.install(sandbox, timeout_s=timeout) + prepared = await sandbox.exec( + f"mkdir -p {quote(_RUNTIME.directory + '/pocketshell')} " + f"{quote(self.session.session_dir)} {quote(self.session.workdir + '/pages')}", + cwd="/", + timeout_s=30, + ) + if prepared.return_code or prepared.error_type: + raise RuntimeError(f"Could not prepare BrowseComp shell paths: {prepared.stderr or prepared.error_type}") + self.ready = True + + async def execute(self, *, command: str, duration: float, workspace: Path, close_timeout: float) -> str: + """Copy new retrieved pages, then interpret the command only inside the sandbox.""" + if not self.ready: + raise RuntimeError("BrowseComp shell setup has not completed") + directory = f"{self.session.session_dir.rsplit('/calls/', 1)[0]}/calls/{uuid4().hex}" + await self.session.begin_activation(session_dir=directory, timeout=close_timeout) + + async def stage() -> SandboxCommand: + prepared = await self.session.sandbox.exec(f"mkdir -p {quote(directory)}", cwd="/", timeout_s=30) + if prepared.return_code or prepared.error_type: + raise RuntimeError("Could not prepare BrowseComp shell activation") + # Pages are immutable after _PageWriter publishes them; only the manifest changes. + # Never copy arbitrary files, symlinks, credentials, or verifier inputs from the host. + for path in sorted((workspace / "pages").glob("*.txt")): + if path.is_symlink() or not path.is_file(): + raise ValueError(f"BrowseComp page is not a regular file: {path.name}") + if path.name not in self.uploaded_pages: + await self.session.sandbox.upload(path, f"{self.session.workdir}/pages/{path.name}") + self.uploaded_pages.add(path.name) + manifest = workspace / "manifest.tsv" + if manifest.is_symlink() or not manifest.is_file(): + raise ValueError("BrowseComp manifest is not a regular file") + await self.session.sandbox.upload(manifest, f"{self.session.workdir}/manifest.tsv") + await upload_text( + self.session.sandbox, + path=f"{directory}/input.json", + text=json.dumps({"command": command, "duration": duration, "workspace": self.session.workdir}), + ) + return SandboxCommand( + argv=[_RUNTIME.python, "-E", "-s", _RUNNER, f"{directory}/input.json", f"{directory}/output.txt"] + ) + + async def collect() -> str: + return await read_text(self.session.sandbox, path=f"{directory}/output.txt") + + return await self.session.execute( + stage_activation=stage, + collect=collect, + timeout=duration + 30, + close_timeout=close_timeout, + ) + + async def close(self, *, timeout: float) -> None: + """Confirm termination and stop the owned sandbox; a failed stop remains retryable.""" + await self.session.close(timeout=timeout) diff --git a/resources_servers/browsecomp_advanced_harness/shell_output.py b/resources_servers/browsecomp_advanced_harness/shell_output.py new file mode 100644 index 00000000000..91b7ea562c1 --- /dev/null +++ b/resources_servers/browsecomp_advanced_harness/shell_output.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Identical rendering for sandbox and explicitly enabled development execution.""" + +_BASH_HEAD_BYTES = 8192 +_BASH_TAIL_BYTES = 4096 + + +def _bash_truncate(s: str) -> str: + if len(s) <= _BASH_HEAD_BYTES + _BASH_TAIL_BYTES + 64: + return s + omitted = len(s) - _BASH_HEAD_BYTES - _BASH_TAIL_BYTES + return ( + f"{s[:_BASH_HEAD_BYTES]}\n" + f"[...truncated {omitted} bytes; head shown above, tail follows...]\n" + f"{s[-_BASH_TAIL_BYTES:]}" + ) + + +def format_shell_output(stdout: str, stderr: str, exit_code: int) -> str: + """Preserve the read-only tool's historical output, including truncation markers.""" + truncated = (len(stdout) + len(stderr)) > (_BASH_HEAD_BYTES + _BASH_TAIL_BYTES) * 2 + parts = [] + if stdout: + parts.append(_bash_truncate(stdout)) + if stderr: + parts.extend(("--- stderr ---", _bash_truncate(stderr))) + parts.append(f"[exit_code={exit_code}{' truncated' if truncated else ''}]") + return "\n".join(parts) diff --git a/resources_servers/browsecomp_advanced_harness/shell_runner.py b/resources_servers/browsecomp_advanced_harness/shell_runner.py new file mode 100644 index 00000000000..c9af000263d --- /dev/null +++ b/resources_servers/browsecomp_advanced_harness/shell_runner.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Run the vendored pocketshell inside its Resources-owned sandbox.""" + +import json +import sys +from pathlib import Path + +from pocketshell import run +from shell_output import format_shell_output + + +def main() -> None: + """Read a JSON command and write its bounded result; command text is never shell source.""" + payload = json.loads(Path(sys.argv[1]).read_text()) + try: + result = run(payload["command"], workspace=payload["workspace"], timeout=payload["duration"]) + output = format_shell_output(result.stdout, result.stderr, result.exit_code) + except Exception as error: + output = f"[bash error: {error}]\n[exit_code=-2]" + Path(sys.argv[2]).write_text(output) + + +if __name__ == "__main__": + main() diff --git a/resources_servers/browsecomp_advanced_harness/tests/test_bash_guard.py b/resources_servers/browsecomp_advanced_harness/tests/test_bash_guard.py index b1ae95373ef..16f697845e5 100644 --- a/resources_servers/browsecomp_advanced_harness/tests/test_bash_guard.py +++ b/resources_servers/browsecomp_advanced_harness/tests/test_bash_guard.py @@ -167,6 +167,7 @@ def _server() -> TavilySearchResourcesServer: port=8080, entrypoint="", name="", + allow_host_execution=True, tavily_api_key="test_api_key", # pragma: allowlist secret exclude_domains_file_path=_DUMMY_EXCLUDE_DOMAINS_FILE, judge_model_server=ModelServerRef(type="responses_api_models", name="judge"), diff --git a/resources_servers/browsecomp_advanced_harness/tests/test_pocketshell_wiring.py b/resources_servers/browsecomp_advanced_harness/tests/test_pocketshell_wiring.py index e5d836398e6..313b5cbbe8d 100644 --- a/resources_servers/browsecomp_advanced_harness/tests/test_pocketshell_wiring.py +++ b/resources_servers/browsecomp_advanced_harness/tests/test_pocketshell_wiring.py @@ -37,6 +37,7 @@ class _Stub: def __init__(self): self._bash_semaphore = asyncio.Semaphore(8) + self.config = types.SimpleNamespace(allow_host_execution=True) def run_bash(keystrokes, workspace, duration=10.0): diff --git a/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py b/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py index 8f6c24b74f8..7cf46840e9a 100644 --- a/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py +++ b/resources_servers/browsecomp_advanced_harness/tests/test_resources_sessions.py @@ -46,7 +46,7 @@ def _server(tmp_path: Path, **overrides) -> TavilySearchResourcesServer: exclude_domains_file_path=_EXCLUDE_DOMAINS_FILE, use_judge=False, workspace_root=str(tmp_path), - **overrides, + **({"allow_host_execution": True} | overrides), ) return TavilySearchResourcesServer(config=config, server_client=MagicMock(spec=ServerClient)) diff --git a/resources_servers/browsecomp_advanced_harness/tests/test_rollout_isolation.py b/resources_servers/browsecomp_advanced_harness/tests/test_rollout_isolation.py index da210ecb1c2..3f829f2ee18 100644 --- a/resources_servers/browsecomp_advanced_harness/tests/test_rollout_isolation.py +++ b/resources_servers/browsecomp_advanced_harness/tests/test_rollout_isolation.py @@ -42,6 +42,7 @@ class _Stub: def __init__(self): self._bash_semaphore = asyncio.Semaphore(8) + self.config = types.SimpleNamespace(allow_host_execution=True) def run_bash(keystrokes, workspace, duration=10.0): diff --git a/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py b/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py new file mode 100644 index 00000000000..9cd94ade4bc --- /dev/null +++ b/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py @@ -0,0 +1,273 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The resources tool boundary, cleanup ownership, and unchanged shell output.""" + +import asyncio +import json +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import Request +from fastapi.testclient import TestClient +from pocketshell import run + +from nemo_gym.client import SESSION_ID_KEY +from nemo_gym.server_runtime import ResourcesCloseSessionRequest, ResourcesSeedSessionRequest +from resources_servers.browsecomp_advanced_harness import app as harness +from resources_servers.browsecomp_advanced_harness import sandbox as shell_module +from resources_servers.browsecomp_advanced_harness.app import BashCommandRequest +from resources_servers.browsecomp_advanced_harness.shell_output import format_shell_output +from resources_servers.browsecomp_advanced_harness.tests.test_resources_sessions import ( + _close, + _seed, + _server, + _verify_body, +) + + +def _request(sid: str = "resources-session") -> Request: + return Request({"type": "http", "session": {SESSION_ID_KEY: sid}}) + + +@pytest.fixture +def sandboxed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + server = _server(tmp_path, workspace="per_session", sandbox_provider="sandbox", allow_host_execution=False) + monkeypatch.setattr(harness, "get_global_config_dict", lambda: {}) + monkeypatch.setattr(harness, "resolve_provider_config", lambda *_: {"docker": {}}) + monkeypatch.setattr(harness, "resolve_provider_metadata", lambda *_: {"owner": "default", "pool": "shared"}) + monkeypatch.setattr(harness, "create_provider", lambda *_: object()) + containers = [] + + def create(_provider): + sandbox = AsyncMock() + containers.append(sandbox) + return sandbox + + async def install(state, *, timeout): + state.ready = True + + monkeypatch.setattr(harness, "AsyncSandbox", create) + monkeypatch.setattr(shell_module.BrowsecompShellSandbox, "install", install) + return server, containers + + +def test_missing_provider_refuses_seed_without_creating_workspace(tmp_path: Path) -> None: + server = _server(tmp_path, workspace="per_session", allow_host_execution=False) + with TestClient(server.setup_webserver()) as client: + response = client.post("/seed_session", json=_seed()) + assert response.status_code == 422 + assert "sandbox_provider" in response.json()["detail"] + assert not server._typed_sessions and not server._shell_sessions + assert list(tmp_path.iterdir()) == [] + + +def test_local_provider_requires_explicit_development_opt_in(sandboxed, monkeypatch: pytest.MonkeyPatch) -> None: + server, containers = sandboxed + monkeypatch.setattr(harness, "resolve_provider_config", lambda *_: {"local": {}}) + with TestClient(server.setup_webserver()) as client: + response = client.post("/seed_session", json=_seed()) + assert response.status_code == 422 + assert "does not isolate" in response.json()["detail"] + assert not containers + + +@pytest.mark.asyncio +async def test_seed_is_idempotent_and_close_stops_each_owned_sandbox(sandboxed) -> None: + server, containers = sandboxed + for sid in ("resources-session", "resources-session", "another-session"): + await server.seed_session(_request(sid), ResourcesSeedSessionRequest.model_validate(_seed(sid))) + assert len(containers) == 2 + assert len({state.session.session_dir for state in server._shell_sessions.values()}) == 2 + for sid in ("resources-session", "another-session"): + await server.close_resources_session(ResourcesCloseSessionRequest.model_validate(_close(sid))) + for container in containers: + container.start.assert_awaited_once() + container.stop.assert_awaited_once() + assert not server._shell_sessions and not server._session_workspaces + + +@pytest.mark.asyncio +async def test_startup_delivers_only_shell_source_and_merges_provider_metadata(sandboxed) -> None: + server, containers = sandboxed + server.config.sandbox_config["metadata"] = {"owner": "configured", "resources_session_id": "override"} + await server.seed_session(_request(), ResourcesSeedSessionRequest.model_validate(_seed())) + spec = containers[0].start.await_args.args[0] + assert spec.metadata == { + "owner": "configured", + "pool": "shared", + "resources_session_id": "resources-session", + "nemo_gym_resources": server.config.name, + } + assert spec.files == shell_module.shell_source_files() + assert any(path.endswith("/pocketshell/__init__.py") for path in spec.files) + assert any(path.endswith("/shell_runner.py") for path in spec.files) + await server.close_resources_session(ResourcesCloseSessionRequest.model_validate(_close())) + + +@pytest.mark.asyncio +async def test_lifespan_stops_remaining_sandbox_on_exception(sandboxed) -> None: + server, containers = sandboxed + app = server.setup_webserver() + with pytest.raises(RuntimeError, match="server failed"): + async with app.router.lifespan_context(app): + await server.seed_session(_request(), ResourcesSeedSessionRequest.model_validate(_seed())) + raise RuntimeError("server failed") + containers[0].stop.assert_awaited_once() + assert not server._shell_sessions and not server._session_workspaces + + +@pytest.mark.asyncio +async def test_bash_uses_sandbox_and_retains_guard_output(sandboxed, monkeypatch: pytest.MonkeyPatch) -> None: + server, _ = sandboxed + await server.seed_session(_request(), ResourcesSeedSessionRequest.model_validate(_seed())) + shell = server._shell_sessions["resources-session"] + shell.execute = AsyncMock(return_value="page text\n[exit_code=0]") + host = MagicMock(side_effect=AssertionError("model command reached the host")) + monkeypatch.setattr(harness, "pocketshell_run", host) + output = await server.bash_command(_request(), BashCommandRequest(keystrokes="cat pages/0001.txt")) + assert output.results_string == "page text\n[exit_code=0]" + assert shell.execute.await_args.kwargs["command"] == "cat pages/0001.txt" + assert shell.execute.await_args.kwargs["workspace"] == Path(server.config.workspace_root) / "resources-session" + blocked = await server.bash_command(_request(), BashCommandRequest(keystrokes="rm -rf pages")) + assert blocked.results_string == "[blocked: command 'rm' is blocked]\n[exit_code=-3]" + shell.execute.assert_awaited_once() + host.assert_not_called() + await server.close_resources_session(ResourcesCloseSessionRequest.model_validate(_close())) + + +@pytest.mark.asyncio +async def test_setup_failure_retains_failed_cleanup_for_retry(sandboxed, monkeypatch: pytest.MonkeyPatch) -> None: + server, containers = sandboxed + + async def fail_install(state, *, timeout): + state.session.sandbox.stop.side_effect = [RuntimeError("stop unavailable"), None] + raise RuntimeError("install unavailable") + + monkeypatch.setattr(shell_module.BrowsecompShellSandbox, "install", fail_install) + with pytest.raises(RuntimeError, match="install unavailable"): + await server.seed_session(_request(), ResourcesSeedSessionRequest.model_validate(_seed())) + assert "resources-session" in server._shell_sessions + assert "resources-session" in server._session_workspaces + await server.close_resources_session(ResourcesCloseSessionRequest.model_validate(_close())) + assert containers[0].stop.await_count == 2 + assert not server._shell_sessions and not server._session_workspaces + + +@pytest.mark.asyncio +async def test_verify_does_not_remove_workspace_until_sandbox_stop_succeeds(sandboxed) -> None: + server, containers = sandboxed + await server.seed_session(_request(), ResourcesSeedSessionRequest.model_validate(_seed())) + workspace = server._session_workspaces["resources-session"].workspace + containers[0].stop.side_effect = [RuntimeError("stop unavailable"), None] + body = harness.TavilySearchVerifyRequest.model_validate(_verify_body("Answer: Paris Confidence: 90%")) + with pytest.raises(RuntimeError, match="stop unavailable"): + await server.verify(_request(), body) + assert workspace.exists() and "resources-session" in server._shell_sessions + assert "resources-session" in server._session_id_to_metrics + output = await server.verify(_request(), body) + assert output.reward == 1 + assert not workspace.exists() and not server._shell_sessions + + +@pytest.mark.asyncio +async def test_close_waits_for_cancelled_tool_cleanup(sandboxed) -> None: + server, containers = sandboxed + await server.seed_session(_request(), ResourcesSeedSessionRequest.model_validate(_seed())) + started, cleanup_started, cleanup_allowed = asyncio.Event(), asyncio.Event(), asyncio.Event() + + async def execute(**kwargs): + started.set() + try: + await asyncio.Event().wait() + finally: + cleanup_started.set() + await cleanup_allowed.wait() + + server._shell_sessions["resources-session"].execute = execute + tool = asyncio.create_task(server.bash_command(_request(), BashCommandRequest(keystrokes="ls"))) + await asyncio.wait_for(started.wait(), 1) + tool.cancel() + await asyncio.wait_for(cleanup_started.wait(), 1) + close = asyncio.create_task(server.close_resources_session(ResourcesCloseSessionRequest.model_validate(_close()))) + await asyncio.sleep(0) + containers[0].stop.assert_not_awaited() + assert "resources-session" in server._session_workspaces + cleanup_allowed.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(tool, 1) + await asyncio.wait_for(close, 1) + containers[0].stop.assert_awaited_once() + + +@pytest.mark.parametrize( + "command", + [ + "cat pages/a.txt", + "grep -P 'Alpha(?= Beta)' pages/a.txt", + "cat pages/missing.txt", + "cat /etc/passwd", + "cat pages/big.txt", + ], +) +def test_real_runner_preserves_pocketshell_output(tmp_path: Path, command: str) -> None: + pages = tmp_path / "pages" + pages.mkdir() + (pages / "a.txt").write_text("Alpha Beta\nsecond line\n") + (pages / "big.txt").write_text("x" * 100_000) + input_file, output_file = tmp_path / "input.json", tmp_path / "output.txt" + input_file.write_text(json.dumps({"command": command, "duration": 5, "workspace": str(tmp_path)})) + runner = Path(shell_module.__file__).with_name("shell_runner.py") + subprocess.run( + [sys.executable, "-E", "-s", str(runner), str(input_file), str(output_file)], check=True, timeout=15 + ) + expected = run(command, str(tmp_path), 5) + assert output_file.read_text() == format_shell_output(expected.stdout, expected.stderr, expected.exit_code) + assert len(output_file.read_text()) < 25_000 + + +@pytest.mark.asyncio +async def test_controller_transfers_only_pages_and_manifest_and_never_interpolates_command( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + workspace = tmp_path / "workspace" + (workspace / "pages").mkdir(parents=True) + (workspace / "pages/a.txt").write_text("page") + (workspace / "manifest.tsv").write_text("manifest") + (workspace / "private-key").write_text("not a retrieved page") + sandbox = AsyncMock() + sandbox.exec.return_value = SimpleNamespace(return_code=0, error_type=None) + session = SimpleNamespace(sandbox=sandbox, session_dir="/private/session", workdir="/workspace") + directories, commands, payloads = [], [], [] + + async def begin(*, session_dir, timeout): + directories.append(session_dir) + session.session_dir = session_dir + + async def execute(*, stage_activation, collect, **kwargs): + commands.append(await stage_activation()) + return "result" + + async def upload(_sandbox, *, path, text): + payloads.append(json.loads(text)) + + session.begin_activation, session.execute = begin, execute + monkeypatch.setattr(shell_module, "upload_text", upload) + state = shell_module.BrowsecompShellSandbox(session=session, ready=True) + command = 'cat pages/a.txt; echo "$(not-shell-source)"' + for _ in range(2): + await state.execute(command=command, duration=5, workspace=workspace, close_timeout=10) + assert directories[0] != directories[1] + assert all(command not in " ".join(call.argv) for call in commands) + assert [payload["command"] for payload in payloads] == [command, command] + sources = [call.args[0] for call in sandbox.upload.await_args_list] + assert sources.count(workspace / "pages/a.txt") == 1 + assert sources.count(workspace / "manifest.tsv") == 2 + assert workspace / "private-key" not in sources + + (workspace / "pages/escape.txt").symlink_to(workspace / "private-key") + with pytest.raises(ValueError, match="not a regular file"): + await state.execute(command="ls", duration=5, workspace=workspace, close_timeout=10) diff --git a/resources_servers/browsecomp_advanced_harness/uv.lock b/resources_servers/browsecomp_advanced_harness/uv.lock index e4a5222405c..f5caeecc3a1 100644 --- a/resources_servers/browsecomp_advanced_harness/uv.lock +++ b/resources_servers/browsecomp_advanced_harness/uv.lock @@ -49,6 +49,15 @@ excludes = [ "werkzeug", ] +[[package]] +name = "aiofiles" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/c3/534eac40372d8ee36ef40df62ec129bee4fdb5ad9706e58a29be53b2c970/aiofiles-25.1.0.tar.gz", hash = "sha256:a8d728f0a29de45dc521f18f07297428d56992a742f0cd2701ba86e44d23d5b2", size = 46354, upload-time = "2025-10-09T20:51:04.358Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/8a/340a1555ae33d7354dbca4faa54948d76d89a27ceef032c8c3bc661d003e/aiofiles-25.1.0-py3-none-any.whl", hash = "sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695", size = 14668, upload-time = "2025-10-09T20:51:03.174Z" }, +] + [[package]] name = "aiohappyeyeballs" version = "2.6.1" @@ -181,6 +190,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2b/27/5e1f8446545be98e9416af59c396b1ed1f9f3294afb0a40ee3a709889273/aiohttp-3.14.4-py3-none-any.whl", hash = "sha256:5c6758ba62aea282c537179cfc8474a90f2b5f7b089cc5ff66d8920d86a9bfdd", size = 279487, upload-time = "2026-10-05T17:43:44.905Z" }, ] +[[package]] +name = "aiohttp-retry" +version = "2.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/61/ebda4d8e3d8cfa1fd3db0fb428db2dd7461d5742cea35178277ad180b033/aiohttp_retry-2.9.1.tar.gz", hash = "sha256:8eb75e904ed4ee5c2ec242fefe85bf04240f685391c4879d8f541d6028ff01f1", size = 13608, upload-time = "2024-11-06T10:44:54.574Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/99/84ba7273339d0f3dfa57901b846489d2e5c2cd731470167757f1935fffbd/aiohttp_retry-2.9.1-py3-none-any.whl", hash = "sha256:66d2759d1921838256a05a3f80ad7e724936f083e35be5abb5e16eed6be6dc54", size = 9981, upload-time = "2024-11-06T10:44:52.917Z" }, +] + [[package]] name = "aiosignal" version = "1.4.0" @@ -279,6 +300,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, ] +[[package]] +name = "bidict" +version = "0.24.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a8/f2/8d2dd8276ca05e1f5157b6a0d34efb2f585f47a0fbed61e8aad04b221f0b/bidict-0.24.1.tar.gz", hash = "sha256:4dca6c17f0b01700e9f24359daa5ebabf7be022d99f4cb2a257b6af2a5076c88", size = 30818, upload-time = "2026-08-25T23:45:52.214Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/53/2a3c7d562271ec6b6e38e7216b3104899f8aa4180c0713cb8aaf69e29cd5/bidict-0.24.1-py3-none-any.whl", hash = "sha256:fd3eaa737917d8a14f4baa391670c433c4e3f6f5fd2cd99d4bf436437f432364", size = 36175, upload-time = "2026-08-25T23:45:51.096Z" }, +] + +[[package]] +name = "boto3" +version = "1.43.123" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/8a/64b46f075149f9160da44aa7e1052dd18431b9ad4435beb0cfb02cc48624/boto3-1.43.123.tar.gz", hash = "sha256:055fabb990c5d2f31fce4ca569255bd640cdd11a3ca68c9ecad35f4f8a5b83b5", size = 112638, upload-time = "2026-10-30T19:18:50.085Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/b2/a7986c327f80aa6f1544f4cb8f2fa7da4e1fccd472379a21cad0508b8a0e/boto3-1.43.123-py3-none-any.whl", hash = "sha256:d18d66e1e13faa0c25d0b1b8c5ce946e29fb5fc0b6456b655a93a678ff4cc313", size = 140044, upload-time = "2026-10-30T19:18:47.734Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.123" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/de/011fc2a47fb066c47064b122498d2fd21f55bc90ea514acbddc37ca0f8c1/botocore-1.43.123.tar.gz", hash = "sha256:80948cb9b1cdf1a1a25809089a42e3a1bc4af0258d198222db6e97dcc660dd0c", size = 16486269, upload-time = "2026-10-30T19:18:44.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/96/4f2dffd304425c07160ab2c12cb13322d852fd956fe9d4386228c93f02ea/botocore-1.43.123-py3-none-any.whl", hash = "sha256:e30580f9c1282c321529238ff8ff75066096a2adbd0a293c8ae592c21d1733a3", size = 16185987, upload-time = "2026-10-30T19:18:39.794Z" }, +] + +[[package]] +name = "bracex" +version = "3.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4d/fc/137095eb2f24ccf0b4db196abd9f5cfe74398a834af51d4bc4876e0beaf0/bracex-3.2.1.tar.gz", hash = "sha256:07a9b2100ceecc0c24afa1a1714bf8d4a6abb43d592939455f662b682ac9baa4", size = 32694, upload-time = "2026-11-03T14:13:43.53Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/ae/30dd8b027b4032b27e81364de5b74bfbef50b3c84c6aa0ead351bbb3749c/bracex-3.2.1-py3-none-any.whl", hash = "sha256:e8622d46b60ef6062b4091033f97a304e1b205e70c0285743c5038b73886a47e", size = 12527, upload-time = "2026-11-03T14:13:42.032Z" }, +] + [[package]] name = "certifi" version = "2026.10.23" @@ -422,6 +489,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "connectrpc" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf-py" }, + { name = "pyqwest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b8/43/025430769a6cc82a1e1c76cd9760639f6bf9b05577f90e38a0bb57b41b39/connectrpc-0.12.1.tar.gz", hash = "sha256:da7ca41c6cd8d9ce640ad86ae325a71cd83dfdd1e43748976315cb4b0546fb69", size = 50556, upload-time = "2026-09-04T15:23:31.327Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/f8/b244edb73395c5a74bfeab13ace9e6ccda89d31fb5e493ca8a8c96106eae/connectrpc-0.12.1-py3-none-any.whl", hash = "sha256:ba85ff49b5b7e0f1b16d07b35a05e39ce92c1a7116a7ed1a271c76393323cd07", size = 70058, upload-time = "2026-09-04T15:23:28.686Z" }, +] + [[package]] name = "coverage" version = "7.10.5" @@ -563,6 +643,134 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/5e/6f8d874366788ad5d549e9ba258037d974dda6e004843be1bda794571701/datasets-4.4.1-py3-none-any.whl", hash = "sha256:c1163de5211e42546079ab355cc0250c7e6db16eb209ac5ac6252f801f596c44", size = 511591, upload-time = "2025-11-05T16:00:36.365Z" }, ] +[[package]] +name = "daytona" +version = "0.210.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiofiles" }, + { name = "aiohttp" }, + { name = "daytona-analytics-api-client" }, + { name = "daytona-analytics-api-client-async" }, + { name = "daytona-api-client" }, + { name = "daytona-api-client-async" }, + { name = "daytona-toolbox-api-client" }, + { name = "daytona-toolbox-api-client-async" }, + { name = "deprecated" }, + { name = "httpx" }, + { name = "httpx-ws" }, + { name = "obstore" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-instrumentation-aiohttp-client" }, + { name = "opentelemetry-sdk" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "python-multipart" }, + { name = "python-socketio", extra = ["asyncio-client", "client"] }, + { name = "toml" }, + { name = "typing-extensions" }, + { name = "urllib3" }, + { name = "wsproto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/dd/9ce4820cb05cc2b7f38f9fd076bc575b1190a3236d69a40a0f5a09ebd14c/daytona-0.210.0.tar.gz", hash = "sha256:313fcec07ec4d04a6ccaa91618ce956612ef9ba78484691aeccdaa36d017d6b4", size = 188817, upload-time = "2026-09-03T21:11:01.028Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/2d/f4c1dec9dc0da86aa79b64a83e6f3c94ff9a25ed78a1bc59c5c2fa3c51db/daytona-0.210.0-py3-none-any.whl", hash = "sha256:0df723203e20a894fd07808e62f13f419e41f6d035016ecdc8f873c50cb32634", size = 226269, upload-time = "2026-09-03T21:11:02.374Z" }, +] + +[[package]] +name = "daytona-analytics-api-client" +version = "0.210.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4f/2d/76721bf383de06bb16ce6dec473d0f18f2b6853602d7410ce0116e1981f9/daytona_analytics_api_client-0.210.0.tar.gz", hash = "sha256:b4230a0a6afe65e2a1c7b0e9bb49bf905a795b1610256afaad956a3475b8569d", size = 30088, upload-time = "2026-09-03T21:10:27.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/31/b6ed690257cfb34b37088919b36ee9f87a8d76aaf08ef110b093580de8ac/daytona_analytics_api_client-0.210.0-py3-none-any.whl", hash = "sha256:2e70b15e8c14bb4e88a12bb3644b6e4f4a296305298f111efac4409f3e92ae37", size = 45013, upload-time = "2026-09-03T21:10:30.978Z" }, +] + +[[package]] +name = "daytona-analytics-api-client-async" +version = "0.210.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "aiohttp-retry" }, + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a5/72/a0d2ced3b786624b95d3059d0fe7641736064e6470018ff023e46ba3c4f4/daytona_analytics_api_client_async-0.210.0.tar.gz", hash = "sha256:762f19d4e8d9eb74b3d5e3cdeec0ebeddae623d7b6a764e6c97a7f5cafa8f8c9", size = 30130, upload-time = "2026-09-03T21:10:35.562Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/a7/8ee5f1fa221c30914aa6f0a9a5a66c7d176d109c3feae3bf61a7b571ac5c/daytona_analytics_api_client_async-0.210.0-py3-none-any.whl", hash = "sha256:258af338e3a95c4a02b3e5c84c8283c0aad1e14151cd2ab4f575700cc200b48a", size = 45284, upload-time = "2026-09-03T21:10:36.483Z" }, +] + +[[package]] +name = "daytona-api-client" +version = "0.210.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b4/fb/3f577dea27c04937ad2cea314877346d2ac5cdf44925bd09eb2b6de2e6f5/daytona_api_client-0.210.0.tar.gz", hash = "sha256:1aea806ca4fe86f0f59b66c043b849d1acf61d9b7db240ec13b7c0678f955590", size = 141803, upload-time = "2026-09-03T21:10:27.399Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/bf/7b6e062bf754707c0a29c5e65de2c98fb1a04b2f298fbceba9ab513b88ea/daytona_api_client-0.210.0-py3-none-any.whl", hash = "sha256:c55f42d13fc8da84380fbdc5bc010c67dca7f9b9d399546ab54e8810c5dc1c38", size = 356132, upload-time = "2026-09-03T21:10:28.713Z" }, +] + +[[package]] +name = "daytona-api-client-async" +version = "0.210.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "aiohttp-retry" }, + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/f4/6707985e647280d34a87de4ed24fd501bbe4999c0165b4c9b374490b773b/daytona_api_client_async-0.210.0.tar.gz", hash = "sha256:0b597f289f992572e776e3616cf5aae89a9b623dbbc911213e69c2d193bfe302", size = 142201, upload-time = "2026-09-03T21:10:31.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/47/7ef49573074d5956578f8c899b528a8107e0dfc9fe0d953c32859d968dc0/daytona_api_client_async-0.210.0-py3-none-any.whl", hash = "sha256:6da7dc68915b216d1a7a821e8dbe3c832b63c63c0e8701cda1216d5c38080e59", size = 358962, upload-time = "2026-09-03T21:10:33.482Z" }, +] + +[[package]] +name = "daytona-toolbox-api-client" +version = "0.210.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/16/83d9b090d8107a95f871ed0154392732918ee6b0606b7117444353b29412/daytona_toolbox_api_client-0.210.0.tar.gz", hash = "sha256:f816c60d01bbd8de4649caa6e6a05bd025a8096e17d6096cbc8b537db900fa9a", size = 88114, upload-time = "2026-09-03T21:10:27.563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/43/5810261a1d57c4eff93960a1848767ca14a2d5fb543d1f32aab74954fdbd/daytona_toolbox_api_client-0.210.0-py3-none-any.whl", hash = "sha256:f346fbfc7db3ad5201dfe35610fd068d3992f8ae5e3bc5a312d1214da528dddb", size = 252413, upload-time = "2026-09-03T21:10:29.504Z" }, +] + +[[package]] +name = "daytona-toolbox-api-client-async" +version = "0.210.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "aiohttp-retry" }, + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c1/b9/419fcacbf90f22679da6f73a2ba6c8dae3c7054e81546589dca28f5c2c0e/daytona_toolbox_api_client_async-0.210.0.tar.gz", hash = "sha256:82e1b0c9cc5db0484be061e39a9810bc7df1e8c0ca0aeb0cd9a3dd6d5eb1101a", size = 82027, upload-time = "2026-09-03T21:10:32.399Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/57/1d2bef2d66f6d212155aeb2ca09e2f817b7eea84df759cd1fab10b4b1f7b/daytona_toolbox_api_client_async-0.210.0-py3-none-any.whl", hash = "sha256:cf4a1c9b64f3a7f8fc8d2d46b5c577824bb43452744fcc4b3874a25e04f52a05", size = 250905, upload-time = "2026-09-03T21:10:34.217Z" }, +] + [[package]] name = "defusedxml" version = "0.7.1" @@ -572,6 +780,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, ] +[[package]] +name = "deprecated" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, +] + [[package]] name = "devtools" version = "0.12.2" @@ -622,6 +842,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, ] +[[package]] +name = "e2b" +version = "2.63.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "connectrpc" }, + { name = "e2b-dockerfile-utils" }, + { name = "h2" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "protobuf-py" }, + { name = "pyqwest" }, + { name = "python-dateutil" }, + { name = "rich" }, + { name = "typing-extensions" }, + { name = "wcmatch" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/f7/4d317f3c22992152a934e1f47ec5c6eea61350635a1de628f2c8f4e34987/e2b-2.63.0.tar.gz", hash = "sha256:01c9aa7c8565e9730c6cfa075a10a67a7560db9e11f67458fb1bce21e0e2e0cc", size = 273466, upload-time = "2026-11-04T16:41:31.849Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/24/4777775082b058be6c531bbeadf22d97897953320b24323b8dd45e902231/e2b-2.63.0-py3-none-any.whl", hash = "sha256:fae437d0de051c633e6854260d94c178d7155711387375b5a447b326efdda065", size = 430158, upload-time = "2026-11-04T16:41:30.173Z" }, +] + +[[package]] +name = "e2b-dockerfile-utils" +version = "0.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/fc/15b2752ff08fcbb7c6b97451581e02dd189ddc9b6eb107bb17c35b04427f/e2b_dockerfile_utils-0.1.1.tar.gz", hash = "sha256:c1d4179bae6f3131fe0bbb1e6787ffb07c6c6dca8bdb3399abb3de9056b2a08f", size = 16844, upload-time = "2026-10-13T15:01:50.959Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/39/8d3f59a073dd1637fccca4c921665935d9591cc1726b452f06ef7cdadabc/e2b_dockerfile_utils-0.1.1-py3-none-any.whl", hash = "sha256:c648ecf56b3ffdb6d91469580850f4691f019cb2c925405e776ad36b761204f1", size = 19901, upload-time = "2026-10-13T15:01:49.942Z" }, +] + [[package]] name = "execnet" version = "2.1.1" @@ -813,7 +1065,7 @@ name = "gym-resources-servers-browsecomp-advanced-harness" version = "0.0.0" source = { virtual = "." } dependencies = [ - { name = "nemo-gym", extra = ["dev"] }, + { name = "nemo-gym", extra = ["dev", "sandbox"] }, { name = "pocketshell", extra = ["pcre"] }, { name = "tavily-python" }, ] @@ -825,7 +1077,7 @@ telemetry = [ [package.metadata] requires-dist = [ - { name = "nemo-gym", extras = ["dev"], editable = "../../" }, + { name = "nemo-gym", extras = ["dev", "sandbox"], editable = "../../" }, { name = "nemo-gym", extras = ["telemetry"], marker = "extra == 'telemetry'", editable = "../../" }, { name = "pocketshell", extras = ["pcre"], editable = "pocketshell" }, { name = "tavily-python", specifier = "==0.7.21" }, @@ -841,6 +1093,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "h2" +version = "4.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "hpack" }, + { name = "hyperframe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/85/7c366e69d84c17bb778fe41419e1fbcce3033d5b7ce29bbffff0a98b859f/h2-4.4.1.tar.gz", hash = "sha256:4e866ffb1a869ae14dd9b5e6beb5c24a13da0495ad72b65925ded182521c1516", size = 2157281, upload-time = "2026-08-03T11:45:09.509Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/22/e85faf23bd72a92d1921e37d674ca56eb298a3c8be31fdecef0ff2b3aaac/h2-4.4.1-py3-none-any.whl", hash = "sha256:0e25f1462b23c9cb82d9eb02e28bc706dac2a68cb457c6a0d74d63c8a2a5d0e6", size = 62636, upload-time = "2026-08-03T11:44:59.164Z" }, +] + [[package]] name = "hf-xet" version = "1.6.0" @@ -865,6 +1130,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, ] +[[package]] +name = "hpack" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/5b/fcabf6028144a8723726318b07a32c2f3314acdff6265743cf08a344b18e/hpack-4.2.0.tar.gz", hash = "sha256:0895cfa3b5531fc65fe439c05eb65144f123bf7a394fcaa56aa423548d8e45c0", size = 51300, upload-time = "2026-06-23T18:34:46.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/b4/4a9fcfb2aef6ba44d9073ecd301443aa00b3dac95de5619f2a7de7ec8a91/hpack-4.2.0-py3-none-any.whl", hash = "sha256:858ac0b02280fa582b5080d68db0899c62a80375e0e5413a74970c5e518b6986", size = 34246, upload-time = "2026-06-23T18:34:45.472Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -931,6 +1205,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" }, ] +[[package]] +name = "httpx-ws" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpcore" }, + { name = "httpx" }, + { name = "wsproto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/cd/ca91a07ae446451f7476bf3fcc909e98cb942ff032ebfda0e3fe449aca7b/httpx_ws-0.9.0.tar.gz", hash = "sha256:797373326f70eec1ae96f6e43ae9f12002fd7d73aee139a4985eaab964338a08", size = 107105, upload-time = "2026-03-28T14:11:10.781Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/f8/a6bc80313a9e93c888fa10534dfce2ad76ff86911b6f485777ce6de6a073/httpx_ws-0.9.0-py3-none-any.whl", hash = "sha256:71640d2fb1bf9a225775015b33cd755cfd4c5f7e21c885192fe3adc4c387b248", size = 15759, upload-time = "2026-03-28T14:11:11.887Z" }, +] + [[package]] name = "huggingface-hub" version = "1.33.0" @@ -965,6 +1254,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/fd/612cc7905787100dc9225ae6ad71b42b60cbc23cd56ca55afcdd1e000f81/hydra_core-1.3.6-py3-none-any.whl", hash = "sha256:55951f49c7e19c29f4281cce2eeaa6e73ef6c5620093a4efb402a7e4b839fc38", size = 163055, upload-time = "2026-08-29T14:36:06.385Z" }, ] +[[package]] +name = "hyperframe" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, +] + [[package]] name = "identify" version = "2.6.13" @@ -1037,6 +1335,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/4a/4175a563579e884192ba6e81725fc0448b042024419be8d83aa8a80a3f44/jiter-0.10.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3aa96f2abba33dc77f79b4cf791840230375f9534e5fac927ccceb58c5e604a5", size = 354213, upload-time = "2025-05-18T19:04:41.894Z" }, ] +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + [[package]] name = "jsonschema" version = "4.25.1" @@ -1261,6 +1568,13 @@ dev = [ { name = "ruff" }, { name = "tomlkit" }, ] +sandbox = [ + { name = "boto3" }, + { name = "daytona" }, + { name = "e2b" }, + { name = "opensandbox" }, + { name = "tenacity" }, +] telemetry = [ { name = "nemo-lens", extra = ["sdk"] }, { name = "opentelemetry-instrumentation-fastapi" }, @@ -1420,6 +1734,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/9e/1652778bce745a67b5fe05adde60ed362d38eb17d919a540e813d30f6874/numpy-2.3.2-cp314-cp314t-win_arm64.whl", hash = "sha256:092aeb3449833ea9c0bf0089d70c29ae480685dd2377ec9cdbbb620257f84631", size = 10544226, upload-time = "2025-07-24T20:56:34.509Z" }, ] +[[package]] +name = "obstore" +version = "0.11.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/27/aa7549157a4a681157e315534ba5ab8f167f77662166e792a6e836938f46/obstore-0.11.1.tar.gz", hash = "sha256:a5afe8b99e3b20cdc9133be7a1b381259acf0d470029f6b2fc79c3f9947ad436", size = 130828, upload-time = "2026-08-21T23:57:27.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/38/438f85772bfbcd2985845a5d00d1c910e98b15f587dae6cb1e435865eba9/obstore-0.11.1-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d5c50b755d781efebe4f9c70ee6d00858e44d95f0229b8b5174358f861d9bd7c", size = 5400146, upload-time = "2026-08-21T23:56:29.168Z" }, + { url = "https://files.pythonhosted.org/packages/e7/62/edd2613649cb6b4400f5d125223ee094d2da1c4c93636453a5360c61b865/obstore-0.11.1-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:093152daa5c32b70a032f231bbc6a7eff76dda4285eed5a81d272b4485032dab", size = 4596203, upload-time = "2026-08-21T23:56:30.7Z" }, + { url = "https://files.pythonhosted.org/packages/c1/52/09edd48251a26a65f786bf2def93994ffce501f3aea6724474d8de0a7f4f/obstore-0.11.1-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b97ee10456e65f166c030b5fbde8380ca508621e23b84efd77aad83afe99315a", size = 5003021, upload-time = "2026-08-21T23:56:32.085Z" }, + { url = "https://files.pythonhosted.org/packages/59/8f/89a28c75d46bbb0552d34a5e84c5bc512526d45f935321f00af2e6275a99/obstore-0.11.1-cp311-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48983a143de69b11de49212caa79b5c39acbff7f592f9e85d156a0133a0e5796", size = 5240376, upload-time = "2026-08-21T23:56:33.658Z" }, + { url = "https://files.pythonhosted.org/packages/50/c3/b4620899003e2472bec7baf5d2bce12cf6f6f11d02d8f17e03e093717a53/obstore-0.11.1-cp311-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b9cb988e03ff963914cf2176aee293eaaff7dcf4efcb905ce6834264b4b0884", size = 5444946, upload-time = "2026-08-21T23:56:35.093Z" }, + { url = "https://files.pythonhosted.org/packages/bd/de/687685ab39ae4a70cc13ac252febfe62a387836a55e046f467c9fb231880/obstore-0.11.1-cp311-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97850f68c8417f7167549bdb705c362c825ed470298c6b04faf52258c5e9234e", size = 5304065, upload-time = "2026-08-21T23:56:36.535Z" }, + { url = "https://files.pythonhosted.org/packages/b4/36/b962ad52031b5bfdd249923db6151ff2b665b08c017611db96838455f337/obstore-0.11.1-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a9f3d66dbf3c073dd6b6033c0787ef14edf78d97bff95595d6f6979692da264", size = 5556770, upload-time = "2026-08-21T23:56:38.377Z" }, + { url = "https://files.pythonhosted.org/packages/e0/04/41a0f2917fcaa1e66b754fc98c92441d5a252ff9c619941ac086f9bdb4f6/obstore-0.11.1-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:2ba3bceec4263b3a70eea873abedb82e03da9599f2326e80d6505cab0c10d401", size = 5337765, upload-time = "2026-08-21T23:56:40.279Z" }, + { url = "https://files.pythonhosted.org/packages/f9/39/0b618efe59ee4b58f16d2b9246674a70c390bebf665cd284b439bade9bd3/obstore-0.11.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e956fa8a953b7eb8580658d68cdf37e410356032c17697a06e44d0e8c4084a0f", size = 5544012, upload-time = "2026-08-21T23:56:41.754Z" }, + { url = "https://files.pythonhosted.org/packages/57/3b/1018f6da240f3529d1cb9a836c79a91ce45349ba3cffcd6baa73296c7e6c/obstore-0.11.1-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:159a50d0f4cc53afe6f5c695313bcaf6e92ac81d7847692c8279153483272bfa", size = 5229851, upload-time = "2026-08-21T23:56:43.367Z" }, + { url = "https://files.pythonhosted.org/packages/35/2c/c2fe082816b255159373b15be3f28b2515d0eb954e248e2f65449c456a14/obstore-0.11.1-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:128da07f3a1b9c70159e2b2be9e27f458a9d531d57b1856dd7c4ddb73b4c9937", size = 5361812, upload-time = "2026-08-21T23:56:44.871Z" }, + { url = "https://files.pythonhosted.org/packages/43/9c/656eb5cce818e7e3985d27f4bd403feb0b6e620494775a55a0943c8d2fff/obstore-0.11.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:67b8acfbbab960c2cb14c34294a96556caa67fcff15ee77c716d5c1bd1558606", size = 5790855, upload-time = "2026-08-21T23:56:46.391Z" }, + { url = "https://files.pythonhosted.org/packages/c1/5c/94ece1c902ff5cdbe1140997d1f720221a0d5defa902fdd93715297604f5/obstore-0.11.1-cp311-abi3-win_amd64.whl", hash = "sha256:e23ea15cebe5f5be5d11005043d7b5ff56848e39499681ef52f8227bd385bd51", size = 5308846, upload-time = "2026-08-21T23:56:47.774Z" }, + { url = "https://files.pythonhosted.org/packages/c0/af/235730a429e55777f15f9d7fef118a6ecf02f430ed4588deb4b7e13964aa/obstore-0.11.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:73284fc8a9804596baf4d80b55c0e71a6007487bfa28d6924acd85264d5be81f", size = 5424585, upload-time = "2026-08-21T23:56:49.532Z" }, + { url = "https://files.pythonhosted.org/packages/e3/3c/432ea70c061fc9401fe399391ed4f04dead4a4cf33ea7c9564ce0e299d24/obstore-0.11.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8a3f93310422b153629af929d9e88d1fc826d5e32a456de4d3a1926a770ae09c", size = 4576791, upload-time = "2026-08-21T23:56:51.014Z" }, + { url = "https://files.pythonhosted.org/packages/43/c7/57dbdfa4ba5339b80fc755359bb232bdba7f82e92df6603ef383396b5f16/obstore-0.11.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d666690f0a53ae3df4be2c18af6820369a74c9c5b8960e8760f0f8c9a8f15b36", size = 4992141, upload-time = "2026-08-21T23:56:52.46Z" }, + { url = "https://files.pythonhosted.org/packages/37/4b/d25a4c129250364b0ca927fbfca5146744fb3bb4de60ae9b302300cbdfef/obstore-0.11.1-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0844ab75c8413c0af2d0fd2d2f6aa42d166809b6ac4be478151cd946ce7e5d69", size = 5217707, upload-time = "2026-08-21T23:56:53.946Z" }, + { url = "https://files.pythonhosted.org/packages/4b/07/0a925c886639d6edce37aafb6eba0c86820cbc9dda42e91ddd784dbe059d/obstore-0.11.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:307b5f9d64a7c00cd13371e376c05e3914216bd0818a19ff2ae05bc0135e01dc", size = 5429440, upload-time = "2026-08-21T23:56:55.507Z" }, + { url = "https://files.pythonhosted.org/packages/5f/c0/38d188a00b9f2de30df26802dd5bc2ecaefdabb882fde786cff800ccf50f/obstore-0.11.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f89647953b4ea50bab3f66591f7f462cd454a5d2fbbeefd885716a2c54e13ce", size = 5308903, upload-time = "2026-08-21T23:56:56.924Z" }, + { url = "https://files.pythonhosted.org/packages/60/d3/815e5a7ca59f901b6d24fb01ad3af46de37dfdfd2971c019f914bdb7f65e/obstore-0.11.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50de3116af69b6f1669cb443cc200ccdc1523a790a41f00854331b48f716cf8f", size = 5547007, upload-time = "2026-08-21T23:56:58.508Z" }, + { url = "https://files.pythonhosted.org/packages/9c/a1/ec4d2291b7eb15e7ddccb2f0646ce560b8a539df4b0ab94d916e7a15a61c/obstore-0.11.1-cp314-cp314t-manylinux_2_24_aarch64.whl", hash = "sha256:eae71e1c5944ade976ce8cd1e780bed9b5c31d6d4f89cb7263e8dcff78f6cd8e", size = 5330062, upload-time = "2026-08-21T23:56:59.954Z" }, + { url = "https://files.pythonhosted.org/packages/0a/09/a4a1301f2474f6569bbc0f2d337108ff2225d0b3d285894ed9a6dc953438/obstore-0.11.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ff634b5edbbf76c56ae81397aa41500a66ad0aa48b1856dcc0cf31b159172dc0", size = 5539870, upload-time = "2026-08-21T23:57:01.485Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8f/7dbbf935a07ff5375c33916307b77aef9d39bded35081530150822898010/obstore-0.11.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cbe509350d66249fc9e65ece4e7b1855d650f18e477721887128452b95c44b24", size = 5222673, upload-time = "2026-08-21T23:57:03.023Z" }, + { url = "https://files.pythonhosted.org/packages/d5/89/4932b3dd7963a3a64fd532c30ccc836f0f4ede1a45bea9f12295299398e2/obstore-0.11.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:46122e585f48ab3f2e4fed51401a4e866279a3a3d1aee2372d598ab85ba2c539", size = 5335696, upload-time = "2026-08-21T23:57:04.439Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ff/f3cf31ac4ebfa648dbd2c69774579fd5da4cf4b05fa1192405dafb6aeb38/obstore-0.11.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7275e75228059b2b30772b2ba3e41562a4a92ebdd3a96619d300f98ef152119f", size = 5783641, upload-time = "2026-08-21T23:57:05.899Z" }, + { url = "https://files.pythonhosted.org/packages/51/31/b352db3450e700c2c51861d139fc92c3284f7e578ca306408b054f6b4a35/obstore-0.11.1-cp314-cp314t-win_amd64.whl", hash = "sha256:13bb0b6a40931ab2da93f93ad843865d483d95acec1bd586df79313daa5a50af", size = 5290060, upload-time = "2026-08-21T23:57:07.632Z" }, +] + [[package]] name = "omegaconf" version = "2.3.0" @@ -1452,6 +1800,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/f4/561ed79fd94876160018a5e75254cfcb9b0e62d4dded9dcb20072e86d623/openai-2.44.0-py3-none-any.whl", hash = "sha256:0a2a3ab2e29aeda368700f662ff9ba0f9df17ba4c54577a64e08b8115a3cc0ad", size = 1366216, upload-time = "2026-06-24T20:55:58.882Z" }, ] +[[package]] +name = "opensandbox" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "httpcore" }, + { name = "httpx" }, + { name = "httpx-sse" }, + { name = "opentelemetry-api" }, + { name = "pydantic" }, + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/2f/6682d18a69043b5ad29e8a411e969e2ee4216127dbc2257752690f502287/opensandbox-1.2.0.tar.gz", hash = "sha256:e7936d0bf3f7defa4e26571b0634089b4c3eb15d023909f6da9d940b95736c54", size = 329030, upload-time = "2026-10-27T12:36:39.214Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/18/2de1578aead2e5da094c352e2c1610d750c399070769feac02cb42ff8941/opensandbox-1.2.0-py3-none-any.whl", hash = "sha256:15a0f3c3245018afa43b75fb14e892a64c2c8807276a0fdaf2a1a469f5e214ea", size = 744130, upload-time = "2026-10-27T12:36:37.805Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.44.0" @@ -1527,6 +1893,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/7b/85eab1215f72adf0e68d3dc4a679b9bff993fa679ff34cd8dd378e2659fd/opentelemetry_instrumentation-0.65b0-py3-none-any.whl", hash = "sha256:ea967a72b9939b5fcfdad572753b4306c59dcb99e3f382d95dae04286805e137", size = 36717, upload-time = "2026-07-16T15:24:51.424Z" }, ] +[[package]] +name = "opentelemetry-instrumentation-aiohttp-client" +version = "0.65b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "opentelemetry-util-http" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/33/3ff7230b035e8b696db6be54f5c52dfa409829d634d91431c076ad789820/opentelemetry_instrumentation_aiohttp_client-0.65b0.tar.gz", hash = "sha256:85906a2806ee5641756b5c33274e9aa75c3cc2441e3b830aa5804cf0e1fa9dd1", size = 19042, upload-time = "2026-07-16T15:25:51.632Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/f9/5c8459224f175829601cabbcffaeab0f9041903c086e4a0368f9971093a5/opentelemetry_instrumentation_aiohttp_client-0.65b0-py3-none-any.whl", hash = "sha256:3a060efa53fa44d02ba7372a7ed2b42cdfa6be6df81b089845067ad840e25729", size = 13677, upload-time = "2026-07-16T15:24:53.361Z" }, +] + [[package]] name = "opentelemetry-instrumentation-asgi" version = "0.65b0" @@ -1801,6 +2183,49 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, ] +[[package]] +name = "protobuf-py" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf-py-ext", marker = "(platform_machine == 'arm64' and platform_python_implementation == 'CPython' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_python_implementation == 'CPython' and sys_platform == 'linux') or (platform_machine == 'AMD64' and platform_python_implementation == 'CPython' and sys_platform == 'win32') or (platform_machine == 'ARM64' and platform_python_implementation == 'CPython' and sys_platform == 'win32')" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/2a/6a08d3a3ddbfd745f987c5ff22129a979ea74319fbfe5b9d59c7a6730d25/protobuf_py-0.6.0.tar.gz", hash = "sha256:be6b8d84ba54380ca5e5e50f20e4dd42df81c845cf9dc8c021f147d8c516b341", size = 161533, upload-time = "2026-09-30T06:45:41.388Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/d2/6271d5682d244b5b771f72fe7536a0c7e680e14300ea58f7d033fce813a8/protobuf_py-0.6.0-py3-none-any.whl", hash = "sha256:2aba52a2d017487901cb4b8a7983e64a7bd369128c2ca86cafa4dd03da31012e", size = 216054, upload-time = "2026-09-30T06:44:46.94Z" }, +] + +[[package]] +name = "protobuf-py-ext" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b4/6c/a1f6a1a50dfc3f14ddb9e54f9a769bc589b85b5e394bab2bb78f67d2dc42/protobuf_py_ext-0.6.0.tar.gz", hash = "sha256:03f61ccfcad52a2d784aec9bb0b6088e0e32f144a6c3ab3deed12f17f1f0d0bb", size = 58124, upload-time = "2026-09-30T06:45:42.602Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/f1/b02c02586e96ad8d4534187713d1fdfca32c471f09ed36dcef7c1debc0d6/protobuf_py_ext-0.6.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:6609132191e992ae7a211bf1988bb02c8fa323c915fb0b8b23914dae12c8cc97", size = 469480, upload-time = "2026-09-30T06:44:57.064Z" }, + { url = "https://files.pythonhosted.org/packages/29/c1/9f3203e0714e021e8d70c95c7a2a048bd4baacf9cf27ae1b5e2dc056fd45/protobuf_py_ext-0.6.0-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64a3075e96fea0107cbe2a0e7103dd35f5566e98088463b4572c70fb737bbede", size = 478330, upload-time = "2026-09-30T06:44:58.588Z" }, + { url = "https://files.pythonhosted.org/packages/07/5b/9520fc2d5be143cc9b54cda777551c60f1a9ea2c1efd0c0fa1701f3cb044/protobuf_py_ext-0.6.0-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e03b2aa768ea7185feab4267eee2a5d5fe5fa39c389f02689a6f5b7869c174c", size = 504049, upload-time = "2026-09-30T06:45:00.025Z" }, + { url = "https://files.pythonhosted.org/packages/6d/6f/7f69494f11c05c0cbc225d391cac2ae2b129be8be7818f48ef6ee4d7e407/protobuf_py_ext-0.6.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:afc419f235195e07d7c095c992935251b9207b2e0f6283c9324f9d6243eef401", size = 657357, upload-time = "2026-09-30T06:45:01.778Z" }, + { url = "https://files.pythonhosted.org/packages/ae/a1/f6048540d29565ebaedf70b5271da23173cc26c23edf09727b789ddd012e/protobuf_py_ext-0.6.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e0d5893a4d26e2c7e535a38c4e34be699469fcb182338626cc7bf4d6ea078", size = 716603, upload-time = "2026-09-30T06:45:03.629Z" }, + { url = "https://files.pythonhosted.org/packages/0c/7b/a0d587cdf856c3413f57a028621f469c749c3d23bbf29a947e9f258f97f0/protobuf_py_ext-0.6.0-cp311-abi3-win_amd64.whl", hash = "sha256:b5be43ebf9ec28aa7e3734bf07c0ea4ae50c2e7554b9299ea08433549052e8d2", size = 440643, upload-time = "2026-09-30T06:45:05.086Z" }, + { url = "https://files.pythonhosted.org/packages/c6/51/f29c03261a837072e8da59956fea097a57a1dab222cf11c07547446f01df/protobuf_py_ext-0.6.0-cp311-abi3-win_arm64.whl", hash = "sha256:a9e50e205f7c366febf1d12ab183b5a3034793441faaa57cab4fe9a8a1c0e1dc", size = 420484, upload-time = "2026-09-30T06:45:06.843Z" }, + { url = "https://files.pythonhosted.org/packages/54/71/131125d553e4bf526965ebc06a354f26d9ca3baf54118c5ad8421978b6e0/protobuf_py_ext-0.6.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a05c984234ad74b84e764fa7eb01b00c2258ed55195202ae67fdfa1b551533b5", size = 471478, upload-time = "2026-09-30T06:45:15.819Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d4/a7fcb16d83b7e37aa2561760faa78faca0be56ecef7bc2a9ce0b646c583f/protobuf_py_ext-0.6.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62f197bec48889c9824bb3f4597aa6b0d45dd2ba89680246d0ddd724aff49281", size = 479518, upload-time = "2026-09-30T06:45:17.16Z" }, + { url = "https://files.pythonhosted.org/packages/9c/69/f385581d4f0f25a9c0271de640be0183e725574ca1166d32bdf798740299/protobuf_py_ext-0.6.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:669519bdf23713494e7247ee89dbd57cf1b9c32071ee69fe0ab5e6eeadd42477", size = 501274, upload-time = "2026-09-30T06:45:18.707Z" }, + { url = "https://files.pythonhosted.org/packages/b3/45/e635f1a7e70438aa2f563321e30a11012cae05aa6ad6c2ac72bbd5369ad6/protobuf_py_ext-0.6.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df1eea6f53d6f78b031e94b08e98e2492d35cfb49d7d2b2639b2d472c48b4de7", size = 657613, upload-time = "2026-09-30T06:45:20.507Z" }, + { url = "https://files.pythonhosted.org/packages/29/a1/e698f967e10c377187064b0d67ea64f561d164e0a55d29385a180495b9a8/protobuf_py_ext-0.6.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8ebcf2610c2cee06f9fbcca8be352848bf4e927e1cf58d902905487a7e249c4b", size = 714222, upload-time = "2026-09-30T06:45:22.073Z" }, + { url = "https://files.pythonhosted.org/packages/6a/26/d09e892ab83420ddd52e5ae86507675c411a3fae038f022dc7e82eba7cf9/protobuf_py_ext-0.6.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:561bda3d588a0ef756cccaba546ef04dbd0bd167556afbd83aea2be76b05aadc", size = 469655, upload-time = "2026-09-30T06:45:23.908Z" }, + { url = "https://files.pythonhosted.org/packages/25/2e/f6c327625cb2ac1ba472452c5ae60563a126b9340ae8d46b7e5a6c2360db/protobuf_py_ext-0.6.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e55f9a1c4329c05638a4adfc1754be6c5aaef790965bf704b431e925f1801d71", size = 477421, upload-time = "2026-09-30T06:45:25.59Z" }, + { url = "https://files.pythonhosted.org/packages/cd/74/0078a5ed1fdebd82772817b8dfd5cc5fbec4c3c51d63356a5a7f9d0c53a7/protobuf_py_ext-0.6.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69aad94e7bda448b3d0f5752ace8f2d745562dba9e50f4c2e3b4fb3d2969d9bc", size = 499937, upload-time = "2026-09-30T06:45:27.392Z" }, + { url = "https://files.pythonhosted.org/packages/fb/92/f7b5a35d2c8804c13edeec22e9cbc084cff2c11ada58aeb896c97f9fc94b/protobuf_py_ext-0.6.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b26888854d3c4b55a161defaacfb2ef895e85662cee3cacd95d56efa29344e3b", size = 655561, upload-time = "2026-09-30T06:45:28.933Z" }, + { url = "https://files.pythonhosted.org/packages/01/c4/3c54994605801ac565f6f156a55e348c5f08cd5592650356e69f68d61c05/protobuf_py_ext-0.6.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:15888690bf4064ecc657623f98a13d8b4ee223cbd790c0b8c48f29530ac47c7e", size = 713156, upload-time = "2026-09-30T06:45:30.493Z" }, + { url = "https://files.pythonhosted.org/packages/41/6a/99667bcfc5a4b914095e8d522a31bdf0cde56655d3d29ff739fa412ed064/protobuf_py_ext-0.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6607d37c86f454f8fa02d54186730befe4f73fa19943d2b09123c9b3747856af", size = 458472, upload-time = "2026-09-30T06:45:31.955Z" }, + { url = "https://files.pythonhosted.org/packages/7a/5e/6b585e3dfaf1efc60c3566b51dcd9a52fb7de990386f041e9bf8e48d9fa8/protobuf_py_ext-0.6.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1cc666423d220b95fae7e6f92c77ed7245b78c71bc50191f671cae021a8db25", size = 466913, upload-time = "2026-09-30T06:45:33.594Z" }, + { url = "https://files.pythonhosted.org/packages/ea/51/ca7535b71b330e1ff8577d0e352ee48250c9df307644319bdced1b743b84/protobuf_py_ext-0.6.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f9a107f2d760b5a5ca7d240d32456bae358455c9195d28237ff6be679d3633e", size = 489980, upload-time = "2026-09-30T06:45:35.103Z" }, + { url = "https://files.pythonhosted.org/packages/45/8b/154c22e66322e99a1ffd94b6d2f7ece89da367fe9b57ad1b1a54a1c4d23c/protobuf_py_ext-0.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0c4827ce9f96a83b72c5db86466f149dcc16b2f15a7ee601e6b1938f9b04e44f", size = 646208, upload-time = "2026-09-30T06:45:36.831Z" }, + { url = "https://files.pythonhosted.org/packages/b7/9b/3a5a66f64a5262cbb2609a02d1ab74b2f4da90ea537ca260b10a8c3bb6e8/protobuf_py_ext-0.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:005b95df22c72366c30a9adc9dc2d962017a2f0779583da70bb29770b09ad346", size = 703607, upload-time = "2026-09-30T06:45:38.512Z" }, +] + [[package]] name = "psutil" version = "7.1.3" @@ -2017,6 +2442,63 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, ] +[[package]] +name = "pyqwest" +version = "0.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/24/2a/f8634142207082b1c559a499451db794c96130e586f1edb22dc1da4b05b2/pyqwest-0.12.0.tar.gz", hash = "sha256:83ff498bd53bee92a2d2bf14476679307a99c78421c02cae08ec113bed2fbd7d", size = 516105, upload-time = "2026-10-09T06:22:44.949Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/e9/bf753f3001ca4cc72986eb9de68c08c1042c010a1dc2e93be32b0290e04d/pyqwest-0.12.0-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:03e8b74a4f6ad32759f0cee9c79d7bae37d3e8e346a17fe6795de992995b23fd", size = 5331355, upload-time = "2026-10-09T06:21:17.876Z" }, + { url = "https://files.pythonhosted.org/packages/b1/a3/68e7e8a173a4b5f8a759d9a5a4a60c2feffe052969e9126f8a3448dc357e/pyqwest-0.12.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:c1af3aad4eef813efc43f40a6e4e2791643d42f9218b6f1e9f9bdaa2a19bdc7d", size = 5189665, upload-time = "2026-10-09T06:21:19.744Z" }, + { url = "https://files.pythonhosted.org/packages/9b/e5/407de30d15456583af485488dcb62c99fa24bcac6fd1521b57ef848aac27/pyqwest-0.12.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dd55f27e4d526babb30c0d0e19c5517d7d6ecb59ab4219a30a5115fac1288698", size = 5751518, upload-time = "2026-10-09T06:21:21.296Z" }, + { url = "https://files.pythonhosted.org/packages/29/9d/d7dcb2f3da4ea43c0e68894acea655063955d0c91401e74fb97f79c8265f/pyqwest-0.12.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:699662fca52a1595a1ed9c169816b7654c0b28308b9284091e96bbc79b8e408f", size = 5722887, upload-time = "2026-10-09T06:21:22.916Z" }, + { url = "https://files.pythonhosted.org/packages/28/dd/dee75b5c39132c49e24313eeb79caa6c178a1975578f7226aac4bed2dbe1/pyqwest-0.12.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1a8815ab7b61dc932379d42424e422a66df9634499095942e3b91bfd8f5e1dfd", size = 5904177, upload-time = "2026-10-09T06:21:24.768Z" }, + { url = "https://files.pythonhosted.org/packages/24/2c/4a75b193f01a70375e6af2e080a12a64fd2057d342f4cd71090adaf6b949/pyqwest-0.12.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:0d4828c8dfbfee936c951cc467f0197891a752eb0d22d7469f1adb02152c400a", size = 6095821, upload-time = "2026-10-09T06:21:26.781Z" }, + { url = "https://files.pythonhosted.org/packages/d9/da/b618b54b17ed111b0bfd3a89d74aa4477ca3cb35dfe73d5b0f9916c6c07d/pyqwest-0.12.0-cp311-abi3-win_amd64.whl", hash = "sha256:3958c9ea1316e9b71290006f044e9c020daec5b5b652629bae9d2b26b6d5294d", size = 4969600, upload-time = "2026-10-09T06:21:28.615Z" }, + { url = "https://files.pythonhosted.org/packages/1c/74/2ef8a770aa6152da60c201e091cd5c692a591641e1c485348caed8591015/pyqwest-0.12.0-cp311-abi3-win_arm64.whl", hash = "sha256:ab6f54ea17a34628f631ca7b1e010f0dd5891f75fd9b743afcbdf1d29740120d", size = 4943859, upload-time = "2026-10-09T06:21:30.151Z" }, + { url = "https://files.pythonhosted.org/packages/88/49/494866a18aecda4593a52ca62397d53fb29938033075d41c009291096fe6/pyqwest-0.12.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:ba88e09d0cb6166b72a4a120a10df863f10c77dac8f53538a7ac266798e14167", size = 5344568, upload-time = "2026-10-09T06:21:31.89Z" }, + { url = "https://files.pythonhosted.org/packages/84/ee/09486016196b4a367286a8819e83185b78f771b01a3ca419e94d5c5589e6/pyqwest-0.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f2843c2f257db63580096f9bd4e83225a7c660a946693ec87777f906f5cdd21d", size = 5192763, upload-time = "2026-10-09T06:21:33.423Z" }, + { url = "https://files.pythonhosted.org/packages/cd/0d/42cb6e771aabb8745ccc7184bd132e4471a8659cdd7bee65f05dc94f0a78/pyqwest-0.12.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:6319e35a3f867a1a57071cfe63364b9f35b38d4ae3a6231fcab5d3a82e9d5705", size = 5751522, upload-time = "2026-10-09T06:21:35.123Z" }, + { url = "https://files.pythonhosted.org/packages/4c/a7/d09ef94ee9811ca7a698f2947a25618a60226c13f3e9193a2bc1880c4178/pyqwest-0.12.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:ba001c5f06a92426635d86c2ccb8a4c732f2bb1424361c68161f8726b5141520", size = 5714017, upload-time = "2026-10-09T06:21:36.672Z" }, + { url = "https://files.pythonhosted.org/packages/f9/b8/0770a8eaa97e845cbdf1e5fd9af36235580aae18f63d7e6ed0f257ffde01/pyqwest-0.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:39a7e94b109bc10b05a6151308bac6ead404a900768861216467544337a971d6", size = 5903886, upload-time = "2026-10-09T06:21:38.202Z" }, + { url = "https://files.pythonhosted.org/packages/df/f6/ad021859fb8380541604822ec6252c6e721f504b108e805b533ae8083a1f/pyqwest-0.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b8527b9795d7e72efaf0f2fcba6b293f85e1720201e44f94889ca33d263079e9", size = 6091049, upload-time = "2026-10-09T06:21:40.333Z" }, + { url = "https://files.pythonhosted.org/packages/0f/25/1167eab0e8fa4c07f8bbec5931749f7932d13bfea1ea3daa6dc78c67b8e1/pyqwest-0.12.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:d9f952e66e848612f5222794112f203a51070f93ca7129c4df5d6c7ef8f4762c", size = 5351261, upload-time = "2026-10-09T06:21:41.989Z" }, + { url = "https://files.pythonhosted.org/packages/d2/10/29daf8cffe7c547121d29b96a5b46d59fbd8f21bd80826c54a28cd89bc75/pyqwest-0.12.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4371bd32687cac6cd31ae61a424b3c072450fb9e1d82ad568c234bc9935e2db8", size = 5195343, upload-time = "2026-10-09T06:21:43.773Z" }, + { url = "https://files.pythonhosted.org/packages/bc/25/905f500e78216a729d69d24ccdf7f7a4c722321755dac97825620a98a366/pyqwest-0.12.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:469ac955bccf7080c6ed92afdd023c633d6d704b4be0653b82741fda3a60c85b", size = 5753112, upload-time = "2026-10-09T06:21:45.194Z" }, + { url = "https://files.pythonhosted.org/packages/34/93/b12629c268e743ed6cf39a08a78a696cbdb10d910fbcf4b0de6b0f89f143/pyqwest-0.12.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:05af8d0c6eb67661d6896824183b714a908321b3d10644594268b51d21a9cb1c", size = 5713673, upload-time = "2026-10-09T06:21:46.687Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c3/b0f689e60356143661c93e7295edd959a738e0d42a101e4c3647045d23c9/pyqwest-0.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:5f1f6599c0f826fd7e6717a3c6709d9fc25e1debc9e70bbb4b613c25377937c8", size = 5905698, upload-time = "2026-10-09T06:21:48.297Z" }, + { url = "https://files.pythonhosted.org/packages/f1/d6/412b69950180abf077c2dbf0b8ddf425a5fd65466e775bfbd63e28aefd1a/pyqwest-0.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d4de6b62293d57153db308897988afdfd22e481c4399279c6332c9efbc183330", size = 6090438, upload-time = "2026-10-09T06:21:50.131Z" }, + { url = "https://files.pythonhosted.org/packages/de/c0/1f6fca89081737afd8842bf6b4a6e6014dedd8d7d44b4da2d73f502c3b4c/pyqwest-0.12.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:34399a6b4c2d03ece732d41ed051710ea35c70f7daed9590ce9916db6cde92e6", size = 5336542, upload-time = "2026-10-09T06:21:51.839Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0f/7f364ac04e01258e0cfc37170a558cb36aaacff110f1a4c9d2f887e64be9/pyqwest-0.12.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1335062111f4da599d58223eeeb051f839a2c588274258facf6091aca92201a3", size = 5174602, upload-time = "2026-10-09T06:21:53.489Z" }, + { url = "https://files.pythonhosted.org/packages/11/9f/cb9c8ad10dc43e0b854041f4b471fde59624e8407c9c6da00f8907933c57/pyqwest-0.12.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:218f7a00d01678c7f5f02a0248d3b601d87d9f882179305f0c8f0545eb88d733", size = 5731531, upload-time = "2026-10-09T06:21:55.061Z" }, + { url = "https://files.pythonhosted.org/packages/52/2e/30a2b41d340d07f8205d02997a24c518c031d09898c8004ad671a8c9bc8b/pyqwest-0.12.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:27843d48330e2a163cf875e05294d9e4ea0e0e91bdbea956e3af630b678bd7d8", size = 5704167, upload-time = "2026-10-09T06:21:56.687Z" }, + { url = "https://files.pythonhosted.org/packages/60/c3/8c870cbf47857abd8490b8da2195cad71c3890556d776ecc01cd744bb35c/pyqwest-0.12.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4cdb0f156e754caead9b8cf88fc2f2806ed81282960c9a39f8a07847110745fe", size = 5882872, upload-time = "2026-10-09T06:21:58.362Z" }, + { url = "https://files.pythonhosted.org/packages/e3/e4/45017cda255c95797c5a4e6da2f87a8035f368aa9db50d9c45abcbd97387/pyqwest-0.12.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3208f670e601ddfbf6ff689c7717236dfeaf175fa934319d30a6c57df0e1c14c", size = 6074173, upload-time = "2026-10-09T06:21:59.918Z" }, + { url = "https://files.pythonhosted.org/packages/a3/d1/955421179cbd93070cd01769e6f937826c42d377fab9fade00172601e138/pyqwest-0.12.0-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:f7476abcdac195e33165e38e4e76c9c6e90785e14033b766912f24ff8f829198", size = 5398661, upload-time = "2026-10-09T06:22:01.5Z" }, + { url = "https://files.pythonhosted.org/packages/e0/b0/c7474b1650d3bafe94f28f87446b0e4371d553acdfa79d2ca7e8151d1982/pyqwest-0.12.0-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:faf377620445e9988b2e290fb805dd3a04ffa62e0f6d1da11e269cecc1d5a837", size = 5241423, upload-time = "2026-10-09T06:22:03.084Z" }, + { url = "https://files.pythonhosted.org/packages/b8/11/3bdeb82f4425ed42d5ee877003974de7a716990f8681c6a637046a3d49eb/pyqwest-0.12.0-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:90fee9cd448324fbdc4ed2dff787749f19ac17315aa9e18d88ced0b843fc8dac", size = 5791319, upload-time = "2026-10-09T06:22:04.628Z" }, + { url = "https://files.pythonhosted.org/packages/9a/b5/024555a08ab26bdadbe63b7acb6115a5d0e536efa94c4c71c4bbce445a45/pyqwest-0.12.0-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:ae7b73b47e3fb2462a5292417d49e7780ce2dc9857e8bce7276ea2f090a90768", size = 5764278, upload-time = "2026-10-09T06:22:06.309Z" }, + { url = "https://files.pythonhosted.org/packages/61/44/57bba169ee17c02dfd0b2a6508ed4c027e0d14ee3765fa3c641399521e2d/pyqwest-0.12.0-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:5f83e15b7bebc761e7b6a0f1cbac334d058e698fb49ba1328306660b00c3c92d", size = 5939856, upload-time = "2026-10-09T06:22:07.915Z" }, + { url = "https://files.pythonhosted.org/packages/69/66/926b90a270e9893958840fa610531b5a0d47604e088e1c60305c13890938/pyqwest-0.12.0-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:8057eddcae424eadbf15c9ab67f4aebdc2ce5f2cf1d8d4f6c34ca9d9ab62fef5", size = 6139873, upload-time = "2026-10-09T06:22:09.498Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6c/a09b970067a60c721ad063107ac10917e31475211bb3d66995d1482331a2/pyqwest-0.12.0-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:6c71f9f32cd344ac4f136010ea6a307201547123bce6ba61272421c93fb4fc6d", size = 5023514, upload-time = "2026-10-09T06:22:11.179Z" }, + { url = "https://files.pythonhosted.org/packages/d7/e5/06c6540c29596fe9230ccc160822309592b1ec85a05482105e1de6b98d14/pyqwest-0.12.0-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:9ca7d29a469326604f7c048bc19d4592970fbc7ac494bc253ef0779a57e1a9aa", size = 4991803, upload-time = "2026-10-09T06:22:12.867Z" }, + { url = "https://files.pythonhosted.org/packages/fb/4e/3f4e924e430cd859df54395b7a9ef21c342f544dc4d4da3af5b3f44c1e08/pyqwest-0.12.0-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:dca14e8c4aa3d79601c91e408e33df0bf463c8166539b394715637f49f90362a", size = 5348621, upload-time = "2026-10-09T06:22:14.718Z" }, + { url = "https://files.pythonhosted.org/packages/90/f2/da5defcc9d231ab9e4ce267704d1e18544b4d93c1cfb4e3c29a9a35892d3/pyqwest-0.12.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:729a81368abcbe6be708a96da1ec8802969082ffaab35d1e08a520e3c0453a49", size = 5195352, upload-time = "2026-10-09T06:22:16.435Z" }, + { url = "https://files.pythonhosted.org/packages/bc/98/871c99fdedaff6458a519e37d49400617beebf9c6560ed5e73bf02b972fe/pyqwest-0.12.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:645072f8e9138556e27f6c9384dd1fe0bdef3f1be50534d2c1d9ffeea9a53de9", size = 5752432, upload-time = "2026-10-09T06:22:18.111Z" }, + { url = "https://files.pythonhosted.org/packages/0c/4e/d1686f7b74c2b5cb3f7939a397aa215054db3cb515a99503fe6e361ddc96/pyqwest-0.12.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:d302f7bf0c072397aafe2cbe6a398743e5c37add3500048bbe0b26bba823075f", size = 5714200, upload-time = "2026-10-09T06:22:19.542Z" }, + { url = "https://files.pythonhosted.org/packages/6f/15/d4a71b8b9c85f74174f3e58ebe6cc34cf2d9175424da84af91088521b74f/pyqwest-0.12.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:ead61088e5226349624965025d297ed893dd947e05ccf315006322c824ef9a4e", size = 5903794, upload-time = "2026-10-09T06:22:21.083Z" }, + { url = "https://files.pythonhosted.org/packages/f4/b1/01928c04841f525748b4b21e46e29a473cc1b9d2c800abcf70d5d6a12b56/pyqwest-0.12.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:db85298d530bcd50234c48dfd18facbe75a3c8f0f16e04ace389617d48a3093f", size = 6091500, upload-time = "2026-10-09T06:22:22.713Z" }, + { url = "https://files.pythonhosted.org/packages/b1/ac/181acdc359a7cc293a4fe579ac315e72676654bd6f5ead732304caa125d0/pyqwest-0.12.0-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:f87b16a83d8e0bc5b639163819c53a9e975ea2fbeec133ecbd2ffce0796dab27", size = 5336607, upload-time = "2026-10-09T06:22:24.437Z" }, + { url = "https://files.pythonhosted.org/packages/74/b1/7365631700810ec294119e3b405bd20d9897774979b021c5dc2c002e4773/pyqwest-0.12.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:fa846050a669d0eacc2c2aa425eda403b3e443a374ff84c5c16c9f82624a2a9d", size = 5174497, upload-time = "2026-10-09T06:22:25.915Z" }, + { url = "https://files.pythonhosted.org/packages/dd/8d/08bc79d4d1c3af8c24b78163c277226cc26c3e751d1bc562db8baf67ee7e/pyqwest-0.12.0-cp315-cp315t-manylinux_2_28_aarch64.whl", hash = "sha256:5f08aeda136ffcb454c11b1e7bb98a85853ea779424cc30e98a9e5e762b84fea", size = 5731691, upload-time = "2026-10-09T06:22:27.408Z" }, + { url = "https://files.pythonhosted.org/packages/39/65/e1a9cc3cf610122149ccd3a1e264ce6b47aa8bcf3d4abd5f27ba7acdd97f/pyqwest-0.12.0-cp315-cp315t-manylinux_2_28_x86_64.whl", hash = "sha256:0b770869bae9e15c13e8823ed71155d2a9e7fa3f4cba3da65d2b4b668e5b3a1e", size = 5704613, upload-time = "2026-10-09T06:22:29.387Z" }, + { url = "https://files.pythonhosted.org/packages/c4/75/a8b3fd3de3eab08377ae71fde7efa5526b7f43cf094eb8b028dd8acee275/pyqwest-0.12.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:a25bdfbdc859d937de818db60d839f523c9d4792c2b85a23667861cc0791246e", size = 5886056, upload-time = "2026-10-09T06:22:31.298Z" }, + { url = "https://files.pythonhosted.org/packages/93/23/6982518b22268151fdb7ae27d3700a3c4fdb0699d4783caaace1e1fa6deb/pyqwest-0.12.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:ac6a7de3327ee56e736108ab72a21efc5dfddd41ce4c19e4469c1e8448a6696e", size = 6074658, upload-time = "2026-10-09T06:22:32.919Z" }, +] + [[package]] name = "pytest" version = "9.0.3" @@ -2093,6 +2575,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61", size = 21230, upload-time = "2025-10-26T15:12:09.109Z" }, ] +[[package]] +name = "python-engineio" +version = "4.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "simple-websocket" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/91/b565807fa922620ed575b412bf978d032e4c5d79558b400fdbfe1008c95a/python_engineio-4.15.1.tar.gz", hash = "sha256:1a412e798a90c64fabe7b20798939ae6d4a03f34da8a33d5a21a25a8bbbc3612", size = 93104, upload-time = "2026-10-28T23:53:30.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/5c/552f1dce2443dc33ce6b2ee7f67599ba5c83ef5582e2235a9878ac239049/python_engineio-4.15.1-py3-none-any.whl", hash = "sha256:745cb3c07701c719a2906439a00b67e44ef042904036568530a2c3a8cf18c89a", size = 60775, upload-time = "2026-10-28T23:53:29.304Z" }, +] + [[package]] name = "python-multipart" version = "0.0.32" @@ -2102,6 +2596,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, ] +[[package]] +name = "python-socketio" +version = "5.18.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "bidict" }, + { name = "python-engineio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/b9/4ab083267c1f8ece9a9e019808412af7c7a03321411a1263cd863321baa2/python_socketio-5.18.1.tar.gz", hash = "sha256:07ab0e7c94a52833a67af1dc05958ad6fb0de69adc87de9198a61b22588c40d4", size = 132106, upload-time = "2026-11-03T12:19:20.723Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/e2/cdc630aef7da00698af4ed061ff873297f806cc049cc5c67bf89077cb7e4/python_socketio-5.18.1-py3-none-any.whl", hash = "sha256:e768c473842be811f6dd24bd0a49cd0d4a0c994182e3eba23cc9d1489c7a697e", size = 83022, upload-time = "2026-11-03T12:19:19.136Z" }, +] + +[package.optional-dependencies] +asyncio-client = [ + { name = "aiohttp" }, +] +client = [ + { name = "requests" }, + { name = "websocket-client" }, +] + [[package]] name = "pytz" version = "2025.2" @@ -2358,6 +2874,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/a8/001d4a7c2b37623a3fd7463208267fb906df40ff31db496157549cfd6e72/ruff-0.12.11-py3-none-win_arm64.whl", hash = "sha256:bae4d6e6a2676f8fb0f98b74594a048bae1b944aab17e9f5d504062303c6dbea", size = 12135290, upload-time = "2025-08-28T13:59:06.933Z" }, ] +[[package]] +name = "s3transfer" +version = "0.19.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/c5/0a5ce655109cb96b8dfad36908a4e108d15f5dc2eff8fbc3146c4312e704/s3transfer-0.19.3.tar.gz", hash = "sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406", size = 166087, upload-time = "2026-10-19T19:19:49.751Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" }, +] + [[package]] name = "scipy" version = "1.18.0" @@ -2399,6 +2927,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/19/969dc072906c84dd0a3b05dcf57ea750936087d7873549e408b35cfc3f97/scipy-1.18.0-cp314-cp314t-win_arm64.whl", hash = "sha256:368e0a705903c466aa5f08eefb39e6b1b6b2d659e7352a31fd9e2438365be0f8", size = 25279661, upload-time = "2026-06-19T15:01:40.817Z" }, ] +[[package]] +name = "simple-websocket" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wsproto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/d4/bfa032f961103eba93de583b161f0e6a5b63cebb8f2c7d0c6e6efe1e3d2e/simple_websocket-1.1.0.tar.gz", hash = "sha256:7939234e7aa067c534abdab3a9ed933ec9ce4691b0713c78acb195560aa52ae4", size = 17300, upload-time = "2024-10-10T22:39:31.412Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/59/0782e51887ac6b07ffd1570e0364cf901ebc36345fea669969d2084baebb/simple_websocket-1.1.0-py3-none-any.whl", hash = "sha256:4af6069630a38ed6c561010f0e11a5bc0d4ca569b36306eb257cd9a192497c8c", size = 13842, upload-time = "2024-10-10T22:39:29.645Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -2456,6 +2996,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/39/85e5be4e9a912022f86f38288d1f4dd2d100b60ec75ebf3da37ca0122375/tavily_python-0.7.21-py3-none-any.whl", hash = "sha256:acfb5b62f2d1053d56321b4fb1ddfd2e98bb975cc4446b86b3fe2d3dd0850288", size = 17957, upload-time = "2026-01-30T16:57:32.278Z" }, ] +[[package]] +name = "tenacity" +version = "9.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/9e/497c1c8ebe5a5b5d1d4a7511aea22c0bb1a97e3170d98abdef0e1b34265a/tenacity-9.2.1.tar.gz", hash = "sha256:a606b5c808d0cded4a359d5b9932d867ff2a6a6b64d37350260fd01bbdf83839", size = 58261, upload-time = "2026-10-07T12:13:01.633Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/26/1ff2b0721ac66a3ec5b1402b333110b352ab0a8724052ac279a7b82d40c4/tenacity-9.2.1-py3-none-any.whl", hash = "sha256:9e56f17539296baab7beabb08b92f6ee3d7be92d8be72d763360677c2ad6580e", size = 32310, upload-time = "2026-10-07T12:13:00.102Z" }, +] + [[package]] name = "tiktoken" version = "0.13.0" @@ -2496,6 +3045,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/5b/f2aa703a4fc5d2dff73460a7d46cc2f3f44aa0f3dd8eeb20d2a0ecf68862/tiktoken-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:85b78cc3a2c3d48723ca751fa981f1fedccd54194ca0471b957364353a898b07", size = 918110, upload-time = "2026-05-15T04:51:17.237Z" }, ] +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, +] + [[package]] name = "tomlkit" version = "0.15.2" @@ -2597,6 +3155,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/06/04c8e804f813cf972e3262f3f8584c232de64f0cde9f703b46cf53a45090/virtualenv-20.34.0-py3-none-any.whl", hash = "sha256:341f5afa7eee943e4984a9207c025feedd768baff6753cd660c857ceb3e36026", size = 5983279, upload-time = "2025-08-13T14:24:05.111Z" }, ] +[[package]] +name = "wcmatch" +version = "11.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "bracex" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bd/07/b7a9162a804bc0f7d03fe84c93e374d3e1c4d828043d5f9b797f7d4e30e0/wcmatch-11.2.tar.gz", hash = "sha256:1605bde546fbde4c063628c7d89eda486909fd4337e51d73a873987fbcb2554c", size = 134939, upload-time = "2026-10-22T00:37:57.622Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/96/b6cebd4bd75320743a291fed494c79f07c01b58fa9b187d12aeb5110f002/wcmatch-11.2-py3-none-any.whl", hash = "sha256:952bab83c75c233836007a69550a3a8753cf06964c1eb3b84ff11ad5bff1c272", size = 44361, upload-time = "2026-10-22T00:37:56.293Z" }, +] + +[[package]] +name = "websocket-client" +version = "1.9.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/cb/a5abcc2891249f393827c650c6296660ce40374ac22d99ab9aea41f9d2a2/websocket_client-1.9.2.tar.gz", hash = "sha256:0fcb57545848be86992e128218fd96dd87a6769ffdb1a968dff79632b85604d0", size = 84110, upload-time = "2026-08-31T14:08:40.964Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/d2/cc4dc1271e464942db7ee278baae2daa99ee77cb2af744025c04da585a3e/websocket_client-1.9.2-py3-none-any.whl", hash = "sha256:e1a673830a9c7bfa47b1cd3d5e4178f4c9651d80a4eab02c9c23a1c3ec6250ce", size = 95786, upload-time = "2026-08-31T14:08:39.899Z" }, +] + [[package]] name = "wrapt" version = "1.17.3" @@ -2636,6 +3215,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, ] +[[package]] +name = "wsproto" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/79/12135bdf8b9c9367b8701c2c19a14c913c120b882d50b014ca0d38083c2c/wsproto-1.3.2.tar.gz", hash = "sha256:b86885dcf294e15204919950f666e06ffc6c7c114ca900b060d6e16293528294", size = 50116, upload-time = "2025-11-20T18:18:01.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/f5/10b68b7b1544245097b2a1b8238f66f2fc6dcaeb24ba5d917f52bd2eed4f/wsproto-1.3.2-py3-none-any.whl", hash = "sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584", size = 24405, upload-time = "2025-11-20T18:18:00.454Z" }, +] + [[package]] name = "xxhash" version = "3.6.0" From d3255ed448c3078a8a45e1d4f2d545042605341b Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 4 Nov 2026 09:50:00 -0800 Subject: [PATCH 08/22] fix(browsecomp_agent): run the research loop in a task sandbox Signed-off-by: yaoyu-33 --- benchmarks/browsecomp/README.md | 10 +- benchmarks/browsecomp/config.yaml | 27 +- .../browsecomp/config_single_agent_turn.yaml | 20 +- .../agent-server/add-or-update-a-harness.mdx | 1 + .../integrate-existing-agents.mdx | 3 +- .../browsecomp_agent/README.md | 43 +- responses_api_agents/browsecomp_agent/app.py | 210 ++++++- .../configs/browsecomp_agent.yaml | 4 +- .../browsecomp_agent/pyproject.toml | 2 +- .../browsecomp_agent/sandbox.py | 126 ++++ .../browsecomp_agent/sandbox_runner.py | 116 ++++ .../tests/test_agent_sessions.py | 51 +- .../browsecomp_agent/tests/test_app.py | 1 + .../tests/test_malformed_tool_arguments.py | 1 + .../tests/test_progress_tracking.py | 1 + .../tests/test_responses_exception_logging.py | 1 + .../tests/test_run_survives_sample_failure.py | 1 + .../browsecomp_agent/tests/test_sandbox.py | 232 +++++++ responses_api_agents/browsecomp_agent/uv.lock | 595 +++++++++++++++++- 19 files changed, 1359 insertions(+), 86 deletions(-) create mode 100644 responses_api_agents/browsecomp_agent/sandbox.py create mode 100644 responses_api_agents/browsecomp_agent/sandbox_runner.py create mode 100644 responses_api_agents/browsecomp_agent/tests/test_sandbox.py diff --git a/benchmarks/browsecomp/README.md b/benchmarks/browsecomp/README.md index 04842cd2064..85145f241d0 100644 --- a/benchmarks/browsecomp/README.md +++ b/benchmarks/browsecomp/README.md @@ -38,16 +38,16 @@ ng_e2e_collect_rollouts \ ## Run through the Environment Server -`config_single_agent_turn.yaml` runs the same benchmark through the `single_agent_turn_legacy` Environment Server instead of the `legacy_agent` relay. A crash then repeats only the current step (seed, agent turn, agent close, verify, close) rather than the whole episode. Use it in place of `config.yaml` in step 3: +`config.yaml` now runs the benchmark through `single_agent_turn_legacy` with sandboxed agent and resources tool execution. It requires a Docker daemon by default; compose another provider config to change the provider. `config_single_agent_turn.yaml` is a compatibility alias. A crash then repeats only the current step (seed, agent turn, agent close, verify, close) rather than the whole episode. Use it in place of `config.yaml` in step 3: ```bash config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ benchmarks/browsecomp/config_single_agent_turn.yaml" ``` -Correct answers are judged the same way on both paths. Errors are handled differently, so the counts can differ: +Tasks, prompts and correct-answer grading are unchanged. Compared with the previous host `/run` recipe, failure handling changes the scored denominator: -| Case | `config.yaml` (`/run`) | `config_single_agent_turn.yaml` | +| Case | Previous host `/run` recipe | Current sandbox session recipe | | --- | --- | --- | | Rollout reaches `rollout_timeout_s` | Reward 0 without a judge call, `timed_out: 1` | The same | | Model call returns 400, 413 or 422 | Reward 0, with `agent_error` | Verified with an empty answer, which scores 0 | @@ -56,4 +56,6 @@ Correct answers are judged the same way on both paths. Errors are handled differ | Model call returns any other 4xx | Reward 0, with `agent_error` | A terminal failure row, not scored and not retried | | Any other error in the agent turn | Reward 0, with `agent_error` | A non-terminal failure row, which a resume retries | -So an error that `config.yaml` scored 0 now leaves the scored set instead of counting as a wrong answer. Check the failure rows and the coverage file alongside the accuracy. +So an error that the previous recipe scored 0 now leaves the scored set instead of counting as a wrong answer. Check the failure rows and the coverage file alongside the accuracy. + +The [BrowseComp adapter README](../../responses_api_agents/browsecomp_agent/README.md) documents Evaluation capability, explicit evidence gaps, sandbox ownership and unsupported controls. For a hosted model without `/tokenize`, set `save_model_call_using_vllm_tokenize_endpoint: false` explicitly. The research agent and read-only shell each run in an independently owned sandbox; the Resources Server keeps search credentials and verification on the host. diff --git a/benchmarks/browsecomp/config.yaml b/benchmarks/browsecomp/config.yaml index ee2ad692fae..dd7bdbb1ceb 100644 --- a/benchmarks/browsecomp/config.yaml +++ b/benchmarks/browsecomp/config.yaml @@ -1,3 +1,8 @@ +config_paths: +- nemo_gym/sandbox/providers/docker/configs/docker.yaml + +use_absolute_ip: true + # Judge: a hosted reasoning model, independent of the policy under test. Keeping # the judge off the policy endpoint stops a weak policy from grading itself. judge_model: @@ -28,6 +33,11 @@ browsecomp_benchmark_resources_server: # terminal mode: search/browse write pages to a per-session disk workspace # and the read-only bash_command tool is exposed for the model to grep/read them. workspace: per_session + sandbox_provider: sandbox + sandbox_config: + image: python:3.13-slim + workdir: /workspace + ttl_s: 7200 use_judge: true judge_model_server: type: responses_api_models @@ -59,9 +69,12 @@ browsecomp_benchmark_agent: # Estimate prompt tokens via the vLLM /tokenize endpoint before each model # call to decide context reset (skips the otherwise-wasted generation). save_model_call_using_vllm_tokenize_endpoint: true - resources_server: - type: resources_servers - name: browsecomp_benchmark_resources_server + resources_server: null + sandbox_provider: sandbox + sandbox_config: + image: python:3.13-slim + workdir: /sandbox + ttl_s: 7200 model_server: type: responses_api_models name: policy_model @@ -75,8 +88,14 @@ browsecomp_benchmark_agent: browsecomp_benchmark_environment_server: environment_servers: - legacy_agent: + single_agent_turn_legacy: entrypoint: app.py + resources_server: + type: resources_servers + name: browsecomp_benchmark_resources_server agent_server: type: responses_api_agents name: browsecomp_benchmark_agent + resources_tool_transports: [direct_http] + default_episode_timeout_seconds: 6000 + cleanup_timeout_seconds: 300 diff --git a/benchmarks/browsecomp/config_single_agent_turn.yaml b/benchmarks/browsecomp/config_single_agent_turn.yaml index dfb7bd2d42b..a7036d8e3f2 100644 --- a/benchmarks/browsecomp/config_single_agent_turn.yaml +++ b/benchmarks/browsecomp/config_single_agent_turn.yaml @@ -1,21 +1,3 @@ -# BrowseComp through the single_agent_turn_legacy Environment Server instead of the legacy_agent relay. -# The servers and dataset are config.yaml's. The Environment Server seeds the harness session and the agent session, -# grants the agent direct HTTP access to the harness tools, runs the agent turn, closes the agent, verifies, and -# closes the harness session, so a crash repeats only the current step. +# Compatibility name: BrowseComp now uses sandboxed agent sessions by default. config_paths: - benchmarks/browsecomp/config.yaml - -browsecomp_benchmark_environment_server: - environment_servers: - _delete_key: legacy_agent - single_agent_turn_legacy: - entrypoint: app.py - resources_server: - type: resources_servers - name: browsecomp_benchmark_resources_server - agent_server: - type: responses_api_agents - name: browsecomp_benchmark_agent - # The agent's search, browse and bash_command calls reach its harness session through this grant. - resources_tool_transports: - - direct_http diff --git a/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx b/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx index f38de2ae882..229103ded41 100644 --- a/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx +++ b/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx @@ -262,6 +262,7 @@ The current adapters support the following tool sources: | Adapter | Built-in tools | Resources Server tool grants | Model API | | --- | --- | --- | --- | +| BrowseComp | `update_progress` when configured. | At most one `direct_http` grant; required MCP grants are rejected. The research tools remain on the Resources Server, which isolates `bash_command` in its own sandbox. | Responses | | Hermes | Hermes toolsets, selected with `enabled_toolsets` and `disabled_toolsets`, for example `[terminal]`. | `mcp` only. Each granted MCP server becomes a Hermes toolset with the same name. Required `direct_http` grants are rejected. | Chat Completions | | Pi | Pi's built-in sandbox tools. `sandbox_bash_timeout_seconds` limits each bash call. | None in agent sessions. Required grants are rejected; keep `resources_tool_transports: []`. | Chat Completions | | Codex | Codex's shell and file tools. Agent sessions require `sandbox_mode: danger-full-access`, because the task sandbox supplies isolation. | `mcp` only. Each granted MCP server becomes a Codex `mcp_servers` entry with the same name. Required `direct_http` grants are rejected. | Responses (streaming) | diff --git a/fern/versions/latest/pages/agent-server/integrate-existing-agents.mdx b/fern/versions/latest/pages/agent-server/integrate-existing-agents.mdx index 18d8bfceb0c..7a94c009e1d 100644 --- a/fern/versions/latest/pages/agent-server/integrate-existing-agents.mdx +++ b/fern/versions/latest/pages/agent-server/integrate-existing-agents.mdx @@ -14,10 +14,11 @@ To use an agent that Gym does not integrate yet, see [Add an agent that is not l ## Sandboxed harnesses -These adapters run a coding harness inside a task sandbox that a Resources server provides, and are driven by the `single_agent_turn` Environment Server. Each row summarizes the adapter's README; the README is the reference for supported features, runtime requirements, and limits. +These adapters run a harness inside a sandbox and are driven by the `single_agent_turn` Environment Server. They borrow a task sandbox when the Resources server supplies one; otherwise a configured provider creates an agent-owned sandbox. Each row summarizes the adapter's README; the README is the reference for supported features, runtime requirements, and limits. | Harness | Model API | Resources tool grants | Token IDs and logprobs | SWE-bench Pro | Terminal Bench 2.1 | | --- | --- | --- | --- | --- | --- | +| BrowseComp (`browsecomp_agent`) | Responses | Direct HTTP | Preserved from Model Server responses; sandbox path qualified for evaluation only | Not declared | Not declared | | Hermes (`hermes_agent`) | Chat Completions | MCP only | Returned in a result file that the model can also write; evaluation only | Declared | Not declared | | Pi (`pi_agent`) | Chat Completions | None | Not wired up; evaluation only | Declared | Declared | | Codex (`codex_agent`) | Responses (streaming) | MCP only | Not wired up; evaluation only | Declared | Declared | diff --git a/responses_api_agents/browsecomp_agent/README.md b/responses_api_agents/browsecomp_agent/README.md index 9ca2a19db2d..eaa021c9058 100644 --- a/responses_api_agents/browsecomp_agent/README.md +++ b/responses_api_agents/browsecomp_agent/README.md @@ -1,21 +1,34 @@ -# Description +# BrowseComp agent -The BrowseComp research agent, paired with the [`browsecomp_advanced_harness`](../../resources_servers/browsecomp_advanced_harness) resources server. Its loop calls the harness's `search`, `browse` and `bash_command` tools. It compacts old tool results (`keep_rounds`) and resets the context past a token threshold (`context_reset_tokens`). With `progress: true` it keeps a progress board that survives the resets. With `rollout_timeout_s` set, it stops at a wall-clock limit. +The repository-native research loop runs inside a task sandbox through an agent session. Its version is the Gym source revision: seed uploads that exact source and a private Python 3.13.14 runtime with dependencies pinned to the Agent Server environment. It preserves the progress board, tool-result compaction, context resets, empty-answer retries, and Responses API model calls. -It runs an episode in one of two ways: +## Execution and composition -- **Through an Environment Server** (`benchmarks/browsecomp/config_single_agent_turn.yaml`). `single_agent_turn_legacy` seeds the harness session and an agent session, calls `/v1/responses` once, closes the agent session, then verifies and closes the harness session. - - The seed must grant exactly one direct HTTP tool access to the harness (`resources_tool_transports: [direct_http]`), and no sandbox or required MCP access. Any other seed is refused with a final 422. - - Tool calls go to the grant's URL with its headers and cookies. The close returns the harness cookies, which the Environment Server sends to `/verify`. - - `max_run_retries` retries an empty final answer (after `` stripping) inside the one activation, keeping the harness session, as `/run` does. - - A model call that fails with an HTTP error returns a `failed` response that names the status (`model_request_error`). The Environment Server's outcome policy then verifies 400, 413 and 422, records 408, 425, 429 and 5xx as non-terminal failures that a resume retries, and fails the episode on any other status. Other errors, such as a dropped connection, fail the activation with a 500, which is also recorded as a non-terminal failure. -- **Through its own `/run`** (`benchmarks/browsecomp/config.yaml`, behind the `legacy_agent` relay). `/run` seeds the harness, retries an empty answer, and verifies. A rollout stopped at `rollout_timeout_s` scores 0 without a verify. An error scores 0 with `agent_error`, except an infrastructure failure (5xx, a dropped connection, a timeout), which is set aside unscored for resume. +`benchmarks/browsecomp/config.yaml` runs `single_agent_turn_legacy`: resources seed → agent seed → one activation → agent close → verify → resources close. `config_single_agent_turn.yaml` remains an alias. Docker is the default provider; compose another provider config to use Enroot or OpenSandbox. Both the Model Server and granted Resources endpoints must be reachable from the agent sandbox (`use_absolute_ip: true`). -On both paths, a rollout stopped at `rollout_timeout_s` scores 0 without a judge call and reports `timed_out: 1`. +The agent borrows `sandbox_access` when supplied. Otherwise `sandbox_provider` and `sandbox_config` create an owned sandbox. Missing sandbox configuration, the unisolated Local provider, required MCP grants, and more than one direct HTTP grant are refused. No sandbox error falls back to host execution. Close stops the harness through `SandboxSession`, captures output, then disconnects a borrowed sandbox or destroys an owned one. -# Licensing information -Code: Apache 2.0 -Data: N/A +The Resources Server remains independent. A direct HTTP grant supplies its tools with per-session headers/cookies; the updated cookie jar returns at agent close. BrowseComp resources execute `bash_command` in their own sandbox, separate from the agent container. This uses two containers per BrowseComp episode and does not require a provider to serialize/connect an existing sandbox. Search credentials, ground truth and verifier code stay on the Resources Server. The model upstream API key stays on the Model Server. -Dependencies -- nemo_gym: Apache 2.0 +For a compatible tool-free benchmark, use `resources_tool_transports: []` and change only the Environment Server's resources binding and dataset. Set `progress: false` when the pairing does not use the progress board. The adapter does not depend on BrowseComp task fields or its verifier. No SWE-bench Pro or Terminal-Bench support is claimed. + +## Capability and limits + +The sandbox path declares **Evaluation** capability. It returns an observation bundle even after failure, with `native_turn_evidence_unavailable` or `observation_capture_failed`. Raw Responses output and model-call captures remain available, but native invocation/turn/tool observations are not emitted. Strict evidence checks are expected to report these gaps; this is not an Observability or Training qualification. Function-calling shell conformance is applicable but optional at this capability level and has not been registered or qualified. Component tests and real pairing runs cover the resources-owned pocketshell boundary. + +- Sampling controls in the request go unchanged to every model call. `max_output_tokens` is per call; `max_steps` and `rollout_timeout_s` bound the loop. `sandbox_runner_timeout_seconds` bounds the full activation, including retries. +- Token IDs/logprobs already returned by the Model Server are preserved; the sandbox path has not been qualified for training. +- With `save_model_call_using_vllm_tokenize_endpoint: true`, a session-authenticated adapter endpoint counts prompts without copying the upstream key into the sandbox. Hosted endpoints without `/tokenize` must explicitly set this option to `false`, using the existing post-call token-count reset path. +- `snap_dir` is rejected for sandbox sessions. Use captured model calls and rollout output; snapshots remain available in explicit host compatibility mode. +- Harness source, runtime and session files live under `/tmp/nemo-gym-*`, outside the task workdir. The runner deletes its staged input after loading scoped grants. The runtime needs no Python or package download access in the task image; seed builds its archives on the Agent Server. +- Agent sessions use one worker, bind the attempt-qualified rollout URL, and replay identical activation retries. A different activation returns 409. Disconnecting an HTTP waiter does not cancel the harness; close owns cancellation. + +## Failure semantics and compatibility + +A native rollout timeout retains partial output, sets `timed_out`, and scores zero without a judge call. Model HTTP errors retain their status in `ng_agent_outcome`; the Environment Server decides verification, failure or retry. Its default policy verifies 400/413/422, retries 408/425/429/5xx, and excludes other model failures from scoring. Read failure rows and coverage alongside accuracy: this differs from the old `/run` path, which counted some errors as zero-reward answers. + +Host `/v1/responses` and `/run` are disabled by default. Development-only compatibility requires `allow_host_execution: true`; `/run` also requires an explicit `resources_server`. The resources server independently requires its own host-execution opt-in if its tool sandbox is disabled. These options do not provide isolation. + +## License + +Apache-2.0. Dependencies: NeMo Gym (Apache-2.0). diff --git a/responses_api_agents/browsecomp_agent/app.py b/responses_api_agents/browsecomp_agent/app.py index 9f8d080c98b..9537c59d012 100644 --- a/responses_api_agents/browsecomp_agent/app.py +++ b/responses_api_agents/browsecomp_agent/app.py @@ -20,12 +20,12 @@ import traceback from dataclasses import dataclass from pathlib import Path -from typing import Any, List, Optional +from typing import Any, ClassVar, List, Literal, Optional from uuid import uuid4 import aiohttp -from fastapi import Request, Response -from pydantic import ConfigDict, Field, ValidationError +from fastapi import HTTPException, Request, Response +from pydantic import ConfigDict, Field, JsonValue, ValidationError from nemo_gym.agents import ( AGENT_OUTCOME_METADATA_KEY, @@ -33,10 +33,13 @@ AgentOutcome, AgentSeedRejectedError, AgentSeedSessionRequest, + AgentSessionSetupError, AgentSessionState, BaseResponsesAPIAgentConfig, Body, + SandboxSession, SimpleResponsesAPIAgent, + assert_model_url_reachable_from_sandbox, ) from nemo_gym.artifacts import NG_FAILURE_CLASS_KEY, NG_NO_PERSIST_KEY from nemo_gym.client import NeMoGymAsyncOpenAI, get_response_json, raise_for_status @@ -60,6 +63,15 @@ NeMoGymResponseOutputText, accumulate_response_usage, ) +from nemo_gym.models import VLLMConverter +from nemo_gym.sandbox import ( + AsyncSandbox, + DirectSandboxConnection, + SandboxSpec, + create_provider, + resolve_provider_config, + resolve_provider_metadata, +) from nemo_gym.server_runtime import ( BaseRunRequest, BaseVerifyRequest, @@ -67,7 +79,8 @@ DirectHTTPToolAccess, MCPToolAccess, ) -from responses_api_models.vllm_model.app import VLLMConverter +from nemo_gym.trajectory import AgentObservationBundle, ObservationGap +from responses_api_agents.browsecomp_agent.sandbox import BrowsecompSandboxSession def _qid(text: str) -> str: @@ -155,8 +168,16 @@ def _qid(text: str) -> str: class BrowsecompAgentConfig(BaseResponsesAPIAgentConfig): - resources_server: ResourcesServerRef + resources_server: ResourcesServerRef | None = None model_server: ModelServerRef + num_workers: Literal[1] = 1 + allow_host_execution: bool = False + sandbox_provider: str | None = None + sandbox_config: dict[str, JsonValue] = Field( + default_factory=lambda: {"image": "python:3.13-slim", "workdir": "/sandbox"} + ) + sandbox_install_timeout_seconds: float = Field(default=900, gt=0) + sandbox_runner_timeout_seconds: float = Field(default=3600, gt=0) max_steps: int = 400 # Wall-clock budget per rollout, measured from the start of the rollout loop. Past it the loop # stops, the response carries `timed_out`, and the harness scores the sample 0 without @@ -248,7 +269,7 @@ def _is_infrastructure_failure(exc: BaseException) -> bool: class BrowsecompAgentSessionState(AgentSessionState): """An Environment Server episode: its grant to the resources server and the cookie jar its tool calls carry.""" - tool_access: DirectHTTPToolAccess + tool_access: DirectHTTPToolAccess | None resources_cookies: dict[str, str] @@ -259,10 +280,12 @@ def _cookie_values(cookies: Any) -> dict[str, str]: class BrowsecompAgent(SimpleResponsesAPIAgent): config: BrowsecompAgentConfig + calls_model_from_sandbox: ClassVar[bool] = True _policy_model_openai_client: Optional[NeMoGymAsyncOpenAI] = None def setup_webserver(self): app = super().setup_webserver() + app.post("/v1/browsecomp/token_count")(self._session_token_count) # For the /tokenize-based pre-call token estimation we need a direct # client to the policy model's vLLM /tokenize endpoint. Built only when # the feature is enabled. (ported from gym-gitlab b66e37c6) @@ -318,19 +341,30 @@ async def responses( try: agent_session_id = self._agent_session_id_from_request(request) if agent_session_id is None: + if not self.config.allow_host_execution: + raise HTTPException( + 422, "BrowseComp requires an agent session and a sandbox; host execution is disabled" + ) return await self._responses_impl(request, response, body) state = self._require_agent_session(agent_session_id) - if not isinstance(state, BrowsecompAgentSessionState): + if not isinstance(state, BrowsecompSandboxSession): raise TypeError("Expected BrowseComp agent session state") - return await self._session_responses(request, response, body, state) + if request.path_params.get("rollout_id") != state.request.episode_id.capture_key: + raise HTTPException(409, "Agent sessions require the seeded attempt-qualified rollout path") + if state.task is None: + state.activation_request = body.model_copy(deep=True) + state.task = asyncio.create_task(self._sandbox_responses(request, body, state)) + elif body != state.activation_request: + raise HTTPException(409, "BrowseComp sessions support one activation; retry the same request") + return (await asyncio.shield(state.task)).model_copy(deep=True) except Exception as e: print(f"[browsecomp][responses_exc] error_type={type(e).__name__} error={str(e)[:300]}", flush=True) traceback.print_exc() raise - async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> BrowsecompAgentSessionState: - if body.sandbox_access is not None: - raise AgentSeedRejectedError("BrowseComp agent does not support sandbox access") + async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> BrowsecompSandboxSession: + if self.config.snap_dir: + raise AgentSeedRejectedError("snap_dir is supported only by the explicit host compatibility path") accesses = self.effective_tool_accesses(body) unsupported = [access.name for access in accesses if isinstance(access, MCPToolAccess) and access.required] if unsupported: @@ -338,24 +372,155 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Brow f"BrowseComp agent does not support required MCP tool access: {', '.join(unsupported)}" ) direct_accesses = [access for access in accesses if isinstance(access, DirectHTTPToolAccess)] - if len(direct_accesses) != 1: + if len(direct_accesses) > 1: + raise AgentSeedRejectedError( + "BrowseComp agent supports at most one direct HTTP tool access; " + "set resources_tool_transports: [direct_http] for a tool-using benchmark" + ) + tool_access = next(iter(direct_accesses), None) + owns_sandbox = body.sandbox_access is None + if owns_sandbox: + provider_ref = self.config.sandbox_provider + workdir = self.config.sandbox_config.get("workdir") + if provider_ref is None: + raise AgentSeedRejectedError("BrowseComp requires sandbox_access or sandbox_provider") + else: + connection = body.sandbox_access.connection + if not isinstance(connection, DirectSandboxConnection): + raise AgentSeedRejectedError("BrowseComp supports only direct sandbox connections") + if body.sandbox_access.user is not None or body.sandbox_access.env: + raise AgentSeedRejectedError("BrowseComp does not support task-specific sandbox users or environment") + provider_ref = connection.provider_config_ref + workdir = body.sandbox_access.workdir + if not isinstance(workdir, str) or not workdir.startswith("/"): + raise AgentSeedRejectedError("BrowseComp requires an absolute sandbox workdir") + provider_config = resolve_provider_config(provider_ref, get_global_config_dict()) + provider_name = next(iter(provider_config), None) + if provider_name == "local": raise AgentSeedRejectedError( - "BrowseComp agent needs exactly one direct HTTP tool access to its resources server, got " - f"{len(direct_accesses)}; set resources_tool_transports: [direct_http] on the Environment Server" + "BrowseComp sessions require an isolated sandbox provider; local is not isolated" ) - [tool_access] = direct_accesses - return BrowsecompAgentSessionState( - request=body, tool_access=tool_access, resources_cookies=dict(tool_access.cookies) + assert_model_url_reachable_from_sandbox( + self.resolve_model_base_url(self.config.model_server.name, body.episode_id.capture_key), + provider_name=provider_name, ) + if tool_access is not None: + assert_model_url_reachable_from_sandbox(str(tool_access.base_url), provider_name=provider_name) + provider = create_provider(provider_config) + try: + if owns_sandbox: + sandbox = AsyncSandbox(provider) + spec = dict(self.config.sandbox_config) + spec["metadata"] = { + **resolve_provider_metadata(provider_ref, get_global_config_dict()), + **(spec.get("metadata") or {}), + "agent_session_id": body.agent_session_id[:63], + } + spec.setdefault( + "ttl_s", + self.config.sandbox_install_timeout_seconds + self.config.sandbox_runner_timeout_seconds + 600, + ) + await sandbox.start(SandboxSpec(**spec)) + else: + sandbox = await AsyncSandbox.connect(connection.descriptor, provider=provider) + except BaseException: + await provider.aclose() + raise + state = BrowsecompSandboxSession( + request=body, + session=SandboxSession( + sandbox=sandbox, + workdir=workdir, + session_dir=f"/tmp/nemo-gym-browsecomp-sessions/{uuid4().hex}", + owns_sandbox=owns_sandbox, + harness="BrowseComp", + ), + tool_access=tool_access, + resources_cookies=dict(tool_access.cookies) if tool_access else {}, + ) + try: + await state.install_runtime(timeout=self.config.sandbox_install_timeout_seconds) + except BaseException as error: + try: + await state.close(timeout=self.config.session_close_timeout_seconds) + except BaseException: + raise AgentSessionSetupError(state, error=error) from error + raise + return state async def _close_agent_session_state(self, state: AgentSessionState) -> AgentCloseSessionResponse: - if not isinstance(state, BrowsecompAgentSessionState): + if not isinstance(state, BrowsecompSandboxSession): raise TypeError("Expected BrowseComp agent session state") + await state.close(timeout=self.config.session_close_timeout_seconds) + artifacts = state.session.artifacts + if artifacts is not None and isinstance(artifacts.get("resources_cookies"), dict): + state.resources_cookies = _cookie_values(artifacts["resources_cookies"]) return AgentCloseSessionResponse( - agent_session_id=state.request.agent_session_id, resources_cookies=state.resources_cookies + agent_session_id=state.request.agent_session_id, + resources_cookies=state.resources_cookies, + agent_observations=AgentObservationBundle( + source="browsecomp", + gaps=[ + ObservationGap( + code="native_turn_evidence_unavailable" if artifacts else "observation_capture_failed", + detail="Evaluation capability: model captures and raw output are retained; native invocation/turn records are not emitted.", + ) + ], + ), ) - async def _session_responses( + async def _session_token_count( + self, request: Request, body: NeMoGymResponseCreateParamsNonStreaming = Body() + ) -> dict[str, int]: + """Count a session's prompt without copying the upstream API credential into its sandbox.""" + session_id = self._agent_session_id_from_request(request) + if session_id is None: + raise HTTPException(409, "Token counting requires an active agent session") + self._require_agent_session(session_id) + if not self.config.save_model_call_using_vllm_tokenize_endpoint: + raise HTTPException(422, "vLLM token counting is disabled") + return {"count": await self._count_prompt_tokens(body)} + + async def _sandbox_responses( + self, request: Request, body: NeMoGymResponseCreateParamsNonStreaming, state: BrowsecompSandboxSession + ) -> NeMoGymResponse: + if body.tools and state.tool_access is None: + raise HTTPException(422, "Tool-using BrowseComp requests require a direct HTTP tool grant") + timeout = self.config.sandbox_runner_timeout_seconds + if state.request.agent_timeout_sec is not None: + timeout = min(timeout, state.request.agent_timeout_sec) + # Keep the native timeout response/zero-reward semantics; the supervisor has a small receipt grace. + config = self.config.model_dump(mode="json", exclude={"tool_accesses", "sandbox_config", "sandbox_provider"}) + config["rollout_timeout_s"] = min( + self.config.rollout_timeout_s if self.config.rollout_timeout_s is not None else timeout, timeout + ) + payload = { + "seed": state.request.model_copy(update={"sandbox_access": None, "tool_accesses": []}).model_dump( + mode="json" + ), + "tool_access": state.tool_access.model_dump(mode="json") if state.tool_access else None, + "body": body.model_dump(mode="json", exclude_unset=True), + "config": config, + "model_url": self.resolve_model_base_url( + self.config.model_server.name, state.request.episode_id.capture_key + ), + "token_count_url": f"{self.server_client._resolve_base_url(self.config.name)}/v1/browsecomp/token_count" + if self.config.save_model_call_using_vllm_tokenize_endpoint + else None, + "agent_cookies": dict(request.cookies), + } + output = await state.session.execute( + stage_activation=lambda: state.stage_activation(payload), + collect=state.collect, + timeout=timeout + 30, + close_timeout=self.config.session_close_timeout_seconds, + ) + if output.get("error"): + raise RuntimeError(f"BrowseComp sandbox failed: {output['error']}") + state.resources_cookies = _cookie_values(output.get("resources_cookies")) + return NeMoGymResponse.model_validate(output["response"]) + + async def _run_session_loop( self, request: Request, response: Response, @@ -764,6 +929,7 @@ async def _tool_call(): resources_server_cookies = api_response.cookies else: resources_server_cookies = resources_server_cookies | _cookie_values(api_response.cookies) + session.resources_cookies = resources_server_cookies # reference-harness parity: the resources server wraps every tool result in a one-field # JSON envelope ({"results_string": "..."}), which JSON-escapes newlines — the # model then sees a single escaped line instead of raw multi-line text. This was @@ -972,6 +1138,10 @@ async def _tool_call(): return model_response async def run(self, request: Request, body: BrowsecompAgentRunRequest) -> BrowsecompAgentVerifyResponse: + if not self.config.allow_host_execution or self.config.resources_server is None: + raise HTTPException( + 422, "Legacy /run requires allow_host_execution=true and resources_server; use agent sessions" + ) cookies = request.cookies question_text = getattr(body, "question", None) or "" diff --git a/responses_api_agents/browsecomp_agent/configs/browsecomp_agent.yaml b/responses_api_agents/browsecomp_agent/configs/browsecomp_agent.yaml index bd3f7c66ab3..2832baa3184 100644 --- a/responses_api_agents/browsecomp_agent/configs/browsecomp_agent.yaml +++ b/responses_api_agents/browsecomp_agent/configs/browsecomp_agent.yaml @@ -2,9 +2,7 @@ browsecomp_agent: responses_api_agents: browsecomp_agent: entrypoint: app.py - resources_server: - type: resources_servers - name: ??? + resources_server: null model_server: type: responses_api_models name: policy_model diff --git a/responses_api_agents/browsecomp_agent/pyproject.toml b/responses_api_agents/browsecomp_agent/pyproject.toml index 169a9088517..e5757ee29fe 100644 --- a/responses_api_agents/browsecomp_agent/pyproject.toml +++ b/responses_api_agents/browsecomp_agent/pyproject.toml @@ -6,7 +6,7 @@ name = "gym-responses-api-agents-browsecomp-agent" version = "0.0.0" requires-python = ">=3.13.14" dependencies = [ - "nemo-gym[dev]", + "nemo-gym[dev,sandbox]", ] [tool.uv] diff --git a/responses_api_agents/browsecomp_agent/sandbox.py b/responses_api_agents/browsecomp_agent/sandbox.py new file mode 100644 index 00000000000..5304442163d --- /dev/null +++ b/responses_api_agents/browsecomp_agent/sandbox.py @@ -0,0 +1,126 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Run the repository's BrowseComp loop in a supervised task sandbox.""" + +import asyncio +import functools +import importlib.metadata +import json +import tempfile +import zipfile +from dataclasses import dataclass +from pathlib import Path +from shlex import quote + +from packaging.requirements import Requirement +from pydantic import JsonValue + +import nemo_gym +from nemo_gym.agents import AgentSessionState, SandboxCommand, SandboxPythonRuntime, SandboxSession +from nemo_gym.core import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming +from nemo_gym.sandbox import read_text, upload_text +from nemo_gym.server_runtime import DirectHTTPToolAccess + + +@functools.cache +def _runtime() -> SandboxPythonRuntime: + # Upload the exact installed Gym sources separately. Resolve their external dependencies using the + # same pinned host environment as every other SandboxPythonRuntime, without dev/provider extras. + requirements = tuple( + raw + for raw in importlib.metadata.requires("nemo-gym") or () + if (requirement := Requirement(raw)).marker is None or requirement.marker.evaluate({"extra": ""}) + ) + return SandboxPythonRuntime( + harness="browsecomp", + requirements=requirements, + import_check="import aiohttp, openai, pydantic, fastapi, omegaconf", + ) + + +def _write_sources(destination: Path) -> None: + """Package code only, never the host configuration, datasets, credentials or caches.""" + library = Path(nemo_gym.__file__).parent + adapter = Path(__file__).parent + with zipfile.ZipFile(destination, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for root, prefix in ((library, "nemo_gym"), (adapter, "responses_api_agents/browsecomp_agent")): + for path in sorted(root.rglob("*")): + relative = path.relative_to(root) + if any(part.startswith(".") or part in {"tests", "__pycache__"} for part in relative.parts): + continue + if path.is_file() and path.suffix in {".py", ".sh"}: + archive.write(path, f"{prefix}/{relative.as_posix()}") + + +@dataclass +class BrowsecompSandboxSession(AgentSessionState): + """Sandbox ownership and retained activation; resource state stays behind its HTTP grant.""" + + session: SandboxSession[dict[str, JsonValue]] + tool_access: DirectHTTPToolAccess | None + resources_cookies: dict[str, str] + task: asyncio.Task[NeMoGymResponse] | None = None + activation_request: NeMoGymResponseCreateParamsNonStreaming | None = None + + async def install_runtime(self, *, timeout: float) -> None: + """Install a private Python, then upload this revision of Gym and the harness.""" + runtime = _runtime() + await runtime.install(self.session.sandbox, timeout_s=timeout) + directory = self.session.session_dir + prepared = await self.session.sandbox.exec( + f"mkdir -p {quote(directory)} {quote(self.session.workdir)}", cwd="/", timeout_s=30 + ) + if prepared.return_code != 0 or prepared.error_type is not None: + raise RuntimeError(prepared.stderr or "Cannot prepare BrowseComp sandbox directory") + with tempfile.TemporaryDirectory(prefix="browsecomp-source-") as temporary: + archive = Path(temporary) / "source.zip" + await asyncio.to_thread(_write_sources, archive) + await self.session.sandbox.upload(archive, f"{directory}/source.zip") + result = await self.session.sandbox.exec( + f"{quote(runtime.python)} -I -m zipfile -e {quote(directory + '/source.zip')} " + f"{quote(directory + '/source')}", + cwd="/", + timeout_s=60, + ) + if result.return_code != 0 or result.error_type is not None: + raise RuntimeError(result.stderr or "Cannot unpack BrowseComp sandbox sources") + + async def stage_activation(self, payload: dict[str, JsonValue]) -> SandboxCommand: + """Write input as data; model-controlled strings never become a launch command.""" + directory = self.session.session_dir + await upload_text(self.session.sandbox, path=f"{directory}/input.json", text=json.dumps(payload)) + return SandboxCommand( + argv=[ + _runtime().python, + "-I", + f"{directory}/source/responses_api_agents/browsecomp_agent/sandbox_runner.py", + f"{directory}/input.json", + f"{directory}/output.json", + ] + ) + + async def collect(self) -> dict[str, JsonValue]: + """Read the runner receipt only after the shared supervisor confirms termination.""" + try: + output = json.loads(await read_text(self.session.sandbox, path=f"{self.session.session_dir}/output.json")) + except Exception as error: + log = await self.session.read_output_log() + raise RuntimeError(f"BrowseComp runner did not produce a receipt: {log}") from error + if not isinstance(output, dict): + raise TypeError("BrowseComp runner receipt must be an object") + return output + + async def close(self, *, timeout: float) -> None: + """Capture and release the sandbox before cancelling an HTTP activation waiter.""" + await self.session.close(timeout=timeout) + if self.task is not None and not self.task.done(): + self.task.cancel() + if self.task is not None: + try: + await asyncio.wait_for(asyncio.shield(self.task), timeout=timeout) + except asyncio.CancelledError: + if not self.task.cancelled(): + raise + except Exception: + if not self.task.done(): + raise diff --git a/responses_api_agents/browsecomp_agent/sandbox_runner.py b/responses_api_agents/browsecomp_agent/sandbox_runner.py new file mode 100644 index 00000000000..453e8e954a8 --- /dev/null +++ b/responses_api_agents/browsecomp_agent/sandbox_runner.py @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Execute the unchanged BrowseComp loop against scoped Gym model and tool endpoints.""" + +import asyncio +import json +import os +import signal +import socket +import sys +import traceback +from pathlib import Path + + +# The source bundle is shipped by the adapter; task PYTHONPATH and user site packages are ignored (-I). +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from aiohttp import ClientResponse +from aiohttp.client import _RequestOptions +from fastapi import Request, Response +from omegaconf import OmegaConf +from typing_extensions import Unpack + +from nemo_gym.agents import AgentSeedSessionRequest +from nemo_gym.client import ServerClient, get_global_aiohttp_client, get_response_json, raise_for_status, request +from nemo_gym.config import BaseServerConfig +from nemo_gym.core import NeMoGymResponseCreateParamsNonStreaming +from nemo_gym.server_runtime import DirectHTTPToolAccess +from responses_api_agents.browsecomp_agent.app import ( + BrowsecompAgent, + BrowsecompAgentConfig, + BrowsecompAgentSessionState, +) + + +class ScopedModelClient(ServerClient): + """Only the rollout-qualified Model Server is visible, never its upstream credentials.""" + + model_name: str + model_url: str + + async def post(self, server_name: str, url_path: str, **kwargs: Unpack[_RequestOptions]) -> ClientResponse: + if server_name != self.model_name or not url_path.endswith("/v1/responses"): + raise ValueError("Sandbox runner attempted an ungranted model route") + body = kwargs.get("json") + if isinstance(body, NeMoGymResponseCreateParamsNonStreaming): + kwargs["json"] = body.model_dump(mode="json", exclude_unset=True) + return await request("POST", f"{self.model_url}/responses", _internal=True, **kwargs) + + +class SandboxBrowsecompAgent(BrowsecompAgent): + """Token counting stays behind the adapter; it never grants upstream model credentials.""" + + token_count_url: str | None = None + agent_cookies: dict[str, str] + + async def _count_prompt_tokens(self, body: NeMoGymResponseCreateParamsNonStreaming) -> int: + if self.token_count_url is None: + raise ValueError("BrowseComp token counting was not granted") + reply = await request( + "POST", self.token_count_url, json=body.model_dump(mode="json"), cookies=self.agent_cookies, _internal=True + ) + await raise_for_status(reply) + return int((await get_response_json(reply))["count"]) + + +async def run(input_path: Path, output_path: Path) -> None: + """Persist a receipt on success, error and supervisor interruption.""" + payload = json.loads(input_path.read_text()) + input_path.unlink() # The runner retains scoped cookies in memory, outside model-readable staged files. + # Deliberately empty: the sandbox never receives the server's secret-bearing global config. + config = OmegaConf.create({}) + os.environ["NEMO_GYM_CONFIG_DICT"] = "{}" + seed = AgentSeedSessionRequest.model_validate(payload["seed"]) + grant = DirectHTTPToolAccess.model_validate(payload["tool_access"]) if payload["tool_access"] else None + state = BrowsecompAgentSessionState( + request=seed, tool_access=grant, resources_cookies=dict(grant.cookies) if grant else {} + ) + agent_config = BrowsecompAgentConfig.model_validate(payload["config"]) + client = ScopedModelClient( + head_server_config=BaseServerConfig(host="127.0.0.1", port=1), + global_config_dict=config, + model_name=agent_config.model_server.name, + model_url=payload["model_url"], + ) + agent = SandboxBrowsecompAgent( + config=agent_config, + server_client=client, + token_count_url=payload.get("token_count_url"), + agent_cookies=payload.get("agent_cookies", {}), + ) + receipt = {"runtime": {"hostname": socket.gethostname(), "pid": os.getpid(), "python": sys.executable}} + task = asyncio.current_task() + loop = asyncio.get_running_loop() + loop.add_signal_handler(signal.SIGTERM, task.cancel) + try: + body = NeMoGymResponseCreateParamsNonStreaming.model_validate(payload["body"]) + http_request = Request({"type": "http", "headers": [], "path": "/v1/responses"}) + result = await agent._run_session_loop(http_request, Response(), body, state) + receipt["response"] = result.model_dump(mode="json") + except asyncio.CancelledError: + receipt["error"] = "BrowseComp harness was interrupted by its supervisor" + receipt["interrupted"] = True + except Exception as error: + receipt["error"] = f"{type(error).__name__}: {error}" + receipt["traceback"] = traceback.format_exc() + finally: + receipt["resources_cookies"] = state.resources_cookies + partial = output_path.with_suffix(".partial") + partial.write_text(json.dumps(receipt)) + partial.replace(output_path) + await get_global_aiohttp_client().close() + + +if __name__ == "__main__": + asyncio.run(run(Path(sys.argv[1]), Path(sys.argv[2]))) diff --git a/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py b/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py index fce211f7799..8e42038f5a4 100644 --- a/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py +++ b/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py @@ -14,6 +14,7 @@ AgentCloseSessionRequest, AgentSeedSessionRequest, OutcomeAction, + SandboxSession, agent_outcome_policy, ) from nemo_gym.client import ServerClient @@ -26,10 +27,10 @@ NeMoGymResponseOutputText, TaskId, ) -from nemo_gym.sandbox import DirectSandboxConnection, SandboxAccess from nemo_gym.server_runtime import DirectHTTPToolAccess, MCPStreamableHTTPConnection, MCPToolAccess from nemo_gym.testing.agent_session_conformance import check_agent_seed_rejected from responses_api_agents.browsecomp_agent.app import BrowsecompAgent, BrowsecompAgentConfig +from responses_api_agents.browsecomp_agent.sandbox import BrowsecompSandboxSession EPISODE = EpisodeId(rollout_id="7-0", attempt=0) @@ -41,6 +42,7 @@ headers={"x-grant": "1"}, ) QUESTION = {"input": [{"role": "user", "content": "What is the capital of France?"}]} +ROUTE = f"/ng-rollout/{EPISODE.capture_key}/v1/responses" def _seed(**overrides) -> AgentSeedSessionRequest: @@ -116,6 +118,31 @@ async def post(server_name=None, url_path=None, json=None, cookies=None): server_client.post = AsyncMock(side_effect=post) harness = _Harness() monkeypatch.setattr("responses_api_agents.browsecomp_agent.app.http_request", harness) + monkeypatch.setattr("responses_api_agents.browsecomp_agent.app.get_global_config_dict", lambda: {}) + monkeypatch.setattr( + "responses_api_agents.browsecomp_agent.app.resolve_provider_config", lambda *args: {"docker": {}} + ) + monkeypatch.setattr("responses_api_agents.browsecomp_agent.app.resolve_provider_metadata", lambda *args: {}) + monkeypatch.setattr( + "responses_api_agents.browsecomp_agent.app.create_provider", lambda *args: MagicMock(aclose=AsyncMock()) + ) + sandbox = MagicMock(start=AsyncMock(), stop=AsyncMock()) + monkeypatch.setattr( + "responses_api_agents.browsecomp_agent.app.AsyncSandbox", + MagicMock(return_value=sandbox, connect=AsyncMock(return_value=sandbox)), + ) + monkeypatch.setattr(BrowsecompSandboxSession, "install_runtime", AsyncMock()) + monkeypatch.setattr(SandboxSession, "close", AsyncMock()) + monkeypatch.setattr(BrowsecompAgent, "resolve_model_base_url", lambda *args: "http://model:9001/v1") + + # Exercise the original loop and HTTP grants independently from provider transport; the runner + # subprocess and supervisor lifecycle have separate tests and real EOS smoke coverage. + async def execute_loop(self, request, body, state): + from fastapi import Response + + return await self._run_session_loop(request, Response(), body, state) + + monkeypatch.setattr(BrowsecompAgent, "_sandbox_responses", execute_loop) agent = BrowsecompAgent( config=BrowsecompAgentConfig( host="0.0.0.0", @@ -125,7 +152,7 @@ async def post(server_name=None, url_path=None, json=None, cookies=None): resources_server=ResourcesServerRef(type="resources_servers", name="configured_harness"), model_server=ModelServerRef(type="responses_api_models", name="policy_model"), nudge_steps=False, - **config, + **({"sandbox_provider": "sandbox"} | config), ), server_client=server_client, ) @@ -135,10 +162,9 @@ async def post(server_name=None, url_path=None, json=None, cookies=None): @pytest.mark.parametrize( ("unsupported", "reason"), [ - ({"tool_accesses": []}, "needs exactly one direct HTTP tool access to its resources server, got 0"), ( {"tool_accesses": [GRANT, GRANT.model_copy(update={"name": "other.direct_http"})]}, - "needs exactly one direct HTTP tool access to its resources server, got 2", + "supports at most one direct HTTP tool access", ), ( { @@ -153,15 +179,6 @@ async def post(server_name=None, url_path=None, json=None, cookies=None): }, "does not support required MCP tool access: resources", ), - ( - { - "sandbox_access": SandboxAccess( - connection=DirectSandboxConnection(provider_config_ref="runtime", descriptor={}), - workdir="/work", - ) - }, - "does not support sandbox access", - ), ], ) def test_a_seed_it_cannot_serve_is_refused_as_final( @@ -177,7 +194,7 @@ def test_tool_calls_use_the_grant_and_close_returns_the_resources_cookies(monkey ) assert client.post("/v1/agent_sessions", json=_seed().model_dump(mode="json")).status_code == 200 - reply = client.post("/v1/responses", json=QUESTION) + reply = client.post(ROUTE, json=QUESTION) assert reply.status_code == 200, reply.text assert [call.kwargs["server_name"] for call in server_client.post.await_args_list] == ["policy_model"] * 2 @@ -211,7 +228,7 @@ def test_an_empty_answer_is_retried_inside_the_activation(monkeypatch: pytest.Mo ) assert client.post("/v1/agent_sessions", json=_seed().model_dump(mode="json")).status_code == 200 - reply = client.post("/v1/responses", json=QUESTION) + reply = client.post(ROUTE, json=QUESTION) assert reply.status_code == 200, reply.text assert reply.json()["output"][-1]["content"][0]["text"] == "Exact Answer: Paris" @@ -224,7 +241,7 @@ def test_the_last_attempt_returns_its_empty_answer(monkeypatch: pytest.MonkeyPat client, server_client, _ = _agent([_model_response(_message("no answer"))], monkeypatch) assert client.post("/v1/agent_sessions", json=_seed().model_dump(mode="json")).status_code == 200 - reply = client.post("/v1/responses", json=QUESTION) + reply = client.post(ROUTE, json=QUESTION) assert reply.status_code == 200, reply.text assert reply.json()["output"][-1]["content"][0]["text"] == "no answer" @@ -241,7 +258,7 @@ def test_a_model_http_error_is_a_failed_response_for_the_outcome_policy( client, _, _ = _agent([_model_response(_tool_call("ls")), status], monkeypatch) assert client.post("/v1/agent_sessions", json=_seed().model_dump(mode="json")).status_code == 200 - reply = client.post("/v1/responses", json=QUESTION) + reply = client.post(ROUTE, json=QUESTION) assert reply.status_code == 200, reply.text response = NeMoGymResponse.model_validate(reply.json()) diff --git a/responses_api_agents/browsecomp_agent/tests/test_app.py b/responses_api_agents/browsecomp_agent/tests/test_app.py index 47009c2df7f..848ae58a027 100644 --- a/responses_api_agents/browsecomp_agent/tests/test_app.py +++ b/responses_api_agents/browsecomp_agent/tests/test_app.py @@ -45,6 +45,7 @@ def _make_config(**kwargs) -> BrowsecompAgentConfig: defaults = dict( + allow_host_execution=True, host="0.0.0.0", port=8080, entrypoint="", diff --git a/responses_api_agents/browsecomp_agent/tests/test_malformed_tool_arguments.py b/responses_api_agents/browsecomp_agent/tests/test_malformed_tool_arguments.py index 5f6ca332fb8..4728883e072 100644 --- a/responses_api_agents/browsecomp_agent/tests/test_malformed_tool_arguments.py +++ b/responses_api_agents/browsecomp_agent/tests/test_malformed_tool_arguments.py @@ -59,6 +59,7 @@ def _make_config(**kwargs) -> BrowsecompAgentConfig: defaults = dict( + allow_host_execution=True, host="0.0.0.0", port=8080, entrypoint="", diff --git a/responses_api_agents/browsecomp_agent/tests/test_progress_tracking.py b/responses_api_agents/browsecomp_agent/tests/test_progress_tracking.py index a08415b667e..e60760f7457 100644 --- a/responses_api_agents/browsecomp_agent/tests/test_progress_tracking.py +++ b/responses_api_agents/browsecomp_agent/tests/test_progress_tracking.py @@ -54,6 +54,7 @@ def _make_config(**kwargs) -> BrowsecompAgentConfig: defaults = dict( + allow_host_execution=True, host="0.0.0.0", port=8080, entrypoint="", diff --git a/responses_api_agents/browsecomp_agent/tests/test_responses_exception_logging.py b/responses_api_agents/browsecomp_agent/tests/test_responses_exception_logging.py index 8c57d2c29fb..252d97552b1 100644 --- a/responses_api_agents/browsecomp_agent/tests/test_responses_exception_logging.py +++ b/responses_api_agents/browsecomp_agent/tests/test_responses_exception_logging.py @@ -37,6 +37,7 @@ def _make_agent() -> BrowsecompAgent: config = BrowsecompAgentConfig( + allow_host_execution=True, host="0.0.0.0", port=8080, entrypoint="", diff --git a/responses_api_agents/browsecomp_agent/tests/test_run_survives_sample_failure.py b/responses_api_agents/browsecomp_agent/tests/test_run_survives_sample_failure.py index 50892bae0e7..6af61dedca1 100644 --- a/responses_api_agents/browsecomp_agent/tests/test_run_survives_sample_failure.py +++ b/responses_api_agents/browsecomp_agent/tests/test_run_survives_sample_failure.py @@ -52,6 +52,7 @@ def _make_config(**kwargs) -> BrowsecompAgentConfig: defaults = dict( + allow_host_execution=True, host="0.0.0.0", port=8080, entrypoint="", diff --git a/responses_api_agents/browsecomp_agent/tests/test_sandbox.py b/responses_api_agents/browsecomp_agent/tests/test_sandbox.py new file mode 100644 index 00000000000..9b300fc1b4f --- /dev/null +++ b/responses_api_agents/browsecomp_agent/tests/test_sandbox.py @@ -0,0 +1,232 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The process/HTTP boundary, source bundle and sandbox session lifecycle.""" + +import asyncio +import json +import sys +import zipfile +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest +from aiohttp import web + +from responses_api_agents.browsecomp_agent.sandbox import BrowsecompSandboxSession, _write_sources +from responses_api_agents.browsecomp_agent.tests.test_agent_sessions import ( + EPISODE, + GRANT, + QUESTION, + ROUTE, + _agent, + _message, + _model_response, + _seed, + _tool_call, +) + + +def test_host_execution_is_an_explicit_opt_in(monkeypatch: pytest.MonkeyPatch) -> None: + client, _, _ = _agent([], monkeypatch) + reply = client.post("/v1/responses", json=QUESTION) + assert reply.status_code == 422 + assert "host execution is disabled" in reply.text + + +def test_seed_requires_a_sandbox_and_retains_no_failed_session(monkeypatch: pytest.MonkeyPatch) -> None: + client, _, _ = _agent([], monkeypatch, sandbox_provider=None) + reply = client.post("/v1/agent_sessions", json=_seed().model_dump(mode="json")) + assert reply.status_code == 422 + assert "sandbox_access or sandbox_provider" in reply.text + + +def test_local_provider_is_not_accepted_as_isolation(monkeypatch: pytest.MonkeyPatch) -> None: + client, _, _ = _agent([], monkeypatch) + monkeypatch.setattr( + "responses_api_agents.browsecomp_agent.app.resolve_provider_config", lambda *args: {"local": {}} + ) + reply = client.post("/v1/agent_sessions", json=_seed().model_dump(mode="json")) + assert reply.status_code == 422 + assert "not isolated" in reply.text + + +def test_retries_replay_and_conflicting_activations_are_rejected(monkeypatch: pytest.MonkeyPatch) -> None: + client, model, _ = _agent([_model_response(_message("Paris"))], monkeypatch) + client.post("/v1/agent_sessions", json=_seed().model_dump(mode="json")) + first = client.post(ROUTE, json=QUESTION) + second = client.post(ROUTE, json=QUESTION) + assert first.status_code == second.status_code == 200 + assert first.json() == second.json() + assert model.post.await_count == 1 + assert client.post(ROUTE, json={"input": "Different request"}).status_code == 409 + assert client.post("/v1/responses", json=QUESTION).status_code == 409 + assert client.post("/ng-rollout/wrong/v1/responses", json=QUESTION).status_code == 409 + + +def test_no_tool_grant_is_valid_for_a_tool_free_pairing(monkeypatch: pytest.MonkeyPatch) -> None: + client, _, _ = _agent([_model_response(_message("42"))], monkeypatch) + assert client.post("/v1/agent_sessions", json=_seed(tool_accesses=[]).model_dump(mode="json")).status_code == 200 + assert client.post(ROUTE, json=QUESTION).status_code == 200 + + +async def test_close_retains_cookies_and_an_explicit_evidence_gap(monkeypatch: pytest.MonkeyPatch) -> None: + client, _, _ = _agent([], monkeypatch) + agent = next(route.endpoint.__self__ for route in client.app.routes if route.path == "/v1/browsecomp/token_count") + state = await agent._seed_agent_session_state(_seed()) + state.session.artifacts = {"resources_cookies": {"harness": "final-cookie"}} + result = await agent._close_agent_session_state(state) + state.session.close.assert_awaited_once() + assert result.resources_cookies == {"harness": "final-cookie"} + assert result.agent_observations.gaps[0].code == "native_turn_evidence_unavailable" + + +def test_source_bundle_contains_only_runtime_code(tmp_path: Path) -> None: + archive = tmp_path / "source.zip" + _write_sources(archive) + with zipfile.ZipFile(archive) as bundle: + names = bundle.namelist() + assert "responses_api_agents/browsecomp_agent/app.py" in names + assert "nemo_gym/core/responses_api_types.py" in names + assert all(Path(name).suffix in {".py", ".sh"} for name in names) + assert not any(part in {".venv", "tests", "__pycache__"} for name in names for part in Path(name).parts) + + +async def test_runner_process_uses_scoped_model_and_tool_urls(tmp_path: Path) -> None: + """Run the actual shim/loop in another interpreter, driving a real HTTP model and tool stub.""" + calls = [] + + async def model(request: web.Request) -> web.Response: + calls.append(await request.json()) + return web.json_response( + _model_response(_tool_call("ls")) if len(calls) == 1 else _model_response(_message("Paris")) + ) + + async def tool(request: web.Request) -> web.Response: + assert request.headers["x-grant"] == "1" + assert request.cookies["harness"] == "seed-cookie" + assert await request.json() == {"keystrokes": "ls"} + response = web.json_response({"results_string": "manifest.tsv\npages"}) + response.set_cookie("harness", "after-tool") + return response + + app = web.Application() + app.router.add_post(ROUTE, model) + app.router.add_post("/tools/bash_command", tool) + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, "127.0.0.1", 0) + await site.start() + port = site._server.sockets[0].getsockname()[1] + origin = f"http://127.0.0.1:{port}" + archive = tmp_path / "source.zip" + _write_sources(archive) + with zipfile.ZipFile(archive) as bundle: + bundle.extractall(tmp_path / "source") + payload = { + "config": { + "name": "browsecomp", + "host": "0.0.0.0", + "port": 1, + "entrypoint": "app.py", + "model_server": {"type": "responses_api_models", "name": "policy"}, + "nudge_steps": False, + }, + "seed": _seed().model_dump(mode="json"), + "tool_access": GRANT.model_copy(update={"base_url": f"{origin}/tools"}).model_dump(mode="json"), + "body": QUESTION, + "model_url": f"{origin}/ng-rollout/{EPISODE.capture_key}/v1", + } + input_path, output_path = tmp_path / "input.json", tmp_path / "output.json" + input_path.write_text(json.dumps(payload)) + process = None + try: + process = await asyncio.create_subprocess_exec( + sys.executable, + "-I", + str(tmp_path / "source/responses_api_agents/browsecomp_agent/sandbox_runner.py"), + str(input_path), + str(output_path), + cwd=tmp_path, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=60) + assert process.returncode == 0, (stdout + stderr).decode(errors="replace") + result = json.loads(output_path.read_text()) + assert "error" not in result, result + assert result["runtime"]["pid"] == process.pid + assert result["resources_cookies"] == {"harness": "after-tool"} + assert result["response"]["output"][1]["output"] == "manifest.tsv\npages" + assert len(calls) == 2 + finally: + if process is not None and process.returncode is None: + process.kill() + await process.wait() + await runner.cleanup() + + +@pytest.mark.parametrize("borrowed", [False, True]) +async def test_seed_selects_ownership_and_propagates_provider_metadata( + monkeypatch: pytest.MonkeyPatch, borrowed: bool +) -> None: + from nemo_gym.sandbox import DirectSandboxConnection, SandboxAccess + from responses_api_agents.browsecomp_agent import app + + client, _, _ = _agent([], monkeypatch, sandbox_config={"workdir": "/sandbox", "metadata": {"priority": "task"}}) + agent = next(route.endpoint.__self__ for route in client.app.routes if route.path == "/v1/browsecomp/token_count") + monkeypatch.setattr(app, "resolve_provider_metadata", lambda *args: {"priority": "provider", "owner": "test"}) + access = ( + SandboxAccess( + connection=DirectSandboxConnection(provider_config_ref="sandbox", descriptor={"id": "borrowed"}), + workdir="/task", + ) + if borrowed + else None + ) + state = await agent._seed_agent_session_state(_seed(sandbox_access=access)) + assert state.session.owns_sandbox is not borrowed + assert state.session.workdir == ("/task" if borrowed else "/sandbox") + if borrowed: + app.AsyncSandbox.connect.assert_awaited_once() + app.AsyncSandbox.return_value.start.assert_not_awaited() + else: + spec = app.AsyncSandbox.return_value.start.await_args.args[0] + assert spec.metadata == {"priority": "task", "owner": "test", "agent_session_id": "agent-session"} + assert spec.ttl_s > agent.config.sandbox_runner_timeout_seconds + + +async def test_failed_install_retains_cleanup_handle_if_release_fails(monkeypatch: pytest.MonkeyPatch) -> None: + from nemo_gym.agents import AgentSessionSetupError + + client, _, _ = _agent([], monkeypatch) + agent = next(route.endpoint.__self__ for route in client.app.routes if route.path == "/v1/browsecomp/token_count") + monkeypatch.setattr( + BrowsecompSandboxSession, "install_runtime", AsyncMock(side_effect=RuntimeError("install failed")) + ) + close = AsyncMock(side_effect=[RuntimeError("provider unavailable"), None]) + monkeypatch.setattr(BrowsecompSandboxSession, "close", close) + with pytest.raises(AgentSessionSetupError) as error: + await agent._seed_agent_session_state(_seed()) + state = error.value.state + assert state.session.owns_sandbox + assert "install failed" in str(error.value) + await agent._close_agent_session_state(state) + assert close.await_count == 2 + + +async def test_close_stops_sandbox_before_cancelling_activation(monkeypatch: pytest.MonkeyPatch) -> None: + from unittest.mock import AsyncMock + + client, _, _ = _agent([], monkeypatch) + agent = next(route.endpoint.__self__ for route in client.app.routes if route.path == "/v1/browsecomp/token_count") + state = await agent._seed_agent_session_state(_seed()) + state.task = asyncio.create_task(asyncio.Event().wait()) + await asyncio.sleep(0) + + async def release(*, timeout: float) -> None: + assert not state.task.cancelled() + + state.session.close = AsyncMock(side_effect=release) + await state.close(timeout=1) + assert state.task.cancelled() + state.session.close.assert_awaited_once() diff --git a/responses_api_agents/browsecomp_agent/uv.lock b/responses_api_agents/browsecomp_agent/uv.lock index 464efadaf03..9efc43b4924 100644 --- a/responses_api_agents/browsecomp_agent/uv.lock +++ b/responses_api_agents/browsecomp_agent/uv.lock @@ -49,6 +49,15 @@ excludes = [ "werkzeug", ] +[[package]] +name = "aiofiles" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/c3/534eac40372d8ee36ef40df62ec129bee4fdb5ad9706e58a29be53b2c970/aiofiles-25.1.0.tar.gz", hash = "sha256:a8d728f0a29de45dc521f18f07297428d56992a742f0cd2701ba86e44d23d5b2", size = 46354, upload-time = "2025-10-09T20:51:04.358Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/8a/340a1555ae33d7354dbca4faa54948d76d89a27ceef032c8c3bc661d003e/aiofiles-25.1.0-py3-none-any.whl", hash = "sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695", size = 14668, upload-time = "2025-10-09T20:51:03.174Z" }, +] + [[package]] name = "aiohappyeyeballs" version = "2.6.1" @@ -181,6 +190,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2b/27/5e1f8446545be98e9416af59c396b1ed1f9f3294afb0a40ee3a709889273/aiohttp-3.14.4-py3-none-any.whl", hash = "sha256:5c6758ba62aea282c537179cfc8474a90f2b5f7b089cc5ff66d8920d86a9bfdd", size = 279487, upload-time = "2026-10-05T17:43:44.905Z" }, ] +[[package]] +name = "aiohttp-retry" +version = "2.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/61/ebda4d8e3d8cfa1fd3db0fb428db2dd7461d5742cea35178277ad180b033/aiohttp_retry-2.9.1.tar.gz", hash = "sha256:8eb75e904ed4ee5c2ec242fefe85bf04240f685391c4879d8f541d6028ff01f1", size = 13608, upload-time = "2024-11-06T10:44:54.574Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/99/84ba7273339d0f3dfa57901b846489d2e5c2cd731470167757f1935fffbd/aiohttp_retry-2.9.1-py3-none-any.whl", hash = "sha256:66d2759d1921838256a05a3f80ad7e724936f083e35be5abb5e16eed6be6dc54", size = 9981, upload-time = "2024-11-06T10:44:52.917Z" }, +] + [[package]] name = "aiosignal" version = "1.4.0" @@ -279,6 +300,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, ] +[[package]] +name = "bidict" +version = "0.24.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a8/f2/8d2dd8276ca05e1f5157b6a0d34efb2f585f47a0fbed61e8aad04b221f0b/bidict-0.24.1.tar.gz", hash = "sha256:4dca6c17f0b01700e9f24359daa5ebabf7be022d99f4cb2a257b6af2a5076c88", size = 30818, upload-time = "2026-08-25T23:45:52.214Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/53/2a3c7d562271ec6b6e38e7216b3104899f8aa4180c0713cb8aaf69e29cd5/bidict-0.24.1-py3-none-any.whl", hash = "sha256:fd3eaa737917d8a14f4baa391670c433c4e3f6f5fd2cd99d4bf436437f432364", size = 36175, upload-time = "2026-08-25T23:45:51.096Z" }, +] + +[[package]] +name = "boto3" +version = "1.43.123" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/8a/64b46f075149f9160da44aa7e1052dd18431b9ad4435beb0cfb02cc48624/boto3-1.43.123.tar.gz", hash = "sha256:055fabb990c5d2f31fce4ca569255bd640cdd11a3ca68c9ecad35f4f8a5b83b5", size = 112638, upload-time = "2026-10-30T19:18:50.085Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/b2/a7986c327f80aa6f1544f4cb8f2fa7da4e1fccd472379a21cad0508b8a0e/boto3-1.43.123-py3-none-any.whl", hash = "sha256:d18d66e1e13faa0c25d0b1b8c5ce946e29fb5fc0b6456b655a93a678ff4cc313", size = 140044, upload-time = "2026-10-30T19:18:47.734Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.123" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/de/011fc2a47fb066c47064b122498d2fd21f55bc90ea514acbddc37ca0f8c1/botocore-1.43.123.tar.gz", hash = "sha256:80948cb9b1cdf1a1a25809089a42e3a1bc4af0258d198222db6e97dcc660dd0c", size = 16486269, upload-time = "2026-10-30T19:18:44.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/96/4f2dffd304425c07160ab2c12cb13322d852fd956fe9d4386228c93f02ea/botocore-1.43.123-py3-none-any.whl", hash = "sha256:e30580f9c1282c321529238ff8ff75066096a2adbd0a293c8ae592c21d1733a3", size = 16185987, upload-time = "2026-10-30T19:18:39.794Z" }, +] + +[[package]] +name = "bracex" +version = "3.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4d/fc/137095eb2f24ccf0b4db196abd9f5cfe74398a834af51d4bc4876e0beaf0/bracex-3.2.1.tar.gz", hash = "sha256:07a9b2100ceecc0c24afa1a1714bf8d4a6abb43d592939455f662b682ac9baa4", size = 32694, upload-time = "2026-11-03T14:13:43.53Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/ae/30dd8b027b4032b27e81364de5b74bfbef50b3c84c6aa0ead351bbb3749c/bracex-3.2.1-py3-none-any.whl", hash = "sha256:e8622d46b60ef6062b4091033f97a304e1b205e70c0285743c5038b73886a47e", size = 12527, upload-time = "2026-11-03T14:13:42.032Z" }, +] + [[package]] name = "certifi" version = "2026.10.23" @@ -422,6 +489,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "connectrpc" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf-py" }, + { name = "pyqwest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b8/43/025430769a6cc82a1e1c76cd9760639f6bf9b05577f90e38a0bb57b41b39/connectrpc-0.12.1.tar.gz", hash = "sha256:da7ca41c6cd8d9ce640ad86ae325a71cd83dfdd1e43748976315cb4b0546fb69", size = 50556, upload-time = "2026-09-04T15:23:31.327Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/f8/b244edb73395c5a74bfeab13ace9e6ccda89d31fb5e493ca8a8c96106eae/connectrpc-0.12.1-py3-none-any.whl", hash = "sha256:ba85ff49b5b7e0f1b16d07b35a05e39ce92c1a7116a7ed1a271c76393323cd07", size = 70058, upload-time = "2026-09-04T15:23:28.686Z" }, +] + [[package]] name = "coverage" version = "7.10.5" @@ -563,6 +643,134 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/5e/6f8d874366788ad5d549e9ba258037d974dda6e004843be1bda794571701/datasets-4.4.1-py3-none-any.whl", hash = "sha256:c1163de5211e42546079ab355cc0250c7e6db16eb209ac5ac6252f801f596c44", size = 511591, upload-time = "2025-11-05T16:00:36.365Z" }, ] +[[package]] +name = "daytona" +version = "0.210.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiofiles" }, + { name = "aiohttp" }, + { name = "daytona-analytics-api-client" }, + { name = "daytona-analytics-api-client-async" }, + { name = "daytona-api-client" }, + { name = "daytona-api-client-async" }, + { name = "daytona-toolbox-api-client" }, + { name = "daytona-toolbox-api-client-async" }, + { name = "deprecated" }, + { name = "httpx" }, + { name = "httpx-ws" }, + { name = "obstore" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-instrumentation-aiohttp-client" }, + { name = "opentelemetry-sdk" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "python-multipart" }, + { name = "python-socketio", extra = ["asyncio-client", "client"] }, + { name = "toml" }, + { name = "typing-extensions" }, + { name = "urllib3" }, + { name = "wsproto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/dd/9ce4820cb05cc2b7f38f9fd076bc575b1190a3236d69a40a0f5a09ebd14c/daytona-0.210.0.tar.gz", hash = "sha256:313fcec07ec4d04a6ccaa91618ce956612ef9ba78484691aeccdaa36d017d6b4", size = 188817, upload-time = "2026-09-03T21:11:01.028Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/2d/f4c1dec9dc0da86aa79b64a83e6f3c94ff9a25ed78a1bc59c5c2fa3c51db/daytona-0.210.0-py3-none-any.whl", hash = "sha256:0df723203e20a894fd07808e62f13f419e41f6d035016ecdc8f873c50cb32634", size = 226269, upload-time = "2026-09-03T21:11:02.374Z" }, +] + +[[package]] +name = "daytona-analytics-api-client" +version = "0.210.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4f/2d/76721bf383de06bb16ce6dec473d0f18f2b6853602d7410ce0116e1981f9/daytona_analytics_api_client-0.210.0.tar.gz", hash = "sha256:b4230a0a6afe65e2a1c7b0e9bb49bf905a795b1610256afaad956a3475b8569d", size = 30088, upload-time = "2026-09-03T21:10:27.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/31/b6ed690257cfb34b37088919b36ee9f87a8d76aaf08ef110b093580de8ac/daytona_analytics_api_client-0.210.0-py3-none-any.whl", hash = "sha256:2e70b15e8c14bb4e88a12bb3644b6e4f4a296305298f111efac4409f3e92ae37", size = 45013, upload-time = "2026-09-03T21:10:30.978Z" }, +] + +[[package]] +name = "daytona-analytics-api-client-async" +version = "0.210.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "aiohttp-retry" }, + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a5/72/a0d2ced3b786624b95d3059d0fe7641736064e6470018ff023e46ba3c4f4/daytona_analytics_api_client_async-0.210.0.tar.gz", hash = "sha256:762f19d4e8d9eb74b3d5e3cdeec0ebeddae623d7b6a764e6c97a7f5cafa8f8c9", size = 30130, upload-time = "2026-09-03T21:10:35.562Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/a7/8ee5f1fa221c30914aa6f0a9a5a66c7d176d109c3feae3bf61a7b571ac5c/daytona_analytics_api_client_async-0.210.0-py3-none-any.whl", hash = "sha256:258af338e3a95c4a02b3e5c84c8283c0aad1e14151cd2ab4f575700cc200b48a", size = 45284, upload-time = "2026-09-03T21:10:36.483Z" }, +] + +[[package]] +name = "daytona-api-client" +version = "0.210.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b4/fb/3f577dea27c04937ad2cea314877346d2ac5cdf44925bd09eb2b6de2e6f5/daytona_api_client-0.210.0.tar.gz", hash = "sha256:1aea806ca4fe86f0f59b66c043b849d1acf61d9b7db240ec13b7c0678f955590", size = 141803, upload-time = "2026-09-03T21:10:27.399Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/bf/7b6e062bf754707c0a29c5e65de2c98fb1a04b2f298fbceba9ab513b88ea/daytona_api_client-0.210.0-py3-none-any.whl", hash = "sha256:c55f42d13fc8da84380fbdc5bc010c67dca7f9b9d399546ab54e8810c5dc1c38", size = 356132, upload-time = "2026-09-03T21:10:28.713Z" }, +] + +[[package]] +name = "daytona-api-client-async" +version = "0.210.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "aiohttp-retry" }, + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/f4/6707985e647280d34a87de4ed24fd501bbe4999c0165b4c9b374490b773b/daytona_api_client_async-0.210.0.tar.gz", hash = "sha256:0b597f289f992572e776e3616cf5aae89a9b623dbbc911213e69c2d193bfe302", size = 142201, upload-time = "2026-09-03T21:10:31.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/47/7ef49573074d5956578f8c899b528a8107e0dfc9fe0d953c32859d968dc0/daytona_api_client_async-0.210.0-py3-none-any.whl", hash = "sha256:6da7dc68915b216d1a7a821e8dbe3c832b63c63c0e8701cda1216d5c38080e59", size = 358962, upload-time = "2026-09-03T21:10:33.482Z" }, +] + +[[package]] +name = "daytona-toolbox-api-client" +version = "0.210.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/16/83d9b090d8107a95f871ed0154392732918ee6b0606b7117444353b29412/daytona_toolbox_api_client-0.210.0.tar.gz", hash = "sha256:f816c60d01bbd8de4649caa6e6a05bd025a8096e17d6096cbc8b537db900fa9a", size = 88114, upload-time = "2026-09-03T21:10:27.563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/43/5810261a1d57c4eff93960a1848767ca14a2d5fb543d1f32aab74954fdbd/daytona_toolbox_api_client-0.210.0-py3-none-any.whl", hash = "sha256:f346fbfc7db3ad5201dfe35610fd068d3992f8ae5e3bc5a312d1214da528dddb", size = 252413, upload-time = "2026-09-03T21:10:29.504Z" }, +] + +[[package]] +name = "daytona-toolbox-api-client-async" +version = "0.210.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "aiohttp-retry" }, + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c1/b9/419fcacbf90f22679da6f73a2ba6c8dae3c7054e81546589dca28f5c2c0e/daytona_toolbox_api_client_async-0.210.0.tar.gz", hash = "sha256:82e1b0c9cc5db0484be061e39a9810bc7df1e8c0ca0aeb0cd9a3dd6d5eb1101a", size = 82027, upload-time = "2026-09-03T21:10:32.399Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/57/1d2bef2d66f6d212155aeb2ca09e2f817b7eea84df759cd1fab10b4b1f7b/daytona_toolbox_api_client_async-0.210.0-py3-none-any.whl", hash = "sha256:cf4a1c9b64f3a7f8fc8d2d46b5c577824bb43452744fcc4b3874a25e04f52a05", size = 250905, upload-time = "2026-09-03T21:10:34.217Z" }, +] + [[package]] name = "defusedxml" version = "0.7.1" @@ -572,6 +780,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, ] +[[package]] +name = "deprecated" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, +] + [[package]] name = "devtools" version = "0.12.2" @@ -622,6 +842,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, ] +[[package]] +name = "e2b" +version = "2.63.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "connectrpc" }, + { name = "e2b-dockerfile-utils" }, + { name = "h2" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "protobuf-py" }, + { name = "pyqwest" }, + { name = "python-dateutil" }, + { name = "rich" }, + { name = "typing-extensions" }, + { name = "wcmatch" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/f7/4d317f3c22992152a934e1f47ec5c6eea61350635a1de628f2c8f4e34987/e2b-2.63.0.tar.gz", hash = "sha256:01c9aa7c8565e9730c6cfa075a10a67a7560db9e11f67458fb1bce21e0e2e0cc", size = 273466, upload-time = "2026-11-04T16:41:31.849Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/24/4777775082b058be6c531bbeadf22d97897953320b24323b8dd45e902231/e2b-2.63.0-py3-none-any.whl", hash = "sha256:fae437d0de051c633e6854260d94c178d7155711387375b5a447b326efdda065", size = 430158, upload-time = "2026-11-04T16:41:30.173Z" }, +] + +[[package]] +name = "e2b-dockerfile-utils" +version = "0.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/fc/15b2752ff08fcbb7c6b97451581e02dd189ddc9b6eb107bb17c35b04427f/e2b_dockerfile_utils-0.1.1.tar.gz", hash = "sha256:c1d4179bae6f3131fe0bbb1e6787ffb07c6c6dca8bdb3399abb3de9056b2a08f", size = 16844, upload-time = "2026-10-13T15:01:50.959Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/39/8d3f59a073dd1637fccca4c921665935d9591cc1726b452f06ef7cdadabc/e2b_dockerfile_utils-0.1.1-py3-none-any.whl", hash = "sha256:c648ecf56b3ffdb6d91469580850f4691f019cb2c925405e776ad36b761204f1", size = 19901, upload-time = "2026-10-13T15:01:49.942Z" }, +] + [[package]] name = "execnet" version = "2.1.1" @@ -813,7 +1065,7 @@ name = "gym-responses-api-agents-browsecomp-agent" version = "0.0.0" source = { virtual = "." } dependencies = [ - { name = "nemo-gym", extra = ["dev"] }, + { name = "nemo-gym", extra = ["dev", "sandbox"] }, ] [package.optional-dependencies] @@ -823,7 +1075,7 @@ telemetry = [ [package.metadata] requires-dist = [ - { name = "nemo-gym", extras = ["dev"], editable = "../../" }, + { name = "nemo-gym", extras = ["dev", "sandbox"], editable = "../../" }, { name = "nemo-gym", extras = ["telemetry"], marker = "extra == 'telemetry'", editable = "../../" }, ] provides-extras = ["telemetry"] @@ -837,6 +1089,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "h2" +version = "4.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "hpack" }, + { name = "hyperframe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/85/7c366e69d84c17bb778fe41419e1fbcce3033d5b7ce29bbffff0a98b859f/h2-4.4.1.tar.gz", hash = "sha256:4e866ffb1a869ae14dd9b5e6beb5c24a13da0495ad72b65925ded182521c1516", size = 2157281, upload-time = "2026-08-03T11:45:09.509Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/22/e85faf23bd72a92d1921e37d674ca56eb298a3c8be31fdecef0ff2b3aaac/h2-4.4.1-py3-none-any.whl", hash = "sha256:0e25f1462b23c9cb82d9eb02e28bc706dac2a68cb457c6a0d74d63c8a2a5d0e6", size = 62636, upload-time = "2026-08-03T11:44:59.164Z" }, +] + [[package]] name = "hf-xet" version = "1.6.0" @@ -861,6 +1126,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, ] +[[package]] +name = "hpack" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/5b/fcabf6028144a8723726318b07a32c2f3314acdff6265743cf08a344b18e/hpack-4.2.0.tar.gz", hash = "sha256:0895cfa3b5531fc65fe439c05eb65144f123bf7a394fcaa56aa423548d8e45c0", size = 51300, upload-time = "2026-06-23T18:34:46.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/b4/4a9fcfb2aef6ba44d9073ecd301443aa00b3dac95de5619f2a7de7ec8a91/hpack-4.2.0-py3-none-any.whl", hash = "sha256:858ac0b02280fa582b5080d68db0899c62a80375e0e5413a74970c5e518b6986", size = 34246, upload-time = "2026-06-23T18:34:45.472Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -927,6 +1201,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" }, ] +[[package]] +name = "httpx-ws" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpcore" }, + { name = "httpx" }, + { name = "wsproto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/cd/ca91a07ae446451f7476bf3fcc909e98cb942ff032ebfda0e3fe449aca7b/httpx_ws-0.9.0.tar.gz", hash = "sha256:797373326f70eec1ae96f6e43ae9f12002fd7d73aee139a4985eaab964338a08", size = 107105, upload-time = "2026-03-28T14:11:10.781Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/f8/a6bc80313a9e93c888fa10534dfce2ad76ff86911b6f485777ce6de6a073/httpx_ws-0.9.0-py3-none-any.whl", hash = "sha256:71640d2fb1bf9a225775015b33cd755cfd4c5f7e21c885192fe3adc4c387b248", size = 15759, upload-time = "2026-03-28T14:11:11.887Z" }, +] + [[package]] name = "huggingface-hub" version = "1.33.0" @@ -961,6 +1250,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/fd/612cc7905787100dc9225ae6ad71b42b60cbc23cd56ca55afcdd1e000f81/hydra_core-1.3.6-py3-none-any.whl", hash = "sha256:55951f49c7e19c29f4281cce2eeaa6e73ef6c5620093a4efb402a7e4b839fc38", size = 163055, upload-time = "2026-08-29T14:36:06.385Z" }, ] +[[package]] +name = "hyperframe" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, +] + [[package]] name = "identify" version = "2.6.13" @@ -1033,6 +1331,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/4a/4175a563579e884192ba6e81725fc0448b042024419be8d83aa8a80a3f44/jiter-0.10.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3aa96f2abba33dc77f79b4cf791840230375f9534e5fac927ccceb58c5e604a5", size = 354213, upload-time = "2025-05-18T19:04:41.894Z" }, ] +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + [[package]] name = "jsonschema" version = "4.25.1" @@ -1257,6 +1564,13 @@ dev = [ { name = "ruff" }, { name = "tomlkit" }, ] +sandbox = [ + { name = "boto3" }, + { name = "daytona" }, + { name = "e2b" }, + { name = "opensandbox" }, + { name = "tenacity" }, +] telemetry = [ { name = "nemo-lens", extra = ["sdk"] }, { name = "opentelemetry-instrumentation-fastapi" }, @@ -1416,6 +1730,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/9e/1652778bce745a67b5fe05adde60ed362d38eb17d919a540e813d30f6874/numpy-2.3.2-cp314-cp314t-win_arm64.whl", hash = "sha256:092aeb3449833ea9c0bf0089d70c29ae480685dd2377ec9cdbbb620257f84631", size = 10544226, upload-time = "2025-07-24T20:56:34.509Z" }, ] +[[package]] +name = "obstore" +version = "0.11.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/27/aa7549157a4a681157e315534ba5ab8f167f77662166e792a6e836938f46/obstore-0.11.1.tar.gz", hash = "sha256:a5afe8b99e3b20cdc9133be7a1b381259acf0d470029f6b2fc79c3f9947ad436", size = 130828, upload-time = "2026-08-21T23:57:27.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/38/438f85772bfbcd2985845a5d00d1c910e98b15f587dae6cb1e435865eba9/obstore-0.11.1-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d5c50b755d781efebe4f9c70ee6d00858e44d95f0229b8b5174358f861d9bd7c", size = 5400146, upload-time = "2026-08-21T23:56:29.168Z" }, + { url = "https://files.pythonhosted.org/packages/e7/62/edd2613649cb6b4400f5d125223ee094d2da1c4c93636453a5360c61b865/obstore-0.11.1-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:093152daa5c32b70a032f231bbc6a7eff76dda4285eed5a81d272b4485032dab", size = 4596203, upload-time = "2026-08-21T23:56:30.7Z" }, + { url = "https://files.pythonhosted.org/packages/c1/52/09edd48251a26a65f786bf2def93994ffce501f3aea6724474d8de0a7f4f/obstore-0.11.1-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b97ee10456e65f166c030b5fbde8380ca508621e23b84efd77aad83afe99315a", size = 5003021, upload-time = "2026-08-21T23:56:32.085Z" }, + { url = "https://files.pythonhosted.org/packages/59/8f/89a28c75d46bbb0552d34a5e84c5bc512526d45f935321f00af2e6275a99/obstore-0.11.1-cp311-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48983a143de69b11de49212caa79b5c39acbff7f592f9e85d156a0133a0e5796", size = 5240376, upload-time = "2026-08-21T23:56:33.658Z" }, + { url = "https://files.pythonhosted.org/packages/50/c3/b4620899003e2472bec7baf5d2bce12cf6f6f11d02d8f17e03e093717a53/obstore-0.11.1-cp311-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b9cb988e03ff963914cf2176aee293eaaff7dcf4efcb905ce6834264b4b0884", size = 5444946, upload-time = "2026-08-21T23:56:35.093Z" }, + { url = "https://files.pythonhosted.org/packages/bd/de/687685ab39ae4a70cc13ac252febfe62a387836a55e046f467c9fb231880/obstore-0.11.1-cp311-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97850f68c8417f7167549bdb705c362c825ed470298c6b04faf52258c5e9234e", size = 5304065, upload-time = "2026-08-21T23:56:36.535Z" }, + { url = "https://files.pythonhosted.org/packages/b4/36/b962ad52031b5bfdd249923db6151ff2b665b08c017611db96838455f337/obstore-0.11.1-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a9f3d66dbf3c073dd6b6033c0787ef14edf78d97bff95595d6f6979692da264", size = 5556770, upload-time = "2026-08-21T23:56:38.377Z" }, + { url = "https://files.pythonhosted.org/packages/e0/04/41a0f2917fcaa1e66b754fc98c92441d5a252ff9c619941ac086f9bdb4f6/obstore-0.11.1-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:2ba3bceec4263b3a70eea873abedb82e03da9599f2326e80d6505cab0c10d401", size = 5337765, upload-time = "2026-08-21T23:56:40.279Z" }, + { url = "https://files.pythonhosted.org/packages/f9/39/0b618efe59ee4b58f16d2b9246674a70c390bebf665cd284b439bade9bd3/obstore-0.11.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e956fa8a953b7eb8580658d68cdf37e410356032c17697a06e44d0e8c4084a0f", size = 5544012, upload-time = "2026-08-21T23:56:41.754Z" }, + { url = "https://files.pythonhosted.org/packages/57/3b/1018f6da240f3529d1cb9a836c79a91ce45349ba3cffcd6baa73296c7e6c/obstore-0.11.1-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:159a50d0f4cc53afe6f5c695313bcaf6e92ac81d7847692c8279153483272bfa", size = 5229851, upload-time = "2026-08-21T23:56:43.367Z" }, + { url = "https://files.pythonhosted.org/packages/35/2c/c2fe082816b255159373b15be3f28b2515d0eb954e248e2f65449c456a14/obstore-0.11.1-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:128da07f3a1b9c70159e2b2be9e27f458a9d531d57b1856dd7c4ddb73b4c9937", size = 5361812, upload-time = "2026-08-21T23:56:44.871Z" }, + { url = "https://files.pythonhosted.org/packages/43/9c/656eb5cce818e7e3985d27f4bd403feb0b6e620494775a55a0943c8d2fff/obstore-0.11.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:67b8acfbbab960c2cb14c34294a96556caa67fcff15ee77c716d5c1bd1558606", size = 5790855, upload-time = "2026-08-21T23:56:46.391Z" }, + { url = "https://files.pythonhosted.org/packages/c1/5c/94ece1c902ff5cdbe1140997d1f720221a0d5defa902fdd93715297604f5/obstore-0.11.1-cp311-abi3-win_amd64.whl", hash = "sha256:e23ea15cebe5f5be5d11005043d7b5ff56848e39499681ef52f8227bd385bd51", size = 5308846, upload-time = "2026-08-21T23:56:47.774Z" }, + { url = "https://files.pythonhosted.org/packages/c0/af/235730a429e55777f15f9d7fef118a6ecf02f430ed4588deb4b7e13964aa/obstore-0.11.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:73284fc8a9804596baf4d80b55c0e71a6007487bfa28d6924acd85264d5be81f", size = 5424585, upload-time = "2026-08-21T23:56:49.532Z" }, + { url = "https://files.pythonhosted.org/packages/e3/3c/432ea70c061fc9401fe399391ed4f04dead4a4cf33ea7c9564ce0e299d24/obstore-0.11.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8a3f93310422b153629af929d9e88d1fc826d5e32a456de4d3a1926a770ae09c", size = 4576791, upload-time = "2026-08-21T23:56:51.014Z" }, + { url = "https://files.pythonhosted.org/packages/43/c7/57dbdfa4ba5339b80fc755359bb232bdba7f82e92df6603ef383396b5f16/obstore-0.11.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d666690f0a53ae3df4be2c18af6820369a74c9c5b8960e8760f0f8c9a8f15b36", size = 4992141, upload-time = "2026-08-21T23:56:52.46Z" }, + { url = "https://files.pythonhosted.org/packages/37/4b/d25a4c129250364b0ca927fbfca5146744fb3bb4de60ae9b302300cbdfef/obstore-0.11.1-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0844ab75c8413c0af2d0fd2d2f6aa42d166809b6ac4be478151cd946ce7e5d69", size = 5217707, upload-time = "2026-08-21T23:56:53.946Z" }, + { url = "https://files.pythonhosted.org/packages/4b/07/0a925c886639d6edce37aafb6eba0c86820cbc9dda42e91ddd784dbe059d/obstore-0.11.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:307b5f9d64a7c00cd13371e376c05e3914216bd0818a19ff2ae05bc0135e01dc", size = 5429440, upload-time = "2026-08-21T23:56:55.507Z" }, + { url = "https://files.pythonhosted.org/packages/5f/c0/38d188a00b9f2de30df26802dd5bc2ecaefdabb882fde786cff800ccf50f/obstore-0.11.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f89647953b4ea50bab3f66591f7f462cd454a5d2fbbeefd885716a2c54e13ce", size = 5308903, upload-time = "2026-08-21T23:56:56.924Z" }, + { url = "https://files.pythonhosted.org/packages/60/d3/815e5a7ca59f901b6d24fb01ad3af46de37dfdfd2971c019f914bdb7f65e/obstore-0.11.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50de3116af69b6f1669cb443cc200ccdc1523a790a41f00854331b48f716cf8f", size = 5547007, upload-time = "2026-08-21T23:56:58.508Z" }, + { url = "https://files.pythonhosted.org/packages/9c/a1/ec4d2291b7eb15e7ddccb2f0646ce560b8a539df4b0ab94d916e7a15a61c/obstore-0.11.1-cp314-cp314t-manylinux_2_24_aarch64.whl", hash = "sha256:eae71e1c5944ade976ce8cd1e780bed9b5c31d6d4f89cb7263e8dcff78f6cd8e", size = 5330062, upload-time = "2026-08-21T23:56:59.954Z" }, + { url = "https://files.pythonhosted.org/packages/0a/09/a4a1301f2474f6569bbc0f2d337108ff2225d0b3d285894ed9a6dc953438/obstore-0.11.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ff634b5edbbf76c56ae81397aa41500a66ad0aa48b1856dcc0cf31b159172dc0", size = 5539870, upload-time = "2026-08-21T23:57:01.485Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8f/7dbbf935a07ff5375c33916307b77aef9d39bded35081530150822898010/obstore-0.11.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cbe509350d66249fc9e65ece4e7b1855d650f18e477721887128452b95c44b24", size = 5222673, upload-time = "2026-08-21T23:57:03.023Z" }, + { url = "https://files.pythonhosted.org/packages/d5/89/4932b3dd7963a3a64fd532c30ccc836f0f4ede1a45bea9f12295299398e2/obstore-0.11.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:46122e585f48ab3f2e4fed51401a4e866279a3a3d1aee2372d598ab85ba2c539", size = 5335696, upload-time = "2026-08-21T23:57:04.439Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ff/f3cf31ac4ebfa648dbd2c69774579fd5da4cf4b05fa1192405dafb6aeb38/obstore-0.11.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7275e75228059b2b30772b2ba3e41562a4a92ebdd3a96619d300f98ef152119f", size = 5783641, upload-time = "2026-08-21T23:57:05.899Z" }, + { url = "https://files.pythonhosted.org/packages/51/31/b352db3450e700c2c51861d139fc92c3284f7e578ca306408b054f6b4a35/obstore-0.11.1-cp314-cp314t-win_amd64.whl", hash = "sha256:13bb0b6a40931ab2da93f93ad843865d483d95acec1bd586df79313daa5a50af", size = 5290060, upload-time = "2026-08-21T23:57:07.632Z" }, +] + [[package]] name = "omegaconf" version = "2.3.0" @@ -1448,6 +1796,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/f4/561ed79fd94876160018a5e75254cfcb9b0e62d4dded9dcb20072e86d623/openai-2.44.0-py3-none-any.whl", hash = "sha256:0a2a3ab2e29aeda368700f662ff9ba0f9df17ba4c54577a64e08b8115a3cc0ad", size = 1366216, upload-time = "2026-06-24T20:55:58.882Z" }, ] +[[package]] +name = "opensandbox" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "httpcore" }, + { name = "httpx" }, + { name = "httpx-sse" }, + { name = "opentelemetry-api" }, + { name = "pydantic" }, + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/2f/6682d18a69043b5ad29e8a411e969e2ee4216127dbc2257752690f502287/opensandbox-1.2.0.tar.gz", hash = "sha256:e7936d0bf3f7defa4e26571b0634089b4c3eb15d023909f6da9d940b95736c54", size = 329030, upload-time = "2026-10-27T12:36:39.214Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/18/2de1578aead2e5da094c352e2c1610d750c399070769feac02cb42ff8941/opensandbox-1.2.0-py3-none-any.whl", hash = "sha256:15a0f3c3245018afa43b75fb14e892a64c2c8807276a0fdaf2a1a469f5e214ea", size = 744130, upload-time = "2026-10-27T12:36:37.805Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.44.0" @@ -1523,6 +1889,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/7b/85eab1215f72adf0e68d3dc4a679b9bff993fa679ff34cd8dd378e2659fd/opentelemetry_instrumentation-0.65b0-py3-none-any.whl", hash = "sha256:ea967a72b9939b5fcfdad572753b4306c59dcb99e3f382d95dae04286805e137", size = 36717, upload-time = "2026-07-16T15:24:51.424Z" }, ] +[[package]] +name = "opentelemetry-instrumentation-aiohttp-client" +version = "0.65b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "opentelemetry-util-http" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/33/3ff7230b035e8b696db6be54f5c52dfa409829d634d91431c076ad789820/opentelemetry_instrumentation_aiohttp_client-0.65b0.tar.gz", hash = "sha256:85906a2806ee5641756b5c33274e9aa75c3cc2441e3b830aa5804cf0e1fa9dd1", size = 19042, upload-time = "2026-07-16T15:25:51.632Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/f9/5c8459224f175829601cabbcffaeab0f9041903c086e4a0368f9971093a5/opentelemetry_instrumentation_aiohttp_client-0.65b0-py3-none-any.whl", hash = "sha256:3a060efa53fa44d02ba7372a7ed2b42cdfa6be6df81b089845067ad840e25729", size = 13677, upload-time = "2026-07-16T15:24:53.361Z" }, +] + [[package]] name = "opentelemetry-instrumentation-asgi" version = "0.65b0" @@ -1779,6 +2161,49 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, ] +[[package]] +name = "protobuf-py" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf-py-ext", marker = "(platform_machine == 'arm64' and platform_python_implementation == 'CPython' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_python_implementation == 'CPython' and sys_platform == 'linux') or (platform_machine == 'AMD64' and platform_python_implementation == 'CPython' and sys_platform == 'win32') or (platform_machine == 'ARM64' and platform_python_implementation == 'CPython' and sys_platform == 'win32')" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/2a/6a08d3a3ddbfd745f987c5ff22129a979ea74319fbfe5b9d59c7a6730d25/protobuf_py-0.6.0.tar.gz", hash = "sha256:be6b8d84ba54380ca5e5e50f20e4dd42df81c845cf9dc8c021f147d8c516b341", size = 161533, upload-time = "2026-09-30T06:45:41.388Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/d2/6271d5682d244b5b771f72fe7536a0c7e680e14300ea58f7d033fce813a8/protobuf_py-0.6.0-py3-none-any.whl", hash = "sha256:2aba52a2d017487901cb4b8a7983e64a7bd369128c2ca86cafa4dd03da31012e", size = 216054, upload-time = "2026-09-30T06:44:46.94Z" }, +] + +[[package]] +name = "protobuf-py-ext" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b4/6c/a1f6a1a50dfc3f14ddb9e54f9a769bc589b85b5e394bab2bb78f67d2dc42/protobuf_py_ext-0.6.0.tar.gz", hash = "sha256:03f61ccfcad52a2d784aec9bb0b6088e0e32f144a6c3ab3deed12f17f1f0d0bb", size = 58124, upload-time = "2026-09-30T06:45:42.602Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/f1/b02c02586e96ad8d4534187713d1fdfca32c471f09ed36dcef7c1debc0d6/protobuf_py_ext-0.6.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:6609132191e992ae7a211bf1988bb02c8fa323c915fb0b8b23914dae12c8cc97", size = 469480, upload-time = "2026-09-30T06:44:57.064Z" }, + { url = "https://files.pythonhosted.org/packages/29/c1/9f3203e0714e021e8d70c95c7a2a048bd4baacf9cf27ae1b5e2dc056fd45/protobuf_py_ext-0.6.0-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64a3075e96fea0107cbe2a0e7103dd35f5566e98088463b4572c70fb737bbede", size = 478330, upload-time = "2026-09-30T06:44:58.588Z" }, + { url = "https://files.pythonhosted.org/packages/07/5b/9520fc2d5be143cc9b54cda777551c60f1a9ea2c1efd0c0fa1701f3cb044/protobuf_py_ext-0.6.0-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e03b2aa768ea7185feab4267eee2a5d5fe5fa39c389f02689a6f5b7869c174c", size = 504049, upload-time = "2026-09-30T06:45:00.025Z" }, + { url = "https://files.pythonhosted.org/packages/6d/6f/7f69494f11c05c0cbc225d391cac2ae2b129be8be7818f48ef6ee4d7e407/protobuf_py_ext-0.6.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:afc419f235195e07d7c095c992935251b9207b2e0f6283c9324f9d6243eef401", size = 657357, upload-time = "2026-09-30T06:45:01.778Z" }, + { url = "https://files.pythonhosted.org/packages/ae/a1/f6048540d29565ebaedf70b5271da23173cc26c23edf09727b789ddd012e/protobuf_py_ext-0.6.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e0d5893a4d26e2c7e535a38c4e34be699469fcb182338626cc7bf4d6ea078", size = 716603, upload-time = "2026-09-30T06:45:03.629Z" }, + { url = "https://files.pythonhosted.org/packages/0c/7b/a0d587cdf856c3413f57a028621f469c749c3d23bbf29a947e9f258f97f0/protobuf_py_ext-0.6.0-cp311-abi3-win_amd64.whl", hash = "sha256:b5be43ebf9ec28aa7e3734bf07c0ea4ae50c2e7554b9299ea08433549052e8d2", size = 440643, upload-time = "2026-09-30T06:45:05.086Z" }, + { url = "https://files.pythonhosted.org/packages/c6/51/f29c03261a837072e8da59956fea097a57a1dab222cf11c07547446f01df/protobuf_py_ext-0.6.0-cp311-abi3-win_arm64.whl", hash = "sha256:a9e50e205f7c366febf1d12ab183b5a3034793441faaa57cab4fe9a8a1c0e1dc", size = 420484, upload-time = "2026-09-30T06:45:06.843Z" }, + { url = "https://files.pythonhosted.org/packages/54/71/131125d553e4bf526965ebc06a354f26d9ca3baf54118c5ad8421978b6e0/protobuf_py_ext-0.6.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a05c984234ad74b84e764fa7eb01b00c2258ed55195202ae67fdfa1b551533b5", size = 471478, upload-time = "2026-09-30T06:45:15.819Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d4/a7fcb16d83b7e37aa2561760faa78faca0be56ecef7bc2a9ce0b646c583f/protobuf_py_ext-0.6.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62f197bec48889c9824bb3f4597aa6b0d45dd2ba89680246d0ddd724aff49281", size = 479518, upload-time = "2026-09-30T06:45:17.16Z" }, + { url = "https://files.pythonhosted.org/packages/9c/69/f385581d4f0f25a9c0271de640be0183e725574ca1166d32bdf798740299/protobuf_py_ext-0.6.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:669519bdf23713494e7247ee89dbd57cf1b9c32071ee69fe0ab5e6eeadd42477", size = 501274, upload-time = "2026-09-30T06:45:18.707Z" }, + { url = "https://files.pythonhosted.org/packages/b3/45/e635f1a7e70438aa2f563321e30a11012cae05aa6ad6c2ac72bbd5369ad6/protobuf_py_ext-0.6.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df1eea6f53d6f78b031e94b08e98e2492d35cfb49d7d2b2639b2d472c48b4de7", size = 657613, upload-time = "2026-09-30T06:45:20.507Z" }, + { url = "https://files.pythonhosted.org/packages/29/a1/e698f967e10c377187064b0d67ea64f561d164e0a55d29385a180495b9a8/protobuf_py_ext-0.6.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8ebcf2610c2cee06f9fbcca8be352848bf4e927e1cf58d902905487a7e249c4b", size = 714222, upload-time = "2026-09-30T06:45:22.073Z" }, + { url = "https://files.pythonhosted.org/packages/6a/26/d09e892ab83420ddd52e5ae86507675c411a3fae038f022dc7e82eba7cf9/protobuf_py_ext-0.6.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:561bda3d588a0ef756cccaba546ef04dbd0bd167556afbd83aea2be76b05aadc", size = 469655, upload-time = "2026-09-30T06:45:23.908Z" }, + { url = "https://files.pythonhosted.org/packages/25/2e/f6c327625cb2ac1ba472452c5ae60563a126b9340ae8d46b7e5a6c2360db/protobuf_py_ext-0.6.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e55f9a1c4329c05638a4adfc1754be6c5aaef790965bf704b431e925f1801d71", size = 477421, upload-time = "2026-09-30T06:45:25.59Z" }, + { url = "https://files.pythonhosted.org/packages/cd/74/0078a5ed1fdebd82772817b8dfd5cc5fbec4c3c51d63356a5a7f9d0c53a7/protobuf_py_ext-0.6.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69aad94e7bda448b3d0f5752ace8f2d745562dba9e50f4c2e3b4fb3d2969d9bc", size = 499937, upload-time = "2026-09-30T06:45:27.392Z" }, + { url = "https://files.pythonhosted.org/packages/fb/92/f7b5a35d2c8804c13edeec22e9cbc084cff2c11ada58aeb896c97f9fc94b/protobuf_py_ext-0.6.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b26888854d3c4b55a161defaacfb2ef895e85662cee3cacd95d56efa29344e3b", size = 655561, upload-time = "2026-09-30T06:45:28.933Z" }, + { url = "https://files.pythonhosted.org/packages/01/c4/3c54994605801ac565f6f156a55e348c5f08cd5592650356e69f68d61c05/protobuf_py_ext-0.6.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:15888690bf4064ecc657623f98a13d8b4ee223cbd790c0b8c48f29530ac47c7e", size = 713156, upload-time = "2026-09-30T06:45:30.493Z" }, + { url = "https://files.pythonhosted.org/packages/41/6a/99667bcfc5a4b914095e8d522a31bdf0cde56655d3d29ff739fa412ed064/protobuf_py_ext-0.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6607d37c86f454f8fa02d54186730befe4f73fa19943d2b09123c9b3747856af", size = 458472, upload-time = "2026-09-30T06:45:31.955Z" }, + { url = "https://files.pythonhosted.org/packages/7a/5e/6b585e3dfaf1efc60c3566b51dcd9a52fb7de990386f041e9bf8e48d9fa8/protobuf_py_ext-0.6.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1cc666423d220b95fae7e6f92c77ed7245b78c71bc50191f671cae021a8db25", size = 466913, upload-time = "2026-09-30T06:45:33.594Z" }, + { url = "https://files.pythonhosted.org/packages/ea/51/ca7535b71b330e1ff8577d0e352ee48250c9df307644319bdced1b743b84/protobuf_py_ext-0.6.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f9a107f2d760b5a5ca7d240d32456bae358455c9195d28237ff6be679d3633e", size = 489980, upload-time = "2026-09-30T06:45:35.103Z" }, + { url = "https://files.pythonhosted.org/packages/45/8b/154c22e66322e99a1ffd94b6d2f7ece89da367fe9b57ad1b1a54a1c4d23c/protobuf_py_ext-0.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0c4827ce9f96a83b72c5db86466f149dcc16b2f15a7ee601e6b1938f9b04e44f", size = 646208, upload-time = "2026-09-30T06:45:36.831Z" }, + { url = "https://files.pythonhosted.org/packages/b7/9b/3a5a66f64a5262cbb2609a02d1ab74b2f4da90ea537ca260b10a8c3bb6e8/protobuf_py_ext-0.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:005b95df22c72366c30a9adc9dc2d962017a2f0779583da70bb29770b09ad346", size = 703607, upload-time = "2026-09-30T06:45:38.512Z" }, +] + [[package]] name = "psutil" version = "7.1.3" @@ -1995,6 +2420,63 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, ] +[[package]] +name = "pyqwest" +version = "0.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/24/2a/f8634142207082b1c559a499451db794c96130e586f1edb22dc1da4b05b2/pyqwest-0.12.0.tar.gz", hash = "sha256:83ff498bd53bee92a2d2bf14476679307a99c78421c02cae08ec113bed2fbd7d", size = 516105, upload-time = "2026-10-09T06:22:44.949Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/e9/bf753f3001ca4cc72986eb9de68c08c1042c010a1dc2e93be32b0290e04d/pyqwest-0.12.0-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:03e8b74a4f6ad32759f0cee9c79d7bae37d3e8e346a17fe6795de992995b23fd", size = 5331355, upload-time = "2026-10-09T06:21:17.876Z" }, + { url = "https://files.pythonhosted.org/packages/b1/a3/68e7e8a173a4b5f8a759d9a5a4a60c2feffe052969e9126f8a3448dc357e/pyqwest-0.12.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:c1af3aad4eef813efc43f40a6e4e2791643d42f9218b6f1e9f9bdaa2a19bdc7d", size = 5189665, upload-time = "2026-10-09T06:21:19.744Z" }, + { url = "https://files.pythonhosted.org/packages/9b/e5/407de30d15456583af485488dcb62c99fa24bcac6fd1521b57ef848aac27/pyqwest-0.12.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dd55f27e4d526babb30c0d0e19c5517d7d6ecb59ab4219a30a5115fac1288698", size = 5751518, upload-time = "2026-10-09T06:21:21.296Z" }, + { url = "https://files.pythonhosted.org/packages/29/9d/d7dcb2f3da4ea43c0e68894acea655063955d0c91401e74fb97f79c8265f/pyqwest-0.12.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:699662fca52a1595a1ed9c169816b7654c0b28308b9284091e96bbc79b8e408f", size = 5722887, upload-time = "2026-10-09T06:21:22.916Z" }, + { url = "https://files.pythonhosted.org/packages/28/dd/dee75b5c39132c49e24313eeb79caa6c178a1975578f7226aac4bed2dbe1/pyqwest-0.12.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1a8815ab7b61dc932379d42424e422a66df9634499095942e3b91bfd8f5e1dfd", size = 5904177, upload-time = "2026-10-09T06:21:24.768Z" }, + { url = "https://files.pythonhosted.org/packages/24/2c/4a75b193f01a70375e6af2e080a12a64fd2057d342f4cd71090adaf6b949/pyqwest-0.12.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:0d4828c8dfbfee936c951cc467f0197891a752eb0d22d7469f1adb02152c400a", size = 6095821, upload-time = "2026-10-09T06:21:26.781Z" }, + { url = "https://files.pythonhosted.org/packages/d9/da/b618b54b17ed111b0bfd3a89d74aa4477ca3cb35dfe73d5b0f9916c6c07d/pyqwest-0.12.0-cp311-abi3-win_amd64.whl", hash = "sha256:3958c9ea1316e9b71290006f044e9c020daec5b5b652629bae9d2b26b6d5294d", size = 4969600, upload-time = "2026-10-09T06:21:28.615Z" }, + { url = "https://files.pythonhosted.org/packages/1c/74/2ef8a770aa6152da60c201e091cd5c692a591641e1c485348caed8591015/pyqwest-0.12.0-cp311-abi3-win_arm64.whl", hash = "sha256:ab6f54ea17a34628f631ca7b1e010f0dd5891f75fd9b743afcbdf1d29740120d", size = 4943859, upload-time = "2026-10-09T06:21:30.151Z" }, + { url = "https://files.pythonhosted.org/packages/88/49/494866a18aecda4593a52ca62397d53fb29938033075d41c009291096fe6/pyqwest-0.12.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:ba88e09d0cb6166b72a4a120a10df863f10c77dac8f53538a7ac266798e14167", size = 5344568, upload-time = "2026-10-09T06:21:31.89Z" }, + { url = "https://files.pythonhosted.org/packages/84/ee/09486016196b4a367286a8819e83185b78f771b01a3ca419e94d5c5589e6/pyqwest-0.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f2843c2f257db63580096f9bd4e83225a7c660a946693ec87777f906f5cdd21d", size = 5192763, upload-time = "2026-10-09T06:21:33.423Z" }, + { url = "https://files.pythonhosted.org/packages/cd/0d/42cb6e771aabb8745ccc7184bd132e4471a8659cdd7bee65f05dc94f0a78/pyqwest-0.12.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:6319e35a3f867a1a57071cfe63364b9f35b38d4ae3a6231fcab5d3a82e9d5705", size = 5751522, upload-time = "2026-10-09T06:21:35.123Z" }, + { url = "https://files.pythonhosted.org/packages/4c/a7/d09ef94ee9811ca7a698f2947a25618a60226c13f3e9193a2bc1880c4178/pyqwest-0.12.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:ba001c5f06a92426635d86c2ccb8a4c732f2bb1424361c68161f8726b5141520", size = 5714017, upload-time = "2026-10-09T06:21:36.672Z" }, + { url = "https://files.pythonhosted.org/packages/f9/b8/0770a8eaa97e845cbdf1e5fd9af36235580aae18f63d7e6ed0f257ffde01/pyqwest-0.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:39a7e94b109bc10b05a6151308bac6ead404a900768861216467544337a971d6", size = 5903886, upload-time = "2026-10-09T06:21:38.202Z" }, + { url = "https://files.pythonhosted.org/packages/df/f6/ad021859fb8380541604822ec6252c6e721f504b108e805b533ae8083a1f/pyqwest-0.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b8527b9795d7e72efaf0f2fcba6b293f85e1720201e44f94889ca33d263079e9", size = 6091049, upload-time = "2026-10-09T06:21:40.333Z" }, + { url = "https://files.pythonhosted.org/packages/0f/25/1167eab0e8fa4c07f8bbec5931749f7932d13bfea1ea3daa6dc78c67b8e1/pyqwest-0.12.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:d9f952e66e848612f5222794112f203a51070f93ca7129c4df5d6c7ef8f4762c", size = 5351261, upload-time = "2026-10-09T06:21:41.989Z" }, + { url = "https://files.pythonhosted.org/packages/d2/10/29daf8cffe7c547121d29b96a5b46d59fbd8f21bd80826c54a28cd89bc75/pyqwest-0.12.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4371bd32687cac6cd31ae61a424b3c072450fb9e1d82ad568c234bc9935e2db8", size = 5195343, upload-time = "2026-10-09T06:21:43.773Z" }, + { url = "https://files.pythonhosted.org/packages/bc/25/905f500e78216a729d69d24ccdf7f7a4c722321755dac97825620a98a366/pyqwest-0.12.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:469ac955bccf7080c6ed92afdd023c633d6d704b4be0653b82741fda3a60c85b", size = 5753112, upload-time = "2026-10-09T06:21:45.194Z" }, + { url = "https://files.pythonhosted.org/packages/34/93/b12629c268e743ed6cf39a08a78a696cbdb10d910fbcf4b0de6b0f89f143/pyqwest-0.12.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:05af8d0c6eb67661d6896824183b714a908321b3d10644594268b51d21a9cb1c", size = 5713673, upload-time = "2026-10-09T06:21:46.687Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c3/b0f689e60356143661c93e7295edd959a738e0d42a101e4c3647045d23c9/pyqwest-0.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:5f1f6599c0f826fd7e6717a3c6709d9fc25e1debc9e70bbb4b613c25377937c8", size = 5905698, upload-time = "2026-10-09T06:21:48.297Z" }, + { url = "https://files.pythonhosted.org/packages/f1/d6/412b69950180abf077c2dbf0b8ddf425a5fd65466e775bfbd63e28aefd1a/pyqwest-0.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d4de6b62293d57153db308897988afdfd22e481c4399279c6332c9efbc183330", size = 6090438, upload-time = "2026-10-09T06:21:50.131Z" }, + { url = "https://files.pythonhosted.org/packages/de/c0/1f6fca89081737afd8842bf6b4a6e6014dedd8d7d44b4da2d73f502c3b4c/pyqwest-0.12.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:34399a6b4c2d03ece732d41ed051710ea35c70f7daed9590ce9916db6cde92e6", size = 5336542, upload-time = "2026-10-09T06:21:51.839Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0f/7f364ac04e01258e0cfc37170a558cb36aaacff110f1a4c9d2f887e64be9/pyqwest-0.12.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1335062111f4da599d58223eeeb051f839a2c588274258facf6091aca92201a3", size = 5174602, upload-time = "2026-10-09T06:21:53.489Z" }, + { url = "https://files.pythonhosted.org/packages/11/9f/cb9c8ad10dc43e0b854041f4b471fde59624e8407c9c6da00f8907933c57/pyqwest-0.12.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:218f7a00d01678c7f5f02a0248d3b601d87d9f882179305f0c8f0545eb88d733", size = 5731531, upload-time = "2026-10-09T06:21:55.061Z" }, + { url = "https://files.pythonhosted.org/packages/52/2e/30a2b41d340d07f8205d02997a24c518c031d09898c8004ad671a8c9bc8b/pyqwest-0.12.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:27843d48330e2a163cf875e05294d9e4ea0e0e91bdbea956e3af630b678bd7d8", size = 5704167, upload-time = "2026-10-09T06:21:56.687Z" }, + { url = "https://files.pythonhosted.org/packages/60/c3/8c870cbf47857abd8490b8da2195cad71c3890556d776ecc01cd744bb35c/pyqwest-0.12.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4cdb0f156e754caead9b8cf88fc2f2806ed81282960c9a39f8a07847110745fe", size = 5882872, upload-time = "2026-10-09T06:21:58.362Z" }, + { url = "https://files.pythonhosted.org/packages/e3/e4/45017cda255c95797c5a4e6da2f87a8035f368aa9db50d9c45abcbd97387/pyqwest-0.12.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3208f670e601ddfbf6ff689c7717236dfeaf175fa934319d30a6c57df0e1c14c", size = 6074173, upload-time = "2026-10-09T06:21:59.918Z" }, + { url = "https://files.pythonhosted.org/packages/a3/d1/955421179cbd93070cd01769e6f937826c42d377fab9fade00172601e138/pyqwest-0.12.0-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:f7476abcdac195e33165e38e4e76c9c6e90785e14033b766912f24ff8f829198", size = 5398661, upload-time = "2026-10-09T06:22:01.5Z" }, + { url = "https://files.pythonhosted.org/packages/e0/b0/c7474b1650d3bafe94f28f87446b0e4371d553acdfa79d2ca7e8151d1982/pyqwest-0.12.0-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:faf377620445e9988b2e290fb805dd3a04ffa62e0f6d1da11e269cecc1d5a837", size = 5241423, upload-time = "2026-10-09T06:22:03.084Z" }, + { url = "https://files.pythonhosted.org/packages/b8/11/3bdeb82f4425ed42d5ee877003974de7a716990f8681c6a637046a3d49eb/pyqwest-0.12.0-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:90fee9cd448324fbdc4ed2dff787749f19ac17315aa9e18d88ced0b843fc8dac", size = 5791319, upload-time = "2026-10-09T06:22:04.628Z" }, + { url = "https://files.pythonhosted.org/packages/9a/b5/024555a08ab26bdadbe63b7acb6115a5d0e536efa94c4c71c4bbce445a45/pyqwest-0.12.0-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:ae7b73b47e3fb2462a5292417d49e7780ce2dc9857e8bce7276ea2f090a90768", size = 5764278, upload-time = "2026-10-09T06:22:06.309Z" }, + { url = "https://files.pythonhosted.org/packages/61/44/57bba169ee17c02dfd0b2a6508ed4c027e0d14ee3765fa3c641399521e2d/pyqwest-0.12.0-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:5f83e15b7bebc761e7b6a0f1cbac334d058e698fb49ba1328306660b00c3c92d", size = 5939856, upload-time = "2026-10-09T06:22:07.915Z" }, + { url = "https://files.pythonhosted.org/packages/69/66/926b90a270e9893958840fa610531b5a0d47604e088e1c60305c13890938/pyqwest-0.12.0-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:8057eddcae424eadbf15c9ab67f4aebdc2ce5f2cf1d8d4f6c34ca9d9ab62fef5", size = 6139873, upload-time = "2026-10-09T06:22:09.498Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6c/a09b970067a60c721ad063107ac10917e31475211bb3d66995d1482331a2/pyqwest-0.12.0-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:6c71f9f32cd344ac4f136010ea6a307201547123bce6ba61272421c93fb4fc6d", size = 5023514, upload-time = "2026-10-09T06:22:11.179Z" }, + { url = "https://files.pythonhosted.org/packages/d7/e5/06c6540c29596fe9230ccc160822309592b1ec85a05482105e1de6b98d14/pyqwest-0.12.0-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:9ca7d29a469326604f7c048bc19d4592970fbc7ac494bc253ef0779a57e1a9aa", size = 4991803, upload-time = "2026-10-09T06:22:12.867Z" }, + { url = "https://files.pythonhosted.org/packages/fb/4e/3f4e924e430cd859df54395b7a9ef21c342f544dc4d4da3af5b3f44c1e08/pyqwest-0.12.0-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:dca14e8c4aa3d79601c91e408e33df0bf463c8166539b394715637f49f90362a", size = 5348621, upload-time = "2026-10-09T06:22:14.718Z" }, + { url = "https://files.pythonhosted.org/packages/90/f2/da5defcc9d231ab9e4ce267704d1e18544b4d93c1cfb4e3c29a9a35892d3/pyqwest-0.12.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:729a81368abcbe6be708a96da1ec8802969082ffaab35d1e08a520e3c0453a49", size = 5195352, upload-time = "2026-10-09T06:22:16.435Z" }, + { url = "https://files.pythonhosted.org/packages/bc/98/871c99fdedaff6458a519e37d49400617beebf9c6560ed5e73bf02b972fe/pyqwest-0.12.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:645072f8e9138556e27f6c9384dd1fe0bdef3f1be50534d2c1d9ffeea9a53de9", size = 5752432, upload-time = "2026-10-09T06:22:18.111Z" }, + { url = "https://files.pythonhosted.org/packages/0c/4e/d1686f7b74c2b5cb3f7939a397aa215054db3cb515a99503fe6e361ddc96/pyqwest-0.12.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:d302f7bf0c072397aafe2cbe6a398743e5c37add3500048bbe0b26bba823075f", size = 5714200, upload-time = "2026-10-09T06:22:19.542Z" }, + { url = "https://files.pythonhosted.org/packages/6f/15/d4a71b8b9c85f74174f3e58ebe6cc34cf2d9175424da84af91088521b74f/pyqwest-0.12.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:ead61088e5226349624965025d297ed893dd947e05ccf315006322c824ef9a4e", size = 5903794, upload-time = "2026-10-09T06:22:21.083Z" }, + { url = "https://files.pythonhosted.org/packages/f4/b1/01928c04841f525748b4b21e46e29a473cc1b9d2c800abcf70d5d6a12b56/pyqwest-0.12.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:db85298d530bcd50234c48dfd18facbe75a3c8f0f16e04ace389617d48a3093f", size = 6091500, upload-time = "2026-10-09T06:22:22.713Z" }, + { url = "https://files.pythonhosted.org/packages/b1/ac/181acdc359a7cc293a4fe579ac315e72676654bd6f5ead732304caa125d0/pyqwest-0.12.0-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:f87b16a83d8e0bc5b639163819c53a9e975ea2fbeec133ecbd2ffce0796dab27", size = 5336607, upload-time = "2026-10-09T06:22:24.437Z" }, + { url = "https://files.pythonhosted.org/packages/74/b1/7365631700810ec294119e3b405bd20d9897774979b021c5dc2c002e4773/pyqwest-0.12.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:fa846050a669d0eacc2c2aa425eda403b3e443a374ff84c5c16c9f82624a2a9d", size = 5174497, upload-time = "2026-10-09T06:22:25.915Z" }, + { url = "https://files.pythonhosted.org/packages/dd/8d/08bc79d4d1c3af8c24b78163c277226cc26c3e751d1bc562db8baf67ee7e/pyqwest-0.12.0-cp315-cp315t-manylinux_2_28_aarch64.whl", hash = "sha256:5f08aeda136ffcb454c11b1e7bb98a85853ea779424cc30e98a9e5e762b84fea", size = 5731691, upload-time = "2026-10-09T06:22:27.408Z" }, + { url = "https://files.pythonhosted.org/packages/39/65/e1a9cc3cf610122149ccd3a1e264ce6b47aa8bcf3d4abd5f27ba7acdd97f/pyqwest-0.12.0-cp315-cp315t-manylinux_2_28_x86_64.whl", hash = "sha256:0b770869bae9e15c13e8823ed71155d2a9e7fa3f4cba3da65d2b4b668e5b3a1e", size = 5704613, upload-time = "2026-10-09T06:22:29.387Z" }, + { url = "https://files.pythonhosted.org/packages/c4/75/a8b3fd3de3eab08377ae71fde7efa5526b7f43cf094eb8b028dd8acee275/pyqwest-0.12.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:a25bdfbdc859d937de818db60d839f523c9d4792c2b85a23667861cc0791246e", size = 5886056, upload-time = "2026-10-09T06:22:31.298Z" }, + { url = "https://files.pythonhosted.org/packages/93/23/6982518b22268151fdb7ae27d3700a3c4fdb0699d4783caaace1e1fa6deb/pyqwest-0.12.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:ac6a7de3327ee56e736108ab72a21efc5dfddd41ce4c19e4469c1e8448a6696e", size = 6074658, upload-time = "2026-10-09T06:22:32.919Z" }, +] + [[package]] name = "pytest" version = "9.0.3" @@ -2071,6 +2553,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61", size = 21230, upload-time = "2025-10-26T15:12:09.109Z" }, ] +[[package]] +name = "python-engineio" +version = "4.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "simple-websocket" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/91/b565807fa922620ed575b412bf978d032e4c5d79558b400fdbfe1008c95a/python_engineio-4.15.1.tar.gz", hash = "sha256:1a412e798a90c64fabe7b20798939ae6d4a03f34da8a33d5a21a25a8bbbc3612", size = 93104, upload-time = "2026-10-28T23:53:30.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/5c/552f1dce2443dc33ce6b2ee7f67599ba5c83ef5582e2235a9878ac239049/python_engineio-4.15.1-py3-none-any.whl", hash = "sha256:745cb3c07701c719a2906439a00b67e44ef042904036568530a2c3a8cf18c89a", size = 60775, upload-time = "2026-10-28T23:53:29.304Z" }, +] + [[package]] name = "python-multipart" version = "0.0.32" @@ -2080,6 +2574,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, ] +[[package]] +name = "python-socketio" +version = "5.18.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "bidict" }, + { name = "python-engineio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/b9/4ab083267c1f8ece9a9e019808412af7c7a03321411a1263cd863321baa2/python_socketio-5.18.1.tar.gz", hash = "sha256:07ab0e7c94a52833a67af1dc05958ad6fb0de69adc87de9198a61b22588c40d4", size = 132106, upload-time = "2026-11-03T12:19:20.723Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/e2/cdc630aef7da00698af4ed061ff873297f806cc049cc5c67bf89077cb7e4/python_socketio-5.18.1-py3-none-any.whl", hash = "sha256:e768c473842be811f6dd24bd0a49cd0d4a0c994182e3eba23cc9d1489c7a697e", size = 83022, upload-time = "2026-11-03T12:19:19.136Z" }, +] + +[package.optional-dependencies] +asyncio-client = [ + { name = "aiohttp" }, +] +client = [ + { name = "requests" }, + { name = "websocket-client" }, +] + [[package]] name = "pytz" version = "2025.2" @@ -2352,6 +2868,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/a8/001d4a7c2b37623a3fd7463208267fb906df40ff31db496157549cfd6e72/ruff-0.12.11-py3-none-win_arm64.whl", hash = "sha256:bae4d6e6a2676f8fb0f98b74594a048bae1b944aab17e9f5d504062303c6dbea", size = 12135290, upload-time = "2025-08-28T13:59:06.933Z" }, ] +[[package]] +name = "s3transfer" +version = "0.19.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/c5/0a5ce655109cb96b8dfad36908a4e108d15f5dc2eff8fbc3146c4312e704/s3transfer-0.19.3.tar.gz", hash = "sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406", size = 166087, upload-time = "2026-10-19T19:19:49.751Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" }, +] + [[package]] name = "scipy" version = "1.18.0" @@ -2393,6 +2921,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/19/969dc072906c84dd0a3b05dcf57ea750936087d7873549e408b35cfc3f97/scipy-1.18.0-cp314-cp314t-win_arm64.whl", hash = "sha256:368e0a705903c466aa5f08eefb39e6b1b6b2d659e7352a31fd9e2438365be0f8", size = 25279661, upload-time = "2026-06-19T15:01:40.817Z" }, ] +[[package]] +name = "simple-websocket" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wsproto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/d4/bfa032f961103eba93de583b161f0e6a5b63cebb8f2c7d0c6e6efe1e3d2e/simple_websocket-1.1.0.tar.gz", hash = "sha256:7939234e7aa067c534abdab3a9ed933ec9ce4691b0713c78acb195560aa52ae4", size = 17300, upload-time = "2024-10-10T22:39:31.412Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/59/0782e51887ac6b07ffd1570e0364cf901ebc36345fea669969d2084baebb/simple_websocket-1.1.0-py3-none-any.whl", hash = "sha256:4af6069630a38ed6c561010f0e11a5bc0d4ca569b36306eb257cd9a192497c8c", size = 13842, upload-time = "2024-10-10T22:39:29.645Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -2436,6 +2976,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, ] +[[package]] +name = "tenacity" +version = "9.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/9e/497c1c8ebe5a5b5d1d4a7511aea22c0bb1a97e3170d98abdef0e1b34265a/tenacity-9.2.1.tar.gz", hash = "sha256:a606b5c808d0cded4a359d5b9932d867ff2a6a6b64d37350260fd01bbdf83839", size = 58261, upload-time = "2026-10-07T12:13:01.633Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/26/1ff2b0721ac66a3ec5b1402b333110b352ab0a8724052ac279a7b82d40c4/tenacity-9.2.1-py3-none-any.whl", hash = "sha256:9e56f17539296baab7beabb08b92f6ee3d7be92d8be72d763360677c2ad6580e", size = 32310, upload-time = "2026-10-07T12:13:00.102Z" }, +] + [[package]] name = "tiktoken" version = "0.14.0" @@ -2483,6 +3032,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/5e/9b01afd037bfa22a0033963fa091e0f75b6fb15cd85bffb42ff86e697323/tiktoken-0.14.0-cp315-cp315t-win_amd64.whl", hash = "sha256:561e7580f84a79859af1ef6f676968e9030fcc3fe195700b15235bca64f009c9", size = 987929, upload-time = "2026-08-17T19:49:38.947Z" }, ] +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, +] + [[package]] name = "tomlkit" version = "0.15.2" @@ -2584,6 +3142,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/06/04c8e804f813cf972e3262f3f8584c232de64f0cde9f703b46cf53a45090/virtualenv-20.34.0-py3-none-any.whl", hash = "sha256:341f5afa7eee943e4984a9207c025feedd768baff6753cd660c857ceb3e36026", size = 5983279, upload-time = "2025-08-13T14:24:05.111Z" }, ] +[[package]] +name = "wcmatch" +version = "11.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "bracex" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bd/07/b7a9162a804bc0f7d03fe84c93e374d3e1c4d828043d5f9b797f7d4e30e0/wcmatch-11.2.tar.gz", hash = "sha256:1605bde546fbde4c063628c7d89eda486909fd4337e51d73a873987fbcb2554c", size = 134939, upload-time = "2026-10-22T00:37:57.622Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/96/b6cebd4bd75320743a291fed494c79f07c01b58fa9b187d12aeb5110f002/wcmatch-11.2-py3-none-any.whl", hash = "sha256:952bab83c75c233836007a69550a3a8753cf06964c1eb3b84ff11ad5bff1c272", size = 44361, upload-time = "2026-10-22T00:37:56.293Z" }, +] + +[[package]] +name = "websocket-client" +version = "1.9.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/cb/a5abcc2891249f393827c650c6296660ce40374ac22d99ab9aea41f9d2a2/websocket_client-1.9.2.tar.gz", hash = "sha256:0fcb57545848be86992e128218fd96dd87a6769ffdb1a968dff79632b85604d0", size = 84110, upload-time = "2026-08-31T14:08:40.964Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/d2/cc4dc1271e464942db7ee278baae2daa99ee77cb2af744025c04da585a3e/websocket_client-1.9.2-py3-none-any.whl", hash = "sha256:e1a673830a9c7bfa47b1cd3d5e4178f4c9651d80a4eab02c9c23a1c3ec6250ce", size = 95786, upload-time = "2026-08-31T14:08:39.899Z" }, +] + [[package]] name = "wrapt" version = "1.17.3" @@ -2623,6 +3202,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, ] +[[package]] +name = "wsproto" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/79/12135bdf8b9c9367b8701c2c19a14c913c120b882d50b014ca0d38083c2c/wsproto-1.3.2.tar.gz", hash = "sha256:b86885dcf294e15204919950f666e06ffc6c7c114ca900b060d6e16293528294", size = 50116, upload-time = "2025-11-20T18:18:01.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/f5/10b68b7b1544245097b2a1b8238f66f2fc6dcaeb24ba5d917f52bd2eed4f/wsproto-1.3.2-py3-none-any.whl", hash = "sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584", size = 24405, upload-time = "2025-11-20T18:18:00.454Z" }, +] + [[package]] name = "xxhash" version = "3.6.0" From e4e40c9a105841989c892a05e61a3f01b191186d Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 4 Nov 2026 09:52:21 -0800 Subject: [PATCH 09/22] fix(browsecomp): route the example pairing through sandbox sessions Signed-off-by: yaoyu-33 --- .../browsecomp_advanced_harness/README.md | 14 ++++++++- .../configs/browsecomp_advanced_harness.yaml | 30 ++++++++++++++++--- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/resources_servers/browsecomp_advanced_harness/README.md b/resources_servers/browsecomp_advanced_harness/README.md index 18cf57af5ae..6559e491b8e 100644 --- a/resources_servers/browsecomp_advanced_harness/README.md +++ b/resources_servers/browsecomp_advanced_harness/README.md @@ -77,14 +77,26 @@ session's metrics after reporting them, unless `dump_session_id_to_metrics_on_ex ## Running +The example pairing uses `single_agent_turn_legacy`: existing flat dataset rows still work, while the environment +server owns resources/agent session setup, verification, and close. The agent loop runs in its own sandbox and +receives the resources session through the `direct_http` transport; its `resources_server` config is `null`. +The config includes the Docker provider and `use_absolute_ip: true` so the sandbox can reach the Gym servers. +Compose another provider config to run on a different backend and set both sandbox image fields for that provider. + +The example dataset exposes only `search` and `browse`, so the example config keeps `workspace: none` and inline +page content. To enable the resources-owned tool sandbox, use `workspace: per_session` with dataset rows exposing +`bash_command`, as the BrowseComp benchmark does. This preserves the example prompts and tool schemas. + ```bash -# If you want to run with browsecomp benchmark instead of the example samples, need to change the datasets part to the one like `benchmarks/browsecomp/config.yaml` in `resources_servers/browsecomp_advanced_harness/configs/browsecomp_advanced_harness.yaml` config_paths="resources_servers/browsecomp_advanced_harness/configs/browsecomp_advanced_harness.yaml,\ responses_api_models/vllm_model/configs/vllm_model.yaml" ng_run "+config_paths=[${config_paths}]" ``` +To run the benchmark instead, use `benchmarks/browsecomp/config.yaml` as the first config path. Its dataset, +workspace, and tool schemas are configured together. + # Licensing information Code: ? diff --git a/resources_servers/browsecomp_advanced_harness/configs/browsecomp_advanced_harness.yaml b/resources_servers/browsecomp_advanced_harness/configs/browsecomp_advanced_harness.yaml index 6e887a99d80..a3e5ef96ade 100644 --- a/resources_servers/browsecomp_advanced_harness/configs/browsecomp_advanced_harness.yaml +++ b/resources_servers/browsecomp_advanced_harness/configs/browsecomp_advanced_harness.yaml @@ -1,3 +1,8 @@ +config_paths: +- nemo_gym/sandbox/providers/docker/configs/docker.yaml + +use_absolute_ip: true + browsecomp_advanced_harness_resources_server: resources_servers: browsecomp_advanced_harness: @@ -7,6 +12,14 @@ browsecomp_advanced_harness_resources_server: debug: false tavily_api_key: ${tavily_api_key} exclude_domains_file_path: ${exclude_domains_file_path} + # The example rows expose search/browse only, so retain inline page content. + # Workspace mode requires rows that also expose bash_command, as in the benchmark. + workspace: none + sandbox_provider: sandbox + sandbox_config: + image: python:3.13-slim + workdir: /workspace + ttl_s: 7200 use_judge: true judge_model_server: type: responses_api_models @@ -31,9 +44,12 @@ browsecomp_advanced_harness_agent: responses_api_agents: browsecomp_agent: entrypoint: app.py - resources_server: - type: resources_servers - name: browsecomp_advanced_harness_resources_server + resources_server: null + sandbox_provider: sandbox + sandbox_config: + image: python:3.13-slim + workdir: /sandbox + ttl_s: 7200 model_server: type: responses_api_models name: policy_model @@ -44,8 +60,14 @@ browsecomp_advanced_harness_agent: browsecomp_advanced_harness_environment_server: environment_servers: - legacy_agent: + single_agent_turn_legacy: entrypoint: app.py + resources_server: + type: resources_servers + name: browsecomp_advanced_harness_resources_server agent_server: type: responses_api_agents name: browsecomp_advanced_harness_agent + resources_tool_transports: [direct_http] + default_episode_timeout_seconds: 6000 + cleanup_timeout_seconds: 300 From 6d27de2931619bb03b1371d885a86b73c242baa5 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 4 Nov 2026 09:54:26 -0800 Subject: [PATCH 10/22] fix(browsecomp_agent): retain ownership after incomplete sandbox startup Signed-off-by: yaoyu-33 --- responses_api_agents/browsecomp_agent/app.py | 38 ++++++++++--------- .../browsecomp_agent/tests/test_sandbox.py | 29 ++++++++++++++ 2 files changed, 49 insertions(+), 18 deletions(-) diff --git a/responses_api_agents/browsecomp_agent/app.py b/responses_api_agents/browsecomp_agent/app.py index 9537c59d012..533215db24b 100644 --- a/responses_api_agents/browsecomp_agent/app.py +++ b/responses_api_agents/browsecomp_agent/app.py @@ -407,25 +407,14 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Brow if tool_access is not None: assert_model_url_reachable_from_sandbox(str(tool_access.base_url), provider_name=provider_name) provider = create_provider(provider_config) - try: - if owns_sandbox: - sandbox = AsyncSandbox(provider) - spec = dict(self.config.sandbox_config) - spec["metadata"] = { - **resolve_provider_metadata(provider_ref, get_global_config_dict()), - **(spec.get("metadata") or {}), - "agent_session_id": body.agent_session_id[:63], - } - spec.setdefault( - "ttl_s", - self.config.sandbox_install_timeout_seconds + self.config.sandbox_runner_timeout_seconds + 600, - ) - await sandbox.start(SandboxSpec(**spec)) - else: + if owns_sandbox: + sandbox = AsyncSandbox(provider) + else: + try: sandbox = await AsyncSandbox.connect(connection.descriptor, provider=provider) - except BaseException: - await provider.aclose() - raise + except BaseException: + await provider.aclose() + raise state = BrowsecompSandboxSession( request=body, session=SandboxSession( @@ -439,6 +428,19 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Brow resources_cookies=dict(tool_access.cookies) if tool_access else {}, ) try: + if owns_sandbox: + spec = dict(self.config.sandbox_config) + spec["metadata"] = { + **resolve_provider_metadata(provider_ref, get_global_config_dict()), + **(spec.get("metadata") or {}), + "agent_session_id": body.agent_session_id[:63], + } + spec.setdefault( + "ttl_s", + self.config.sandbox_install_timeout_seconds + self.config.sandbox_runner_timeout_seconds + 600, + ) + # Retain ownership before start: failed file delivery can also fail its automatic stop. + await sandbox.start(SandboxSpec(**spec)) await state.install_runtime(timeout=self.config.sandbox_install_timeout_seconds) except BaseException as error: try: diff --git a/responses_api_agents/browsecomp_agent/tests/test_sandbox.py b/responses_api_agents/browsecomp_agent/tests/test_sandbox.py index 9b300fc1b4f..cf383a5e744 100644 --- a/responses_api_agents/browsecomp_agent/tests/test_sandbox.py +++ b/responses_api_agents/browsecomp_agent/tests/test_sandbox.py @@ -230,3 +230,32 @@ async def release(*, timeout: float) -> None: await state.close(timeout=1) assert state.task.cancelled() state.session.close.assert_awaited_once() + + +async def test_start_upload_failure_retains_owned_sandbox_for_cleanup_retry(monkeypatch: pytest.MonkeyPatch) -> None: + from unittest.mock import MagicMock + + from nemo_gym.agents import AgentSessionSetupError, SandboxSession + from nemo_gym.sandbox import AsyncSandbox + from responses_api_agents.browsecomp_agent import app + + # Restore the real sandbox/close after the transport-focused test factory patches them. + real_close = SandboxSession.close + client, _, _ = _agent([], monkeypatch, sandbox_config={"workdir": "/sandbox", "files": {"/task.txt": "task"}}) + agent = next(route.endpoint.__self__ for route in client.app.routes if route.path == "/v1/browsecomp/token_count") + provider = MagicMock() + handle = MagicMock() + provider.create = AsyncMock(return_value=handle) + provider.upload_file = AsyncMock(side_effect=RuntimeError("file delivery failed")) + provider.close = AsyncMock(side_effect=[RuntimeError("stop failed"), RuntimeError("stop failed again"), None]) + provider.aclose = AsyncMock() + monkeypatch.setattr(app, "create_provider", lambda *args: provider) + monkeypatch.setattr(app, "AsyncSandbox", AsyncSandbox) + monkeypatch.setattr(SandboxSession, "close", real_close) + with pytest.raises(AgentSessionSetupError) as error: + await agent._seed_agent_session_state(_seed()) + assert provider.close.await_count == 2 + assert error.value.state.session.sandbox._handle is handle + await agent._close_agent_session_state(error.value.state) + assert provider.close.await_count == 3 + provider.aclose.assert_awaited() From 8257c232e3d4de1082cb59af27f1ac8305943485 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 4 Nov 2026 09:56:56 -0800 Subject: [PATCH 11/22] docs(browsecomp): clarify sandbox provider replacement Signed-off-by: yaoyu-33 --- benchmarks/browsecomp/README.md | 2 +- resources_servers/browsecomp_advanced_harness/README.md | 2 +- responses_api_agents/browsecomp_agent/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/browsecomp/README.md b/benchmarks/browsecomp/README.md index 85145f241d0..683a6e8218f 100644 --- a/benchmarks/browsecomp/README.md +++ b/benchmarks/browsecomp/README.md @@ -38,7 +38,7 @@ ng_e2e_collect_rollouts \ ## Run through the Environment Server -`config.yaml` now runs the benchmark through `single_agent_turn_legacy` with sandboxed agent and resources tool execution. It requires a Docker daemon by default; compose another provider config to change the provider. `config_single_agent_turn.yaml` is a compatibility alias. A crash then repeats only the current step (seed, agent turn, agent close, verify, close) rather than the whole episode. Use it in place of `config.yaml` in step 3: +`config.yaml` now runs the benchmark through `single_agent_turn_legacy` with sandboxed agent and resources tool execution. It requires a Docker daemon by default. Replace the included Docker provider config and both sandbox images when selecting another provider; do not merge two provider blocks under `sandbox`. `config_single_agent_turn.yaml` is a compatibility alias. A crash then repeats only the current step (seed, agent turn, agent close, verify, close) rather than the whole episode. Use it in place of `config.yaml` in step 3: ```bash config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ diff --git a/resources_servers/browsecomp_advanced_harness/README.md b/resources_servers/browsecomp_advanced_harness/README.md index 6559e491b8e..08161d2ed2a 100644 --- a/resources_servers/browsecomp_advanced_harness/README.md +++ b/resources_servers/browsecomp_advanced_harness/README.md @@ -81,7 +81,7 @@ The example pairing uses `single_agent_turn_legacy`: existing flat dataset rows server owns resources/agent session setup, verification, and close. The agent loop runs in its own sandbox and receives the resources session through the `direct_http` transport; its `resources_server` config is `null`. The config includes the Docker provider and `use_absolute_ip: true` so the sandbox can reach the Gym servers. -Compose another provider config to run on a different backend and set both sandbox image fields for that provider. +Replace the included Docker provider config to run on a different backend, and set both sandbox image fields for that provider. Do not merge two provider blocks under `sandbox`. The example dataset exposes only `search` and `browse`, so the example config keeps `workspace: none` and inline page content. To enable the resources-owned tool sandbox, use `workspace: per_session` with dataset rows exposing diff --git a/responses_api_agents/browsecomp_agent/README.md b/responses_api_agents/browsecomp_agent/README.md index eaa021c9058..c710e1e5052 100644 --- a/responses_api_agents/browsecomp_agent/README.md +++ b/responses_api_agents/browsecomp_agent/README.md @@ -4,7 +4,7 @@ The repository-native research loop runs inside a task sandbox through an agent ## Execution and composition -`benchmarks/browsecomp/config.yaml` runs `single_agent_turn_legacy`: resources seed → agent seed → one activation → agent close → verify → resources close. `config_single_agent_turn.yaml` remains an alias. Docker is the default provider; compose another provider config to use Enroot or OpenSandbox. Both the Model Server and granted Resources endpoints must be reachable from the agent sandbox (`use_absolute_ip: true`). +`benchmarks/browsecomp/config.yaml` runs `single_agent_turn_legacy`: resources seed → agent seed → one activation → agent close → verify → resources close. `config_single_agent_turn.yaml` remains an alias. Docker is the default provider. To use Enroot or OpenSandbox, replace the included Docker provider config and set the matching sandbox images; do not merge two provider blocks under `sandbox`. Both the Model Server and granted Resources endpoints must be reachable from the agent sandbox (`use_absolute_ip: true`). The agent borrows `sandbox_access` when supplied. Otherwise `sandbox_provider` and `sandbox_config` create an owned sandbox. Missing sandbox configuration, the unisolated Local provider, required MCP grants, and more than one direct HTTP grant are refused. No sandbox error falls back to host execution. Close stops the harness through `SandboxSession`, captures output, then disconnects a borrowed sandbox or destroys an owned one. From 39c7616b8da0ba51d5f79f1e06cf26007ab16ed7 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 4 Nov 2026 10:01:10 -0800 Subject: [PATCH 12/22] docs(browsecomp): record the sandbox activation time limit Signed-off-by: yaoyu-33 --- benchmarks/browsecomp/README.md | 2 +- responses_api_agents/browsecomp_agent/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/browsecomp/README.md b/benchmarks/browsecomp/README.md index 683a6e8218f..e85773d8f46 100644 --- a/benchmarks/browsecomp/README.md +++ b/benchmarks/browsecomp/README.md @@ -45,7 +45,7 @@ config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ benchmarks/browsecomp/config_single_agent_turn.yaml" ``` -Tasks, prompts and correct-answer grading are unchanged. Compared with the previous host `/run` recipe, failure handling changes the scored denominator: +Tasks, prompts and correct-answer grading are unchanged. The sandbox path adds a 3600-second activation cap by default (`sandbox_runner_timeout_seconds`); when `rollout_timeout_s` is unset, that cap also bounds the loop. Account for this limit when comparing with previous step-only runs. Compared with the previous host `/run` recipe, failure handling changes the scored denominator: | Case | Previous host `/run` recipe | Current sandbox session recipe | | --- | --- | --- | diff --git a/responses_api_agents/browsecomp_agent/README.md b/responses_api_agents/browsecomp_agent/README.md index c710e1e5052..fccdd8b4dd8 100644 --- a/responses_api_agents/browsecomp_agent/README.md +++ b/responses_api_agents/browsecomp_agent/README.md @@ -16,7 +16,7 @@ For a compatible tool-free benchmark, use `resources_tool_transports: []` and ch The sandbox path declares **Evaluation** capability. It returns an observation bundle even after failure, with `native_turn_evidence_unavailable` or `observation_capture_failed`. Raw Responses output and model-call captures remain available, but native invocation/turn/tool observations are not emitted. Strict evidence checks are expected to report these gaps; this is not an Observability or Training qualification. Function-calling shell conformance is applicable but optional at this capability level and has not been registered or qualified. Component tests and real pairing runs cover the resources-owned pocketshell boundary. -- Sampling controls in the request go unchanged to every model call. `max_output_tokens` is per call; `max_steps` and `rollout_timeout_s` bound the loop. `sandbox_runner_timeout_seconds` bounds the full activation, including retries. +- Sampling controls in the request go unchanged to every model call. `max_output_tokens` is per call; `max_steps` and `rollout_timeout_s` bound the loop. `sandbox_runner_timeout_seconds` bounds the full activation, including retries (3600 seconds by default). If `rollout_timeout_s` is unset, the sandbox runner cap becomes the loop deadline; this adds a wall-clock limit to previously step-only runs. - Token IDs/logprobs already returned by the Model Server are preserved; the sandbox path has not been qualified for training. - With `save_model_call_using_vllm_tokenize_endpoint: true`, a session-authenticated adapter endpoint counts prompts without copying the upstream key into the sandbox. Hosted endpoints without `/tokenize` must explicitly set this option to `false`, using the existing post-call token-count reset path. - `snap_dir` is rejected for sandbox sessions. Use captured model calls and rollout output; snapshots remain available in explicit host compatibility mode. From be3925b94589eca0685dcd746fda50492476d2dd Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 4 Nov 2026 10:03:11 -0800 Subject: [PATCH 13/22] fix(browsecomp_agent): build the pinned pure Python parser for sandbox bootstrap Signed-off-by: yaoyu-33 --- responses_api_agents/browsecomp_agent/sandbox.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/responses_api_agents/browsecomp_agent/sandbox.py b/responses_api_agents/browsecomp_agent/sandbox.py index 5304442163d..4824508d81c 100644 --- a/responses_api_agents/browsecomp_agent/sandbox.py +++ b/responses_api_agents/browsecomp_agent/sandbox.py @@ -31,9 +31,18 @@ def _runtime() -> SandboxPythonRuntime: for raw in importlib.metadata.requires("nemo-gym") or () if (requirement := Requirement(raw)).marker is None or requirement.marker.evaluate({"extra": ""}) ) + # Hydra/OmegaConf need this pure-Python dependency, whose pinned release has no wheel. + # The direct, hash-checked sdist opts only this package into the shared runtime's host build path. + # Version and digest match this adapter's uv.lock; all other packages still require target wheels. + antlr = ( + "antlr4-python3-runtime @ https://files.pythonhosted.org/packages/3e/38/" + "7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/" + "antlr4-python3-runtime-4.9.3.tar.gz" + "#sha256=f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b" + ) return SandboxPythonRuntime( harness="browsecomp", - requirements=requirements, + requirements=(*requirements, antlr), import_check="import aiohttp, openai, pydantic, fastapi, omegaconf", ) From 57a70ce32269eb5c17500a6e83607d17ebbbd3cb Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 4 Nov 2026 10:07:14 -0800 Subject: [PATCH 14/22] docs(browsecomp): specify the empty domain filter schema Signed-off-by: yaoyu-33 --- benchmarks/browsecomp/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/browsecomp/README.md b/benchmarks/browsecomp/README.md index e85773d8f46..53bcbf7e63f 100644 --- a/benchmarks/browsecomp/README.md +++ b/benchmarks/browsecomp/README.md @@ -1,5 +1,5 @@ # Run instructions -1. Update your env.yaml file with the necessary keys and configs. You can leave `exclude_domains_file_path` as a path to an empty json. +1. Update your env.yaml file with the necessary keys and configs. Set `exclude_domains_file_path` to a JSON file containing `{"notices": []}` when no additional domain exclusions are needed; an empty list or `{}` does not match the parser schema. ```yaml browsecomp_benchmark_resources_server: resources_servers: From 95d66b58184a0754c2cac59bf20da7301d2b7114 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 4 Nov 2026 10:44:53 -0800 Subject: [PATCH 15/22] fix(browsecomp): require Enroot process isolation before tools Signed-off-by: yaoyu-33 --- .../browsecomp_advanced_harness/README.md | 6 ++ .../browsecomp_advanced_harness/sandbox.py | 34 ++++++- .../tests/test_sandbox_shell.py | 99 +++++++++++++++++++ 3 files changed, 137 insertions(+), 2 deletions(-) diff --git a/resources_servers/browsecomp_advanced_harness/README.md b/resources_servers/browsecomp_advanced_harness/README.md index 18cf57af5ae..b49961efb32 100644 --- a/resources_servers/browsecomp_advanced_harness/README.md +++ b/resources_servers/browsecomp_advanced_harness/README.md @@ -47,6 +47,12 @@ For example, composing `nemo_gym/sandbox/providers/docker/configs/docker.yaml` d The shared sandbox API also supports other providers; Enroot needs an image available on its cluster. Provider `default_metadata` is merged with `sandbox_config.metadata`, then the resource server and session labels. +Enroot must provide a PID namespace separate from the resources-server host, with `/proc` mounted for that same +namespace. Setup checks the running sandbox's namespace IDs before permitting model commands; a version string +or `ENROOT_UNSHARE_PID` setting alone is insufficient because older Enroot builds may ignore that setting. If the +check fails, upgrade or configure Enroot with working PID isolation, or select another isolating provider. The +failed session follows the usual sandbox cleanup and close-retry path. + Each resources session owns one tool sandbox. An agent that runs its loop in a sandbox owns a separate sandbox; neither component borrows the other's container. This works with providers such as Enroot that do not support serializing and reconnecting to a sandbox. Search and browse retain their existing responses and write immutable diff --git a/resources_servers/browsecomp_advanced_harness/sandbox.py b/resources_servers/browsecomp_advanced_harness/sandbox.py index 2cad0339d26..34a9384da18 100644 --- a/resources_servers/browsecomp_advanced_harness/sandbox.py +++ b/resources_servers/browsecomp_advanced_harness/sandbox.py @@ -3,6 +3,8 @@ """Resources-owned, supervised execution of the existing read-only shell.""" import json +import os +import re from dataclasses import dataclass, field from pathlib import Path from shlex import quote @@ -14,6 +16,10 @@ _RUNTIME = SandboxPythonRuntime(harness="browsecomp-shell", requirements=("regex",), import_check="import regex") _RUNNER = f"{_RUNTIME.directory}/shell_runner.py" +_ENROOT_PID_ERROR = ( + "BrowseComp requires an Enroot PID namespace separate from the host and a matching /proc mount. " + "Upgrade or configure Enroot with PID isolation; older Enroot builds can ignore ENROOT_UNSHARE_PID." +) def shell_source_files() -> dict[str, str]: @@ -41,16 +47,40 @@ async def install(self, *, timeout: float) -> None: sandbox = self.session.sandbox if not await _RUNTIME.is_installed(sandbox): await _RUNTIME.install(sandbox, timeout_s=timeout) + await self._require_enroot_pid_namespace() prepared = await sandbox.exec( f"mkdir -p {quote(_RUNTIME.directory + '/pocketshell')} " f"{quote(self.session.session_dir)} {quote(self.session.workdir + '/pages')}", cwd="/", timeout_s=30, ) - if prepared.return_code or prepared.error_type: + if prepared.return_code != 0 or prepared.error_type is not None: raise RuntimeError(f"Could not prepare BrowseComp shell paths: {prepared.stderr or prepared.error_type}") self.ready = True + async def _require_enroot_pid_namespace(self) -> None: + """Check actual process isolation before any model-directed command can run.""" + sandbox = self.session.sandbox + if sandbox.provider_name != "enroot": + return + try: + host_namespace = os.readlink("/proc/self/ns/pid") + except OSError as error: + raise RuntimeError(f"{_ENROOT_PID_ERROR} Cannot read the host PID namespace.") from error + probe = "import os; print(os.readlink('/proc/self/ns/pid')); print(os.readlink('/proc/1/ns/pid'))" + result = await sandbox.exec(f"{quote(_RUNTIME.python)} -I -c {quote(probe)}", cwd="/", timeout_s=30) + if result.return_code != 0 or result.error_type is not None: + raise RuntimeError(f"{_ENROOT_PID_ERROR} The namespace probe did not complete successfully.") + namespaces = (result.stdout or "").splitlines() + if ( + not re.fullmatch(r"pid:\[[0-9]+\]", host_namespace) + or len(namespaces) != 2 + or any(not re.fullmatch(r"pid:\[[0-9]+\]", value) for value in namespaces) + or namespaces[0] != namespaces[1] + or namespaces[0] == host_namespace + ): + raise RuntimeError(f"{_ENROOT_PID_ERROR} The namespace probe did not confirm isolation.") + async def execute(self, *, command: str, duration: float, workspace: Path, close_timeout: float) -> str: """Copy new retrieved pages, then interpret the command only inside the sandbox.""" if not self.ready: @@ -60,7 +90,7 @@ async def execute(self, *, command: str, duration: float, workspace: Path, close async def stage() -> SandboxCommand: prepared = await self.session.sandbox.exec(f"mkdir -p {quote(directory)}", cwd="/", timeout_s=30) - if prepared.return_code or prepared.error_type: + if prepared.return_code != 0 or prepared.error_type is not None: raise RuntimeError("Could not prepare BrowseComp shell activation") # Pages are immutable after _PageWriter publishes them; only the manifest changes. # Never copy arbitrary files, symlinks, credentials, or verifier inputs from the host. diff --git a/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py b/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py index 9cd94ade4bc..806020d41c0 100644 --- a/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py +++ b/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py @@ -55,6 +55,105 @@ async def install(state, *, timeout): return server, containers +@pytest.fixture +def shell_install(monkeypatch: pytest.MonkeyPatch): + sandbox = AsyncMock(spec=harness.AsyncSandbox) + sandbox.provider_name = "enroot" + runtime = SimpleNamespace( + python="/private/runtime/python/bin/python3", + directory="/private/runtime", + is_installed=AsyncMock(return_value=False), + install=AsyncMock(), + ) + monkeypatch.setattr(shell_module, "_RUNTIME", runtime) + readlink = MagicMock(return_value="pid:[100]") + monkeypatch.setattr(shell_module, "os", SimpleNamespace(readlink=readlink)) + session = SimpleNamespace(sandbox=sandbox, session_dir="/private/session", workdir="/workspace") + return shell_module.BrowsecompShellSandbox(session=session), sandbox, runtime, readlink + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("stdout", "return_code", "error_type"), + [ + ("pid:[100]\npid:[100]\n", 0, None), + ("pid:[200]\npid:[100]\n", 0, None), + ("pid:[200]\npid:[200]\n", 1, None), + ("pid:[200]\npid:[200]\n", 0, "timeout"), + (None, 0, None), + ("unavailable\nunavailable\n", 0, None), + ], +) +async def test_enroot_install_refuses_unproven_pid_isolation(shell_install, stdout, return_code, error_type) -> None: + state, sandbox, runtime, readlink = shell_install + sandbox.exec.return_value = SimpleNamespace(stdout=stdout, return_code=return_code, error_type=error_type) + with pytest.raises(RuntimeError, match="older Enroot builds can ignore ENROOT_UNSHARE_PID"): + await state.install(timeout=20) + runtime.install.assert_awaited_once_with(sandbox, timeout_s=20) + readlink.assert_called_once_with("/proc/self/ns/pid") + assert not state.ready + sandbox.exec.assert_awaited_once() # Failed isolation prevents workspace activation. + command = sandbox.exec.await_args.args[0] + assert f"{runtime.python} -I -c " in command + assert "/proc/self/ns/pid" in command and "/proc/1/ns/pid" in command + + +@pytest.mark.asyncio +async def test_enroot_install_requires_readable_host_pid_namespace(shell_install) -> None: + state, sandbox, _, readlink = shell_install + readlink.side_effect = OSError("no proc mount") + with pytest.raises(RuntimeError, match="Cannot read the host PID namespace"): + await state.install(timeout=20) + assert not state.ready + sandbox.exec.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_enroot_install_accepts_distinct_namespace_with_matching_proc_mount(shell_install) -> None: + state, sandbox, _, _ = shell_install + sandbox.exec.side_effect = [ + SimpleNamespace(stdout="pid:[200]\npid:[200]\n", return_code=0, error_type=None), + SimpleNamespace(return_code=0, error_type=None), + ] + await state.install(timeout=20) + assert state.ready + assert sandbox.exec.await_count == 2 + + +@pytest.mark.asyncio +async def test_other_providers_do_not_probe_host_pid_namespace(shell_install) -> None: + state, sandbox, _, readlink = shell_install + sandbox.provider_name = "docker" + sandbox.exec.return_value = SimpleNamespace(return_code=0, error_type=None) + await state.install(timeout=20) + assert state.ready + readlink.assert_not_called() + sandbox.exec.assert_awaited_once() + assert "ns/pid" not in sandbox.exec.await_args.args[0] + + +@pytest.mark.asyncio +async def test_pid_guard_failure_keeps_failed_stop_retryable( + shell_install, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _, sandbox, _, _ = shell_install + server = _server(tmp_path, workspace="per_session", sandbox_provider="sandbox", allow_host_execution=False) + monkeypatch.setattr(harness, "get_global_config_dict", lambda: {}) + monkeypatch.setattr(harness, "resolve_provider_config", lambda *_: {"enroot": {}}) + monkeypatch.setattr(harness, "resolve_provider_metadata", lambda *_: {}) + monkeypatch.setattr(harness, "create_provider", lambda *_: object()) + monkeypatch.setattr(harness, "AsyncSandbox", lambda *_: sandbox) + sandbox.exec.return_value = SimpleNamespace(stdout="pid:[100]\npid:[100]\n", return_code=0, error_type=None) + sandbox.stop.side_effect = [RuntimeError("stop unavailable"), None] + with pytest.raises(RuntimeError, match="namespace probe did not confirm isolation"): + await server.seed_session(_request(), ResourcesSeedSessionRequest.model_validate(_seed())) + assert "resources-session" in server._shell_sessions + assert not server._shell_sessions["resources-session"].ready + await server.close_resources_session(ResourcesCloseSessionRequest.model_validate(_close())) + assert sandbox.stop.await_count == 2 + assert not server._shell_sessions and not server._session_workspaces + + def test_missing_provider_refuses_seed_without_creating_workspace(tmp_path: Path) -> None: server = _server(tmp_path, workspace="per_session", allow_host_execution=False) with TestClient(server.setup_webserver()) as client: From 1b2db4f797f93b04f7f45c56b257036f617c8d25 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 4 Nov 2026 10:45:18 -0800 Subject: [PATCH 16/22] fix(browsecomp_agent): reject Enroot sessions without PID isolation Signed-off-by: yaoyu-33 --- .../browsecomp_agent/README.md | 2 + .../browsecomp_agent/sandbox.py | 27 ++++++++ .../browsecomp_agent/tests/test_sandbox.py | 64 ++++++++++++++++++- 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/responses_api_agents/browsecomp_agent/README.md b/responses_api_agents/browsecomp_agent/README.md index fccdd8b4dd8..395070a1c85 100644 --- a/responses_api_agents/browsecomp_agent/README.md +++ b/responses_api_agents/browsecomp_agent/README.md @@ -8,6 +8,8 @@ The repository-native research loop runs inside a task sandbox through an agent The agent borrows `sandbox_access` when supplied. Otherwise `sandbox_provider` and `sandbox_config` create an owned sandbox. Missing sandbox configuration, the unisolated Local provider, required MCP grants, and more than one direct HTTP grant are refused. No sandbox error falls back to host execution. Close stops the harness through `SandboxSession`, captures output, then disconnects a borrowed sandbox or destroys an owned one. +Enroot must provide PID isolation (upstream 4.2 or newer). Startup checks the actual exec process namespace and its `/proc` mount before staging rollout input; a shared host namespace or failed probe rejects the session. Use a complete matching Enroot installation, including its helper binaries. Merely setting `ENROOT_UNSHARE_PID` on an older installation is insufficient. + The Resources Server remains independent. A direct HTTP grant supplies its tools with per-session headers/cookies; the updated cookie jar returns at agent close. BrowseComp resources execute `bash_command` in their own sandbox, separate from the agent container. This uses two containers per BrowseComp episode and does not require a provider to serialize/connect an existing sandbox. Search credentials, ground truth and verifier code stay on the Resources Server. The model upstream API key stays on the Model Server. For a compatible tool-free benchmark, use `resources_tool_transports: []` and change only the Environment Server's resources binding and dataset. Set `progress: false` when the pairing does not use the progress board. The adapter does not depend on BrowseComp task fields or its verifier. No SWE-bench Pro or Terminal-Bench support is claimed. diff --git a/responses_api_agents/browsecomp_agent/sandbox.py b/responses_api_agents/browsecomp_agent/sandbox.py index 4824508d81c..3a1c02bebc9 100644 --- a/responses_api_agents/browsecomp_agent/sandbox.py +++ b/responses_api_agents/browsecomp_agent/sandbox.py @@ -6,6 +6,8 @@ import functools import importlib.metadata import json +import os +import re import tempfile import zipfile from dataclasses import dataclass @@ -75,6 +77,7 @@ async def install_runtime(self, *, timeout: float) -> None: """Install a private Python, then upload this revision of Gym and the harness.""" runtime = _runtime() await runtime.install(self.session.sandbox, timeout_s=timeout) + await self._check_process_isolation(python=runtime.python) directory = self.session.session_dir prepared = await self.session.sandbox.exec( f"mkdir -p {quote(directory)} {quote(self.session.workdir)}", cwd="/", timeout_s=30 @@ -94,6 +97,30 @@ async def install_runtime(self, *, timeout: float) -> None: if result.return_code != 0 or result.error_type is not None: raise RuntimeError(result.stderr or "Cannot unpack BrowseComp sandbox sources") + async def _check_process_isolation(self, *, python: str) -> None: + # Older Enroot silently ignores ENROOT_UNSHARE_PID. Check the actual exec + # namespace and its proc mount before delivering any rollout input. + sandbox = self.session.sandbox + if sandbox.provider_name != "enroot": + return + host_namespace = os.readlink("/proc/self/ns/pid") + probe = ( + "import os; namespace = os.readlink('/proc/self/ns/pid'); " + "assert namespace == os.readlink('/proc/1/ns/pid'); print(namespace)" + ) + result = await sandbox.exec(f"{quote(python)} -I -c {quote(probe)}", cwd="/", timeout_s=30) + namespace = result.stdout.strip() + if ( + result.return_code != 0 + or result.error_type is not None + or re.fullmatch(r"pid:\[\d+\]", namespace) is None + or namespace == host_namespace + ): + raise RuntimeError( + "BrowseComp requires an isolated PID namespace and matching /proc mount. " + "Install a complete Enroot distribution with PID isolation support (4.2 or newer)." + ) + async def stage_activation(self, payload: dict[str, JsonValue]) -> SandboxCommand: """Write input as data; model-controlled strings never become a launch command.""" directory = self.session.session_dir diff --git a/responses_api_agents/browsecomp_agent/tests/test_sandbox.py b/responses_api_agents/browsecomp_agent/tests/test_sandbox.py index cf383a5e744..d6611a1d7c5 100644 --- a/responses_api_agents/browsecomp_agent/tests/test_sandbox.py +++ b/responses_api_agents/browsecomp_agent/tests/test_sandbox.py @@ -7,7 +7,7 @@ import sys import zipfile from pathlib import Path -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock import pytest from aiohttp import web @@ -26,6 +26,68 @@ ) +@pytest.mark.parametrize( + ("stdout", "return_code", "error_type"), + [ + ("pid:[100]\n", 0, None), # Same namespace as the host. + ("", 1, None), # Includes a different /proc/1 namespace: the in-container assertion fails. + ("pid:[200]\n", 0, "timeout"), + ("", 0, None), + ("not a namespace", 0, None), + ], +) +async def test_enroot_rejects_missing_process_isolation( + monkeypatch: pytest.MonkeyPatch, stdout: str, return_code: int, error_type: str | None +) -> None: + session = MagicMock() + session.sandbox.provider_name = "enroot" + session.sandbox.exec = AsyncMock( + return_value=MagicMock(stdout=stdout, return_code=return_code, error_type=error_type) + ) + state = BrowsecompSandboxSession(request=_seed(), session=session, tool_access=None, resources_cookies={}) + monkeypatch.setattr("responses_api_agents.browsecomp_agent.sandbox.os.readlink", lambda path: "pid:[100]") + with pytest.raises(RuntimeError, match="isolated PID namespace"): + await state._check_process_isolation(python="/private/python") + + +async def test_enroot_accepts_a_distinct_process_namespace(monkeypatch: pytest.MonkeyPatch) -> None: + session = MagicMock() + session.sandbox.provider_name = "enroot" + session.sandbox.exec = AsyncMock(return_value=MagicMock(stdout="pid:[200]\n", return_code=0, error_type=None)) + state = BrowsecompSandboxSession(request=_seed(), session=session, tool_access=None, resources_cookies={}) + monkeypatch.setattr("responses_api_agents.browsecomp_agent.sandbox.os.readlink", lambda path: "pid:[100]") + await state._check_process_isolation(python="/private/python") + assert session.sandbox.exec.await_args.kwargs["timeout_s"] == 30 + + +async def test_process_probe_does_not_run_on_other_providers(monkeypatch: pytest.MonkeyPatch) -> None: + def no_host_proc(path: str) -> str: + raise AssertionError("Other providers must not inspect host /proc") + + monkeypatch.setattr("responses_api_agents.browsecomp_agent.sandbox.os.readlink", no_host_proc) + session = MagicMock() + session.sandbox.provider_name = "docker" + session.sandbox.exec = AsyncMock() + state = BrowsecompSandboxSession(request=_seed(), session=session, tool_access=None, resources_cookies={}) + await state._check_process_isolation(python="/private/python") + session.sandbox.exec.assert_not_awaited() + + +async def test_isolation_failure_prevents_source_and_input_delivery(monkeypatch: pytest.MonkeyPatch) -> None: + from responses_api_agents.browsecomp_agent import sandbox + + session = MagicMock() + session.sandbox.upload = AsyncMock() + state = BrowsecompSandboxSession(request=_seed(), session=session, tool_access=None, resources_cookies={}) + runtime = MagicMock(install=AsyncMock()) + monkeypatch.setattr(sandbox, "_runtime", lambda: runtime) + monkeypatch.setattr(state, "_check_process_isolation", AsyncMock(side_effect=RuntimeError("isolation failed"))) + with pytest.raises(RuntimeError, match="isolation failed"): + await state.install_runtime(timeout=60) + runtime.install.assert_awaited_once() + session.sandbox.upload.assert_not_awaited() + + def test_host_execution_is_an_explicit_opt_in(monkeypatch: pytest.MonkeyPatch) -> None: client, _, _ = _agent([], monkeypatch) reply = client.post("/v1/responses", json=QUESTION) From ac48bc4461f00d8f7769a92c03fd3b513e684a6e Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 4 Nov 2026 13:00:25 -0800 Subject: [PATCH 17/22] fix(browsecomp_advanced_harness): start /run sandboxes on first use; keep the shell's error envelope - benchmarks/browsecomp/config.yaml kept workspace: per_session with no sandbox_provider, so every seed got 422 until #5489 lands. Give the benchmark's resources server the Docker provider, as #5489 does. - An agent's /run can end without /verify (browsecomp_agent does after a timeout), and only /verify releases that path's sandbox. Start the /run sandbox with the first bash_command instead of at the seed, so only a run that used the shell can hold one, and a failed start no longer strands the seed's workspace (the failed seed's cookie was dropped, so nothing could release it). Document the remaining gap. - A command killed at the supervisor deadline, or a provider exec failure, answered HTTP 500. Answer with the host interpreter's [exit_code=-1] / [exit_code=-2] envelopes, as the README promises, and log the detail. - Say that only Enroot is probed for isolation, and that providers such as Apptainer bind-mount the host's home and /tmp by default. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- benchmarks/browsecomp/config.yaml | 8 ++++ .../browsecomp_advanced_harness/README.md | 10 +++-- .../browsecomp_advanced_harness/app.py | 26 +++++++++---- .../tests/test_sandbox_shell.py | 38 +++++++++++++++++++ 4 files changed, 71 insertions(+), 11 deletions(-) diff --git a/benchmarks/browsecomp/config.yaml b/benchmarks/browsecomp/config.yaml index ee2ad692fae..f5f94e08303 100644 --- a/benchmarks/browsecomp/config.yaml +++ b/benchmarks/browsecomp/config.yaml @@ -1,3 +1,6 @@ +config_paths: +- nemo_gym/sandbox/providers/docker/configs/docker.yaml + # Judge: a hosted reasoning model, independent of the policy under test. Keeping # the judge off the policy endpoint stops a weak policy from grading itself. judge_model: @@ -28,6 +31,11 @@ browsecomp_benchmark_resources_server: # terminal mode: search/browse write pages to a per-session disk workspace # and the read-only bash_command tool is exposed for the model to grep/read them. workspace: per_session + sandbox_provider: sandbox + sandbox_config: + image: python:3.13-slim + workdir: /workspace + ttl_s: 7200 use_judge: true judge_model_server: type: responses_api_models diff --git a/resources_servers/browsecomp_advanced_harness/README.md b/resources_servers/browsecomp_advanced_harness/README.md index b49961efb32..818f2f71dd7 100644 --- a/resources_servers/browsecomp_advanced_harness/README.md +++ b/resources_servers/browsecomp_advanced_harness/README.md @@ -51,7 +51,9 @@ Enroot must provide a PID namespace separate from the resources-server host, wit namespace. Setup checks the running sandbox's namespace IDs before permitting model commands; a version string or `ENROOT_UNSHARE_PID` setting alone is insufficient because older Enroot builds may ignore that setting. If the check fails, upgrade or configure Enroot with working PID isolation, or select another isolating provider. The -failed session follows the usual sandbox cleanup and close-retry path. +failed session follows the usual sandbox cleanup and close-retry path. Only Enroot is probed. Choose any other +provider for sandboxes that cannot see the host filesystem: Apptainer, for example, bind-mounts the host's home +directory and `/tmp` unless configured otherwise. Each resources session owns one tool sandbox. An agent that runs its loop in a sandbox owns a separate sandbox; neither component borrows the other's container. This works with providers such as Enroot that do not support @@ -63,7 +65,9 @@ The shell uses the checked-out pocketshell implementation and `SandboxPythonRunt version pinned into its isolated Python runtime. The first seed builds that runtime if needed; subsequent sessions reuse its host-side artifact cache. Setup failures surface to the caller and retain ownership when cleanup fails, so a later close can retry. Commands are supervised inside the sandbox; their guard, supported commands, output -truncation, and exit-code envelope are unchanged. There is no fallback to host execution after a sandbox failure. +truncation, and exit-code envelope are unchanged. A command that outlives the supervisor deadline answers +`[exit_code=-1]` and a sandbox failure answers `[exit_code=-2]`, as on the host. There is no fallback to host +execution after a sandbox failure. `allow_host_execution: true` explicitly enables the former host interpreter for development when no provider is configured, or permits the non-isolating `local` provider. It defaults to `false`; production and benchmark configs @@ -74,7 +78,7 @@ must provide an isolating provider. Configurations with `workspace: none` do not Each rollout has its own session: the provider-call metrics that `/verify` reports and, with `workspace: per_session`, the disk workspace that `search` and `browse` write pages to and `bash_command` reads. - **Environment Server.** `/seed_session` takes a `ResourcesSeedSessionRequest` and keys the session by its `resources_session_id`. The seed is refused with 422 when `task_data` lacks the `question` and `ground_truth` that `/verify` needs, or, with a workspace, when the ID is not a plain directory name. `/close_session` removes the workspace and the metrics, and is idempotent. A close that cannot remove the workspace fails and keeps the session, so the caller can retry it. After a close, a seed for the same ID is refused with 409, and a tool call for it gets 404 (`session_lost`) instead of recreating the workspace. `/verify` of a closed session returns a masked sample (`reward: 0`, `mask_sample: true`, `failure_kind: session_lost`). The reward depends only on the response and the task, so the mask is kept on purpose: a closed session reads the same on every server under the resources session contract. `single_agent_turn` verifies before it closes, so only an out-of-order verify gets the mask. -- **An agent's `/run`.** `/seed_session` takes the flat row and keys the session by the cookie. `/verify` releases the sandbox and removes the workspace, since this path sends no close. +- **An agent's `/run`.** `/seed_session` takes the flat row and keys the session by the cookie. `/verify` releases the sandbox and removes the workspace, since this path sends no close. The sandbox starts with the first `bash_command`, not at the seed. A run whose agent never sends `/verify`, as `browsecomp_agent` does after a timeout, keeps its sandbox until the provider's `ttl_s` reclaims it or the server shuts down; Enroot ignores `ttl_s`. Sandboxed runs should use the Environment Server path, which always closes. `/verify` and `/close_session` stop the owned sandbox before deleting its workspace and metrics. A failed stop leaves the session available for a close retry. Shutdown also stops any remaining tool sandboxes. Tool calls, verification, diff --git a/resources_servers/browsecomp_advanced_harness/app.py b/resources_servers/browsecomp_advanced_harness/app.py index ff8325b2a15..871c729a98b 100644 --- a/resources_servers/browsecomp_advanced_harness/app.py +++ b/resources_servers/browsecomp_advanced_harness/app.py @@ -1065,13 +1065,14 @@ async def seed_session( if self.config.workspace == "per_session": self._validate_shell_config() if not isinstance(body, ResourcesSeedSessionRequest): - # An agent's /run seeds with its row; /verify owns its eventual cleanup. + # An agent's /run seeds with its row; /verify owns its eventual cleanup. Such an agent can end a run + # without /verify (after a timeout, for example), so its sandbox starts with the first bash_command: only + # a run that used the shell can leave one behind, and a failed start cannot strand this seed's workspace. sid = request.session[SESSION_ID_KEY] async with self._fence.locked(sid): if self.config.workspace == "per_session": await self._release_session(sid) self._session_workspaces[sid] = _PageWriter(Path(self._workspace_root) / sid) - await self._ensure_shell(sid) return BaseSeedSessionResponse() sid = body.resources_session_id @@ -1233,12 +1234,21 @@ async def bash_command(self, request: Request, body: BashCommandRequest) -> Bash out = await self._run_bash_readonly(command, duration, str(page_writer.workspace)) else: async with self._bash_semaphore: - out = await shell.execute( - command=command, - duration=duration, - workspace=page_writer.workspace, - close_timeout=self.config.sandbox_close_timeout_seconds, - ) + try: + out = await shell.execute( + command=command, + duration=duration, + workspace=page_writer.workspace, + close_timeout=self.config.sandbox_close_timeout_seconds, + ) + except TimeoutError: + # The supervisor deadline passed without output: answer as the host interpreter did. + LOG.warning("BrowseComp shell command timed out in session %s: %r", sid, command[:160]) + out = f"[command timed out after {duration:.0f}s]\n[exit_code=-1]" + except Exception: + # The model gets the usual error envelope; the provider's detail stays in the server log. + LOG.exception("BrowseComp shell command failed in session %s", sid) + out = "[bash error: the sandbox could not run the command]\n[exit_code=-2]" return BashCommandResponse(results_string=out) async def _run_bash_readonly(self, keystrokes: str, duration: float, cwd: str) -> str: diff --git a/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py b/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py index 806020d41c0..b5fa551123c 100644 --- a/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py +++ b/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py @@ -238,6 +238,44 @@ async def test_bash_uses_sandbox_and_retains_guard_output(sandboxed, monkeypatch await server.close_resources_session(ResourcesCloseSessionRequest.model_validate(_close())) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("error", "expected"), + [ + (TimeoutError("supervisor deadline"), "[command timed out after 10s]\n[exit_code=-1]"), + ( + RuntimeError("exec failed: provider detail"), + "[bash error: the sandbox could not run the command]\n[exit_code=-2]", + ), + ], +) +async def test_sandbox_failures_answer_with_the_host_error_envelope(sandboxed, error, expected) -> None: + server, _ = sandboxed + await server.seed_session(_request(), ResourcesSeedSessionRequest.model_validate(_seed())) + server._shell_sessions["resources-session"].execute = AsyncMock(side_effect=error) + output = await server.bash_command(_request(), BashCommandRequest(keystrokes="cat pages/0001.txt")) + assert output.results_string == expected + await server.close_resources_session(ResourcesCloseSessionRequest.model_validate(_close())) + + +@pytest.mark.asyncio +async def test_run_seed_starts_the_sandbox_with_the_first_shell_command( + sandboxed, monkeypatch: pytest.MonkeyPatch +) -> None: + # A /run agent can skip /verify, so only a run that used the shell may hold a sandbox. + server, containers = sandboxed + monkeypatch.setattr(shell_module.BrowsecompShellSandbox, "execute", AsyncMock(return_value="text\n[exit_code=0]")) + await server.seed_session(_request(), harness.BaseSeedSessionRequest()) + assert not containers and "resources-session" in server._session_workspaces + output = await server.bash_command(_request(), BashCommandRequest(keystrokes="cat pages/0001.txt")) + assert output.results_string == "text\n[exit_code=0]" + containers[0].start.assert_awaited_once() + body = harness.TavilySearchVerifyRequest.model_validate(_verify_body("Answer: Paris Confidence: 90%")) + assert (await server.verify(_request(), body)).reward == 1 + containers[0].stop.assert_awaited_once() + assert not server._shell_sessions and not server._session_workspaces + + @pytest.mark.asyncio async def test_setup_failure_retains_failed_cleanup_for_retry(sandboxed, monkeypatch: pytest.MonkeyPatch) -> None: server, containers = sandboxed From 8f7891c09892d10da58e082484260227bd9ed973 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 4 Nov 2026 13:07:32 -0800 Subject: [PATCH 18/22] fix(browsecomp_agent): fail fast on unreachable endpoints; refuse unfixable seeds with 422 - The runner checks that the sandbox can connect to the model, tool-grant and token-count endpoints before the loop. Gym's client resends a refused internal request without limit, so an unreachable host spun until the activation cap and came back as a scored timed_out zero instead of the unscored failure row the README promises. - A tool-free session (no direct HTTP grant) crashed with AttributeError when the model called a tool the loop does not handle; answer it with a tool error instead. - rollout_timeout_s now counts from the first attempt, so empty-answer retries stay inside the activation cap instead of being killed by the supervisor. - An unusable provider block, a loopback model or tool URL, and an Enroot sandbox without PID isolation raise AgentSeedRejectedError (422) after cleanup, instead of a 500 the Environment Server retries on every task. - The Enroot probe reads stdout with `or ""`, since a timed-out exec reports none. - Print the runner's traceback when it reports an error. - Correct the benchmark README: BrowseComp sessions restart after a crash (restart_only), and say what happens to unreachable endpoints and later connection loss. - Import check_agent_seed_rejected from nemo_gym.agents, not its moved path. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- benchmarks/browsecomp/README.md | 6 +- responses_api_agents/browsecomp_agent/app.py | 42 ++++++-- .../browsecomp_agent/sandbox.py | 8 +- .../browsecomp_agent/sandbox_runner.py | 25 +++++ .../tests/test_agent_sessions.py | 2 +- .../browsecomp_agent/tests/test_sandbox.py | 96 +++++++++++++++++++ 6 files changed, 164 insertions(+), 15 deletions(-) diff --git a/benchmarks/browsecomp/README.md b/benchmarks/browsecomp/README.md index 53bcbf7e63f..f2835af25b4 100644 --- a/benchmarks/browsecomp/README.md +++ b/benchmarks/browsecomp/README.md @@ -38,7 +38,7 @@ ng_e2e_collect_rollouts \ ## Run through the Environment Server -`config.yaml` now runs the benchmark through `single_agent_turn_legacy` with sandboxed agent and resources tool execution. It requires a Docker daemon by default. Replace the included Docker provider config and both sandbox images when selecting another provider; do not merge two provider blocks under `sandbox`. `config_single_agent_turn.yaml` is a compatibility alias. A crash then repeats only the current step (seed, agent turn, agent close, verify, close) rather than the whole episode. Use it in place of `config.yaml` in step 3: +`config.yaml` now runs the benchmark through `single_agent_turn_legacy` with sandboxed agent and resources tool execution. It requires a Docker daemon by default. Replace the included Docker provider config and both sandbox images when selecting another provider; do not merge two provider blocks under `sandbox`. `config_single_agent_turn.yaml` is a compatibility alias. BrowseComp sessions are not exported to checkpoints, so after a crash an episode starts over from its input. Use it in place of `config.yaml` in step 3: ```bash config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ @@ -51,11 +51,13 @@ Tasks, prompts and correct-answer grading are unchanged. The sandbox path adds a | --- | --- | --- | | Rollout reaches `rollout_timeout_s` | Reward 0 without a judge call, `timed_out: 1` | The same | | Model call returns 400, 413 or 422 | Reward 0, with `agent_error` | Verified with an empty answer, which scores 0 | -| Model call returns 5xx after the client's retries, or a connection drops or times out | Set aside unscored (`kill_shaped`) for resume | A non-terminal failure row (`_ng_failure_*`), unscored, which `gym eval run --resume` retries | +| Model call returns 5xx after the client's retries, or the sandbox cannot connect to the model or tools when the activation starts | Set aside unscored (`kill_shaped`) for resume | A non-terminal failure row (`_ng_failure_*`), unscored, which `gym eval run --resume` retries | | Model call returns 408, 425 or 429 after the client's retries | Reward 0, with `agent_error` | A non-terminal failure row, which a resume retries | | Model call returns any other 4xx | Reward 0, with `agent_error` | A terminal failure row, not scored and not retried | | Any other error in the agent turn | Reward 0, with `agent_error` | A non-terminal failure row, which a resume retries | +The sandbox checks that it can connect to the model, tool and token-count endpoints before the loop starts. A connection lost later is resent until it recovers or the activation cap ends the loop, which scores as a timeout. + So an error that the previous recipe scored 0 now leaves the scored set instead of counting as a wrong answer. Check the failure rows and the coverage file alongside the accuracy. The [BrowseComp adapter README](../../responses_api_agents/browsecomp_agent/README.md) documents Evaluation capability, explicit evidence gaps, sandbox ownership and unsupported controls. For a hosted model without `/tokenize`, set `save_model_call_using_vllm_tokenize_endpoint: false` explicitly. The research agent and read-only shell each run in an independently owned sandbox; the Resources Server keeps search credentials and verification on the host. diff --git a/responses_api_agents/browsecomp_agent/app.py b/responses_api_agents/browsecomp_agent/app.py index 533215db24b..ba1985182ee 100644 --- a/responses_api_agents/browsecomp_agent/app.py +++ b/responses_api_agents/browsecomp_agent/app.py @@ -80,7 +80,7 @@ MCPToolAccess, ) from nemo_gym.trajectory import AgentObservationBundle, ObservationGap -from responses_api_agents.browsecomp_agent.sandbox import BrowsecompSandboxSession +from responses_api_agents.browsecomp_agent.sandbox import BrowsecompSandboxSession, SandboxIsolationError def _qid(text: str) -> str: @@ -394,18 +394,24 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Brow workdir = body.sandbox_access.workdir if not isinstance(workdir, str) or not workdir.startswith("/"): raise AgentSeedRejectedError("BrowseComp requires an absolute sandbox workdir") - provider_config = resolve_provider_config(provider_ref, get_global_config_dict()) + try: + provider_config = resolve_provider_config(provider_ref, get_global_config_dict()) + except (TypeError, ValueError) as error: + raise AgentSeedRejectedError(f"BrowseComp sandbox provider is not usable: {error}") from error provider_name = next(iter(provider_config), None) if provider_name == "local": raise AgentSeedRejectedError( "BrowseComp sessions require an isolated sandbox provider; local is not isolated" ) - assert_model_url_reachable_from_sandbox( - self.resolve_model_base_url(self.config.model_server.name, body.episode_id.capture_key), - provider_name=provider_name, - ) - if tool_access is not None: - assert_model_url_reachable_from_sandbox(str(tool_access.base_url), provider_name=provider_name) + try: + assert_model_url_reachable_from_sandbox( + self.resolve_model_base_url(self.config.model_server.name, body.episode_id.capture_key), + provider_name=provider_name, + ) + if tool_access is not None: + assert_model_url_reachable_from_sandbox(str(tool_access.base_url), provider_name=provider_name) + except ValueError as error: + raise AgentSeedRejectedError(str(error)) from error provider = create_provider(provider_config) if owns_sandbox: sandbox = AsyncSandbox(provider) @@ -447,6 +453,9 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Brow await state.close(timeout=self.config.session_close_timeout_seconds) except BaseException: raise AgentSessionSetupError(state, error=error) from error + if isinstance(error, SandboxIsolationError): + # Every retry would get a sandbox from the same provider. + raise AgentSeedRejectedError(str(error)) from error raise return state @@ -518,6 +527,11 @@ async def _sandbox_responses( close_timeout=self.config.session_close_timeout_seconds, ) if output.get("error"): + if output.get("traceback"): + print( + f"[browsecomp][sandbox_error][{state.request.agent_session_id}]\n{output['traceback']}", + flush=True, + ) raise RuntimeError(f"BrowseComp sandbox failed: {output['error']}") state.resources_cookies = _cookie_values(output.get("resources_cookies")) return NeMoGymResponse.model_validate(output["response"]) @@ -535,6 +549,8 @@ async def _run_session_loop( an HTTP error returns a failed response that names its status, for the Environment Server's outcome policy. """ attempts = max(1, self.config.max_run_retries) + # rollout_timeout_s bounds the whole activation, so every retry counts from the first attempt's start. + started_at = time.monotonic() for attempt in range(attempts): attempt_body = body if self.config.snap_dir: @@ -543,7 +559,9 @@ async def _run_session_loop( metadata.update(task_index=state.request.episode_id.capture_key, attempt=str(attempt)) attempt_body = body.model_copy(update={"metadata": metadata}) try: - model_response = await self._responses_impl(request, response, attempt_body, session=state) + model_response = await self._responses_impl( + request, response, attempt_body, session=state, started_at=started_at + ) except aiohttp.ClientResponseError as error: return self._model_request_failure(body, error) answer = re.sub(r".*?", "", self._last_message_text(model_response), flags=re.DOTALL) @@ -592,6 +610,7 @@ async def _responses_impl( body: NeMoGymResponseCreateParamsNonStreaming, *, session: Optional[BrowsecompAgentSessionState] = None, + started_at: Optional[float] = None, ) -> NeMoGymResponse: body = body.model_copy(deep=True) @@ -609,7 +628,7 @@ async def _responses_impl( resources_server_cookies = request.cookies if session is None else dict(session.resources_cookies) reset_threshold = self._reset_threshold(self.config) - loop_started = time.monotonic() + loop_started = time.monotonic() if started_at is None else started_at deadline = loop_started + self.config.rollout_timeout_s if self.config.rollout_timeout_s is not None else None timed_out = False model_response = None @@ -898,6 +917,9 @@ async def _model_call(): "Progress board updated (%d chars). It persists in the " "system prompt across context resets." % len(progress_board) ) + elif session is not None and session.tool_access is None: + # A tool-free pairing has no grant to send the call to; answer as an unknown tool would. + tool_output = f"Tool '{output_function_call.name}' is not available in this session." else: async def _tool_call(): diff --git a/responses_api_agents/browsecomp_agent/sandbox.py b/responses_api_agents/browsecomp_agent/sandbox.py index 3a1c02bebc9..ec711dcc726 100644 --- a/responses_api_agents/browsecomp_agent/sandbox.py +++ b/responses_api_agents/browsecomp_agent/sandbox.py @@ -63,6 +63,10 @@ def _write_sources(destination: Path) -> None: archive.write(path, f"{prefix}/{relative.as_posix()}") +class SandboxIsolationError(RuntimeError): + """The provider's sandbox does not isolate the harness's processes; no retry can fix it.""" + + @dataclass class BrowsecompSandboxSession(AgentSessionState): """Sandbox ownership and retained activation; resource state stays behind its HTTP grant.""" @@ -109,14 +113,14 @@ async def _check_process_isolation(self, *, python: str) -> None: "assert namespace == os.readlink('/proc/1/ns/pid'); print(namespace)" ) result = await sandbox.exec(f"{quote(python)} -I -c {quote(probe)}", cwd="/", timeout_s=30) - namespace = result.stdout.strip() + namespace = (result.stdout or "").strip() if ( result.return_code != 0 or result.error_type is not None or re.fullmatch(r"pid:\[\d+\]", namespace) is None or namespace == host_namespace ): - raise RuntimeError( + raise SandboxIsolationError( "BrowseComp requires an isolated PID namespace and matching /proc mount. " "Install a complete Enroot distribution with PID isolation support (4.2 or newer)." ) diff --git a/responses_api_agents/browsecomp_agent/sandbox_runner.py b/responses_api_agents/browsecomp_agent/sandbox_runner.py index 453e8e954a8..df2c565cd38 100644 --- a/responses_api_agents/browsecomp_agent/sandbox_runner.py +++ b/responses_api_agents/browsecomp_agent/sandbox_runner.py @@ -10,6 +10,7 @@ import sys import traceback from pathlib import Path +from urllib.parse import urlsplit # The source bundle is shipped by the adapter; task PYTHONPATH and user site packages are ignored (-I). @@ -64,6 +65,27 @@ async def _count_prompt_tokens(self, body: NeMoGymResponseCreateParamsNonStreami return int((await get_response_json(reply))["count"]) +async def check_reachable(urls: list[str | None], *, attempts: int = 3, timeout_s: float = 10.0) -> None: + """Fail the activation when the sandbox cannot open a connection to a granted endpoint. + + Gym's client resends a refused internal request without limit, so an endpoint the sandbox cannot reach would + otherwise spin until the loop deadline and come back as a scored, timed-out zero. + """ + for url in filter(None, urls): + parsed = urlsplit(url) + host, port = parsed.hostname, parsed.port or (443 if parsed.scheme == "https" else 80) + for attempt in range(attempts): + try: + _, writer = await asyncio.wait_for(asyncio.open_connection(host, port), timeout=timeout_s) + except (OSError, TimeoutError) as error: + if attempt == attempts - 1: + raise ConnectionError(f"The sandbox cannot connect to {host}:{port}: {error!r}") from error + await asyncio.sleep(1) + else: + writer.close() + break + + async def run(input_path: Path, output_path: Path) -> None: """Persist a receipt on success, error and supervisor interruption.""" payload = json.loads(input_path.read_text()) @@ -94,6 +116,9 @@ async def run(input_path: Path, output_path: Path) -> None: loop = asyncio.get_running_loop() loop.add_signal_handler(signal.SIGTERM, task.cancel) try: + await check_reachable( + [payload["model_url"], str(grant.base_url) if grant else None, payload.get("token_count_url")] + ) body = NeMoGymResponseCreateParamsNonStreaming.model_validate(payload["body"]) http_request = Request({"type": "http", "headers": [], "path": "/v1/responses"}) result = await agent._run_session_loop(http_request, Response(), body, state) diff --git a/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py b/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py index 8e42038f5a4..76f9f70bde0 100644 --- a/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py +++ b/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py @@ -16,6 +16,7 @@ OutcomeAction, SandboxSession, agent_outcome_policy, + check_agent_seed_rejected, ) from nemo_gym.client import ServerClient from nemo_gym.config import ModelServerRef, ResourcesServerRef @@ -28,7 +29,6 @@ TaskId, ) from nemo_gym.server_runtime import DirectHTTPToolAccess, MCPStreamableHTTPConnection, MCPToolAccess -from nemo_gym.testing.agent_session_conformance import check_agent_seed_rejected from responses_api_agents.browsecomp_agent.app import BrowsecompAgent, BrowsecompAgentConfig from responses_api_agents.browsecomp_agent.sandbox import BrowsecompSandboxSession diff --git a/responses_api_agents/browsecomp_agent/tests/test_sandbox.py b/responses_api_agents/browsecomp_agent/tests/test_sandbox.py index d6611a1d7c5..0fbfcd7b2d8 100644 --- a/responses_api_agents/browsecomp_agent/tests/test_sandbox.py +++ b/responses_api_agents/browsecomp_agent/tests/test_sandbox.py @@ -4,6 +4,7 @@ import asyncio import json +import socket import sys import zipfile from pathlib import Path @@ -12,6 +13,7 @@ import pytest from aiohttp import web +from nemo_gym.core import NeMoGymResponse from responses_api_agents.browsecomp_agent.sandbox import BrowsecompSandboxSession, _write_sources from responses_api_agents.browsecomp_agent.tests.test_agent_sessions import ( EPISODE, @@ -321,3 +323,97 @@ async def test_start_upload_failure_retains_owned_sandbox_for_cleanup_retry(monk await agent._close_agent_session_state(error.value.state) assert provider.close.await_count == 3 provider.aclose.assert_awaited() + + +async def test_an_unreachable_endpoint_fails_the_activation_before_the_loop(tmp_path: Path) -> None: + """The runner reports an endpoint it cannot connect to instead of spinning until its deadline.""" + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + closed_port = probe.getsockname()[1] + archive = tmp_path / "source.zip" + _write_sources(archive) + with zipfile.ZipFile(archive) as bundle: + bundle.extractall(tmp_path / "source") + payload = { + "config": { + "name": "browsecomp", + "host": "0.0.0.0", + "port": 1, + "entrypoint": "app.py", + "model_server": {"type": "responses_api_models", "name": "policy"}, + "rollout_timeout_s": 600, + }, + "seed": _seed(tool_accesses=[]).model_dump(mode="json"), + "tool_access": None, + "body": QUESTION, + "model_url": f"http://127.0.0.1:{closed_port}/ng-rollout/{EPISODE.capture_key}/v1", + } + input_path, output_path = tmp_path / "input.json", tmp_path / "output.json" + input_path.write_text(json.dumps(payload)) + runner = str(tmp_path / "source/responses_api_agents/browsecomp_agent/sandbox_runner.py") + process = await asyncio.create_subprocess_exec( + sys.executable, "-I", runner, str(input_path), str(output_path), cwd=tmp_path + ) + try: + assert await asyncio.wait_for(process.wait(), timeout=60) == 0 + finally: + if process.returncode is None: + process.kill() + await process.wait() + receipt = json.loads(output_path.read_text()) + assert receipt["error"].startswith(f"ConnectionError: The sandbox cannot connect to 127.0.0.1:{closed_port}") + assert "response" not in receipt + + +def test_a_tool_free_session_answers_a_tool_call_without_a_grant(monkeypatch: pytest.MonkeyPatch) -> None: + client, _, harness = _agent([_model_response(_tool_call("ls")), _model_response(_message("42"))], monkeypatch) + assert client.post("/v1/agent_sessions", json=_seed(tool_accesses=[]).model_dump(mode="json")).status_code == 200 + reply = client.post(ROUTE, json=QUESTION) + assert reply.status_code == 200, reply.text + assert reply.json()["output"][1]["output"] == "Tool 'bash_command' is not available in this session." + assert harness.calls == [] + + +def test_retries_count_from_the_first_attempts_start(monkeypatch: pytest.MonkeyPatch) -> None: + from responses_api_agents.browsecomp_agent.app import BrowsecompAgent + + client, _, _ = _agent([], monkeypatch, max_run_retries=3, rollout_timeout_s=60) + starts = [] + + async def empty_attempt(self, request, response, body, *, session=None, started_at=None): + starts.append(started_at) + return NeMoGymResponse.model_validate(_model_response(_message(""))) + + monkeypatch.setattr(BrowsecompAgent, "_responses_impl", empty_attempt) + assert client.post("/v1/agent_sessions", json=_seed().model_dump(mode="json")).status_code == 200 + assert client.post(ROUTE, json=QUESTION).status_code == 200 + assert len(starts) == 3 and starts[0] is not None and len(set(starts)) == 1 + + +def test_a_loopback_model_url_refuses_the_seed(monkeypatch: pytest.MonkeyPatch) -> None: + from responses_api_agents.browsecomp_agent.app import BrowsecompAgent + + client, _, _ = _agent([], monkeypatch) + monkeypatch.setattr("responses_api_agents.browsecomp_agent.app.resolve_provider_config", lambda *args: {"e2b": {}}) + monkeypatch.setattr(BrowsecompAgent, "resolve_model_base_url", lambda *args: "http://127.0.0.1:9001/v1") + reply = client.post("/v1/agent_sessions", json=_seed().model_dump(mode="json")) + assert reply.status_code == 422 + assert "loopback" in reply.text + + +def test_missing_process_isolation_refuses_the_seed_after_stopping_the_sandbox( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from nemo_gym.agents import SandboxSession + from responses_api_agents.browsecomp_agent.sandbox import SandboxIsolationError + + client, _, _ = _agent([], monkeypatch) + monkeypatch.setattr( + BrowsecompSandboxSession, + "install_runtime", + AsyncMock(side_effect=SandboxIsolationError("BrowseComp requires an isolated PID namespace")), + ) + reply = client.post("/v1/agent_sessions", json=_seed().model_dump(mode="json")) + assert reply.status_code == 422 + assert "isolated PID namespace" in reply.text + SandboxSession.close.assert_awaited_once() From 33eeb92226828859b9591acb749788a616cc3c2d Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 4 Nov 2026 17:15:29 -0800 Subject: [PATCH 19/22] fix(browsecomp_advanced_harness): answer and mask sandbox start failures; refuse bad sandbox config - A /run shell sandbox that cannot start now answers the [exit_code=-2] envelope instead of HTTP 500 with the provider's error text, and a blocked command answers before any sandbox starts. - A session whose tool sandbox failed to start or run a command gets a masked sample at /verify (failure_kind provider_unavailable). The judged reward is kept. - A command the supervisor ends at its deadline answers [exit_code=-1], as the README says, instead of -2. - A missing provider block or an unknown sandbox_config key refuses the seed with 422 instead of a retryable 500. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- .../browsecomp_advanced_harness/README.md | 7 +- .../browsecomp_advanced_harness/app.py | 35 ++++++-- .../browsecomp_advanced_harness/sandbox.py | 18 ++-- .../tests/test_sandbox_shell.py | 84 ++++++++++++++++++- 4 files changed, 127 insertions(+), 17 deletions(-) diff --git a/resources_servers/browsecomp_advanced_harness/README.md b/resources_servers/browsecomp_advanced_harness/README.md index 818f2f71dd7..f3dbc4199ca 100644 --- a/resources_servers/browsecomp_advanced_harness/README.md +++ b/resources_servers/browsecomp_advanced_harness/README.md @@ -67,7 +67,10 @@ reuse its host-side artifact cache. Setup failures surface to the caller and ret so a later close can retry. Commands are supervised inside the sandbox; their guard, supported commands, output truncation, and exit-code envelope are unchanged. A command that outlives the supervisor deadline answers `[exit_code=-1]` and a sandbox failure answers `[exit_code=-2]`, as on the host. There is no fallback to host -execution after a sandbox failure. +execution after a sandbox failure. A sandbox that cannot start or cannot run a command marks its session, and +`/verify` keeps the judged reward but masks the sample (`mask_sample: true`, `failure_kind: +provider_unavailable`), since the model lost its tool to the infrastructure. A blocked command answers without +starting a sandbox. A missing provider block or an unknown `sandbox_config` key refuses the seed with 422. `allow_host_execution: true` explicitly enables the former host interpreter for development when no provider is configured, or permits the non-isolating `local` provider. It defaults to `false`; production and benchmark configs @@ -78,7 +81,7 @@ must provide an isolating provider. Configurations with `workspace: none` do not Each rollout has its own session: the provider-call metrics that `/verify` reports and, with `workspace: per_session`, the disk workspace that `search` and `browse` write pages to and `bash_command` reads. - **Environment Server.** `/seed_session` takes a `ResourcesSeedSessionRequest` and keys the session by its `resources_session_id`. The seed is refused with 422 when `task_data` lacks the `question` and `ground_truth` that `/verify` needs, or, with a workspace, when the ID is not a plain directory name. `/close_session` removes the workspace and the metrics, and is idempotent. A close that cannot remove the workspace fails and keeps the session, so the caller can retry it. After a close, a seed for the same ID is refused with 409, and a tool call for it gets 404 (`session_lost`) instead of recreating the workspace. `/verify` of a closed session returns a masked sample (`reward: 0`, `mask_sample: true`, `failure_kind: session_lost`). The reward depends only on the response and the task, so the mask is kept on purpose: a closed session reads the same on every server under the resources session contract. `single_agent_turn` verifies before it closes, so only an out-of-order verify gets the mask. -- **An agent's `/run`.** `/seed_session` takes the flat row and keys the session by the cookie. `/verify` releases the sandbox and removes the workspace, since this path sends no close. The sandbox starts with the first `bash_command`, not at the seed. A run whose agent never sends `/verify`, as `browsecomp_agent` does after a timeout, keeps its sandbox until the provider's `ttl_s` reclaims it or the server shuts down; Enroot ignores `ttl_s`. Sandboxed runs should use the Environment Server path, which always closes. +- **An agent's `/run`.** `/seed_session` takes the flat row and keys the session by the cookie. `/verify` releases the sandbox and removes the workspace, since this path sends no close. The sandbox starts with the first `bash_command`, not at the seed. A run whose agent never sends `/verify`, as `browsecomp_agent` does after a timeout or an aborted run, keeps its sandbox until the provider's `ttl_s` reclaims it or the server shuts down; Enroot ignores `ttl_s`. Sandboxed runs should use the Environment Server path, which always closes. `/verify` and `/close_session` stop the owned sandbox before deleting its workspace and metrics. A failed stop leaves the session available for a close retry. Shutdown also stops any remaining tool sandboxes. Tool calls, verification, diff --git a/resources_servers/browsecomp_advanced_harness/app.py b/resources_servers/browsecomp_advanced_harness/app.py index 871c729a98b..589196c9249 100644 --- a/resources_servers/browsecomp_advanced_harness/app.py +++ b/resources_servers/browsecomp_advanced_harness/app.py @@ -41,6 +41,7 @@ from nemo_gym.client import RATE_LIMIT_ERROR_CODES, RETRY_ERROR_CODES, SESSION_ID_KEY, raise_for_status, request from nemo_gym.config import ModelServerRef, get_global_config_dict from nemo_gym.core import ( + PROVIDER_UNAVAILABLE, EpisodeId, NeMoGymEasyInputMessage, NeMoGymResponse, @@ -968,6 +969,8 @@ class TavilySearchResourcesServer(SimpleResourcesServer): _bash_semaphore: Optional[asyncio.Semaphore] = PrivateAttr(default=None) _workspace_root: Optional[str] = PrivateAttr(default=None) _shell_sessions: dict[str, BrowsecompShellSandbox] = PrivateAttr(default_factory=dict) + # Why the session's tool sandbox failed, if it did: /verify masks that sample instead of scoring it. + _shell_failures: dict[str, str] = PrivateAttr(default_factory=dict) JUDGE_PROMPT_TEMPLATE: ClassVar[str] = JUDGE_PROMPT_TEMPLATE JUDGE_MAX_ATTEMPTS: ClassVar[int] = 15 @@ -1125,6 +1128,7 @@ async def _release_session(self, sid: str) -> None: if self.config.workspace == "per_session": await asyncio.to_thread(self._cleanup_workspace, sid) self._release_metrics(sid) + self._shell_failures.pop(sid, None) def _validate_shell_config(self) -> None: """Refuse a missing isolation boundary before creating a workspace or provider.""" @@ -1135,7 +1139,12 @@ def _validate_shell_config(self) -> None: "allow_host_execution=true is only for development" ) return - provider_config = resolve_provider_config(self.config.sandbox_provider, get_global_config_dict()) + try: + provider_config = resolve_provider_config(self.config.sandbox_provider, get_global_config_dict()) + SandboxSpec(**self.config.sandbox_config) + except (TypeError, ValueError) as error: + # A missing provider block or an unknown sandbox_config key fails every seed alike, so no retry can help. + raise ResourcesSeedRejectedError(f"BrowseComp sandbox configuration is invalid: {error}") from error if "local" in provider_config and not self.config.allow_host_execution: raise ResourcesSeedRejectedError("The local provider does not isolate BrowseComp workspace tools") workdir = self.config.sandbox_config.get("workdir", "/workspace") @@ -1224,12 +1233,16 @@ async def bash_command(self, request: Request, body: BashCommandRequest) -> Bash return BashCommandResponse( results_string="[bash error: terminal workspace is not enabled for this server]" ) - shell = await self._ensure_shell(sid) command = body.keystrokes or "" + # A blocked command needs no sandbox, so it neither starts one nor waits for one. blocked = _bash_denylisted(command) or _bash_allowlisted(command) if blocked is not None: return BashCommandResponse(results_string=f"[blocked: {blocked}]\n[exit_code=-3]") duration = max(1.0, min(self.config.bash_timeout_s, float(body.duration or 10))) + try: + shell = await self._ensure_shell(sid) + except Exception: + return BashCommandResponse(results_string=self._shell_failure(sid, "start")) if shell is None: out = await self._run_bash_readonly(command, duration, str(page_writer.workspace)) else: @@ -1246,11 +1259,16 @@ async def bash_command(self, request: Request, body: BashCommandRequest) -> Bash LOG.warning("BrowseComp shell command timed out in session %s: %r", sid, command[:160]) out = f"[command timed out after {duration:.0f}s]\n[exit_code=-1]" except Exception: - # The model gets the usual error envelope; the provider's detail stays in the server log. - LOG.exception("BrowseComp shell command failed in session %s", sid) - out = "[bash error: the sandbox could not run the command]\n[exit_code=-2]" + out = self._shell_failure(sid, "run a command") return BashCommandResponse(results_string=out) + def _shell_failure(self, sid: str, action: str) -> str: + """Log the current sandbox error, mark the session's sample for masking, and return the model's envelope.""" + # The model gets the usual error envelope; the provider's detail stays in the server log. + LOG.exception("BrowseComp shell sandbox could not %s in session %s", action, sid) + self._shell_failures.setdefault(sid, f"The BrowseComp tool sandbox could not {action}") + return "[bash error: the sandbox could not run the command]\n[exit_code=-2]" + async def _run_bash_readonly(self, keystrokes: str, duration: float, cwd: str) -> str: """Development-only pocketshell execution; HTTP routes require explicit host opt-in. @@ -1718,6 +1736,13 @@ async def _verify(self, request: Request, body: TavilySearchVerifyRequest) -> Ta num_provider_other_retries=num_provider_other_retries, ) + shell_failure = self._shell_failures.get(sid) + if shell_failure is not None: + # The model lost its shell to the sandbox, not to its own commands, so the reward does not measure it. + verify_response = verify_response.model_copy( + update={"mask_sample": True, "failure_kind": PROVIDER_UNAVAILABLE, "failure_reason": shell_failure} + ) + # The verify response carries the session's metrics, and an agent's /run sends no close, so release here. await self._release_session(sid) diff --git a/resources_servers/browsecomp_advanced_harness/sandbox.py b/resources_servers/browsecomp_advanced_harness/sandbox.py index 34a9384da18..cbeb6b94fa3 100644 --- a/resources_servers/browsecomp_advanced_harness/sandbox.py +++ b/resources_servers/browsecomp_advanced_harness/sandbox.py @@ -116,12 +116,18 @@ async def stage() -> SandboxCommand: async def collect() -> str: return await read_text(self.session.sandbox, path=f"{directory}/output.txt") - return await self.session.execute( - stage_activation=stage, - collect=collect, - timeout=duration + 30, - close_timeout=close_timeout, - ) + try: + return await self.session.execute( + stage_activation=stage, + collect=collect, + timeout=duration + 30, + close_timeout=close_timeout, + ) + except Exception as error: + # The supervisor ends a command at its deadline and records timed_out; the command wrote no output. + if self.session.cleanup is not None and self.session.cleanup["timed_out"]: + raise TimeoutError("BrowseComp shell command passed its supervisor deadline") from error + raise async def close(self, *, timeout: float) -> None: """Confirm termination and stop the owned sandbox; a failed stop remains retryable.""" diff --git a/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py b/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py index b5fa551123c..9c1e849eb21 100644 --- a/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py +++ b/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py @@ -174,6 +174,30 @@ def test_local_provider_requires_explicit_development_opt_in(sandboxed, monkeypa assert not containers +def _missing_provider_block(*_) -> dict: + raise ValueError("Sandbox provider config 'sandbox' was not found") + + +@pytest.mark.parametrize( + ("sandbox_config", "resolve"), + [({"ttl": 7200}, None), ({}, _missing_provider_block)], + ids=["unknown sandbox_config key", "missing provider block"], +) +def test_invalid_sandbox_configuration_refuses_the_seed_for_good( + sandboxed, monkeypatch: pytest.MonkeyPatch, sandbox_config, resolve +) -> None: + # Every seed fails the same way, so a retryable 500 would only repeat the failure for each episode. + server, containers = sandboxed + server.config.sandbox_config.update(sandbox_config) + if resolve is not None: + monkeypatch.setattr(harness, "resolve_provider_config", resolve) + with TestClient(server.setup_webserver()) as client: + response = client.post("/seed_session", json=_seed()) + assert response.status_code == 422 + assert "sandbox configuration is invalid" in response.json()["detail"] + assert not containers and not server._typed_sessions + + @pytest.mark.asyncio async def test_seed_is_idempotent_and_close_stops_each_owned_sandbox(sandboxed) -> None: server, containers = sandboxed @@ -240,21 +264,29 @@ async def test_bash_uses_sandbox_and_retains_guard_output(sandboxed, monkeypatch @pytest.mark.asyncio @pytest.mark.parametrize( - ("error", "expected"), + ("error", "expected", "masked"), [ - (TimeoutError("supervisor deadline"), "[command timed out after 10s]\n[exit_code=-1]"), + (TimeoutError("supervisor deadline"), "[command timed out after 10s]\n[exit_code=-1]", False), ( RuntimeError("exec failed: provider detail"), "[bash error: the sandbox could not run the command]\n[exit_code=-2]", + True, ), ], ) -async def test_sandbox_failures_answer_with_the_host_error_envelope(sandboxed, error, expected) -> None: +async def test_sandbox_failures_answer_with_the_host_error_envelope(sandboxed, error, expected, masked) -> None: server, _ = sandboxed await server.seed_session(_request(), ResourcesSeedSessionRequest.model_validate(_seed())) server._shell_sessions["resources-session"].execute = AsyncMock(side_effect=error) output = await server.bash_command(_request(), BashCommandRequest(keystrokes="cat pages/0001.txt")) assert output.results_string == expected + # A slow command is the model's doing; a sandbox that cannot run one is not, so only that sample is masked. + body = harness.TavilySearchVerifyRequest.model_validate(_verify_body("Answer: Paris Confidence: 90%")) + verified = await server.verify(_request(), body) + assert verified.reward == 1 + assert verified.mask_sample is masked + assert verified.failure_kind == ("provider_unavailable" if masked else None) + assert not server._shell_failures await server.close_resources_session(ResourcesCloseSessionRequest.model_validate(_close())) @@ -276,6 +308,50 @@ async def test_run_seed_starts_the_sandbox_with_the_first_shell_command( assert not server._shell_sessions and not server._session_workspaces +@pytest.mark.asyncio +async def test_a_run_whose_sandbox_cannot_start_gets_the_envelope_and_a_masked_sample( + sandboxed, monkeypatch: pytest.MonkeyPatch +) -> None: + server, containers = sandboxed + + async def fail_install(state, *, timeout): + raise RuntimeError("docker daemon unavailable: provider detail") + + monkeypatch.setattr(shell_module.BrowsecompShellSandbox, "install", fail_install) + await server.seed_session(_request(), harness.BaseSeedSessionRequest()) + blocked = await server.bash_command(_request(), BashCommandRequest(keystrokes="rm -rf pages")) + assert blocked.results_string == "[blocked: command 'rm' is blocked]\n[exit_code=-3]" + assert not containers # A blocked command needs no sandbox. + + output = await server.bash_command(_request(), BashCommandRequest(keystrokes="cat pages/0001.txt")) + assert output.results_string == "[bash error: the sandbox could not run the command]\n[exit_code=-2]" + containers[0].stop.assert_awaited_once() + assert not server._shell_sessions + + body = harness.TavilySearchVerifyRequest.model_validate(_verify_body("Answer: Paris Confidence: 90%")) + verified = await server.verify(_request(), body) + assert (verified.reward, verified.mask_sample, verified.failure_kind) == (1, True, "provider_unavailable") + assert "provider detail" not in verified.failure_reason + assert not server._shell_failures and not server._session_workspaces + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("timed_out", "raised"), [(True, TimeoutError), (False, RuntimeError)]) +async def test_a_command_the_supervisor_ends_at_its_deadline_reads_as_a_timeout(timed_out, raised) -> None: + # The supervisor kills the runner and exits 0 with a timed_out receipt, so only the missing output fails. + session = SimpleNamespace( + session_dir="/private/session/calls/first", + workdir="/workspace", + begin_activation=AsyncMock(), + execute=AsyncMock(side_effect=RuntimeError("output.txt is missing")), + cleanup={"cleanup_confirmed": True, "return_code": None, "error": None, "timed_out": timed_out}, + ) + state = shell_module.BrowsecompShellSandbox(session=session) + state.ready = True + with pytest.raises(raised): + await state.execute(command="grep -r x pages", duration=10, workspace=Path("/unused"), close_timeout=5) + + @pytest.mark.asyncio async def test_setup_failure_retains_failed_cleanup_for_retry(sandboxed, monkeypatch: pytest.MonkeyPatch) -> None: server, containers = sandboxed @@ -377,7 +453,7 @@ async def test_controller_transfers_only_pages_and_manifest_and_never_interpolat (workspace / "private-key").write_text("not a retrieved page") sandbox = AsyncMock() sandbox.exec.return_value = SimpleNamespace(return_code=0, error_type=None) - session = SimpleNamespace(sandbox=sandbox, session_dir="/private/session", workdir="/workspace") + session = SimpleNamespace(sandbox=sandbox, session_dir="/private/session", workdir="/workspace", cleanup=None) directories, commands, payloads = [], [], [] async def begin(*, session_dir, timeout): From 1adaa74def1daccf09be7d5e70218001650cde3e Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 4 Nov 2026 17:21:03 -0800 Subject: [PATCH 20/22] fix(browsecomp_agent): retry unfinished isolation probes; refuse unusable sandbox config and loopback token counting - An Enroot isolation probe that times out or hits a provider error now fails the seed as retryable. Before, it was refused for good with 422. A probe that answers with a shared namespace is still refused. - An unknown provider or an unknown sandbox_config key refuses the seed with 422 instead of a retryable 500. - The seed checks this agent's own token-count URL for loopback, as it already did for the model and tool URLs. - The sandboxed loop's deadline counts from the runner's start, so the endpoint checks spend the rollout's budget, not the supervisor's 30-second grace. - Test the host activation path (deadline clamp, payload, error receipt), and drop a cluster name from a test comment. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- .../browsecomp_agent/README.md | 6 +- responses_api_agents/browsecomp_agent/app.py | 38 +++-- .../browsecomp_agent/sandbox.py | 10 +- .../browsecomp_agent/sandbox_runner.py | 5 +- .../tests/test_agent_sessions.py | 2 +- .../browsecomp_agent/tests/test_sandbox.py | 139 +++++++++++++++++- 6 files changed, 176 insertions(+), 24 deletions(-) diff --git a/responses_api_agents/browsecomp_agent/README.md b/responses_api_agents/browsecomp_agent/README.md index 395070a1c85..ea60074a2e2 100644 --- a/responses_api_agents/browsecomp_agent/README.md +++ b/responses_api_agents/browsecomp_agent/README.md @@ -4,11 +4,11 @@ The repository-native research loop runs inside a task sandbox through an agent ## Execution and composition -`benchmarks/browsecomp/config.yaml` runs `single_agent_turn_legacy`: resources seed → agent seed → one activation → agent close → verify → resources close. `config_single_agent_turn.yaml` remains an alias. Docker is the default provider. To use Enroot or OpenSandbox, replace the included Docker provider config and set the matching sandbox images; do not merge two provider blocks under `sandbox`. Both the Model Server and granted Resources endpoints must be reachable from the agent sandbox (`use_absolute_ip: true`). +`benchmarks/browsecomp/config.yaml` runs `single_agent_turn_legacy`: resources seed → agent seed → one activation → agent close → verify → resources close. `config_single_agent_turn.yaml` remains an alias. Docker is the default provider. To use Enroot or OpenSandbox, replace the included Docker provider config and set the matching sandbox images; do not merge two provider blocks under `sandbox`. Both the Model Server and granted Resources endpoints must be reachable from the agent sandbox (`use_absolute_ip: true`), and so must this agent's own token-count endpoint when `save_model_call_using_vllm_tokenize_endpoint` is on. A loopback address for any of them, an unknown provider or an unknown `sandbox_config` key refuses the seed with 422, since every retry would fail the same way. The agent borrows `sandbox_access` when supplied. Otherwise `sandbox_provider` and `sandbox_config` create an owned sandbox. Missing sandbox configuration, the unisolated Local provider, required MCP grants, and more than one direct HTTP grant are refused. No sandbox error falls back to host execution. Close stops the harness through `SandboxSession`, captures output, then disconnects a borrowed sandbox or destroys an owned one. -Enroot must provide PID isolation (upstream 4.2 or newer). Startup checks the actual exec process namespace and its `/proc` mount before staging rollout input; a shared host namespace or failed probe rejects the session. Use a complete matching Enroot installation, including its helper binaries. Merely setting `ENROOT_UNSHARE_PID` on an older installation is insufficient. +Enroot must provide PID isolation (upstream 4.2 or newer). Startup checks the actual exec process namespace and its `/proc` mount before staging rollout input; a shared host namespace or a failed probe assertion refuses the seed with 422. A probe that times out or hits a provider error fails the seed as retryable instead, because it says nothing about isolation. Use a complete matching Enroot installation, including its helper binaries. Merely setting `ENROOT_UNSHARE_PID` on an older installation is insufficient. The Resources Server remains independent. A direct HTTP grant supplies its tools with per-session headers/cookies; the updated cookie jar returns at agent close. BrowseComp resources execute `bash_command` in their own sandbox, separate from the agent container. This uses two containers per BrowseComp episode and does not require a provider to serialize/connect an existing sandbox. Search credentials, ground truth and verifier code stay on the Resources Server. The model upstream API key stays on the Model Server. @@ -27,7 +27,7 @@ The sandbox path declares **Evaluation** capability. It returns an observation b ## Failure semantics and compatibility -A native rollout timeout retains partial output, sets `timed_out`, and scores zero without a judge call. Model HTTP errors retain their status in `ng_agent_outcome`; the Environment Server decides verification, failure or retry. Its default policy verifies 400/413/422, retries 408/425/429/5xx, and excludes other model failures from scoring. Read failure rows and coverage alongside accuracy: this differs from the old `/run` path, which counted some errors as zero-reward answers. +A native rollout timeout retains partial output, sets `timed_out`, and scores zero without a judge call. Inside the sandbox the timeout counts from the runner's start, so its endpoint checks spend the rollout's budget and the supervisor's 30-second grace is left for startup and the receipt. Model HTTP errors retain their status in `ng_agent_outcome`; the Environment Server decides verification, failure or retry. Its default policy verifies 400/413/422, retries 408/425/429/5xx, and excludes other model failures from scoring. Read failure rows and coverage alongside accuracy: this differs from the old `/run` path, which counted some errors as zero-reward answers. Host `/v1/responses` and `/run` are disabled by default. Development-only compatibility requires `allow_host_execution: true`; `/run` also requires an explicit `resources_server`. The resources server independently requires its own host-execution opt-in if its tool sandbox is disabled. These options do not provide isolation. diff --git a/responses_api_agents/browsecomp_agent/app.py b/responses_api_agents/browsecomp_agent/app.py index ba1985182ee..129b66b8d1a 100644 --- a/responses_api_agents/browsecomp_agent/app.py +++ b/responses_api_agents/browsecomp_agent/app.py @@ -396,7 +396,10 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Brow raise AgentSeedRejectedError("BrowseComp requires an absolute sandbox workdir") try: provider_config = resolve_provider_config(provider_ref, get_global_config_dict()) + if owns_sandbox: + SandboxSpec(**self.config.sandbox_config) except (TypeError, ValueError) as error: + # Every seed would fail the same way, so a retryable 500 would only repeat it for each episode. raise AgentSeedRejectedError(f"BrowseComp sandbox provider is not usable: {error}") from error provider_name = next(iter(provider_config), None) if provider_name == "local": @@ -404,15 +407,20 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Brow "BrowseComp sessions require an isolated sandbox provider; local is not isolated" ) try: - assert_model_url_reachable_from_sandbox( + # Every URL the sandboxed loop calls: the model, the tool grant and this agent's token counting. + sandbox_urls = [ self.resolve_model_base_url(self.config.model_server.name, body.episode_id.capture_key), - provider_name=provider_name, - ) - if tool_access is not None: - assert_model_url_reachable_from_sandbox(str(tool_access.base_url), provider_name=provider_name) + str(tool_access.base_url) if tool_access is not None else None, + self._token_count_url(), + ] + for url in filter(None, sandbox_urls): + assert_model_url_reachable_from_sandbox(url, provider_name=provider_name) except ValueError as error: raise AgentSeedRejectedError(str(error)) from error - provider = create_provider(provider_config) + try: + provider = create_provider(provider_config) + except (TypeError, ValueError) as error: + raise AgentSeedRejectedError(f"BrowseComp sandbox provider is not usable: {error}") from error if owns_sandbox: sandbox = AsyncSandbox(provider) else: @@ -492,6 +500,12 @@ async def _session_token_count( raise HTTPException(422, "vLLM token counting is disabled") return {"count": await self._count_prompt_tokens(body)} + def _token_count_url(self) -> str | None: + """This agent's token-count endpoint, which a sandboxed loop calls instead of the upstream model.""" + if not self.config.save_model_call_using_vllm_tokenize_endpoint: + return None + return f"{self.server_client._resolve_base_url(self.config.name)}/v1/browsecomp/token_count" + async def _sandbox_responses( self, request: Request, body: NeMoGymResponseCreateParamsNonStreaming, state: BrowsecompSandboxSession ) -> NeMoGymResponse: @@ -515,9 +529,7 @@ async def _sandbox_responses( "model_url": self.resolve_model_base_url( self.config.model_server.name, state.request.episode_id.capture_key ), - "token_count_url": f"{self.server_client._resolve_base_url(self.config.name)}/v1/browsecomp/token_count" - if self.config.save_model_call_using_vllm_tokenize_endpoint - else None, + "token_count_url": self._token_count_url(), "agent_cookies": dict(request.cookies), } output = await state.session.execute( @@ -542,15 +554,19 @@ async def _run_session_loop( response: Response, body: NeMoGymResponseCreateParamsNonStreaming, state: BrowsecompAgentSessionState, + *, + started_at: float | None = None, ) -> NeMoGymResponse: """Run an Environment Server's one activation, with the empty-answer retries run() makes around it. Like run(), a retry keeps the session's resources state; it does not seed again. A model call that fails with an HTTP error returns a failed response that names its status, for the Environment Server's outcome policy. + ``started_at`` is when the activation began, if before this call (``time.monotonic()``). """ attempts = max(1, self.config.max_run_retries) - # rollout_timeout_s bounds the whole activation, so every retry counts from the first attempt's start. - started_at = time.monotonic() + # rollout_timeout_s bounds the whole activation, so every retry counts from the activation's start. + if started_at is None: + started_at = time.monotonic() for attempt in range(attempts): attempt_body = body if self.config.snap_dir: diff --git a/responses_api_agents/browsecomp_agent/sandbox.py b/responses_api_agents/browsecomp_agent/sandbox.py index ec711dcc726..cc1c66f17e0 100644 --- a/responses_api_agents/browsecomp_agent/sandbox.py +++ b/responses_api_agents/browsecomp_agent/sandbox.py @@ -113,13 +113,11 @@ async def _check_process_isolation(self, *, python: str) -> None: "assert namespace == os.readlink('/proc/1/ns/pid'); print(namespace)" ) result = await sandbox.exec(f"{quote(python)} -I -c {quote(probe)}", cwd="/", timeout_s=30) + if result.error_type is not None: + # The probe did not run to an answer, which says nothing about isolation; a retry may get one. + raise RuntimeError(f"BrowseComp could not finish its process isolation probe: {result.error_type}") namespace = (result.stdout or "").strip() - if ( - result.return_code != 0 - or result.error_type is not None - or re.fullmatch(r"pid:\[\d+\]", namespace) is None - or namespace == host_namespace - ): + if result.return_code != 0 or re.fullmatch(r"pid:\[\d+\]", namespace) is None or namespace == host_namespace: raise SandboxIsolationError( "BrowseComp requires an isolated PID namespace and matching /proc mount. " "Install a complete Enroot distribution with PID isolation support (4.2 or newer)." diff --git a/responses_api_agents/browsecomp_agent/sandbox_runner.py b/responses_api_agents/browsecomp_agent/sandbox_runner.py index df2c565cd38..48bdda2766a 100644 --- a/responses_api_agents/browsecomp_agent/sandbox_runner.py +++ b/responses_api_agents/browsecomp_agent/sandbox_runner.py @@ -8,6 +8,7 @@ import signal import socket import sys +import time import traceback from pathlib import Path from urllib.parse import urlsplit @@ -88,6 +89,8 @@ async def check_reachable(urls: list[str | None], *, attempts: int = 3, timeout_ async def run(input_path: Path, output_path: Path) -> None: """Persist a receipt on success, error and supervisor interruption.""" + # The supervisor's deadline counts from launch, so the loop's deadline includes the endpoint checks below. + started_at = time.monotonic() payload = json.loads(input_path.read_text()) input_path.unlink() # The runner retains scoped cookies in memory, outside model-readable staged files. # Deliberately empty: the sandbox never receives the server's secret-bearing global config. @@ -121,7 +124,7 @@ async def run(input_path: Path, output_path: Path) -> None: ) body = NeMoGymResponseCreateParamsNonStreaming.model_validate(payload["body"]) http_request = Request({"type": "http", "headers": [], "path": "/v1/responses"}) - result = await agent._run_session_loop(http_request, Response(), body, state) + result = await agent._run_session_loop(http_request, Response(), body, state, started_at=started_at) receipt["response"] = result.model_dump(mode="json") except asyncio.CancelledError: receipt["error"] = "BrowseComp harness was interrupted by its supervisor" diff --git a/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py b/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py index 76f9f70bde0..0782f392f94 100644 --- a/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py +++ b/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py @@ -136,7 +136,7 @@ async def post(server_name=None, url_path=None, json=None, cookies=None): monkeypatch.setattr(BrowsecompAgent, "resolve_model_base_url", lambda *args: "http://model:9001/v1") # Exercise the original loop and HTTP grants independently from provider transport; the runner - # subprocess and supervisor lifecycle have separate tests and real EOS smoke coverage. + # subprocess and supervisor lifecycle have separate tests and real cluster smoke coverage. async def execute_loop(self, request, body, state): from fastapi import Response diff --git a/responses_api_agents/browsecomp_agent/tests/test_sandbox.py b/responses_api_agents/browsecomp_agent/tests/test_sandbox.py index 0fbfcd7b2d8..da1ed4e34c2 100644 --- a/responses_api_agents/browsecomp_agent/tests/test_sandbox.py +++ b/responses_api_agents/browsecomp_agent/tests/test_sandbox.py @@ -6,6 +6,7 @@ import json import socket import sys +import time import zipfile from pathlib import Path from unittest.mock import AsyncMock, MagicMock @@ -13,7 +14,7 @@ import pytest from aiohttp import web -from nemo_gym.core import NeMoGymResponse +from nemo_gym.core import NeMoGymResponse, NgFailureEvent from responses_api_agents.browsecomp_agent.sandbox import BrowsecompSandboxSession, _write_sources from responses_api_agents.browsecomp_agent.tests.test_agent_sessions import ( EPISODE, @@ -33,7 +34,6 @@ [ ("pid:[100]\n", 0, None), # Same namespace as the host. ("", 1, None), # Includes a different /proc/1 namespace: the in-container assertion fails. - ("pid:[200]\n", 0, "timeout"), ("", 0, None), ("not a namespace", 0, None), ], @@ -52,6 +52,20 @@ async def test_enroot_rejects_missing_process_isolation( await state._check_process_isolation(python="/private/python") +async def test_an_unfinished_isolation_probe_is_retryable_not_a_refusal(monkeypatch: pytest.MonkeyPatch) -> None: + from responses_api_agents.browsecomp_agent.sandbox import SandboxIsolationError + + session = MagicMock() + session.sandbox.provider_name = "enroot" + session.sandbox.exec = AsyncMock(return_value=MagicMock(stdout="", return_code=None, error_type="timeout")) + state = BrowsecompSandboxSession(request=_seed(), session=session, tool_access=None, resources_cookies={}) + monkeypatch.setattr("responses_api_agents.browsecomp_agent.sandbox.os.readlink", lambda path: "pid:[100]") + with pytest.raises(RuntimeError, match="could not finish its process isolation probe: timeout") as error: + await state._check_process_isolation(python="/private/python") + # SandboxIsolationError refuses the seed for good (422); a probe that timed out may pass on a retry. + assert not isinstance(error.value, SandboxIsolationError) + + async def test_enroot_accepts_a_distinct_process_namespace(monkeypatch: pytest.MonkeyPatch) -> None: session = MagicMock() session.sandbox.provider_name = "enroot" @@ -401,6 +415,127 @@ def test_a_loopback_model_url_refuses_the_seed(monkeypatch: pytest.MonkeyPatch) assert "loopback" in reply.text +def test_a_loopback_token_count_url_refuses_the_seed(monkeypatch: pytest.MonkeyPatch) -> None: + from responses_api_agents.browsecomp_agent.app import BrowsecompAgent + + monkeypatch.setattr( + "responses_api_agents.browsecomp_agent.app.get_first_server_config_dict", + lambda *args: {"base_url": "http://10.0.0.5:9001/v1", "api_key": "unused"}, + ) + client, server_client, _ = _agent([], monkeypatch, save_model_call_using_vllm_tokenize_endpoint=True) + monkeypatch.setattr("responses_api_agents.browsecomp_agent.app.resolve_provider_config", lambda *args: {"e2b": {}}) + monkeypatch.setattr(BrowsecompAgent, "resolve_model_base_url", lambda *args: "http://10.0.0.5:9001/v1") + server_client._resolve_base_url.return_value = "http://127.0.0.1:8080" + reply = client.post("/v1/agent_sessions", json=_seed(tool_accesses=[]).model_dump(mode="json")) + assert reply.status_code == 422 + assert "127.0.0.1:8080/v1/browsecomp/token_count" in reply.text + + +def _unknown_provider(*args): + raise ValueError("Unknown sandbox provider 'dokcer'") + + +@pytest.mark.parametrize( + ("config", "create_provider"), + [({"sandbox_config": {"ttl": 7200, "workdir": "/sandbox"}}, None), ({}, _unknown_provider)], + ids=["unknown sandbox_config key", "unknown provider"], +) +def test_an_unusable_sandbox_configuration_refuses_the_seed( + monkeypatch: pytest.MonkeyPatch, config: dict, create_provider +) -> None: + # Every seed fails the same way, so a retryable 500 would only repeat the failure for each episode. + client, _, _ = _agent([], monkeypatch, **config) + if create_provider is not None: + monkeypatch.setattr("responses_api_agents.browsecomp_agent.app.create_provider", create_provider) + reply = client.post("/v1/agent_sessions", json=_seed().model_dump(mode="json")) + assert reply.status_code == 422 + assert "sandbox provider is not usable" in reply.text + + +async def test_the_loop_deadline_counts_from_the_runners_start( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The supervisor counts from launch, so the endpoint checks must spend the loop's budget, not its grace.""" + from responses_api_agents.browsecomp_agent import sandbox_runner + + calls = [] + + async def check_reachable(urls): + calls.append(("check", time.monotonic())) + + async def run_session_loop(self, request, response, body, state, *, started_at=None): + calls.append(("loop", started_at)) + return NeMoGymResponse.model_validate(_model_response(_message("Paris"))) + + monkeypatch.setattr(sandbox_runner, "check_reachable", check_reachable) + monkeypatch.setattr(sandbox_runner.SandboxBrowsecompAgent, "_run_session_loop", run_session_loop) + monkeypatch.setattr(sandbox_runner, "get_global_aiohttp_client", lambda: MagicMock(close=AsyncMock())) + monkeypatch.setattr(asyncio.get_running_loop(), "add_signal_handler", lambda *args: None) + monkeypatch.setenv("NEMO_GYM_CONFIG_DICT", "{}") + payload = { + "config": { + "name": "browsecomp", + "host": "0.0.0.0", + "port": 1, + "entrypoint": "app.py", + "model_server": {"type": "responses_api_models", "name": "policy"}, + "rollout_timeout_s": 600, + }, + "seed": _seed(tool_accesses=[]).model_dump(mode="json"), + "tool_access": None, + "body": QUESTION, + "model_url": f"http://model:9001/ng-rollout/{EPISODE.capture_key}/v1", + } + input_path, output_path = tmp_path / "input.json", tmp_path / "output.json" + input_path.write_text(json.dumps(payload)) + await sandbox_runner.run(input_path, output_path) + assert [name for name, _ in calls] == ["check", "loop"] + assert calls[1][1] is not None and calls[1][1] <= calls[0][1] + assert "response" in json.loads(output_path.read_text()) + + +def test_the_sandbox_activation_clamps_its_deadline_and_fails_on_an_error_receipt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from nemo_gym.agents import SandboxSession + from responses_api_agents.browsecomp_agent.app import BrowsecompAgent + + activation = BrowsecompAgent._sandbox_responses # _agent() runs the loop on the host instead. + payloads, timeouts = [], [] + receipts = [ + {"response": _model_response(_message("Paris")), "resources_cookies": {"harness": "next-cookie"}}, + {"error": "ConnectionError: The sandbox cannot connect to model:9001", "resources_cookies": {}}, + ] + + async def stage_activation(self, payload): + payloads.append(payload) + + async def execute(self, *, stage_activation, collect, timeout, close_timeout): + await stage_activation() + timeouts.append(timeout) + return receipts.pop(0) + + def activate(): + client, _, _ = _agent([], monkeypatch, sandbox_runner_timeout_seconds=900) + monkeypatch.setattr(BrowsecompAgent, "_sandbox_responses", activation) + monkeypatch.setattr(BrowsecompSandboxSession, "stage_activation", stage_activation) + monkeypatch.setattr(SandboxSession, "execute", execute) + assert client.post("/v1/agent_sessions", json=_seed(agent_timeout_sec=120).model_dump(mode="json")).is_success + return client.post(ROUTE, json=QUESTION) + + reply = activate() + assert reply.status_code == 200, reply.text + assert reply.json()["output"][0]["content"][0]["text"] == "Paris" + # The task's own limit wins over the runner's, and the supervisor gets its grace on top. + assert payloads[0]["config"]["rollout_timeout_s"] == 120 and timeouts == [150] + assert payloads[0]["token_count_url"] is None + assert payloads[0]["seed"]["tool_accesses"] == [] and payloads[0]["seed"]["sandbox_access"] is None + + with pytest.raises(NgFailureEvent) as failed: # The served app answers it with 500. + activate() + assert "BrowseComp sandbox failed: ConnectionError" in str(failed.value.__cause__) + + def test_missing_process_isolation_refuses_the_seed_after_stopping_the_sandbox( monkeypatch: pytest.MonkeyPatch, ) -> None: From 81e7e2f6c6e865ba7bd9a22230173df578abf203 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 4 Nov 2026 21:53:27 -0800 Subject: [PATCH 21/22] fix(browsecomp_advanced_harness): mask provider timeouts and keep the mask until close - Only the supervisor's deadline answers a shell command with [exit_code=-1]. A provider timeout (docker cp, the exec, reading the output) is a sandbox failure: [exit_code=-2] and a masked sample. - /verify of an Environment Server session no longer releases it; the close does. A re-sent or retried verify still finds the shell failure and the metrics. A judge failure on a masked sample returns the masked sample instead of raising, on both paths. - A typed seed whose provider cannot be created, or whose Enroot sandbox shows the host's PID namespace or a mismatched /proc mount, answers 422 once the sandbox is stopped. A probe that does not finish stays retryable. - After a failed start, later commands answer [exit_code=-2] without starting another sandbox. - The shell runner writes UTF-8 with errors="replace", so a lone surrogate no longer loses the output and its exit-code envelope. - The sandbox timeouts must be finite. - README: shutdown cleanup is best-effort; verify and close ownership. - Docstrings for seed_session, bash_command and verify, and tests for the failure paths above. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- .../browsecomp_advanced_harness/README.md | 30 ++- .../browsecomp_advanced_harness/app.py | 75 +++++- .../browsecomp_advanced_harness/sandbox.py | 15 +- .../shell_runner.py | 5 +- .../tests/test_sandbox_shell.py | 251 ++++++++++++++++-- 5 files changed, 330 insertions(+), 46 deletions(-) diff --git a/resources_servers/browsecomp_advanced_harness/README.md b/resources_servers/browsecomp_advanced_harness/README.md index f3dbc4199ca..0af27aa3e77 100644 --- a/resources_servers/browsecomp_advanced_harness/README.md +++ b/resources_servers/browsecomp_advanced_harness/README.md @@ -51,7 +51,9 @@ Enroot must provide a PID namespace separate from the resources-server host, wit namespace. Setup checks the running sandbox's namespace IDs before permitting model commands; a version string or `ENROOT_UNSHARE_PID` setting alone is insufficient because older Enroot builds may ignore that setting. If the check fails, upgrade or configure Enroot with working PID isolation, or select another isolating provider. The -failed session follows the usual sandbox cleanup and close-retry path. Only Enroot is probed. Choose any other +failed session follows the usual sandbox cleanup and close-retry path. A probe that shows the host's namespace or a +mismatched `/proc` mount refuses the seed with 422 once the sandbox is stopped, since no retry can pass it; a probe +that does not finish fails the seed with a retryable error. Only Enroot is probed. Choose any other provider for sandboxes that cannot see the host filesystem: Apptainer, for example, bind-mounts the host's home directory and `/tmp` unless configured otherwise. @@ -66,11 +68,14 @@ version pinned into its isolated Python runtime. The first seed builds that runt reuse its host-side artifact cache. Setup failures surface to the caller and retain ownership when cleanup fails, so a later close can retry. Commands are supervised inside the sandbox; their guard, supported commands, output truncation, and exit-code envelope are unchanged. A command that outlives the supervisor deadline answers -`[exit_code=-1]` and a sandbox failure answers `[exit_code=-2]`, as on the host. There is no fallback to host -execution after a sandbox failure. A sandbox that cannot start or cannot run a command marks its session, and -`/verify` keeps the judged reward but masks the sample (`mask_sample: true`, `failure_kind: -provider_unavailable`), since the model lost its tool to the infrastructure. A blocked command answers without -starting a sandbox. A missing provider block or an unknown `sandbox_config` key refuses the seed with 422. +`[exit_code=-1]` and a sandbox failure answers `[exit_code=-2]`, as on the host. Only the supervisor's deadline +reads as the command's timeout; a provider timeout (an upload, the exec itself, or reading the output) is a sandbox +failure. There is no fallback to host execution after a sandbox failure. A sandbox that cannot start or cannot run a +command marks its session, and `/verify` keeps the judged reward but masks the sample (`mask_sample: true`, +`failure_kind: provider_unavailable`), since the model lost its tool to the infrastructure. A masked sample is +returned even when the judge cannot be reached. After a failed start, later commands answer `[exit_code=-2]` without +starting another sandbox. A blocked command answers without starting a sandbox. A missing provider block, an +unknown provider, or an unknown `sandbox_config` key refuses the seed with 422. `allow_host_execution: true` explicitly enables the former host interpreter for development when no provider is configured, or permits the non-isolating `local` provider. It defaults to `false`; production and benchmark configs @@ -83,10 +88,15 @@ Each rollout has its own session: the provider-call metrics that `/verify` repor - **Environment Server.** `/seed_session` takes a `ResourcesSeedSessionRequest` and keys the session by its `resources_session_id`. The seed is refused with 422 when `task_data` lacks the `question` and `ground_truth` that `/verify` needs, or, with a workspace, when the ID is not a plain directory name. `/close_session` removes the workspace and the metrics, and is idempotent. A close that cannot remove the workspace fails and keeps the session, so the caller can retry it. After a close, a seed for the same ID is refused with 409, and a tool call for it gets 404 (`session_lost`) instead of recreating the workspace. `/verify` of a closed session returns a masked sample (`reward: 0`, `mask_sample: true`, `failure_kind: session_lost`). The reward depends only on the response and the task, so the mask is kept on purpose: a closed session reads the same on every server under the resources session contract. `single_agent_turn` verifies before it closes, so only an out-of-order verify gets the mask. - **An agent's `/run`.** `/seed_session` takes the flat row and keys the session by the cookie. `/verify` releases the sandbox and removes the workspace, since this path sends no close. The sandbox starts with the first `bash_command`, not at the seed. A run whose agent never sends `/verify`, as `browsecomp_agent` does after a timeout or an aborted run, keeps its sandbox until the provider's `ttl_s` reclaims it or the server shuts down; Enroot ignores `ttl_s`. Sandboxed runs should use the Environment Server path, which always closes. -`/verify` and `/close_session` stop the owned sandbox before deleting its workspace and metrics. A failed stop leaves -the session available for a close retry. Shutdown also stops any remaining tool sandboxes. Tool calls, verification, -and close are serialized per session, so cleanup cannot delete files while a tool is running. `/verify` drops the -session's metrics after reporting them, unless `dump_session_id_to_metrics_on_exit` keeps them for the dump at shutdown. +`/close_session`, and `/verify` on the `/run` path, stop the owned sandbox before deleting its workspace and metrics. +A failed stop leaves the session available for a retry. `/verify` of an Environment Server session releases nothing, +so a re-sent verify still finds the session's metrics and its sandbox failure; its close releases them. Tool calls, +verification, and close are serialized per session, so cleanup cannot delete files while a tool is running. Releasing +a session drops its metrics, unless `dump_session_id_to_metrics_on_exit` keeps them for the dump at shutdown. + +Shutdown tries to stop any remaining tool sandboxes, but only on a best-effort basis. `gym env` kills the server one +second after its interrupt unless `server_shutdown_timeout_seconds` is raised to cover the closes, and Enroot ignores +`ttl_s`, so an Enroot sandbox left behind stays until `enroot remove` deletes it. ## Running diff --git a/resources_servers/browsecomp_advanced_harness/app.py b/resources_servers/browsecomp_advanced_harness/app.py index 589196c9249..82f34a0e1c2 100644 --- a/resources_servers/browsecomp_advanced_harness/app.py +++ b/resources_servers/browsecomp_advanced_harness/app.py @@ -75,7 +75,12 @@ ) from nemo_gym.verification import JUDGE_FAILURES, JudgeError, call_judge, is_judge_failure from resources_servers.browsecomp_advanced_harness.judge_prompt import JUDGE_PROMPT_TEMPLATE -from resources_servers.browsecomp_advanced_harness.sandbox import BrowsecompShellSandbox, shell_source_files +from resources_servers.browsecomp_advanced_harness.sandbox import ( + BrowsecompShellSandbox, + SandboxIsolationError, + ShellCommandTimedOut, + shell_source_files, +) from resources_servers.browsecomp_advanced_harness.shell_output import format_shell_output @@ -119,8 +124,8 @@ class TavilySearchResourcesServerConfig(BaseResourcesServerConfig): sandbox_config: dict[str, JsonValue] = Field( default_factory=lambda: {"image": "python:3.13-slim", "workdir": "/workspace", "ttl_s": 7200} ) - sandbox_install_timeout_seconds: float = Field(default=900, gt=0) - sandbox_close_timeout_seconds: float = Field(default=60, gt=0) + sandbox_install_timeout_seconds: float = Field(default=900, gt=0, allow_inf_nan=False) + sandbox_close_timeout_seconds: float = Field(default=60, gt=0, allow_inf_nan=False) # Development-only opt-in. Never selected after a sandbox setup/execution failure. allow_host_execution: bool = False max_page_bytes: int = 2_000_000 # cap per-page bytes written to disk @@ -914,6 +919,8 @@ def write_browse_page(self, url, title, content) -> str: # A typed seed's resources_session_id names its workspace directory, so it must be one plain path component. _WORKSPACE_SESSION_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") +# What the model reads when the tool sandbox fails; the provider's detail stays in the server log. +_SHELL_FAILURE_OUTPUT = "[bash error: the sandbox could not run the command]\n[exit_code=-2]" def _remove_workspace(workspace: Path) -> None: @@ -1038,6 +1045,10 @@ def setup_webserver(self) -> FastAPI: @asynccontextmanager async def lifespan_wrapper(app): async def close_shell(sid: str) -> None: + # Best effort. Five close timeouts leave room for a running tool's cleanup under the fence and for + # the release's phases, each bounded by one close timeout. gym env kills this process one second + # after its interrupt unless server_shutdown_timeout_seconds is raised, and Enroot ignores ttl_s, so + # a sandbox left behind there needs `enroot remove`. try: async with asyncio.timeout(self.config.sandbox_close_timeout_seconds * 5): async with self._fence.locked(sid): @@ -1065,6 +1076,15 @@ async def close_shell(sid: str) -> None: async def seed_session( self, request: Request, body: ResourcesSeedSessionRequest | BaseSeedSessionRequest ) -> ResourcesSeedSessionResponse | BaseSeedSessionResponse: + """Open a session's workspace and, for an Environment Server session, its tool sandbox. + + A typed seed is keyed by ``resources_session_id`` and validates ``task_data``. With ``workspace: per_session`` + it also starts the tool sandbox. A repeat for the same episode and task reuses the session, and another + episode or a closed session answers 409. A flat ``/run`` seed is keyed by the cookie, replaces earlier state + for it, and defers the sandbox to the first ``bash_command``. A missing or invalid sandbox configuration + answers 422, and so does a typed seed whose provider cannot be created or whose sandbox shares the host's PID + namespace. + """ if self.config.workspace == "per_session": self._validate_shell_config() if not isinstance(body, ResourcesSeedSessionRequest): @@ -1164,7 +1184,11 @@ async def _ensure_shell(self, sid: str) -> BrowsecompShellSandbox | None: raise RuntimeError("BrowseComp shell setup/close is incomplete; close this resources session") return shell global_config = get_global_config_dict() - provider = create_provider(resolve_provider_config(self.config.sandbox_provider, global_config)) + try: + provider = create_provider(resolve_provider_config(self.config.sandbox_provider, global_config)) + except (TypeError, ValueError) as error: + # An unknown provider name or option fails every seed alike, so no retry can help. + raise ResourcesSeedRejectedError(f"BrowseComp sandbox provider cannot be created: {error}") from error sandbox = AsyncSandbox(provider) config = {"image": "python:3.13-slim", "workdir": "/workspace", "ttl_s": 7200, **self.config.sandbox_config} config["metadata"] = ( @@ -1187,13 +1211,16 @@ async def _ensure_shell(self, sid: str) -> BrowsecompShellSandbox | None: try: await sandbox.start(SandboxSpec(**config)) await shell.install(timeout=self.config.sandbox_install_timeout_seconds) - except BaseException: + except BaseException as error: try: await shell.close(timeout=self.config.sandbox_close_timeout_seconds) except BaseException: LOG.exception("BrowseComp shell setup cleanup failed for %s; retaining it for close", sid) else: self._shell_sessions.pop(sid, None) + if isinstance(error, SandboxIsolationError): + # Every sandbox from this provider shares the host's namespace, so no retry can help. + raise ResourcesSeedRejectedError(str(error)) from error raise return shell @@ -1225,6 +1252,16 @@ def _cleanup_workspace(self, sid: str) -> None: self._session_workspaces.pop(sid, None) async def bash_command(self, request: Request, body: BashCommandRequest) -> BashCommandResponse: + """Run one read-only shell command over this session's retrieved pages. + + Serialized with the session's other tool calls, verify and close. A command the guard blocks answers + ``[exit_code=-3]`` without starting a sandbox. Otherwise it runs in the session's Resources-owned sandbox, + started at seed for an Environment Server session and on the first unblocked command for an agent's + ``/run``; with no ``sandbox_provider`` and ``allow_host_execution`` it runs in-process. A command the + supervisor ends at its deadline answers ``[exit_code=-1]``. A sandbox that cannot start or run it answers + ``[exit_code=-2]`` and marks the session so ``/verify`` masks the sample (``provider_unavailable``); after a + failed start, later commands get the same answer without another start. A closed session answers 404. + """ sid = self._open_session_id(request) async with self._fence.locked(sid): self._open_session_id(request) @@ -1238,6 +1275,9 @@ async def bash_command(self, request: Request, body: BashCommandRequest) -> Bash blocked = _bash_denylisted(command) or _bash_allowlisted(command) if blocked is not None: return BashCommandResponse(results_string=f"[blocked: {blocked}]\n[exit_code=-3]") + if sid in self._shell_failures and sid not in self._shell_sessions: + # The start already failed and the sample is masked; another start would only repeat the failure. + return BashCommandResponse(results_string=_SHELL_FAILURE_OUTPUT) duration = max(1.0, min(self.config.bash_timeout_s, float(body.duration or 10))) try: shell = await self._ensure_shell(sid) @@ -1254,8 +1294,9 @@ async def bash_command(self, request: Request, body: BashCommandRequest) -> Bash workspace=page_writer.workspace, close_timeout=self.config.sandbox_close_timeout_seconds, ) - except TimeoutError: - # The supervisor deadline passed without output: answer as the host interpreter did. + except ShellCommandTimedOut: + # The supervisor deadline passed without output: answer as the host interpreter did. Any + # other timeout (an upload, the provider's exec, a read) is the sandbox's, not the model's. LOG.warning("BrowseComp shell command timed out in session %s: %r", sid, command[:160]) out = f"[command timed out after {duration:.0f}s]\n[exit_code=-1]" except Exception: @@ -1264,10 +1305,9 @@ async def bash_command(self, request: Request, body: BashCommandRequest) -> Bash def _shell_failure(self, sid: str, action: str) -> str: """Log the current sandbox error, mark the session's sample for masking, and return the model's envelope.""" - # The model gets the usual error envelope; the provider's detail stays in the server log. LOG.exception("BrowseComp shell sandbox could not %s in session %s", action, sid) self._shell_failures.setdefault(sid, f"The BrowseComp tool sandbox could not {action}") - return "[bash error: the sandbox could not run the command]\n[exit_code=-2]" + return _SHELL_FAILURE_OUTPUT async def _run_bash_readonly(self, keystrokes: str, duration: float, cwd: str) -> str: """Development-only pocketshell execution; HTTP routes require explicit host opt-in. @@ -1686,6 +1726,13 @@ async def _browse(self, request: Request, body: BrowseRequest) -> BrowseResponse return BrowseResponse(results_string=results_string) async def verify(self, request: Request, body: TavilySearchVerifyRequest) -> TavilySearchVerifyResponse: + """Grade the final answer and report the session's provider metrics. + + Masks the sample when the session's tool sandbox failed (``provider_unavailable``), even if the judge could + not be reached, or when the session is already closed (``session_lost``). A judge failure on an unmasked + sample raises. For an agent's ``/run``, which sends no close, verify then releases the session's sandbox, + workspace and metrics; an Environment Server session is released by ``/close_session``. + """ async with self._fence.locked(request.session[SESSION_ID_KEY]): return await self._verify(request, body) @@ -1743,10 +1790,14 @@ async def _verify(self, request: Request, body: TavilySearchVerifyRequest) -> Ta update={"mask_sample": True, "failure_kind": PROVIDER_UNAVAILABLE, "failure_reason": shell_failure} ) - # The verify response carries the session's metrics, and an agent's /run sends no close, so release here. - await self._release_session(sid) + if sid not in self._typed_sessions: + # An agent's /run sends no close, so its verify releases the session. An Environment Server session is + # released by its close, so a re-sent or retried verify still finds the shell failure and the metrics. + await self._release_session(sid) - if judge_error is not None: + # A masked sample does not need the judge's verdict, and a retry after the judge error may no longer find + # the shell failure (a /run session is released above; a reverify has a new session). + if judge_error is not None and shell_failure is None: raise judge_error return verify_response diff --git a/resources_servers/browsecomp_advanced_harness/sandbox.py b/resources_servers/browsecomp_advanced_harness/sandbox.py index cbeb6b94fa3..e8df5b8a6d8 100644 --- a/resources_servers/browsecomp_advanced_harness/sandbox.py +++ b/resources_servers/browsecomp_advanced_harness/sandbox.py @@ -22,6 +22,14 @@ ) +class ShellCommandTimedOut(TimeoutError): + """The supervisor ended the model's command at its deadline; any other TimeoutError is a sandbox failure.""" + + +class SandboxIsolationError(RuntimeError): + """A completed probe showed the host's PID namespace or a mismatched /proc mount; no retry can change that.""" + + def shell_source_files() -> dict[str, str]: """Deliver the checked-out shell implementation through SandboxSpec's cleanup-aware startup.""" source = Path(__file__).parent @@ -72,14 +80,15 @@ async def _require_enroot_pid_namespace(self) -> None: if result.return_code != 0 or result.error_type is not None: raise RuntimeError(f"{_ENROOT_PID_ERROR} The namespace probe did not complete successfully.") namespaces = (result.stdout or "").splitlines() + # Output that names no namespace says nothing about isolation, so it stays retryable like an unfinished probe. if ( not re.fullmatch(r"pid:\[[0-9]+\]", host_namespace) or len(namespaces) != 2 or any(not re.fullmatch(r"pid:\[[0-9]+\]", value) for value in namespaces) - or namespaces[0] != namespaces[1] - or namespaces[0] == host_namespace ): raise RuntimeError(f"{_ENROOT_PID_ERROR} The namespace probe did not confirm isolation.") + if namespaces[0] != namespaces[1] or namespaces[0] == host_namespace: + raise SandboxIsolationError(f"{_ENROOT_PID_ERROR} The namespace probe did not confirm isolation.") async def execute(self, *, command: str, duration: float, workspace: Path, close_timeout: float) -> str: """Copy new retrieved pages, then interpret the command only inside the sandbox.""" @@ -126,7 +135,7 @@ async def collect() -> str: except Exception as error: # The supervisor ends a command at its deadline and records timed_out; the command wrote no output. if self.session.cleanup is not None and self.session.cleanup["timed_out"]: - raise TimeoutError("BrowseComp shell command passed its supervisor deadline") from error + raise ShellCommandTimedOut("BrowseComp shell command passed its supervisor deadline") from error raise async def close(self, *, timeout: float) -> None: diff --git a/resources_servers/browsecomp_advanced_harness/shell_runner.py b/resources_servers/browsecomp_advanced_harness/shell_runner.py index c9af000263d..c2c7c5f7ae2 100644 --- a/resources_servers/browsecomp_advanced_harness/shell_runner.py +++ b/resources_servers/browsecomp_advanced_harness/shell_runner.py @@ -12,13 +12,14 @@ def main() -> None: """Read a JSON command and write its bounded result; command text is never shell source.""" - payload = json.loads(Path(sys.argv[1]).read_text()) + payload = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) try: result = run(payload["command"], workspace=payload["workspace"], timeout=payload["duration"]) output = format_shell_output(result.stdout, result.stderr, result.exit_code) except Exception as error: output = f"[bash error: {error}]\n[exit_code=-2]" - Path(sys.argv[2]).write_text(output) + # A lone surrogate in the output must not cost the model the whole result and its exit code. + Path(sys.argv[2]).write_text(output, encoding="utf-8", errors="replace") if __name__ == "__main__": diff --git a/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py b/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py index 9c1e849eb21..dca4b95e008 100644 --- a/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py +++ b/resources_servers/browsecomp_advanced_harness/tests/test_sandbox_shell.py @@ -14,12 +14,15 @@ from fastapi import Request from fastapi.testclient import TestClient from pocketshell import run +from pydantic import ValidationError from nemo_gym.client import SESSION_ID_KEY +from nemo_gym.sandbox import create_provider from nemo_gym.server_runtime import ResourcesCloseSessionRequest, ResourcesSeedSessionRequest +from nemo_gym.verification import JudgeError from resources_servers.browsecomp_advanced_harness import app as harness from resources_servers.browsecomp_advanced_harness import sandbox as shell_module -from resources_servers.browsecomp_advanced_harness.app import BashCommandRequest +from resources_servers.browsecomp_advanced_harness.app import BashCommandRequest, JudgeEvaluation from resources_servers.browsecomp_advanced_harness.shell_output import format_shell_output from resources_servers.browsecomp_advanced_harness.tests.test_resources_sessions import ( _close, @@ -74,21 +77,25 @@ def shell_install(monkeypatch: pytest.MonkeyPatch): @pytest.mark.asyncio @pytest.mark.parametrize( - ("stdout", "return_code", "error_type"), + ("stdout", "return_code", "error_type", "final"), [ - ("pid:[100]\npid:[100]\n", 0, None), - ("pid:[200]\npid:[100]\n", 0, None), - ("pid:[200]\npid:[200]\n", 1, None), - ("pid:[200]\npid:[200]\n", 0, "timeout"), - (None, 0, None), - ("unavailable\nunavailable\n", 0, None), + ("pid:[100]\npid:[100]\n", 0, None, True), + ("pid:[200]\npid:[100]\n", 0, None, True), + ("pid:[200]\npid:[200]\n", 1, None, False), + ("pid:[200]\npid:[200]\n", 0, "timeout", False), + (None, 0, None, False), + ("unavailable\nunavailable\n", 0, None, False), ], ) -async def test_enroot_install_refuses_unproven_pid_isolation(shell_install, stdout, return_code, error_type) -> None: +async def test_enroot_install_refuses_unproven_pid_isolation( + shell_install, stdout, return_code, error_type, final +) -> None: state, sandbox, runtime, readlink = shell_install sandbox.exec.return_value = SimpleNamespace(stdout=stdout, return_code=return_code, error_type=error_type) - with pytest.raises(RuntimeError, match="older Enroot builds can ignore ENROOT_UNSHARE_PID"): + with pytest.raises(RuntimeError, match="older Enroot builds can ignore ENROOT_UNSHARE_PID") as refused: await state.install(timeout=20) + # Only a finished probe that names the host's namespace, or a mismatched /proc, is final; the rest may retry. + assert isinstance(refused.value, shell_module.SandboxIsolationError) is final runtime.install.assert_awaited_once_with(sandbox, timeout_s=20) readlink.assert_called_once_with("/proc/self/ns/pid") assert not state.ready @@ -154,6 +161,45 @@ async def test_pid_guard_failure_keeps_failed_stop_retryable( assert not server._shell_sessions and not server._session_workspaces +def test_a_sandbox_in_the_host_pid_namespace_refuses_the_seed_for_good( + shell_install, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Every sandbox from this provider shares the namespace alike, so a retryable 500 would only repeat the probe. + _, sandbox, _, _ = shell_install + server = _server(tmp_path, workspace="per_session", sandbox_provider="sandbox", allow_host_execution=False) + monkeypatch.setattr(harness, "get_global_config_dict", lambda: {}) + monkeypatch.setattr(harness, "resolve_provider_config", lambda *_: {"enroot": {}}) + monkeypatch.setattr(harness, "resolve_provider_metadata", lambda *_: {}) + monkeypatch.setattr(harness, "create_provider", lambda *_: object()) + monkeypatch.setattr(harness, "AsyncSandbox", lambda *_: sandbox) + sandbox.exec.return_value = SimpleNamespace(stdout="pid:[100]\npid:[100]\n", return_code=0, error_type=None) + with TestClient(server.setup_webserver()) as client: + response = client.post("/seed_session", json=_seed()) + assert response.status_code == 422 + assert "namespace probe did not confirm isolation" in response.json()["detail"] + sandbox.stop.assert_awaited_once() + assert not server._shell_sessions + assert client.post("/close_session", json=_close()).status_code == 200 + assert not server._session_workspaces + + +def test_an_unknown_provider_refuses_the_seed_for_good(sandboxed, monkeypatch: pytest.MonkeyPatch) -> None: + server, containers = sandboxed + monkeypatch.setattr(harness, "resolve_provider_config", lambda *_: {"no-such-provider": {}}) + monkeypatch.setattr(harness, "create_provider", create_provider) + with TestClient(server.setup_webserver()) as client: + response = client.post("/seed_session", json=_seed()) + assert response.status_code == 422 + assert "Unknown sandbox provider 'no-such-provider'" in response.json()["detail"] + assert not containers and not server._shell_sessions + + +def test_sandbox_timeouts_must_be_finite(tmp_path: Path) -> None: + for field in ("sandbox_install_timeout_seconds", "sandbox_close_timeout_seconds"): + with pytest.raises(ValidationError, match="finite number"): + _server(tmp_path, **{field: float("inf")}) + + def test_missing_provider_refuses_seed_without_creating_workspace(tmp_path: Path) -> None: server = _server(tmp_path, workspace="per_session", allow_host_execution=False) with TestClient(server.setup_webserver()) as client: @@ -266,13 +312,24 @@ async def test_bash_uses_sandbox_and_retains_guard_output(sandboxed, monkeypatch @pytest.mark.parametrize( ("error", "expected", "masked"), [ - (TimeoutError("supervisor deadline"), "[command timed out after 10s]\n[exit_code=-1]", False), + ( + shell_module.ShellCommandTimedOut("supervisor deadline"), + "[command timed out after 10s]\n[exit_code=-1]", + False, + ), ( RuntimeError("exec failed: provider detail"), "[bash error: the sandbox could not run the command]\n[exit_code=-2]", True, ), + # A provider timeout (docker cp, the exec, a read) is not the command's deadline. + ( + TimeoutError("docker command timed out after 60s"), + "[bash error: the sandbox could not run the command]\n[exit_code=-2]", + True, + ), ], + ids=["supervisor deadline", "provider error", "provider timeout"], ) async def test_sandbox_failures_answer_with_the_host_error_envelope(sandboxed, error, expected, masked) -> None: server, _ = sandboxed @@ -286,8 +343,8 @@ async def test_sandbox_failures_answer_with_the_host_error_envelope(sandboxed, e assert verified.reward == 1 assert verified.mask_sample is masked assert verified.failure_kind == ("provider_unavailable" if masked else None) - assert not server._shell_failures await server.close_resources_session(ResourcesCloseSessionRequest.model_validate(_close())) + assert not server._shell_failures @pytest.mark.asyncio @@ -336,7 +393,138 @@ async def fail_install(state, *, timeout): @pytest.mark.asyncio -@pytest.mark.parametrize(("timed_out", "raised"), [(True, TimeoutError), (False, RuntimeError)]) +async def test_commands_after_a_failed_start_do_not_start_another_sandbox( + sandboxed, monkeypatch: pytest.MonkeyPatch +) -> None: + server, containers = sandboxed + install = AsyncMock(side_effect=RuntimeError("docker daemon unavailable")) + monkeypatch.setattr(shell_module.BrowsecompShellSandbox, "install", install) + await server.seed_session(_request(), harness.BaseSeedSessionRequest()) + for _ in range(3): + output = await server.bash_command(_request(), BashCommandRequest(keystrokes="cat pages/0001.txt")) + assert output.results_string == "[bash error: the sandbox could not run the command]\n[exit_code=-2]" + assert len(containers) == 1 and install.await_count == 1 + body = harness.TavilySearchVerifyRequest.model_validate(_verify_body("Answer: Paris Confidence: 90%")) + assert (await server.verify(_request(), body)).mask_sample is True + + +@pytest.mark.asyncio +async def test_a_run_whose_start_and_first_stop_fail_is_masked_and_stopped_by_verify( + sandboxed, monkeypatch: pytest.MonkeyPatch +) -> None: + server, containers = sandboxed + + async def fail_install(state, *, timeout): + state.session.sandbox.stop.side_effect = [RuntimeError("stop unavailable"), None] + raise RuntimeError("install unavailable") + + monkeypatch.setattr(shell_module.BrowsecompShellSandbox, "install", fail_install) + await server.seed_session(_request(), harness.BaseSeedSessionRequest()) + for _ in range(2): + output = await server.bash_command(_request(), BashCommandRequest(keystrokes="ls")) + assert output.results_string == "[bash error: the sandbox could not run the command]\n[exit_code=-2]" + # The failed stop retains the sandbox, so the second command neither starts another nor reaches the host. + assert "resources-session" in server._shell_sessions and len(containers) == 1 + body = harness.TavilySearchVerifyRequest.model_validate(_verify_body("Answer: Paris Confidence: 90%")) + verified = await server.verify(_request(), body) + assert (verified.mask_sample, verified.failure_kind) == (True, "provider_unavailable") + assert containers[0].stop.await_count == 2 + assert not server._shell_sessions and not server._session_workspaces + + +@pytest.mark.asyncio +async def test_concurrent_commands_on_a_run_seed_start_one_sandbox(sandboxed, monkeypatch: pytest.MonkeyPatch) -> None: + server, containers = sandboxed + entered, release = asyncio.Event(), asyncio.Event() + running = peak = 0 + + async def execute(state, **kwargs): + nonlocal running, peak + running += 1 + peak = max(peak, running) + entered.set() + await release.wait() + running -= 1 + return f"{kwargs['command']}\n[exit_code=0]" + + monkeypatch.setattr(shell_module.BrowsecompShellSandbox, "execute", execute) + await server.seed_session(_request(), harness.BaseSeedSessionRequest()) + calls = [ + asyncio.create_task(server.bash_command(_request(), BashCommandRequest(keystrokes=f"cat pages/{n}.txt"))) + for n in range(3) + ] + await asyncio.wait_for(entered.wait(), 1) + await asyncio.sleep(0.01) + release.set() + outputs = await asyncio.wait_for(asyncio.gather(*calls), 1) + assert sorted(output.results_string for output in outputs) == [ + f"cat pages/{n}.txt\n[exit_code=0]" for n in range(3) + ] + # The session's fence serializes the lazy start and the commands. + assert len(containers) == 1 and peak == 1 + containers[0].start.assert_awaited_once() + body = harness.TavilySearchVerifyRequest.model_validate(_verify_body("Answer: Paris Confidence: 90%")) + await server.verify(_request(), body) + containers[0].stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_a_provider_upload_timeout_is_a_masked_sandbox_failure(sandboxed) -> None: + # Through the real SandboxSession: docker cp timed out before launch, so no supervisor deadline passed. + server, containers = sandboxed + await server.seed_session(_request(), ResourcesSeedSessionRequest.model_validate(_seed())) + containers[0].exec.return_value = SimpleNamespace(return_code=0, error_type=None, stdout="", stderr="") + containers[0].upload.side_effect = TimeoutError("docker command timed out after 60s: docker cp") + output = await server.bash_command(_request(), BashCommandRequest(keystrokes="cat manifest.tsv")) + assert output.results_string == "[bash error: the sandbox could not run the command]\n[exit_code=-2]" + body = harness.TavilySearchVerifyRequest.model_validate(_verify_body("Answer: Paris Confidence: 90%")) + verified = await server.verify(_request(), body) + assert (verified.mask_sample, verified.failure_kind) == (True, "provider_unavailable") + await server.close_resources_session(ResourcesCloseSessionRequest.model_validate(_close())) + + +@pytest.mark.asyncio +async def test_a_provider_exec_timeout_without_a_timed_out_receipt_is_a_masked_sandbox_failure(sandboxed) -> None: + server, _ = sandboxed + await server.seed_session(_request(), ResourcesSeedSessionRequest.model_validate(_seed())) + shell = server._shell_sessions["resources-session"] + shell.session.begin_activation = AsyncMock() + shell.session.execute = AsyncMock(side_effect=TimeoutError("supervisor exceeded its execution deadline")) + shell.session.cleanup = {"cleanup_confirmed": True, "return_code": None, "error": None, "timed_out": False} + output = await server.bash_command(_request(), BashCommandRequest(keystrokes="cat manifest.tsv")) + assert output.results_string == "[bash error: the sandbox could not run the command]\n[exit_code=-2]" + body = harness.TavilySearchVerifyRequest.model_validate(_verify_body("Answer: Paris Confidence: 90%")) + assert (await server.verify(_request(), body)).mask_sample is True + await server.close_resources_session(ResourcesCloseSessionRequest.model_validate(_close())) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("shell_failed", [True, False], ids=["masked", "unmasked"]) +async def test_a_judge_failure_raises_only_for_an_unmasked_sample(sandboxed, shell_failed: bool) -> None: + # A re-sent verify, or one retried after the judge error, must read the same session state as the first. + server, _ = sandboxed + server.config.use_judge = True + await server.seed_session(_request(), ResourcesSeedSessionRequest.model_validate(_seed())) + if shell_failed: + server._shell_sessions["resources-session"].execute = AsyncMock(side_effect=RuntimeError("exec failed")) + await server.bash_command(_request(), BashCommandRequest(keystrokes="ls")) + judged = JudgeEvaluation(reasoning="", extracted_final_answer="Paris", reward=1.0) + server._verify_answer_with_judge = AsyncMock(side_effect=[(judged, JudgeError("judge down")), (judged, None)]) + body = harness.TavilySearchVerifyRequest.model_validate(_verify_body("Answer: Paris Confidence: 90%")) + if shell_failed: + first = await server.verify(_request(), body) + assert (first.mask_sample, first.failure_kind) == (True, "provider_unavailable") + else: + with pytest.raises(JudgeError, match="judge down"): + await server.verify(_request(), body) + retried = await server.verify(_request(), body) + assert (retried.reward, retried.mask_sample) == (1, shell_failed) + await server.close_resources_session(ResourcesCloseSessionRequest.model_validate(_close())) + assert not server._shell_failures and "resources-session" not in server._session_id_to_metrics + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("timed_out", "raised"), [(True, shell_module.ShellCommandTimedOut), (False, RuntimeError)]) async def test_a_command_the_supervisor_ends_at_its_deadline_reads_as_a_timeout(timed_out, raised) -> None: # The supervisor kills the runner and exits 0 with a timed_out receipt, so only the missing output fails. session = SimpleNamespace( @@ -371,19 +559,26 @@ async def fail_install(state, *, timeout): @pytest.mark.asyncio -async def test_verify_does_not_remove_workspace_until_sandbox_stop_succeeds(sandboxed) -> None: +async def test_typed_verify_keeps_the_session_and_close_retries_a_failed_stop(sandboxed) -> None: + # The Environment Server closes every session it seeds, so the close, not verify, stops the sandbox. server, containers = sandboxed await server.seed_session(_request(), ResourcesSeedSessionRequest.model_validate(_seed())) workspace = server._session_workspaces["resources-session"].workspace containers[0].stop.side_effect = [RuntimeError("stop unavailable"), None] body = harness.TavilySearchVerifyRequest.model_validate(_verify_body("Answer: Paris Confidence: 90%")) - with pytest.raises(RuntimeError, match="stop unavailable"): - await server.verify(_request(), body) + assert (await server.verify(_request(), body)).reward == 1 + containers[0].stop.assert_not_awaited() assert workspace.exists() and "resources-session" in server._shell_sessions assert "resources-session" in server._session_id_to_metrics - output = await server.verify(_request(), body) - assert output.reward == 1 + with pytest.raises(RuntimeError, match="stop unavailable"): + await server.close_resources_session(ResourcesCloseSessionRequest.model_validate(_close())) + assert workspace.exists() and "resources-session" in server._shell_sessions + assert server._fence.closed_episode("resources-session") is None + await server.close_resources_session(ResourcesCloseSessionRequest.model_validate(_close())) + assert containers[0].stop.await_count == 2 assert not workspace.exists() and not server._shell_sessions + assert "resources-session" not in server._session_id_to_metrics + assert server._fence.closed_episode("resources-session") is not None @pytest.mark.asyncio @@ -406,7 +601,9 @@ async def execute(**kwargs): tool.cancel() await asyncio.wait_for(cleanup_started.wait(), 1) close = asyncio.create_task(server.close_resources_session(ResourcesCloseSessionRequest.model_validate(_close()))) - await asyncio.sleep(0) + # Long enough for a close that ignored the fence to finish. + await asyncio.sleep(0.05) + assert not close.done() containers[0].stop.assert_not_awaited() assert "resources-session" in server._session_workspaces cleanup_allowed.set() @@ -442,6 +639,22 @@ def test_real_runner_preserves_pocketshell_output(tmp_path: Path, command: str) assert len(output_file.read_text()) < 25_000 +def test_real_runner_keeps_the_envelope_for_a_lone_surrogate(tmp_path: Path) -> None: + # A tool argument JSON-escaped as "\ud83d" (half an emoji) reaches echo as a lone surrogate. + command = "echo a\ud83db" + input_file, output_file = tmp_path / "input.json", tmp_path / "output.txt" + input_file.write_text(json.dumps({"command": command, "duration": 5, "workspace": str(tmp_path)})) + runner = Path(shell_module.__file__).with_name("shell_runner.py") + subprocess.run( + [sys.executable, "-E", "-s", str(runner), str(input_file), str(output_file)], check=True, timeout=15 + ) + expected = run(command, str(tmp_path), 5) + assert "\ud83d" in expected.stdout + rendered = format_shell_output(expected.stdout, expected.stderr, expected.exit_code).replace("\ud83d", "?") + assert output_file.read_text(encoding="utf-8") == rendered + assert rendered.endswith("[exit_code=0]") + + @pytest.mark.asyncio async def test_controller_transfers_only_pages_and_manifest_and_never_interpolates_command( tmp_path: Path, monkeypatch: pytest.MonkeyPatch From c0b4ac1439b92be7e1d02072862b47fa3a8b5f36 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 4 Nov 2026 22:06:24 -0800 Subject: [PATCH 22/22] fix(browsecomp_agent): fail a lost endpoint as retryable; keep the head server off the network - The sandboxed loop's model, tool and token-count calls stop resending a refused connection after about 90 seconds. The error reaches the runner's receipt, so the activation fails as retryable instead of running into the rollout deadline and scoring as a timed-out zero. - The benchmark and example configs set absolute_ip_server_types to the model, resources and agent servers, so the head server, whose config endpoint returns the merged config, stays on loopback. - update_progress arguments that are valid JSON but not an object answer the model with a tool error instead of failing the activation. - The Enroot probe follows the resources server's rule: only a finished probe that shows the host's namespace or a mismatched /proc mount refuses the seed; a non-zero exit or unparsable output is retryable. - Upstream error text in a failed response keeps its first 4000 characters, and a missing receipt reports the runner log's last 4000. - The sandbox timeouts must be finite. - Removed the unreachable snap_dir branch of the session loop and the config_single_agent_turn.yaml alias, which main never had. - README: loopback URLs only warn under Docker; lost connections. - Docstrings for responses and run; tests for the paths above, the loopback tool grant, receipt parsing, a cancelled waiter and the token-count endpoint. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- benchmarks/browsecomp/README.md | 11 +- benchmarks/browsecomp/config.yaml | 2 + .../browsecomp/config_single_agent_turn.yaml | 3 - .../browsecomp_advanced_harness/README.md | 4 +- .../configs/browsecomp_advanced_harness.yaml | 2 + .../browsecomp_agent/README.md | 4 +- responses_api_agents/browsecomp_agent/app.py | 56 ++++- .../browsecomp_agent/sandbox.py | 31 ++- .../browsecomp_agent/sandbox_runner.py | 20 +- .../tests/test_agent_sessions.py | 76 +++++++ .../tests/test_malformed_tool_arguments.py | 18 ++ .../browsecomp_agent/tests/test_sandbox.py | 195 ++++++++++++++++-- 12 files changed, 365 insertions(+), 57 deletions(-) delete mode 100644 benchmarks/browsecomp/config_single_agent_turn.yaml diff --git a/benchmarks/browsecomp/README.md b/benchmarks/browsecomp/README.md index f2835af25b4..48d309592c1 100644 --- a/benchmarks/browsecomp/README.md +++ b/benchmarks/browsecomp/README.md @@ -38,12 +38,7 @@ ng_e2e_collect_rollouts \ ## Run through the Environment Server -`config.yaml` now runs the benchmark through `single_agent_turn_legacy` with sandboxed agent and resources tool execution. It requires a Docker daemon by default. Replace the included Docker provider config and both sandbox images when selecting another provider; do not merge two provider blocks under `sandbox`. `config_single_agent_turn.yaml` is a compatibility alias. BrowseComp sessions are not exported to checkpoints, so after a crash an episode starts over from its input. Use it in place of `config.yaml` in step 3: - -```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/browsecomp/config_single_agent_turn.yaml" -``` +`config.yaml` now runs the benchmark through `single_agent_turn_legacy` with sandboxed agent and resources tool execution. It requires a Docker daemon by default. Replace the included Docker provider config and both sandbox images when selecting another provider; do not merge two provider blocks under `sandbox`. The sandboxes reach the model, resources and agent servers at the host's address (`use_absolute_ip: true`); `absolute_ip_server_types` keeps the head server, whose config endpoint returns the merged config, on loopback. BrowseComp sessions are not exported to checkpoints, so after a crash an episode starts over from its input. Tasks, prompts and correct-answer grading are unchanged. The sandbox path adds a 3600-second activation cap by default (`sandbox_runner_timeout_seconds`); when `rollout_timeout_s` is unset, that cap also bounds the loop. Account for this limit when comparing with previous step-only runs. Compared with the previous host `/run` recipe, failure handling changes the scored denominator: @@ -51,12 +46,12 @@ Tasks, prompts and correct-answer grading are unchanged. The sandbox path adds a | --- | --- | --- | | Rollout reaches `rollout_timeout_s` | Reward 0 without a judge call, `timed_out: 1` | The same | | Model call returns 400, 413 or 422 | Reward 0, with `agent_error` | Verified with an empty answer, which scores 0 | -| Model call returns 5xx after the client's retries, or the sandbox cannot connect to the model or tools when the activation starts | Set aside unscored (`kill_shaped`) for resume | A non-terminal failure row (`_ng_failure_*`), unscored, which `gym eval run --resume` retries | +| Model call returns 5xx after the client's retries, or the sandbox cannot connect to the model or tools, at the activation's start or for about 90 seconds later | Set aside unscored (`kill_shaped`) for resume | A non-terminal failure row (`_ng_failure_*`), unscored, which `gym eval run --resume` retries | | Model call returns 408, 425 or 429 after the client's retries | Reward 0, with `agent_error` | A non-terminal failure row, which a resume retries | | Model call returns any other 4xx | Reward 0, with `agent_error` | A terminal failure row, not scored and not retried | | Any other error in the agent turn | Reward 0, with `agent_error` | A non-terminal failure row, which a resume retries | -The sandbox checks that it can connect to the model, tool and token-count endpoints before the loop starts. A connection lost later is resent until it recovers or the activation cap ends the loop, which scores as a timeout. +The sandbox checks that it can connect to the model, tool and token-count endpoints before the loop starts. A connection lost later is resent for about 90 seconds; if the endpoint does not come back, the activation fails with a non-terminal failure row instead of scoring as a timeout. So an error that the previous recipe scored 0 now leaves the scored set instead of counting as a wrong answer. Check the failure rows and the coverage file alongside the accuracy. diff --git a/benchmarks/browsecomp/config.yaml b/benchmarks/browsecomp/config.yaml index dd7bdbb1ceb..f30b734720e 100644 --- a/benchmarks/browsecomp/config.yaml +++ b/benchmarks/browsecomp/config.yaml @@ -2,6 +2,8 @@ config_paths: - nemo_gym/sandbox/providers/docker/configs/docker.yaml use_absolute_ip: true +# Only the servers the sandboxes call get the host address; the head server's config endpoint stays on loopback. +absolute_ip_server_types: [responses_api_models, resources_servers, responses_api_agents] # Judge: a hosted reasoning model, independent of the policy under test. Keeping # the judge off the policy endpoint stops a weak policy from grading itself. diff --git a/benchmarks/browsecomp/config_single_agent_turn.yaml b/benchmarks/browsecomp/config_single_agent_turn.yaml deleted file mode 100644 index a7036d8e3f2..00000000000 --- a/benchmarks/browsecomp/config_single_agent_turn.yaml +++ /dev/null @@ -1,3 +0,0 @@ -# Compatibility name: BrowseComp now uses sandboxed agent sessions by default. -config_paths: -- benchmarks/browsecomp/config.yaml diff --git a/resources_servers/browsecomp_advanced_harness/README.md b/resources_servers/browsecomp_advanced_harness/README.md index 5c2d4997044..ee4e2ef5821 100644 --- a/resources_servers/browsecomp_advanced_harness/README.md +++ b/resources_servers/browsecomp_advanced_harness/README.md @@ -103,7 +103,9 @@ second after its interrupt unless `server_shutdown_timeout_seconds` is raised to The example pairing uses `single_agent_turn_legacy`: existing flat dataset rows still work, while the environment server owns resources/agent session setup, verification, and close. The agent loop runs in its own sandbox and receives the resources session through the `direct_http` transport; its `resources_server` config is `null`. -The config includes the Docker provider and `use_absolute_ip: true` so the sandbox can reach the Gym servers. +The config includes the Docker provider and `use_absolute_ip: true` so the sandbox can reach the Gym servers; +`absolute_ip_server_types` limits that to the model, resources and agent servers, so the head server, whose config +endpoint returns the merged config, stays on loopback. Replace the included Docker provider config to run on a different backend, and set both sandbox image fields for that provider. Do not merge two provider blocks under `sandbox`. The example dataset exposes only `search` and `browse`, so the example config keeps `workspace: none` and inline diff --git a/resources_servers/browsecomp_advanced_harness/configs/browsecomp_advanced_harness.yaml b/resources_servers/browsecomp_advanced_harness/configs/browsecomp_advanced_harness.yaml index a3e5ef96ade..33397f4cafc 100644 --- a/resources_servers/browsecomp_advanced_harness/configs/browsecomp_advanced_harness.yaml +++ b/resources_servers/browsecomp_advanced_harness/configs/browsecomp_advanced_harness.yaml @@ -2,6 +2,8 @@ config_paths: - nemo_gym/sandbox/providers/docker/configs/docker.yaml use_absolute_ip: true +# Only the servers the sandboxes call get the host address; the head server's config endpoint stays on loopback. +absolute_ip_server_types: [responses_api_models, resources_servers, responses_api_agents] browsecomp_advanced_harness_resources_server: resources_servers: diff --git a/responses_api_agents/browsecomp_agent/README.md b/responses_api_agents/browsecomp_agent/README.md index ea60074a2e2..7a85dfc8fe6 100644 --- a/responses_api_agents/browsecomp_agent/README.md +++ b/responses_api_agents/browsecomp_agent/README.md @@ -4,11 +4,11 @@ The repository-native research loop runs inside a task sandbox through an agent ## Execution and composition -`benchmarks/browsecomp/config.yaml` runs `single_agent_turn_legacy`: resources seed → agent seed → one activation → agent close → verify → resources close. `config_single_agent_turn.yaml` remains an alias. Docker is the default provider. To use Enroot or OpenSandbox, replace the included Docker provider config and set the matching sandbox images; do not merge two provider blocks under `sandbox`. Both the Model Server and granted Resources endpoints must be reachable from the agent sandbox (`use_absolute_ip: true`), and so must this agent's own token-count endpoint when `save_model_call_using_vllm_tokenize_endpoint` is on. A loopback address for any of them, an unknown provider or an unknown `sandbox_config` key refuses the seed with 422, since every retry would fail the same way. +`benchmarks/browsecomp/config.yaml` runs `single_agent_turn_legacy`: resources seed → agent seed → one activation → agent close → verify → resources close. Docker is the default provider. To use Enroot or OpenSandbox, replace the included Docker provider config and set the matching sandbox images; do not merge two provider blocks under `sandbox`. Both the Model Server and granted Resources endpoints must be reachable from the agent sandbox (`use_absolute_ip: true`), and so must this agent's own token-count endpoint when `save_model_call_using_vllm_tokenize_endpoint` is on. The benchmark config sets `absolute_ip_server_types` to those three server types, so the head server, whose config endpoint returns the merged config, stays on loopback. An unknown provider or an unknown `sandbox_config` key refuses the seed with 422, since every retry would fail the same way. So does a loopback address for any of these endpoints, except under Docker: a container on the host network can reach the host's loopback, so the shared check only warns, and an activation that cannot connect then fails its reachability check as a retryable failure. The agent borrows `sandbox_access` when supplied. Otherwise `sandbox_provider` and `sandbox_config` create an owned sandbox. Missing sandbox configuration, the unisolated Local provider, required MCP grants, and more than one direct HTTP grant are refused. No sandbox error falls back to host execution. Close stops the harness through `SandboxSession`, captures output, then disconnects a borrowed sandbox or destroys an owned one. -Enroot must provide PID isolation (upstream 4.2 or newer). Startup checks the actual exec process namespace and its `/proc` mount before staging rollout input; a shared host namespace or a failed probe assertion refuses the seed with 422. A probe that times out or hits a provider error fails the seed as retryable instead, because it says nothing about isolation. Use a complete matching Enroot installation, including its helper binaries. Merely setting `ENROOT_UNSHARE_PID` on an older installation is insufficient. +Enroot must provide PID isolation (upstream 4.2 or newer). Startup checks the actual exec process namespace and its `/proc` mount before staging rollout input; a probe that shows the host's namespace or a `/proc` mount of another namespace refuses the seed with 422. A probe that does not finish (a non-zero exit, a timeout or a provider error) or names no namespace fails the seed as retryable instead, because it says nothing about isolation. The resources server's shell sandbox applies the same rule. Use a complete matching Enroot installation, including its helper binaries. Merely setting `ENROOT_UNSHARE_PID` on an older installation is insufficient. The Resources Server remains independent. A direct HTTP grant supplies its tools with per-session headers/cookies; the updated cookie jar returns at agent close. BrowseComp resources execute `bash_command` in their own sandbox, separate from the agent container. This uses two containers per BrowseComp episode and does not require a provider to serialize/connect an existing sandbox. Search credentials, ground truth and verifier code stay on the Resources Server. The model upstream API key stays on the Model Server. diff --git a/responses_api_agents/browsecomp_agent/app.py b/responses_api_agents/browsecomp_agent/app.py index 129b66b8d1a..aa74bc8a0b7 100644 --- a/responses_api_agents/browsecomp_agent/app.py +++ b/responses_api_agents/browsecomp_agent/app.py @@ -80,7 +80,17 @@ MCPToolAccess, ) from nemo_gym.trajectory import AgentObservationBundle, ObservationGap -from responses_api_agents.browsecomp_agent.sandbox import BrowsecompSandboxSession, SandboxIsolationError +from responses_api_agents.browsecomp_agent.sandbox import ( + _ERROR_DETAIL_CHARS, + BrowsecompSandboxSession, + SandboxIsolationError, +) + + +# Connection retries for the sandboxed loop's model, tool and token-count calls. Gym's client waits 0.5 s between +# resends, so this rides out about 90 s of outage; a longer one ends in an error receipt, a retryable activation +# failure, instead of running into the rollout deadline and scoring as a timed-out zero. +_SANDBOX_LOOP_CONNECTION_RETRIES = 180 def _qid(text: str) -> str: @@ -176,8 +186,8 @@ class BrowsecompAgentConfig(BaseResponsesAPIAgentConfig): sandbox_config: dict[str, JsonValue] = Field( default_factory=lambda: {"image": "python:3.13-slim", "workdir": "/sandbox"} ) - sandbox_install_timeout_seconds: float = Field(default=900, gt=0) - sandbox_runner_timeout_seconds: float = Field(default=3600, gt=0) + sandbox_install_timeout_seconds: float = Field(default=900, gt=0, allow_inf_nan=False) + sandbox_runner_timeout_seconds: float = Field(default=3600, gt=0, allow_inf_nan=False) max_steps: int = 400 # Wall-clock budget per rollout, measured from the start of the rollout loop. Past it the loop # stops, the response carries `timed_out`, and the harness scores the sample 0 without @@ -331,6 +341,13 @@ async def responses( response: Response, body: NeMoGymResponseCreateParamsNonStreaming = Body(), ) -> NeMoGymResponse: + """Serve an agent session's single activation in its sandbox, or the host loop when explicitly allowed. + + A session request must use the seeded attempt-qualified rollout path (else 409). The first activation starts + the sandboxed loop; an identical retry waits for and replays its result, and a different request answers 409. + Disconnecting does not cancel the loop; close does. A request without a session answers 422 unless + ``allow_host_execution`` is true. + """ # Thin diagnostic wrapper around the real implementation. A full-400 eval lost 6 # samples to a 500 out of THIS endpoint with NO traceback anywhere in the log — not # even the `🚨` marker that exception_handling_middleware prints for ordinary app @@ -568,15 +585,9 @@ async def _run_session_loop( if started_at is None: started_at = time.monotonic() for attempt in range(attempts): - attempt_body = body - if self.config.snap_dir: - # The keys run() sets: the rollout's capture key is unique across tasks, repeats and attempts. - metadata = dict(body.metadata or {}) - metadata.update(task_index=state.request.episode_id.capture_key, attempt=str(attempt)) - attempt_body = body.model_copy(update={"metadata": metadata}) try: model_response = await self._responses_impl( - request, response, attempt_body, session=state, started_at=started_at + request, response, body, session=state, started_at=started_at ) except aiohttp.ClientResponseError as error: return self._model_request_failure(body, error) @@ -600,6 +611,8 @@ def _model_request_failure( """ content = getattr(error, "response_content", b"") detail = content.decode(errors="replace") if isinstance(content, bytes) else str(content) + # The rollout row carries this message, so an upstream error page stays bounded. + detail = detail[:_ERROR_DETAIL_CHARS] return NeMoGymResponse( id=f"resp_{uuid4().hex}", created_at=time.time(), @@ -915,6 +928,21 @@ async def _model_call(): f"Invalid JSON in the arguments of your '{output_function_call.name}' tool " f"call: {tool_args_error}. Re-issue the call with valid JSON arguments." ) + elif ( + self.config.progress + and output_function_call.name == "update_progress" + and not isinstance(tool_args, dict) + ): + # Valid JSON that is not an object (a list, a string, null) is the model's mistake, like + # invalid JSON above: it reads the error and may retry, so the rollout keeps its score. + print( + f"[browsecomp][tool_fail][{qid}] step={step} tool=update_progress status=bad_arguments_type", + flush=True, + ) + tool_output = ( + "Arguments of 'update_progress' must be a JSON object with a 'board' string; " + "re-issue the call." + ) elif self.config.progress and output_function_call.name == "update_progress": # Board writes are handled by the agent itself — the board is # per-trajectory agent state, not a resources-server tool. @@ -955,6 +983,7 @@ async def _tool_call(): cookies=resources_server_cookies, headers=dict(session.tool_access.headers), _internal=True, + _max_connection_retries=_SANDBOX_LOOP_CONNECTION_RETRIES, ) return response, (await response.content.read()).decode() @@ -1178,6 +1207,13 @@ async def _tool_call(): return model_response async def run(self, request: Request, body: BrowsecompAgentRunRequest) -> BrowsecompAgentVerifyResponse: + """Seed, answer and verify one row on the host; a development-only path. + + Answers 422 unless ``allow_host_execution`` is true and ``resources_server`` is set. Benchmark configs use + agent sessions through ``single_agent_turn_legacy`` instead. An empty answer is retried up to + ``max_run_retries`` times, and a timed-out rollout scores 0 without the judge. Any error scores 0 with + ``agent_error``; an infrastructure error is also marked ``kill_shaped`` for resume. + """ if not self.config.allow_host_execution or self.config.resources_server is None: raise HTTPException( 422, "Legacy /run requires allow_host_execution=true and resources_server; use agent sessions" diff --git a/responses_api_agents/browsecomp_agent/sandbox.py b/responses_api_agents/browsecomp_agent/sandbox.py index cc1c66f17e0..b1fdd93daaa 100644 --- a/responses_api_agents/browsecomp_agent/sandbox.py +++ b/responses_api_agents/browsecomp_agent/sandbox.py @@ -24,6 +24,10 @@ from nemo_gym.server_runtime import DirectHTTPToolAccess +# Most characters of upstream error text that an error message keeps; failure rows carry these messages. +_ERROR_DETAIL_CHARS = 4000 + + @functools.cache def _runtime() -> SandboxPythonRuntime: # Upload the exact installed Gym sources separately. Resolve their external dependencies using the @@ -103,21 +107,27 @@ async def install_runtime(self, *, timeout: float) -> None: async def _check_process_isolation(self, *, python: str) -> None: # Older Enroot silently ignores ENROOT_UNSHARE_PID. Check the actual exec - # namespace and its proc mount before delivering any rollout input. + # namespace and its proc mount before delivering any rollout input. The resources server's shell sandbox + # (resources_servers/browsecomp_advanced_harness/sandbox.py) applies the same rule: only a finished probe + # that names the host's namespace, or a /proc mount of another one, is final. sandbox = self.session.sandbox if sandbox.provider_name != "enroot": return host_namespace = os.readlink("/proc/self/ns/pid") - probe = ( - "import os; namespace = os.readlink('/proc/self/ns/pid'); " - "assert namespace == os.readlink('/proc/1/ns/pid'); print(namespace)" - ) + probe = "import os; print(os.readlink('/proc/self/ns/pid')); print(os.readlink('/proc/1/ns/pid'))" result = await sandbox.exec(f"{quote(python)} -I -c {quote(probe)}", cwd="/", timeout_s=30) - if result.error_type is not None: + if result.return_code != 0 or result.error_type is not None: # The probe did not run to an answer, which says nothing about isolation; a retry may get one. - raise RuntimeError(f"BrowseComp could not finish its process isolation probe: {result.error_type}") - namespace = (result.stdout or "").strip() - if result.return_code != 0 or re.fullmatch(r"pid:\[\d+\]", namespace) is None or namespace == host_namespace: + outcome = result.error_type or f"exit code {result.return_code}" + raise RuntimeError(f"BrowseComp could not finish its process isolation probe: {outcome}") + namespaces = (result.stdout or "").splitlines() + if ( + re.fullmatch(r"pid:\[\d+\]", host_namespace) is None + or len(namespaces) != 2 + or any(re.fullmatch(r"pid:\[\d+\]", namespace) is None for namespace in namespaces) + ): + raise RuntimeError("BrowseComp's process isolation probe did not name a PID namespace") + if namespaces[0] != namespaces[1] or namespaces[0] == host_namespace: raise SandboxIsolationError( "BrowseComp requires an isolated PID namespace and matching /proc mount. " "Install a complete Enroot distribution with PID isolation support (4.2 or newer)." @@ -142,7 +152,8 @@ async def collect(self) -> dict[str, JsonValue]: try: output = json.loads(await read_text(self.session.sandbox, path=f"{self.session.session_dir}/output.json")) except Exception as error: - log = await self.session.read_output_log() + # The tail holds the runner's last words; the whole log of a long run would bloat failure rows. + log = (await self.session.read_output_log())[-_ERROR_DETAIL_CHARS:] raise RuntimeError(f"BrowseComp runner did not produce a receipt: {log}") from error if not isinstance(output, dict): raise TypeError("BrowseComp runner receipt must be an object") diff --git a/responses_api_agents/browsecomp_agent/sandbox_runner.py b/responses_api_agents/browsecomp_agent/sandbox_runner.py index 48bdda2766a..7bb2951a1ae 100644 --- a/responses_api_agents/browsecomp_agent/sandbox_runner.py +++ b/responses_api_agents/browsecomp_agent/sandbox_runner.py @@ -29,6 +29,7 @@ from nemo_gym.core import NeMoGymResponseCreateParamsNonStreaming from nemo_gym.server_runtime import DirectHTTPToolAccess from responses_api_agents.browsecomp_agent.app import ( + _SANDBOX_LOOP_CONNECTION_RETRIES, BrowsecompAgent, BrowsecompAgentConfig, BrowsecompAgentSessionState, @@ -47,7 +48,13 @@ async def post(self, server_name: str, url_path: str, **kwargs: Unpack[_RequestO body = kwargs.get("json") if isinstance(body, NeMoGymResponseCreateParamsNonStreaming): kwargs["json"] = body.model_dump(mode="json", exclude_unset=True) - return await request("POST", f"{self.model_url}/responses", _internal=True, **kwargs) + return await request( + "POST", + f"{self.model_url}/responses", + _internal=True, + _max_connection_retries=_SANDBOX_LOOP_CONNECTION_RETRIES, + **kwargs, + ) class SandboxBrowsecompAgent(BrowsecompAgent): @@ -60,7 +67,12 @@ async def _count_prompt_tokens(self, body: NeMoGymResponseCreateParamsNonStreami if self.token_count_url is None: raise ValueError("BrowseComp token counting was not granted") reply = await request( - "POST", self.token_count_url, json=body.model_dump(mode="json"), cookies=self.agent_cookies, _internal=True + "POST", + self.token_count_url, + json=body.model_dump(mode="json"), + cookies=self.agent_cookies, + _internal=True, + _max_connection_retries=_SANDBOX_LOOP_CONNECTION_RETRIES, ) await raise_for_status(reply) return int((await get_response_json(reply))["count"]) @@ -69,8 +81,8 @@ async def _count_prompt_tokens(self, body: NeMoGymResponseCreateParamsNonStreami async def check_reachable(urls: list[str | None], *, attempts: int = 3, timeout_s: float = 10.0) -> None: """Fail the activation when the sandbox cannot open a connection to a granted endpoint. - Gym's client resends a refused internal request without limit, so an endpoint the sandbox cannot reach would - otherwise spin until the loop deadline and come back as a scored, timed-out zero. + The loop's calls resend a refused connection for about 90 seconds before they fail the activation, so checking + first fails an endpoint the sandbox cannot reach at all without spending the rollout's budget. """ for url in filter(None, urls): parsed = urlsplit(url) diff --git a/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py b/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py index 0782f392f94..67f884bc352 100644 --- a/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py +++ b/responses_api_agents/browsecomp_agent/tests/test_agent_sessions.py @@ -2,15 +2,18 @@ # SPDX-License-Identifier: Apache-2.0 """The agent sessions an Environment Server (single_agent_turn) seeds, activates once, and closes.""" +import asyncio import json as jsonlib from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock import pytest from aiohttp import ClientResponseError +from fastapi import Request, Response from fastapi.testclient import TestClient from nemo_gym.agents import ( + AGENT_SESSION_COOKIE_KEY, AgentCloseSessionRequest, AgentSeedSessionRequest, OutcomeAction, @@ -23,6 +26,7 @@ from nemo_gym.core import ( EpisodeId, NeMoGymResponse, + NeMoGymResponseCreateParamsNonStreaming, NeMoGymResponseFunctionToolCall, NeMoGymResponseOutputMessage, NeMoGymResponseOutputText, @@ -266,3 +270,75 @@ def test_a_model_http_error_is_a_failed_response_for_the_outcome_policy( assert response.output == [] assert f"Model HTTP {status}: model refused the request" in response.error.message assert agent_outcome_policy(response) == action + + +def _agent_of(client: TestClient) -> BrowsecompAgent: + return next(route.endpoint.__self__ for route in client.app.routes if route.path == "/v1/browsecomp/token_count") + + +def test_a_model_http_error_keeps_only_the_start_of_a_long_upstream_body(monkeypatch: pytest.MonkeyPatch) -> None: + client, _, _ = _agent([], monkeypatch) + request_info = SimpleNamespace(url="http://model", method="POST", headers={}, real_url="http://model") + error = ClientResponseError(request_info, (), status=502, message="Bad Gateway") + error.response_content = b"" + b"x" * 10_000 + body = NeMoGymResponseCreateParamsNonStreaming.model_validate(QUESTION) + response = _agent_of(client)._model_request_failure(body, error) + # The rollout row carries this message, so an upstream error page cannot bloat it. + assert response.error.message == "Model HTTP 502: " + "x" * 3994 + + +async def test_a_cancelled_waiter_leaves_the_activation_running_for_its_retry(monkeypatch: pytest.MonkeyPatch) -> None: + """A disconnected HTTP request must not cancel the shared activation; the identical retry gets its result.""" + client, _, _ = _agent([], monkeypatch) + agent = _agent_of(client) + seed_scope = {"type": "http", "headers": [], "session": {}} + await agent.seed_agent_session(Request(seed_scope), _seed(), Response()) + started, finish = asyncio.Event(), asyncio.Event() + runs = [] + + async def activation(self, request, body, state): + runs.append(body) + started.set() + await finish.wait() + return NeMoGymResponse.model_validate(_model_response(_message("Paris"))) + + monkeypatch.setattr(BrowsecompAgent, "_sandbox_responses", activation) + scope = { + "type": "http", + "headers": [], + "session": {AGENT_SESSION_COOKIE_KEY: "agent-session"}, + "path_params": {"rollout_id": EPISODE.capture_key}, + } + body = NeMoGymResponseCreateParamsNonStreaming.model_validate(QUESTION) + waiter = asyncio.create_task(agent.responses(Request(scope), Response(), body)) + await asyncio.wait_for(started.wait(), 1) + waiter.cancel() + with pytest.raises(asyncio.CancelledError): + await waiter + state = agent._require_agent_session("agent-session") + assert state.task is not None and not state.task.done() + finish.set() + retried = await asyncio.wait_for(agent.responses(Request(scope), Response(), body), 1) + assert retried.output[0].content[0].text == "Paris" + assert len(runs) == 1 + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_the_token_count_endpoint_serves_only_a_seeded_session(monkeypatch: pytest.MonkeyPatch, enabled: bool) -> None: + """The sandbox's only path to token counting: refused without the session cookie, and when disabled.""" + monkeypatch.setattr( + "responses_api_agents.browsecomp_agent.app.get_first_server_config_dict", + lambda *args: {"base_url": "http://10.0.0.5:9001/v1", "api_key": "unused"}, + ) + client, _, _ = _agent([], monkeypatch, save_model_call_using_vllm_tokenize_endpoint=enabled) + counted = AsyncMock(return_value=1234) + monkeypatch.setattr(BrowsecompAgent, "_count_prompt_tokens", counted) + assert client.post("/v1/browsecomp/token_count", json=QUESTION).status_code == 409 + assert client.post("/v1/agent_sessions", json=_seed().model_dump(mode="json")).status_code == 200 + reply = client.post("/v1/browsecomp/token_count", json=QUESTION) + if enabled: + assert reply.status_code == 200 and reply.json() == {"count": 1234} + counted.assert_awaited_once() + else: + assert reply.status_code == 422 + counted.assert_not_awaited() diff --git a/responses_api_agents/browsecomp_agent/tests/test_malformed_tool_arguments.py b/responses_api_agents/browsecomp_agent/tests/test_malformed_tool_arguments.py index 4728883e072..0623c8a60fc 100644 --- a/responses_api_agents/browsecomp_agent/tests/test_malformed_tool_arguments.py +++ b/responses_api_agents/browsecomp_agent/tests/test_malformed_tool_arguments.py @@ -34,6 +34,8 @@ import json as jsonlib from unittest.mock import AsyncMock, MagicMock +import pytest + from nemo_gym.client import ServerClient from nemo_gym.config import ModelServerRef, ResourcesServerRef from nemo_gym.core import ( @@ -228,6 +230,22 @@ async def test_malformed_update_progress_args_do_not_raise(self) -> None: assert len(outputs) == 1 assert "JSON" in outputs[0] + @pytest.mark.parametrize("arguments", ["[]", "null", '"board"']) + async def test_non_object_update_progress_args_are_reported_back_to_the_model(self, arguments: str) -> None: + """Valid JSON that is not an object is the model's mistake too, so the rollout keeps its score.""" + agent, fake = _make_agent( + [ + _model_response([_raw_fn_call("update_progress", arguments)]), + _model_response([_make_msg("Exact Answer: X")], response_id="resp_002"), + ], + progress=True, + ) + result = await _run(agent, [SYSTEM, USER]) + assert _fn_call_outputs(result) == [ + "Arguments of 'update_progress' must be a JSON object with a 'board' string; re-issue the call." + ] + assert fake.tool_paths == [] and len(fake.model_bodies) == 2 + class TestWellFormedArgumentsUnaffected: """Regression guard: the happy paths must behave exactly as before.""" diff --git a/responses_api_agents/browsecomp_agent/tests/test_sandbox.py b/responses_api_agents/browsecomp_agent/tests/test_sandbox.py index da1ed4e34c2..19b63e6b3c0 100644 --- a/responses_api_agents/browsecomp_agent/tests/test_sandbox.py +++ b/responses_api_agents/browsecomp_agent/tests/test_sandbox.py @@ -30,46 +30,56 @@ @pytest.mark.parametrize( - ("stdout", "return_code", "error_type"), - [ - ("pid:[100]\n", 0, None), # Same namespace as the host. - ("", 1, None), # Includes a different /proc/1 namespace: the in-container assertion fails. - ("", 0, None), - ("not a namespace", 0, None), - ], + "stdout", + ["pid:[100]\npid:[100]\n", "pid:[200]\npid:[100]\n"], + ids=["the host's namespace", "a /proc mount of another namespace"], ) -async def test_enroot_rejects_missing_process_isolation( - monkeypatch: pytest.MonkeyPatch, stdout: str, return_code: int, error_type: str | None -) -> None: +async def test_enroot_rejects_missing_process_isolation(monkeypatch: pytest.MonkeyPatch, stdout: str) -> None: + from responses_api_agents.browsecomp_agent.sandbox import SandboxIsolationError + session = MagicMock() session.sandbox.provider_name = "enroot" - session.sandbox.exec = AsyncMock( - return_value=MagicMock(stdout=stdout, return_code=return_code, error_type=error_type) - ) + session.sandbox.exec = AsyncMock(return_value=MagicMock(stdout=stdout, return_code=0, error_type=None)) state = BrowsecompSandboxSession(request=_seed(), session=session, tool_access=None, resources_cookies={}) monkeypatch.setattr("responses_api_agents.browsecomp_agent.sandbox.os.readlink", lambda path: "pid:[100]") - with pytest.raises(RuntimeError, match="isolated PID namespace"): + with pytest.raises(SandboxIsolationError, match="isolated PID namespace"): await state._check_process_isolation(python="/private/python") -async def test_an_unfinished_isolation_probe_is_retryable_not_a_refusal(monkeypatch: pytest.MonkeyPatch) -> None: +@pytest.mark.parametrize( + ("stdout", "return_code", "error_type", "message"), + [ + ("", None, "timeout", "could not finish its process isolation probe: timeout"), + ("", 137, None, "could not finish its process isolation probe: exit code 137"), + ("", 0, None, "did not name a PID namespace"), + ("not a namespace\nnot a namespace\n", 0, None, "did not name a PID namespace"), + ], + ids=["timeout", "killed", "no output", "no namespace"], +) +async def test_an_unfinished_isolation_probe_is_retryable_not_a_refusal( + monkeypatch: pytest.MonkeyPatch, stdout: str, return_code: int | None, error_type: str | None, message: str +) -> None: from responses_api_agents.browsecomp_agent.sandbox import SandboxIsolationError session = MagicMock() session.sandbox.provider_name = "enroot" - session.sandbox.exec = AsyncMock(return_value=MagicMock(stdout="", return_code=None, error_type="timeout")) + session.sandbox.exec = AsyncMock( + return_value=MagicMock(stdout=stdout, return_code=return_code, error_type=error_type) + ) state = BrowsecompSandboxSession(request=_seed(), session=session, tool_access=None, resources_cookies={}) monkeypatch.setattr("responses_api_agents.browsecomp_agent.sandbox.os.readlink", lambda path: "pid:[100]") - with pytest.raises(RuntimeError, match="could not finish its process isolation probe: timeout") as error: + with pytest.raises(RuntimeError, match=message) as error: await state._check_process_isolation(python="/private/python") - # SandboxIsolationError refuses the seed for good (422); a probe that timed out may pass on a retry. + # SandboxIsolationError refuses the seed for good (422); a probe that gave no answer may pass on a retry. assert not isinstance(error.value, SandboxIsolationError) async def test_enroot_accepts_a_distinct_process_namespace(monkeypatch: pytest.MonkeyPatch) -> None: session = MagicMock() session.sandbox.provider_name = "enroot" - session.sandbox.exec = AsyncMock(return_value=MagicMock(stdout="pid:[200]\n", return_code=0, error_type=None)) + session.sandbox.exec = AsyncMock( + return_value=MagicMock(stdout="pid:[200]\npid:[200]\n", return_code=0, error_type=None) + ) state = BrowsecompSandboxSession(request=_seed(), session=session, tool_access=None, resources_cookies={}) monkeypatch.setattr("responses_api_agents.browsecomp_agent.sandbox.os.readlink", lambda path: "pid:[100]") await state._check_process_isolation(python="/private/python") @@ -431,6 +441,25 @@ def test_a_loopback_token_count_url_refuses_the_seed(monkeypatch: pytest.MonkeyP assert "127.0.0.1:8080/v1/browsecomp/token_count" in reply.text +@pytest.mark.parametrize(("provider", "status"), [("e2b", 422), ("docker", 200)]) +def test_a_loopback_tool_grant_url_refuses_the_seed_except_under_docker( + monkeypatch: pytest.MonkeyPatch, provider: str, status: int +) -> None: + # A Docker container on the host network reaches the host's loopback, so the shared check only warns there. + from responses_api_agents.browsecomp_agent.app import BrowsecompAgent + + client, _, _ = _agent([], monkeypatch) + monkeypatch.setattr( + "responses_api_agents.browsecomp_agent.app.resolve_provider_config", lambda *args: {provider: {}} + ) + monkeypatch.setattr(BrowsecompAgent, "resolve_model_base_url", lambda *args: "http://10.0.0.5:9001/v1") + grant = GRANT.model_copy(update={"base_url": "http://127.0.0.1:9000"}) + reply = client.post("/v1/agent_sessions", json=_seed(tool_accesses=[grant]).model_dump(mode="json")) + assert reply.status_code == status, reply.text + if status == 422: + assert "127.0.0.1:9000" in reply.text + + def _unknown_provider(*args): raise ValueError("Unknown sandbox provider 'dokcer'") @@ -494,6 +523,77 @@ async def run_session_loop(self, request, response, body, state, *, started_at=N assert "response" in json.loads(output_path.read_text()) +@pytest.mark.parametrize("lost", ["model", "tool", "token count"]) +async def test_an_endpoint_lost_mid_episode_fails_the_activation_instead_of_timing_out( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, lost: str +) -> None: + """The loop's calls stop resending a refused connection, so an outage leaves an error receipt (retryable) + instead of spinning until the rollout deadline and scoring as a timed-out zero.""" + from responses_api_agents.browsecomp_agent import app, sandbox_runner + + # Production rides out about 90 s of outage; two resends keep this test fast. + monkeypatch.setattr(app, "_SANDBOX_LOOP_CONNECTION_RETRIES", 2) + monkeypatch.setattr(sandbox_runner, "_SANDBOX_LOOP_CONNECTION_RETRIES", 2) + runners: dict[str, web.AppRunner] = {} + origins: dict[str, str] = {} + + async def model(request: web.Request) -> web.Response: + if lost == "tool": + await runners["tool"].cleanup() + return web.json_response(_model_response(_tool_call("ls"))) + + async def tool(request: web.Request) -> web.Response: + if lost != "tool": + await runners[lost].cleanup() + return web.json_response({"results_string": "pages"}) + + async def token_count(request: web.Request) -> web.Response: + return web.json_response({"count": 1}) + + for name, path, handler in ( + ("model", ROUTE, model), + ("tool", "/tools/bash_command", tool), + ("token count", "/token_count", token_count), + ): + application = web.Application() + application.router.add_post(path, handler) + runners[name] = web.AppRunner(application) + await runners[name].setup() + site = web.TCPSite(runners[name], "127.0.0.1", 0) + await site.start() + origins[name] = f"http://127.0.0.1:{site._server.sockets[0].getsockname()[1]}" + monkeypatch.setattr(asyncio.get_running_loop(), "add_signal_handler", lambda *args: None) + monkeypatch.setattr(sandbox_runner, "get_global_aiohttp_client", lambda: MagicMock(close=AsyncMock())) + monkeypatch.setenv("NEMO_GYM_CONFIG_DICT", "{}") + payload = { + "config": { + "name": "browsecomp", + "host": "0.0.0.0", + "port": 1, + "entrypoint": "app.py", + "model_server": {"type": "responses_api_models", "name": "policy"}, + "nudge_steps": False, + "rollout_timeout_s": 60, + "save_model_call_using_vllm_tokenize_endpoint": True, + }, + "seed": _seed().model_dump(mode="json"), + "tool_access": GRANT.model_copy(update={"base_url": f"{origins['tool']}/tools"}).model_dump(mode="json"), + "body": QUESTION, + "model_url": f"{origins['model']}/ng-rollout/{EPISODE.capture_key}/v1", + "token_count_url": f"{origins['token count']}/token_count", + } + input_path, output_path = tmp_path / "input.json", tmp_path / "output.json" + input_path.write_text(json.dumps(payload)) + try: + await asyncio.wait_for(sandbox_runner.run(input_path, output_path), timeout=30) + finally: + for runner in runners.values(): + await runner.cleanup() + receipt = json.loads(output_path.read_text()) + assert "response" not in receipt + assert receipt["error"].startswith(("ClientConnectorError", "ServerDisconnectedError")), receipt["error"] + + def test_the_sandbox_activation_clamps_its_deadline_and_fails_on_an_error_receipt( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -552,3 +652,60 @@ def test_missing_process_isolation_refuses_the_seed_after_stopping_the_sandbox( assert reply.status_code == 422 assert "isolated PID namespace" in reply.text SandboxSession.close.assert_awaited_once() + + +def _receipt_session(downloaded: bytes | Exception, log: str = "runner log tail") -> BrowsecompSandboxSession: + async def download(remote_path: str, local_path: Path) -> None: + if isinstance(downloaded, Exception): + raise downloaded + Path(local_path).write_bytes(downloaded) + + session = MagicMock(session_dir="/private/session") + session.sandbox.download = download + session.read_output_log = AsyncMock(return_value=log) + return BrowsecompSandboxSession(request=_seed(), session=session, tool_access=None, resources_cookies={}) + + +@pytest.mark.parametrize( + ("downloaded", "raised", "message"), + [ + (FileNotFoundError("output.json"), RuntimeError, "did not produce a receipt: runner log tail"), + (b'{"response": {"id": "resp", "out', RuntimeError, "did not produce a receipt: runner log tail"), + (b"\xff\xfe not json", RuntimeError, "did not produce a receipt: runner log tail"), + (b'["not", "an", "object"]', TypeError, "must be an object"), + ], + ids=["missing", "truncated", "garbled", "not an object"], +) +async def test_an_unusable_receipt_fails_the_collection_with_the_runner_log(downloaded, raised, message) -> None: + with pytest.raises(raised, match=message): + await _receipt_session(downloaded).collect() + + +async def test_a_well_formed_receipt_is_returned_as_is() -> None: + receipt = {"response": {"id": "resp"}, "resources_cookies": {"harness": "c"}} + assert await _receipt_session(json.dumps(receipt).encode()).collect() == receipt + + +async def test_a_missing_receipt_reports_only_the_end_of_a_long_runner_log() -> None: + log = "early step\n" * 1000 + "Traceback: the runner's last words" + with pytest.raises(RuntimeError) as failed: + await _receipt_session(FileNotFoundError("output.json"), log=log).collect() + assert str(failed.value) == f"BrowseComp runner did not produce a receipt: {log[-4000:]}" + assert str(failed.value).endswith("the runner's last words") + + +@pytest.mark.parametrize("field", ["sandbox_install_timeout_seconds", "sandbox_runner_timeout_seconds"]) +def test_sandbox_timeouts_must_be_finite(field: str) -> None: + from pydantic import ValidationError + + from responses_api_agents.browsecomp_agent.app import BrowsecompAgentConfig + + with pytest.raises(ValidationError, match="finite number"): + BrowsecompAgentConfig( + host="0.0.0.0", + port=8080, + entrypoint="", + name="browsecomp_agent", + model_server={"type": "responses_api_models", "name": "policy_model"}, + **{field: float("inf")}, + )