From d211645fc2f736d0f1bc9db79f1cc39df9b05f85 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 22 Sep 2026 16:30:24 -0700 Subject: [PATCH 01/44] feat(agent): run OpenClaw inside native task sandboxes Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/README.md | 102 ++- responses_api_agents/openclaw_agent/app.py | 500 +++++++++- .../install_openclaw_runtime.sh | 89 ++ .../openclaw_agent/sandbox.py | 140 +++ .../openclaw_agent/sandbox_runner.py | 139 +++ .../openclaw_agent/tests/test_app.py | 15 +- .../tests/test_install_runtime.py | 195 ++++ .../openclaw_agent/tests/test_sandbox.py | 857 ++++++++++++++++++ .../tests/test_sandbox_runner.py | 132 +++ 9 files changed, 2153 insertions(+), 16 deletions(-) create mode 100644 responses_api_agents/openclaw_agent/install_openclaw_runtime.sh create mode 100644 responses_api_agents/openclaw_agent/sandbox.py create mode 100644 responses_api_agents/openclaw_agent/sandbox_runner.py create mode 100644 responses_api_agents/openclaw_agent/tests/test_install_runtime.py create mode 100644 responses_api_agents/openclaw_agent/tests/test_sandbox.py create mode 100644 responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index 15c1cc92a5d..c946c7adcf1 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -8,7 +8,8 @@ Minimal, meant to be extended, and currently eval-only. ## Quick start -OpenClaw must be installed (or it is auto-installed on first start). +OpenClaw is auto-installed on the first local CLI invocation. Starting an agent server for native +sessions does not install or execute OpenClaw on the agent-server host. Make sure `env.yaml` is also set. ```bash @@ -60,6 +61,103 @@ provider configuration is unchanged. - `extra_args`: extra flags appended to `openclaw agent` - `env`: extra env vars for the subprocess (e.g. provider API keys) - `openclaw_config`: deep-merged into the generated `openclaw.json` -- `openclaw_version`: npm version to pin on install (null means latest) +- `openclaw_version`: exact npm version to install (required; native validation uses `2026.6.11`) See `configs/openclaw_agent.yaml`. + + +## Native EnvironmentServer sessions + +The native path runs OpenClaw and its file/process tools inside the task sandbox created by +Resources. The agent server borrows `SandboxAccess`, installs the pinned runtime, launches one +invocation in `SandboxAccess.workdir`, and disconnects after confirmed cleanup. Resources retains +ownership of task preparation, verification, and sandbox destruction. + +Bind the agent, Resources, model server, and sandbox provider in your environment config. For +example, the following agent/environment blocks refer to an existing Resources server named +`task_resources` and a Gym model server named `policy`: + +```yaml +openclaw_native: + responses_api_agents: + openclaw_agent: + entrypoint: app.py + num_workers: 1 + openclaw_version: 2026.6.11 + model_server: {type: responses_api_models, name: policy} + model: your-served-model-id + timeout: 900 + sandbox_install_timeout_seconds: 900 + session_close_timeout_seconds: 60 + session_close_retry_window_seconds: 300 + +native_environment: + environment_servers: + single_agent: + entrypoint: app.py + resources_server: {type: resources_servers, name: task_resources} + agent_server: {type: responses_api_agents, name: openclaw_native} +``` + +Submit episodes to **EnvironmentServer `/run`**. It seeds Resources, opens the agent session, +calls the rollout-scoped Responses endpoint, closes the agent, verifies, and closes Resources. +The model server address must be reachable from inside the task sandbox. Its `/ng-rollout/.../v1` +route is embedded in OpenClaw's Chat Completions provider configuration to retain model-call linkage. + +The installer supports Linux glibc on x86_64/aarch64 and requires Python 3.9 or later for the +supervisor. Node 22.19.0 and OpenClaw are installed under +`/tmp/nemo-gym-openclaw-node-22.19.0-`. Node archives are checked against upstream SHA-256 +sums; the installed package version is verified before caching and when reusing a cached runtime. +A version-scoped `flock` serializes runtime installation and cache validation across session setup +attempts. Missing `flock`, `curl`, CA certificates, `tar`, `xz`, `sha256sum`, and `awk` are installed with `apt-get` only when +running as root. Other images receive an actionable prerequisite error. Installation errors include +the failed command, exit status, and stderr. Supporting these architectures does not establish +validation on every task image; record the actual image digest and runtime versions with each run. + +Per-session HOME, config, caches, prompt, transcript, and supervisor output are isolated under +`/tmp/nemo-gym-openclaw-sessions/`. OpenClaw reads the generated config directly without onboarding +or creating bootstrap files in the task repository. The task's PATH and dependencies are preserved. +The native adapter enables `read`, `write`, `edit`, `exec`, and `process`; subagent/channel/plugin +execution and required external HTTP/MCP tools are unsupported. OpenClaw's internal `gateway` +exec host refers to the embedded CLI process inside the borrowed sandbox. + +Native request support is deliberately explicit: + +- One activation per session; one worker per agent server. Concurrent sessions are independent. +- A text user prompt, optionally preceded by one system message. Text-part arrays are accepted. + Configured `system_prompt`, request `instructions`, and the optional system message are joined + and prepended to OpenClaw's user prompt. The harness retains its own system prompt. +- The request `model`, when supplied, must match the configured model. +- `max_output_tokens`, `temperature`, `top_p`, reasoning overrides, tool-selection controls, output + schemas, history replay, and other unsupported Responses options are rejected before activation. + The native config also rejects `max_output_tokens`: model metadata does not prove an effective + inference limit. Configure sampling and per-call limits on the Gym model server and inspect the + effective captured requests. An episode-wide token budget is not implemented. +- Custom command, environment, OpenClaw config, Node path, extra arguments, and agent-ID overrides + are rejected for native sessions. Existing callers without an agent-session cookie retain the + local CLI behavior and configuration. + +Responses retain reasoning, tool calls/results, final or partial text, and terminal failure status. +Usage sums all retained assistant model calls, including cache reads/writes and failed final calls; +missing per-call usage and unavailable reasoning-token details are reported as observation gaps. +A timeout returns partial output as `incomplete`; model errors remain `failed` even when a useful +patch exists. Do not infer rollout success from reward alone. Close returns the captured observations, +including salvaged transcript evidence after cancellation. + +A Linux child subreaper supervises each activation and reaps detached/background descendants before +writing an explicit cleanup receipt. Close is serialized and checks the receipt, process handle, +adapter-owned file removal, and disconnect before succeeding. Unknown launch outcomes and missing +or negative cleanup evidence fail closed and block verification. Successful close receipts remain +retryable for `session_close_retry_window_seconds` after cleanup (300 seconds by default); traffic +and retries do not shorten or extend the window. Stale cookies continue to reject activations after +receipt expiry and never enter the host CLI path. + +Session state is process-local. Keep requests on one worker. Failed cleanup sessions are retained +until owner/provider recovery and agent restart; they are never relabeled as successfully closed. +Cached runtime files remain until Resources destroys the sandbox. Supervisor cleanup prevents +verification races; it is not a security boundary against hostile task code. + +This path is eval-only. Working inference does not establish training token-ID/logprob support. +A tool-and-verifier smoke is required for the actual image/model configuration before review; +unit tests alone do not establish runtime compatibility or benchmark accuracy. Process/memory/setup +and close-latency overhead have not been benchmarked at scale. diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index 3b0462c2325..b7964a924aa 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -19,27 +19,34 @@ import json import logging import os +import re import shlex import shutil import signal from asyncio import Semaphore +from collections import OrderedDict from collections.abc import Mapping from pathlib import Path -from time import time +from time import monotonic, time from typing import Any, Callable, ClassVar, Optional from uuid import uuid4 import psutil -from fastapi import Request -from pydantic import ConfigDict, Field +from fastapi import HTTPException, Request +from pydantic import ConfigDict, Field, PrivateAttr from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import ( + AgentCloseSessionRequest, + AgentCloseSessionResponse, + AgentSeedSessionRequest, + AgentSeedSessionResponse, BaseResponsesAPIAgentConfig, Body, SimpleResponsesAPIAgent, ) from nemo_gym.config_types import ModelServerRef, ResourcesServerRef +from nemo_gym.episode_types import EpisodeId from nemo_gym.openai_utils import ( NeMoGymEasyInputMessage, NeMoGymFunctionCallOutput, @@ -59,7 +66,10 @@ AgentObservationBundle, ObservationGap, ) -from nemo_gym.server_utils import get_response_json, raise_for_status +from nemo_gym.sandbox import AsyncSandbox, create_provider +from nemo_gym.sandbox.access import DirectSandboxConnection +from nemo_gym.sandbox.config import resolve_provider_config +from nemo_gym.server_utils import get_global_config_dict, get_response_json, raise_for_status from responses_api_agents.openclaw_agent.observability import ( OPENCLAW_OBSERVATION_SOURCE, OpenClawSessionTree, @@ -67,11 +77,13 @@ build_openclaw_observations, discover_openclaw_session_tree, ) +from responses_api_agents.openclaw_agent.sandbox import _SANDBOX_PATH_CHECK, OpenClawSandboxSession from responses_api_agents.openclaw_agent.setup_openclaw import ensure_openclaw LOG = logging.getLogger(__name__) _INTERNAL_OBSERVATIONS_KEY = "_ng_agent_observations" +_SANDBOX_SESSION_KEY = "nemo_gym_openclaw_sandbox_session" def _decode_last_json_dict_suffix(raw: str) -> Optional[dict[str, Any]]: @@ -341,7 +353,7 @@ def _extract_instruction(body_input) -> tuple[str, Optional[str]]: class OpenClawAgentConfig(BaseResponsesAPIAgentConfig): - resources_server: ResourcesServerRef + resources_server: Optional[ResourcesServerRef] = None model_server: Optional[ModelServerRef] = None concurrency: int = 32 command: str = "openclaw" @@ -361,6 +373,14 @@ class OpenClawAgentConfig(BaseResponsesAPIAgentConfig): max_output_tokens: Optional[int] = None # required: every config must pin an explicit version so runs are reproducible and cannot silently drift openclaw_version: str + sandbox_install_timeout_seconds: float = Field(default=600, gt=0) + session_close_timeout_seconds: float = Field(default=60, gt=0) + session_close_retry_window_seconds: float = Field( + default=300.0, + gt=0, + allow_inf_nan=False, + description="Keep successful close receipts for this many seconds; cover the caller's retry horizon.", + ) @property def command_parts(self) -> list[str]: @@ -395,12 +415,427 @@ class OpenClawAgent(SimpleResponsesAPIAgent): # deny the interactive "message" channel so the headless agent finishes _HEADLESS_TOOL_DENY: ClassVar[tuple[str, ...]] = ("message",) + _sandbox_sessions: dict[str, OpenClawSandboxSession] = PrivateAttr(default_factory=dict) + _closed_sandbox_sessions: OrderedDict[str, tuple[EpisodeId, AgentCloseSessionResponse, float]] = PrivateAttr( + default_factory=OrderedDict + ) + _local_setup_task: asyncio.Task[None] | None = PrivateAttr(default=None) + + async def seed_agent_session(self, request: Request, body: AgentSeedSessionRequest) -> AgentSeedSessionResponse: + """Borrow the Resources-owned task sandbox and install a pinned OpenClaw runtime inside it.""" + self._expire_closed_agent_sessions() + if request.session.get(_SANDBOX_SESSION_KEY) in self._sandbox_sessions: + raise HTTPException(409, "OpenClaw session already exists") + session_id = f"openclaw-{uuid4().hex}" + state = await self._initialize_agent_session_state(session_id, body) + self._sandbox_sessions[session_id] = state + request.session[_SANDBOX_SESSION_KEY] = session_id + return AgentSeedSessionResponse(agent_session_id=session_id) + + async def _initialize_agent_session_state( + self, agent_session_id: str, body: AgentSeedSessionRequest + ) -> OpenClawSandboxSession: + # Match Hermes: session initialization owns the sandbox runtime setup, + # with the same AgentSeedSessionRequest/SandboxAccess wire contracts. + if self.config.num_workers not in (None, 1): + raise HTTPException(422, "Native OpenClaw sessions require num_workers=1") + if body.sandbox_access is None or not isinstance(body.sandbox_access.connection, DirectSandboxConnection): + raise HTTPException(422, "Native OpenClaw requires direct, Resources-owned SandboxAccess") + if not body.sandbox_access.workdir.startswith("/") or body.sandbox_access.workdir in ("/", "/tmp"): + raise HTTPException(422, "OpenClaw workdir must be absolute and separate from /tmp runtime storage") + if any(access.required for access in self.effective_tool_accesses(body)): + raise HTTPException(422, "Native OpenClaw supports its own sandbox tools, not required HTTP/MCP tools") + if self.config.model_server is None: + raise HTTPException(422, "Native OpenClaw requires a sandbox-reachable Gym model_server") + if not self.config.openclaw_version or not re.fullmatch( + r"\d+\.\d+\.\d+(?:-\d+)?", self.config.openclaw_version + ): + raise HTTPException(422, "Native OpenClaw requires an exact openclaw_version, for example 2026.6.11") + if self.config.openclaw_config or self.config.max_output_tokens is not None or self.config.node_bin_dir: + raise HTTPException( + 422, "Native OpenClaw does not support openclaw_config, max_output_tokens, or node_bin_dir overrides" + ) + if self.config.openclaw_agent_id != "main": + raise HTTPException(422, "Native OpenClaw currently requires openclaw_agent_id=main") + if self.config.command != "openclaw" or self.config.extra_args or self.config.env: + raise HTTPException(422, "Native OpenClaw does not support command, extra_args, or env overrides") + + connection = body.sandbox_access.connection + provider = create_provider(resolve_provider_config(connection.provider_config_ref, get_global_config_dict())) + try: + sandbox = await AsyncSandbox.connect(connection.descriptor, provider=provider) + except BaseException: + await provider.aclose() + raise + directory = f"/tmp/nemo-gym-openclaw-sessions/{agent_session_id}" + runtime = f"/tmp/nemo-gym-openclaw-node-22.19.0-{self.config.openclaw_version}" + try: + check_paths = shlex.join( + [ + "python3", + "-c", + _SANDBOX_PATH_CHECK, + body.sandbox_access.workdir, + "/tmp/nemo-gym-openclaw-sessions", + runtime, + ] + ) + prepared = await sandbox.exec( + "command -v python3 >/dev/null 2>&1 || " + "{ echo 'Native OpenClaw requires Python >=3.9 in the task image' >&2; exit 1; }; " + f"{check_paths} && mkdir -p {shlex.quote(directory + '/home/.openclaw')}", + timeout_s=30, + ) + if prepared.return_code != 0 or getattr(prepared, "error_type", None): + raise RuntimeError(prepared.stderr or "Cannot create OpenClaw sandbox session directory") + # Install only the agent runtime in the existing task sandbox. + # Resources has already prepared the task repository and its dependencies. + installer = "install_openclaw_runtime.sh" + await sandbox.upload(Path(__file__).with_name(installer), f"{directory}/{installer}") + installed = await sandbox.exec( + f"bash {shlex.quote(directory + '/' + installer)} {shlex.quote(runtime)} " + f"{shlex.quote(self.config.openclaw_version)}", + cwd=body.sandbox_access.workdir, + timeout_s=self.config.sandbox_install_timeout_seconds, + ) + if installed.return_code != 0 or getattr(installed, "error_type", None): + raise RuntimeError( + f"OpenClaw runtime setup failed (exit {installed.return_code}): " + f"bash {directory}/{installer} {runtime} {self.config.openclaw_version}\n" + f"{installed.stderr or installed.stdout}" + ) + await sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), f"{directory}/sandbox_runner.py") + except BaseException: + await sandbox.disconnect() + raise + return OpenClawSandboxSession(body, sandbox, directory, runtime) + + async def close_agent_session(self, request: Request, body: AgentCloseSessionRequest) -> AgentCloseSessionResponse: + """Confirm OpenClaw teardown before allowing verification; never destroy the borrowed sandbox.""" + self._expire_closed_agent_sessions() + session_id = request.session.get(_SANDBOX_SESSION_KEY) + closed = self._closed_sandbox_sessions.get(session_id) + if closed is not None and body.agent_session_id == session_id and body.episode_id == closed[0]: + return closed[1] + state = self._sandbox_sessions.get(session_id) + if state is None or body.agent_session_id != session_id or body.episode_id != state.seed.episode_id: + raise HTTPException(409, "OpenClaw close does not match the seeded session and episode") + await state.close(self.config.session_close_timeout_seconds) + # Another close may have completed while this caller waited for the + # session's cleanup lock. Reuse its receipt without renewing its expiry. + self._expire_closed_agent_sessions() + closed = self._closed_sandbox_sessions.get(session_id) + if closed is not None: + return closed[1] + if self._sandbox_sessions.get(session_id) is not state: + raise HTTPException(409, "OpenClaw close receipt has expired") + observations = state.observations or AgentObservationBundle( + source="openclaw", gaps=[ObservationGap(code="agent_activation_interrupted")] + ) + self._sandbox_sessions.pop(session_id, None) + result = AgentCloseSessionResponse(agent_session_id=session_id, agent_observations=observations) + # Keep the cookie as a tombstone so later activations cannot fall back + # to host execution. Other sessions must not shorten the retry window. + self._closed_sandbox_sessions[session_id] = ( + body.episode_id, + result, + monotonic() + self.config.session_close_retry_window_seconds, + ) + return result + + def _expire_closed_agent_sessions(self) -> None: + """Prune receipts by close time, never by traffic or retry access order.""" + now = monotonic() + while self._closed_sandbox_sessions: + if next(iter(self._closed_sandbox_sessions.values()))[2] > now: + break + self._closed_sandbox_sessions.popitem(last=False) + + def _sandbox_input(self, body: NeMoGymResponseCreateParamsNonStreaming) -> tuple[str, str]: + """Validate and normalize input before consuming the session's activation.""" + unsupported = ( + "include", + "store", + "service_tier", + "prompt_cache_key", + "prompt_cache_retention", + "safety_identifier", + "stream_options", + "user", + "max_output_tokens", + "temperature", + "top_p", + "reasoning", + "max_tool_calls", + "previous_response_id", + "prompt", + "text", + "context_management", + "conversation", + "moderation", + "top_logprobs", + "truncation", + ) + if body.model is not None and body.model != self.config.model: + raise HTTPException(422, "Native OpenClaw model must match the configured model") + extras = set(body.model_extra or {}) + if extras: + raise HTTPException(422, f"Native OpenClaw does not support extra request fields: {sorted(extras)}") + values = body.model_dump(mode="json") + for name in unsupported: + if values.get(name) is not None: + raise HTTPException(422, f"Native OpenClaw does not support request field {name}") + if body.tools or body.tool_choice != "auto" or not body.parallel_tool_calls or body.background: + raise HTTPException(422, "OpenClaw owns tool selection and execution policy") + if (body.metadata or {}).get("chat_template_kwargs") is not None: + raise HTTPException(422, "Configure chat_template_kwargs on the Gym model server for OpenClaw") + items = ( + [NeMoGymEasyInputMessage(role="user", content=body.input)] if isinstance(body.input, str) else body.input + ) + roles = [getattr(item, "role", None) for item in items] + if roles not in (["user"], ["system", "user"]): + raise HTTPException(422, "Native OpenClaw accepts one text user prompt with an optional system message") + for item in items: + if not isinstance(item.content, str) and any( + (part.get("type") if isinstance(part, dict) else getattr(part, "type", None)) != "input_text" + for part in item.content + ): + raise HTTPException(422, "Native OpenClaw only supports text input") + prompt, input_system = _extract_instruction(items) + system = "\n\n".join(part for part in (self.config.system_prompt, body.instructions, input_system) if part) + if not prompt.strip(): + raise HTTPException(422, "Native OpenClaw requires a nonempty user prompt") + return prompt, system + + async def _sandbox_response( + self, + state: OpenClawSandboxSession, + body: NeMoGymResponseCreateParamsNonStreaming, + *, + prompt: str, + system: str, + ) -> NeMoGymResponse: + # The pinned CLI reads configuration directly; onboarding would create + # workspace bootstrap files and unrelated provider/channel state. + native_config = self._build_openclaw_config({}, state.seed.episode_id.capture_key) + # OpenClaw conservatively disables stream usage for custom origins. Gym + # supports it; request the final usage chunk instead of retaining zero counters. + native_config["models"]["providers"]["nemo"]["models"][0]["compat"] = { + "supportsUsageInStreaming": True, + } + native_config.update( + { + "agents": { + "defaults": { + "workspace": state.seed.sandbox_access.workdir, + "skipBootstrap": True, + "skills": [], + "model": {"primary": self._effective_model()}, + "sandbox": {"mode": "off"}, + "memorySearch": {"enabled": False}, + } + }, + "tools": { + "allow": ["read", "write", "edit", "exec", "process"], + "exec": {"host": "gateway", "security": "full", "ask": "off"}, + }, + "plugins": {"enabled": False}, + } + ) + await state.upload_json("home/.openclaw/openclaw.json", native_config) + command = [ + f"{state.runtime}/node/bin/node", + f"{state.runtime}/openclaw/node_modules/openclaw/openclaw.mjs", + "agent", + "--local", + "--json", + "--agent", + "main", + "--session-id", + state.directory.rsplit("/", 1)[-1], + "--thinking", + self.config.thinking, + "--model", + self._effective_model(), + "--message-file", + f"{state.directory}/prompt.txt", + "--timeout", + str(self.config.timeout), + ] + payload = { + "directory": state.directory, + "command": command, + "prompt": f"{system}\n\n{prompt}" if system else prompt, + "cwd": state.seed.sandbox_access.workdir, + "env": { + "HOME": f"{state.directory}/home", + "XDG_CACHE_HOME": f"{state.directory}/home/.cache", + "OPENCLAW_STATE_DIR": f"{state.directory}/home/.openclaw", + "OPENCLAW_CONFIG_PATH": f"{state.directory}/home/.openclaw/openclaw.json", + "OPENCLAW_TELEMETRY": "0", + "CLAWHUB_DISABLE_TELEMETRY": "1", + "OPENCLAW_EXEC_SHELL_SNAPSHOT": "0", + }, + "timeout": self.config.timeout, + "cleanup_timeout": self.config.session_close_timeout_seconds / 3, + } + try: + async with self.sem: + stdout = await state.execute( + payload, timeout=self.config.timeout, close_timeout=self.config.session_close_timeout_seconds + ) + except BaseException: + # Cancellation/cleanup failure still leaves useful transcript evidence + # for the subsequent close. Never convert uncertain cleanup to success. + try: + await self._collect_sandbox_response(state, body, prompt=prompt, system=system) + except Exception: + LOG.exception("Could not salvage interrupted OpenClaw observations") + raise + return await self._collect_sandbox_response(state, body, prompt=prompt, system=system, stdout=stdout) + + async def _collect_sandbox_response( + self, + state: OpenClawSandboxSession, + body: NeMoGymResponseCreateParamsNonStreaming, + *, + prompt: str, + system: str, + stdout: str = "", + ) -> NeMoGymResponse: + gaps = [] + if not stdout: + try: + stdout = await state.read_text("stdout.log") + except Exception: + gaps.append(ObservationGap(code="agent_stdout_unavailable")) + session_id = state.directory.rsplit("/", 1)[-1] + events = [] + try: + transcript = await state.read_text(f"home/.openclaw/agents/main/sessions/{session_id}.jsonl") + events = parse_openclaw_session_events(transcript) + except Exception: + gaps.append(ObservationGap(code="agent_transcript_unavailable")) + # include_input=True preserves reasoning fields; only remove the input + # messages when projecting the transcript into Responses output. + output = [ + item + for item in parse_openclaw_session_items(events, include_input=True) + if getattr(item, "role", None) not in {"user", "system", "developer"} + ] + try: + fallback, envelope_usage = parse_openclaw_output(stdout) + except (TypeError, ValueError, AttributeError): + fallback, envelope_usage = [], {"input_tokens": 0, "output_tokens": 0} + gaps.append(ObservationGap(code="agent_stdout_unparseable")) + if not output: + output = fallback + assistants = [ + event["message"] + for event in events + if event.get("type") == "message" + and isinstance(event.get("message"), dict) + and event["message"].get("role") == "assistant" + ] + input_tokens = output_tokens = cached_tokens = 0 + for message in assistants: + usage = message.get("usage") + if not isinstance(usage, dict): + gaps.append(ObservationGap(code="model_call_usage_unavailable")) + continue + if any(type(usage.get(key)) is not int or usage[key] < 0 for key in ("input", "output")): + gaps.append(ObservationGap(code="model_call_usage_unavailable", detail="Incomplete usage counters")) + if not any(usage.get(key, 0) for key in ("input", "output", "cacheRead", "cacheWrite")): + gaps.append( + ObservationGap(code="model_call_usage_unavailable", detail="Harness reported only zero counters") + ) + if "cacheRead" not in usage: + gaps.append(ObservationGap(code="cached_token_usage_unavailable")) + + def count(name: str) -> int: + value = usage.get(name) + return value if type(value) is int and value >= 0 else 0 + + # OpenClaw/Pi input excludes cache reads and writes. Sum every + # assistant call, including failed final calls, rather than envelope last-call totals. + input_tokens += count("input") + count("cacheRead") + count("cacheWrite") + output_tokens += count("output") + cached_tokens += count("cacheRead") + if not assistants: + input_tokens = envelope_usage["input_tokens"] + output_tokens = envelope_usage["output_tokens"] + cached_tokens = envelope_usage.get("cached_tokens", 0) + gaps.append( + ObservationGap(code="model_call_usage_unavailable", detail="Only CLI envelope totals available") + ) + gaps.append(ObservationGap(code="reasoning_token_usage_unavailable")) + result = state.result + error = result.error if result else "OpenClaw activation ended without a cleanup receipt" + last = assistants[-1] if assistants else {} + if last.get("stopReason") in {"error", "aborted"}: + error = error or last.get("errorMessage") or f"OpenClaw model call {last['stopReason']}" + if result and result.return_code and not result.timed_out: + try: + stderr = (await state.read_text("stderr.log"))[-4000:] + except Exception: + stderr = "stderr unavailable" + error = error or f"OpenClaw exited {result.return_code}: {stderr}" + incomplete = bool(result and result.timed_out) or last.get("stopReason") == "length" + if not incomplete and not error and last.get("stopReason") != "stop": + error = "OpenClaw produced no terminal assistant result" + status = "failed" if error else "incomplete" if incomplete else "completed" + response = NeMoGymResponse( + id=f"resp_{uuid4().hex}", + created_at=int(time()), + model=self.config.model, + object="response", + output=output, + status=status, + error={"code": "server_error", "message": error} if error else None, + tool_choice=body.tool_choice, + tools=body.tools, + parallel_tool_calls=body.parallel_tool_calls, + usage=NeMoGymResponseUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=input_tokens + output_tokens, + input_tokens_details=NeMoGymResponseInputTokensDetails(cached_tokens=cached_tokens), + output_tokens_details=NeMoGymResponseOutputTokensDetails(reasoning_tokens=0), + ), + metadata={ + "harness_execution": "sandbox", + "harness_hostname": result.hostname if result else "unknown", + "harness_pid": str(result.pid) if result else "unknown", + "openclaw_version": self.config.openclaw_version, + }, + ) + input_items = [NeMoGymEasyInputMessage(role="system", content=system)] if system else [] + input_items.append(NeMoGymEasyInputMessage(role="user", content=prompt)) + try: + state.observations = build_openclaw_observations( + session_id, + openclaw_session_conversation(events, input_items=input_items, fallback_output=output), + events, + transcript_available=bool(events), + model_ref=self.config.model_server, + ) + # Native sessions enable only local file/process tools, so there is + # exactly one invocation; child sessions cannot be spawned. + state.observations.gaps = [ + gap for gap in state.observations.gaps if gap.code != "subagent_hierarchy_unavailable" + ] + state.observations.records[0].status = status + state.observations.gaps.extend(gaps) + except Exception: + LOG.exception("Could not build native OpenClaw observations") + state.observations = AgentObservationBundle( + source=OPENCLAW_OBSERVATION_SOURCE, + gaps=[ObservationGap(code="observation_capture_failed"), *gaps], + ) + return response + def model_post_init(self, __context: Any) -> None: self.sem = Semaphore(self.config.concurrency) - ensure_openclaw(self.config.openclaw_version) - command = self.config.command_parts[0] if self.config.command_parts else "" - if not command or shutil.which(command) is None: - LOG.warning("openclaw command %r is not on PATH yet", self.config.command) @staticmethod def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: @@ -574,6 +1009,21 @@ async def _run_openclaw( ] = None, ) -> tuple[list[Any], dict[str, int], str]: """setup and run agent. returns (output_items, usage, model_name).""" + # Local callers still get automatic installation. Native sessions never + # enter this path, so their host does not need OpenClaw, Node, or npm. + if self._local_setup_task is None: + self._local_setup_task = asyncio.create_task( + asyncio.to_thread(ensure_openclaw, self.config.openclaw_version) + ) + setup_task = self._local_setup_task + try: + # Share one installation across concurrent local calls. Cancelling + # a caller must not release another caller to start a second install. + await asyncio.shield(setup_task) + except Exception: + if self._local_setup_task is setup_task: + self._local_setup_task = None + raise prompt = instruction if not system_prompt else f"{system_prompt}\n\n{instruction}" work_dir = self._workspace_root() home = work_dir / ".openclaw-home" @@ -749,6 +1199,29 @@ async def responses( request: Request, body: NeMoGymResponseCreateParamsNonStreaming = Body(), ) -> NeMoGymResponse: + try: + session_id = request.session.get(_SANDBOX_SESSION_KEY) + has_session = isinstance(request.session, Mapping) and _SANDBOX_SESSION_KEY in request.session + except (AssertionError, AttributeError): + session_id, has_session = None, False + if has_session: + if not isinstance(session_id, str): + raise HTTPException(409, "Invalid OpenClaw session marker") + state = self._sandbox_sessions.get(session_id) + rollout_id = request.path_params.get("rollout_id") + if state is None or state.seed.episode_id.capture_key != rollout_id: + raise HTTPException(409, "OpenClaw activation does not match the seeded session and rollout route") + if state.activated or state.closing: + raise HTTPException(409, "OpenClaw sandbox sessions support one activation") + prompt, system = self._sandbox_input(body) + state.activated = True + state.task = asyncio.create_task(self._sandbox_response(state, body, prompt=prompt, system=system)) + try: + return await asyncio.shield(state.task) + except asyncio.CancelledError: + if not state.task.done() and not state.task.cancelling(): + state.task.cancel() + raise path_params = getattr(request, "path_params", None) rollout_id = path_params.get("rollout_id") if isinstance(path_params, Mapping) else None if not isinstance(rollout_id, str): @@ -838,6 +1311,15 @@ def collect_output(value: list[Any]) -> None: return AgentEpisode(response=response, observations=observations) async def run(self, request: Request, body: OpenClawAgentRunRequest) -> OpenClawAgentVerifyResponse: + try: + if isinstance(request.session, Mapping) and _SANDBOX_SESSION_KEY in request.session: + raise HTTPException(409, "Native OpenClaw sessions must use EnvironmentServer /run") + except (AssertionError, AttributeError): + pass + if self.config.resources_server is None: + raise HTTPException( + 422, "Direct OpenClaw /run requires resources_server; use EnvironmentServer /run for native sessions" + ) async with self.sem: cookies = request.cookies diff --git a/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh b/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh new file mode 100644 index 00000000000..311ca8b2d18 --- /dev/null +++ b/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +trap 'status=$?; echo "OpenClaw installer failed (exit $status) at line $LINENO: $BASH_COMMAND" >&2; exit "$status"' ERR + +runtime=$1 +openclaw_version=$2 +node_version=22.19.0 +if [ "$(uname -s)" != Linux ] || ! getconf GNU_LIBC_VERSION >/dev/null 2>&1; then + echo 'Native OpenClaw requires a Linux glibc sandbox; Alpine/musl is not supported.' >&2 + exit 1 +fi +if ! command -v python3 >/dev/null 2>&1; then + echo 'Native OpenClaw requires Python >=3.9 in the task image for descendant supervision.' >&2 + exit 1 +fi +python3 -c 'import sys; assert sys.version_info >= (3, 9), "Native OpenClaw requires Python >=3.9"' +case "$(uname -m)" in + x86_64) arch=x64 ;; + aarch64) arch=arm64 ;; + *) echo 'Native OpenClaw supports Linux x86_64/aarch64 glibc only.' >&2; exit 1 ;; +esac + +# Never change the task repository or its dependency environment. +python3 - "$runtime" "$PWD" <<'PY' +import pathlib, sys +runtime, workdir = (pathlib.Path(p).resolve() for p in sys.argv[1:]) +if workdir == runtime or workdir in runtime.parents or runtime in workdir.parents: + raise SystemExit('OpenClaw runtime must be outside the task repository') +PY +# One sandbox can receive overlapping session setup attempts. Keep the ready +# check and all cache writes under a version-scoped advisory lock. +if ! command -v flock >/dev/null 2>&1; then + if [ "$(id -u)" -ne 0 ] || ! command -v apt-get >/dev/null 2>&1; then + echo 'Native OpenClaw requires flock: preinstall util-linux in the task image (automatic installation requires root and apt-get).' >&2 + exit 1 + fi + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends util-linux +fi +mkdir -p "$runtime" +exec 9>"$runtime/.install.lock" +flock -x 9 +verify_version() { + "$runtime/node/bin/node" -e \ + 'const p=require(process.argv[1]); if(p.version!==process.argv[2])throw Error("OpenClaw version mismatch")' \ + "$runtime/openclaw/node_modules/openclaw/package.json" "$openclaw_version" +} +if [ -f "$runtime/ready" ]; then + verify_version + "$runtime/node/bin/node" "$runtime/openclaw/node_modules/openclaw/openclaw.mjs" --version + exit 0 +fi + +missing=() +for prerequisite in curl tar xz sha256sum awk; do + command -v "$prerequisite" >/dev/null 2>&1 || missing+=("$prerequisite") +done +if [ ! -s /etc/ssl/certs/ca-certificates.crt ] && [ ! -s /etc/pki/tls/certs/ca-bundle.crt ]; then + missing+=(ca-certificates) +fi +if [ "${#missing[@]}" -gt 0 ]; then + if [ "$(id -u)" -ne 0 ] || ! command -v apt-get >/dev/null 2>&1; then + echo "Native OpenClaw prerequisites missing: ${missing[*]}. Preinstall curl, ca-certificates, tar, xz-utils, coreutils, gawk (automatic installation requires root and apt-get)." >&2 + exit 1 + fi + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends curl ca-certificates tar xz-utils coreutils gawk +fi +mkdir -p "$runtime/home" "$runtime/cache" +export HOME="$runtime/home" +export npm_config_cache="$runtime/cache/npm" +export XDG_CACHE_HOME="$runtime/cache" +cd "$runtime" +archive="node-v${node_version}-linux-${arch}.tar.xz" +curl -fsSL --retry 3 "https://nodejs.org/dist/v${node_version}/${archive}" -o "$archive" +curl -fsSL --retry 3 "https://nodejs.org/dist/v${node_version}/SHASUMS256.txt" -o SHASUMS256.txt +awk -v archive="$archive" '$2 == archive' SHASUMS256.txt | sha256sum --check --strict - +mkdir -p node +tar -xJf "$archive" --strip-components=1 -C node +PATH="$runtime/node/bin:$PATH" "$runtime/node/bin/node" \ + "$runtime/node/lib/node_modules/npm/bin/npm-cli.js" install \ + --prefix "$runtime/openclaw" --no-audit --no-fund "openclaw@${openclaw_version}" +"$runtime/node/bin/node" -e \ + 'const p=require(process.argv[1]); if(p.version!==process.argv[2])throw Error("OpenClaw version mismatch")' \ + "$runtime/openclaw/node_modules/openclaw/package.json" "$openclaw_version" +"$runtime/node/bin/node" "$runtime/openclaw/node_modules/openclaw/openclaw.mjs" --version +printf '%s\n' "$openclaw_version" > "$runtime/ready" diff --git a/responses_api_agents/openclaw_agent/sandbox.py b/responses_api_agents/openclaw_agent/sandbox.py new file mode 100644 index 00000000000..35a5869dcfa --- /dev/null +++ b/responses_api_agents/openclaw_agent/sandbox.py @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""OpenClaw-specific borrowed-sandbox execution; Resources retains sandbox ownership.""" + +import asyncio +import json +import tempfile +from dataclasses import dataclass, field +from pathlib import Path +from shlex import quote + +from pydantic import BaseModel, ConfigDict, JsonValue + +from nemo_gym.base_responses_api_agent import AgentSeedSessionRequest +from nemo_gym.openai_utils import NeMoGymResponse +from nemo_gym.rollout_observability import AgentObservationBundle +from nemo_gym.sandbox import AsyncSandbox +from nemo_gym.sandbox.providers.base import SandboxPtySession + + +# Executed by sandbox Python before uploading any adapter-owned files. Resolve +# symlinks and '..' on the sandbox filesystem, not on the agent-server host. +_SANDBOX_PATH_CHECK = """ +from pathlib import Path +import sys +workdir, session_root, runtime_root = (Path(value).resolve() for value in sys.argv[1:]) +if not workdir.is_dir(): + raise SystemExit("OpenClaw task workdir is missing or is not a directory: " + str(workdir)) +for owned in (session_root, runtime_root): + if workdir == owned or workdir in owned.parents or owned in workdir.parents: + raise SystemExit("OpenClaw task workdir overlaps adapter-owned storage: " + str(owned)) +""" + + +class OpenClawSandboxResult(BaseModel): + """Require an explicit cleanup acknowledgement, not just process exit.""" + + model_config = ConfigDict(extra="forbid", strict=True) + return_code: int + timed_out: bool + cleanup_confirmed: bool + error: str | None + hostname: str + pid: int + + +@dataclass +class OpenClawSandboxSession: + """Worker-local session with retryable, fail-closed runner teardown.""" + + seed: AgentSeedSessionRequest + sandbox: AsyncSandbox + directory: str + runtime: str + task: asyncio.Task[NeMoGymResponse] | None = None + runner: SandboxPtySession | None = None + exit_task: asyncio.Task[int] | None = None + result: OpenClawSandboxResult | None = None + observations: AgentObservationBundle | None = None + activated: bool = False + closing: bool = False + launch_started: bool = False + runner_closed: bool = False + closed: bool = False + close_lock: asyncio.Lock = field(default_factory=asyncio.Lock) + + async def upload_json(self, name: str, payload: JsonValue) -> None: + """Upload adapter-owned data beneath this session's directory.""" + with tempfile.TemporaryDirectory(prefix="openclaw-session-upload-") as directory: + path = Path(directory) / "payload.json" + path.write_text(json.dumps(payload)) + await self.sandbox.upload(path, f"{self.directory}/{name}") + + async def read_text(self, name: str) -> str: + """Read an adapter-owned result without interpreting it as a shell command.""" + with tempfile.TemporaryDirectory(prefix="openclaw-session-download-") as directory: + path = Path(directory) / "payload" + await self.sandbox.download(f"{self.directory}/{name}", path) + return path.read_text(errors="replace") + + async def stop_runner(self, timeout: float) -> None: + """Wait for the supervisor's cleanup receipt; retain handles on any failure.""" + if not self.launch_started: + return + if self.runner is None or self.exit_task is None: + raise RuntimeError("OpenClaw launch outcome is unknown; cannot authorize verification") + if not self.exit_task.done(): + await self.runner.send_signal("SIGTERM") + await asyncio.wait_for(asyncio.shield(self.exit_task), timeout=timeout) + if self.result is None: + self.result = OpenClawSandboxResult.model_validate_json(await self.read_text("result.json")) + if not self.result.cleanup_confirmed: + raise RuntimeError(f"OpenClaw sandbox cleanup was not confirmed: {self.result.error}") + if not self.runner_closed: + await self.runner.close() + self.runner_closed = True + # Keep the successful receipt for retries if file cleanup/disconnect fails. + + async def close(self, timeout: float) -> None: + """Stop only OpenClaw-owned work and detach; never call sandbox.stop().""" + async with self.close_lock: + if self.closed: + return + self.closing = True + if self.task is not None: + if not self.task.done() and not self.task.cancelling(): + self.task.cancel() + try: + await asyncio.wait_for(asyncio.shield(self.task), timeout=timeout) + except asyncio.CancelledError: + if not self.task.cancelled(): + raise + except Exception: + if not self.task.done(): + raise + # A response error does not establish cleanup; stop_runner below must. + await self.stop_runner(timeout) + result = await self.sandbox.exec(f"rm -rf -- {quote(self.directory)}", timeout_s=timeout) + if result.return_code != 0 or getattr(result, "error_type", None): + raise RuntimeError(f"Could not remove OpenClaw session files: {result.stderr}") + await self.sandbox.disconnect() + self.closed = True + + async def execute(self, payload: dict[str, JsonValue], *, timeout: float, close_timeout: float) -> str: + """Start the supervisor and OpenClaw inside the borrowed task sandbox.""" + await self.upload_json("input.json", payload) + self.launch_started = True + self.runner = await self.sandbox.pty.create( + command=f"exec python3 -I {quote(self.directory + '/sandbox_runner.py')} {quote(self.directory + '/input.json')}", + cwd=self.seed.sandbox_access.workdir, + pty=False, + ) + self.exit_task = asyncio.create_task(self.runner.wait_exit()) + try: + # The supervisor owns the execution timeout; this bounds missing receipts/provider failures too. + await asyncio.wait_for(asyncio.shield(self.exit_task), timeout=timeout + close_timeout * 3) + self.result = OpenClawSandboxResult.model_validate_json(await self.read_text("result.json")) + return await self.read_text("stdout.log") + finally: + await self.stop_runner(close_timeout) diff --git a/responses_api_agents/openclaw_agent/sandbox_runner.py b/responses_api_agents/openclaw_agent/sandbox_runner.py new file mode 100644 index 00000000000..b4ea0b086c5 --- /dev/null +++ b/responses_api_agents/openclaw_agent/sandbox_runner.py @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Supervise one OpenClaw invocation and reap all Linux tool descendants.""" + +from __future__ import annotations + +import ctypes +import json +import os +import signal +import subprocess +import sys +from pathlib import Path +from time import monotonic, sleep +from typing import TypedDict + + +class RunnerRequest(TypedDict): + """Invocation data uploaded by the adapter.""" + + directory: str + prompt: str + command: list[str] + cwd: str + env: dict[str, str] + timeout: float + cleanup_timeout: float + + +class RunnerResult(TypedDict): + """Receipt proving whether verification can safely inspect the task.""" + + return_code: int + timed_out: bool + cleanup_confirmed: bool + error: str | None + hostname: str + pid: int + + +def enable_subreaper() -> None: + """Adopt detached children that escape the original process group.""" + if sys.platform != "linux": + raise RuntimeError("Native OpenClaw sessions require Linux") + if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), "Cannot establish OpenClaw child-subreaper boundary") + + +def drain_children(timeout: float) -> None: + """Kill and reap descendants before acknowledging cleanup.""" + children = Path(f"/proc/self/task/{os.getpid()}/children") + deadline = monotonic() + timeout + while True: + for child in children.read_text().split(): + try: + os.kill(int(child), signal.SIGKILL) + except ProcessLookupError: + pass + try: + while os.waitpid(-1, os.WNOHANG)[0]: + pass + except ChildProcessError: + return + if monotonic() >= deadline: + raise TimeoutError("OpenClaw descendants remain alive; verification must not proceed") + sleep(0.01) + + +def run(params: RunnerRequest) -> RunnerResult: + """Write stdout incrementally; the transcript remains available after failures.""" + directory = Path(params["directory"]) + process = None + error = None + timed_out = False + cleanup_confirmed = False + stopping = False + + def interrupt(_signum: int, _frame: object) -> None: + nonlocal stopping + # Signals must not interrupt Popen between child creation and handle assignment. + stopping = True + + signal.signal(signal.SIGTERM, interrupt) + (directory / "prompt.txt").write_text(params["prompt"]) + with (directory / "stdout.log").open("wb") as stdout, (directory / "stderr.log").open("wb") as stderr: + try: + enable_subreaper() + process = subprocess.Popen( + params["command"], + cwd=params["cwd"], + env=dict(os.environ) | params["env"], + stdin=subprocess.DEVNULL, + stdout=stdout, + stderr=stderr, + start_new_session=True, + ) + deadline = monotonic() + params["timeout"] + while process.poll() is None: + if stopping or monotonic() >= deadline: + timed_out = True + break + sleep(0.05) + except Exception as exc: + error = str(exc) + finally: + signal.signal(signal.SIGTERM, signal.SIG_IGN) + try: + if process is not None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=params["cleanup_timeout"]) + drain_children(params["cleanup_timeout"]) + cleanup_confirmed = True + except Exception as exc: + error = f"cleanup: {exc}" + return { + "return_code": process.returncode if process is not None and process.returncode is not None else 1, + "timed_out": timed_out, + "cleanup_confirmed": cleanup_confirmed, + "error": error, + "hostname": os.uname().nodename, + "pid": os.getpid(), + } + + +def main() -> None: + """Atomically publish the cleanup receipt after all descendants have exited.""" + params = json.loads(Path(sys.argv[1]).read_text()) + result = run(params) + output = Path(params["directory"]) / "result.json" + temporary = output.with_suffix(".tmp") + temporary.write_text(json.dumps(result)) + temporary.replace(output) + + +if __name__ == "__main__": + main() diff --git a/responses_api_agents/openclaw_agent/tests/test_app.py b/responses_api_agents/openclaw_agent/tests/test_app.py index 3ca94bb07f0..02c519b7d2f 100644 --- a/responses_api_agents/openclaw_agent/tests/test_app.py +++ b/responses_api_agents/openclaw_agent/tests/test_app.py @@ -24,6 +24,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import psutil +import pytest import yaml from nemo_gym.config_types import ModelServerRef @@ -52,6 +53,12 @@ ) +@pytest.fixture(autouse=True) +def no_host_install(): + with patch("responses_api_agents.openclaw_agent.app.ensure_openclaw"): + yield + + class _FakeResponse: ok = True @@ -80,9 +87,7 @@ def _config(**kwargs) -> OpenClawAgentConfig: def _make_agent(**kwargs) -> OpenClawAgent: - with patch("responses_api_agents.openclaw_agent.app.OpenClawAgent.model_post_init"): - agent = OpenClawAgent(config=_config(**kwargs), server_client=MagicMock(spec=ServerClient)) - agent.sem = asyncio.Semaphore(agent.config.concurrency) + agent = OpenClawAgent(config=_config(**kwargs), server_client=MagicMock(spec=ServerClient)) return agent @@ -834,7 +839,7 @@ async def _main(): for _ in range(100): if signal.SIGTERM in registered: break - await asyncio.sleep(0) + await asyncio.sleep(0.01) assert signal.SIGTERM in registered, "SIGTERM handler was never installed" registered[signal.SIGTERM](signal.SIGTERM, None) return await task @@ -896,7 +901,7 @@ def _install_signal(sig, handler): for _ in range(100): if len(agent.sigterm_events) == 2: break - await asyncio.sleep(0) + await asyncio.sleep(0.01) assert len(agent.sigterm_events) == 2 os.kill(os.getpid(), signal.SIGTERM) return await asyncio.gather(task_a, task_b) diff --git a/responses_api_agents/openclaw_agent/tests/test_install_runtime.py b/responses_api_agents/openclaw_agent/tests/test_install_runtime.py new file mode 100644 index 00000000000..9d0ee1e4524 --- /dev/null +++ b/responses_api_agents/openclaw_agent/tests/test_install_runtime.py @@ -0,0 +1,195 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + + +INSTALLER = Path(__file__).parents[1] / "install_openclaw_runtime.sh" +pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Native OpenClaw requires Linux/glibc") + + +@pytest.fixture +def sandbox(tmp_path: Path) -> tuple[Path, dict[str, str]]: + """Run the real installer with an isolated PATH and no access to the host package manager.""" + bindir = tmp_path / "bin" + bindir.mkdir() + for name in ("bash", "uname", "getconf", "mkdir", "cp", "tar", "xz", "sha256sum", "flock", "awk", "sleep"): + executable = shutil.which(name) + if executable is None: + pytest.skip(f"Installer test requires {name}") + (bindir / name).symlink_to(executable) + (bindir / "python3").symlink_to(sys.executable) + scripts = { + "id": '#!/bin/bash\necho "${TEST_UID:-0}"\n', + "apt-get": """#!/bin/bash +echo "$*" >> "$TEST_ROOT/packages.log" +if [ "$1" = install ]; then + [ "${TEST_INSTALL_FAIL:-0}" = 0 ] || exit 100 + cp "$TEST_ROOT/curl" "$TEST_ROOT/bin/curl" +fi +""", + } + for name, script in scripts.items(): + (bindir / name).write_text(script) + (bindir / name).chmod(0o755) + # Stop at the first download: neither apt nor external network requests are real in these tests. + curl = tmp_path / "curl" + curl.write_text('#!/bin/bash\necho "$*" > "$TEST_ROOT/download.log"\nexit 19\n') + curl.chmod(0o755) + return tmp_path, os.environ | {"PATH": str(bindir), "TEST_ROOT": str(tmp_path)} + + +def run_installer(root: Path, env: dict[str, str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [str(root / "bin/bash"), str(INSTALLER), str(root / "runtime"), "2026.6.11"], + env=env, + capture_output=True, + text=True, + errors="replace", + timeout=10, + ) + + +def test_missing_curl_is_installed_before_downloading_node(sandbox: tuple[Path, dict[str, str]]) -> None: + root, env = sandbox + result = run_installer(root, env) + assert result.returncode == 19, result.stderr + assert (root / "packages.log").read_text().splitlines() == [ + "update", + "install -y --no-install-recommends curl ca-certificates tar xz-utils coreutils gawk", + ] + assert "https://nodejs.org/dist/v22.19.0/node-v22.19.0-linux-" in (root / "download.log").read_text() + assert not (root / "runtime/ready").exists() + + +def test_existing_curl_does_not_install_packages(sandbox: tuple[Path, dict[str, str]]) -> None: + root, env = sandbox + shutil.copy(root / "curl", root / "bin/curl") + result = run_installer(root, env | {"TEST_UID": "1000"}) + assert result.returncode == 19, result.stderr + assert (root / "download.log").exists() + assert not (root / "packages.log").exists() + + +@pytest.mark.parametrize("missing", ["root", "apt-get"]) +def test_missing_curl_without_bootstrap_support_explains_remedy( + sandbox: tuple[Path, dict[str, str]], missing: str +) -> None: + root, env = sandbox + if missing == "root": + env["TEST_UID"] = "1000" + else: + (root / "bin/apt-get").unlink() + result = run_installer(root, env) + assert result.returncode == 1 + assert "Preinstall curl, ca-certificates" in result.stderr + assert not (root / "packages.log").exists() + assert not (root / "download.log").exists() + + +def test_package_failure_stops_before_download(sandbox: tuple[Path, dict[str, str]]) -> None: + root, env = sandbox + result = run_installer(root, env | {"TEST_INSTALL_FAIL": "1"}) + assert result.returncode == 100 + assert not (root / "download.log").exists() + assert not (root / "runtime/ready").exists() + + +def test_cached_runtime_needs_no_package_installation(sandbox: tuple[Path, dict[str, str]]) -> None: + root, env = sandbox + (root / "runtime").mkdir() + (root / "runtime/ready").write_text("2026.6.11") + (root / "runtime/node/bin").mkdir(parents=True) + node = root / "runtime/node/bin/node" + node.write_text('#!/bin/bash\necho "2026.6.11"\n') + node.chmod(0o755) + result = run_installer(root, env) + assert result.returncode == 0, result.stderr + assert not (root / "packages.log").exists() + assert not (root / "download.log").exists() + + +def test_missing_supervisor_python_explains_task_image_requirement(sandbox): + root, env = sandbox + (root / "bin/python3").unlink() + result = run_installer(root, env) + assert result.returncode == 1 + assert "requires Python >=3.9 in the task image" in result.stderr + assert not (root / "download.log").exists() + + +def test_download_failure_identifies_command_and_exit_status(sandbox): + root, env = sandbox + shutil.copy(root / "curl", root / "bin/curl") + result = run_installer(root, env) + assert result.returncode == 19 + assert "failed (exit 19)" in result.stderr + assert "curl -fsSL --retry 3" in result.stderr + assert not (root / "runtime/ready").exists() + + +def test_missing_install_lock_explains_prerequisite(sandbox): + root, env = sandbox + (root / "bin/flock").unlink() + result = run_installer(root, env | {"TEST_UID": "1000"}) + assert result.returncode == 1 + assert "preinstall util-linux" in result.stderr + assert not (root / "download.log").exists() + + +def test_concurrent_installs_share_one_verified_runtime(sandbox): + """Run two real shell installers/locks with deterministic download/npm substitutes.""" + root, env = sandbox + bindir = root / "bin" + replacements = { + "curl": '#!/bin/bash\nfor last; do :; done\n: > "$last"\necho download >> "$TEST_ROOT/downloads"\n', + "sha256sum": "#!/bin/bash\nwhile read -r line; do :; done\n", + "tar": "#!/bin/bash\nexit 0\n", + } + for name, script in replacements.items(): + path = bindir / name + path.unlink(missing_ok=True) + path.write_text(script) + path.chmod(0o755) + node = root / "runtime/node/bin/node" + node.parent.mkdir(parents=True) + node.write_text("""#!/bin/bash +case "$1" in + */npm-cli.js) echo install >> "$TEST_ROOT/npm-installs"; sleep 0.3 ;; + -e) echo version-check >> "$TEST_ROOT/version-checks" ;; + *) echo 2026.6.11 ;; +esac +""") + node.chmod(0o755) + command = [str(bindir / "bash"), str(INSTALLER), str(root / "runtime"), "2026.6.11"] + processes = [subprocess.Popen(command, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) for _ in range(2)] + try: + for process in processes: + stdout, stderr = process.communicate(timeout=10) + assert process.returncode == 0, (stdout, stderr) + finally: + for process in processes: + if process.poll() is None: + process.kill() + process.wait() + assert (root / "npm-installs").read_text().splitlines() == ["install"] + assert len((root / "downloads").read_text().splitlines()) == 2 + assert len((root / "version-checks").read_text().splitlines()) == 2 + assert (root / "runtime/ready").read_text().strip() == "2026.6.11" + + +def test_missing_awk_explains_prerequisite_before_download(sandbox): + root, env = sandbox + shutil.copy(root / "curl", root / "bin/curl") + (root / "bin/awk").unlink() + result = run_installer(root, env | {"TEST_UID": "1000"}) + assert result.returncode == 1 + assert "prerequisites missing: awk" in result.stderr + assert "gawk" in result.stderr + assert not (root / "download.log").exists() diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py new file mode 100644 index 00000000000..fbfad379868 --- /dev/null +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -0,0 +1,857 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +import json +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi import HTTPException, Request +from fastapi.testclient import TestClient +from omegaconf import OmegaConf +from pydantic import ValidationError + +from nemo_gym.base_responses_api_agent import AgentCloseSessionRequest, AgentSeedSessionRequest +from nemo_gym.episode_types import EpisodeId, TaskId +from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming +from nemo_gym.server_utils import ServerClient +from responses_api_agents.openclaw_agent.app import OpenClawAgent, OpenClawAgentConfig +from responses_api_agents.openclaw_agent.sandbox import OpenClawSandboxResult + + +def seed() -> AgentSeedSessionRequest: + return AgentSeedSessionRequest( + episode_id=EpisodeId(rollout_id="openclaw-smoke", attempt=2), + task_id=TaskId(taskset="swe-pro", task_id="task"), + sandbox_access={ + "connection": { + "kind": "direct", + "provider_config_ref": "sandbox", + "descriptor": {"sandbox_id": "resources-owned"}, + }, + "workdir": "/app", + }, + ) + + +def events(*, stop_reason="stop") -> str: + return "\n".join( + json.dumps(event) + for i, event in enumerate( + [ + { + "type": "message", + "message": { + "role": "assistant", + "responseId": "call-1", + "content": [ + {"type": "thinking", "thinking": "Inspect the repository"}, + {"type": "toolCall", "id": "tool-1", "name": "bash", "arguments": {"command": "pwd"}}, + ], + "usage": {"input": 10, "output": 3, "cacheRead": 2}, + "stopReason": "toolUse", + }, + }, + { + "type": "message", + "message": { + "role": "toolResult", + "toolCallId": "tool-1", + "toolName": "bash", + "content": [{"type": "text", "text": "/app"}], + }, + }, + { + "type": "message", + "message": { + "role": "assistant", + "responseId": "call-2", + "content": [{"type": "text", "text": "Fixed"}], + "usage": {"input": 5, "output": 2}, + "stopReason": stop_reason, + "errorMessage": "model error" if stop_reason == "error" else None, + }, + }, + {"type": "agent_end", "messages": [{"role": "assistant", "stopReason": stop_reason}]}, + ] + ) + ) + + +class Sandbox: + def __init__(self): + self.files = {} + self.result = { + "return_code": 0, + "timed_out": False, + "cleanup_confirmed": True, + "error": None, + "hostname": "task-container", + "pid": 123, + } + self.events = events() + self.blocked = False + self.started = asyncio.Event() + self.exited = asyncio.Event() + self.exec = AsyncMock(return_value=SimpleNamespace(return_code=0, stdout="", stderr="")) + self.stop = AsyncMock() + self.disconnect = AsyncMock() + self.runner = SimpleNamespace( + wait_exit=AsyncMock(side_effect=self.wait_exit), + send_signal=AsyncMock(side_effect=self.signal), + close=AsyncMock(), + ) + self.pty = SimpleNamespace(create=AsyncMock(side_effect=self.create)) + + async def upload(self, source, destination): + self.files[destination] = Path(source).read_text() + + async def download(self, source, destination): + Path(destination).write_text(self.files[source]) + + async def create(self, **kwargs): + payload_path = next(path for path in self.files if path.endswith("/input.json")) + payload = json.loads(self.files[payload_path]) + assert payload["cwd"] == "/app" + assert kwargs["cwd"] == "/app" + assert "sandbox_runner.py" in kwargs["command"] + self.directory = payload["directory"] + self.started.set() + if not self.blocked: + self.exited.set() + return self.runner + + async def signal(self, name): + assert name == "SIGTERM" + self.result["timed_out"] = True + self.exited.set() + + async def wait_exit(self): + await self.exited.wait() + self.files[f"{self.directory}/result.json"] = json.dumps(self.result) + self.files[f"{self.directory}/stdout.log"] = "" + self.files[f"{self.directory}/stderr.log"] = "model error" + session_id = self.directory.rsplit("/", 1)[-1] + self.files[f"{self.directory}/home/.openclaw/agents/main/sessions/{session_id}.jsonl"] = self.events + return 0 + + +@pytest.fixture +def setup(): + sandbox = Sandbox() + client = MagicMock(spec=ServerClient) + client.global_config_dict = OmegaConf.create( + {"policy": {"responses_api_models": {"openai_model": {"host": "model.example", "port": 9000}}}} + ) + client._build_server_base_url.return_value = "http://model.example:9000" + config = OpenClawAgentConfig( + name="openclaw", + host="localhost", + port=8001, + entrypoint="app.py", + num_workers=1, + model_server={"type": "responses_api_models", "name": "policy"}, + model="test-model", + openclaw_version="2026.6.11", + session_close_timeout_seconds=1, + ) + module = "responses_api_agents.openclaw_agent.app" + with ( + patch( + f"{module}.ensure_openclaw", side_effect=AssertionError("native sessions must not install host OpenClaw") + ), + patch(f"{module}.resolve_provider_config"), + patch(f"{module}.get_global_config_dict", return_value={}), + patch(f"{module}.create_provider"), + patch(f"{module}.AsyncSandbox.connect", AsyncMock(return_value=sandbox)), + ): + agent = OpenClawAgent(config=config, server_client=client) + yield agent, sandbox + + +def close_body(session_id): + return {"agent_session_id": session_id, "episode_id": seed().episode_id.model_dump()} + + +def test_http_native_flow_runs_openclaw_in_borrowed_sandbox(setup): + agent, sandbox = setup + with patch.object(agent, "_run_openclaw", AsyncMock(side_effect=AssertionError("host OpenClaw must not run"))): + with TestClient(agent.setup_webserver()) as client: + created = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")) + assert created.status_code == 200, created.text + session_id = created.json()["agent_session_id"] + directory = f"/tmp/nemo-gym-openclaw-sessions/{session_id}" + installer = f"{directory}/install_openclaw_runtime.sh" + assert installer in sandbox.files + assert agent.config.resources_server is None + assert not sandbox.pty.create.called + assert not any(path.startswith("/app/") for path in sandbox.files) + install_call = sandbox.exec.await_args_list[1] + assert install_call.args[0] == ( + f"bash {installer} /tmp/nemo-gym-openclaw-node-22.19.0-2026.6.11 2026.6.11" + ) + assert install_call.kwargs["timeout_s"] == agent.config.sandbox_install_timeout_seconds + result = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "Fix the code"}) + assert result.status_code == 200, result.text + body = result.json() + assert body["status"] == "completed" + assert [item["type"] for item in body["output"]] == [ + "reasoning", + "function_call", + "function_call_output", + "message", + ] + assert body["usage"]["total_tokens"] == 22 + assert body["usage"]["input_tokens_details"]["cached_tokens"] == 2 + assert body["metadata"]["harness_execution"] == "sandbox" + assert "_ng_agent_observations" not in body + payload = json.loads(sandbox.files[f"{sandbox.directory}/input.json"]) + assert payload["prompt"] == "Fix the code" + assert payload["command"][0].endswith("/node/bin/node") + assert "PATH" not in payload["env"] + models = json.loads(sandbox.files[f"{sandbox.directory}/home/.openclaw/openclaw.json"]) + assert models["agents"]["defaults"]["workspace"] == "/app" + assert models["models"]["providers"]["nemo"]["models"][0]["compat"]["supportsUsageInStreaming"] is True + assert ( + models["models"]["providers"]["nemo"]["baseUrl"] + == "http://model.example:9000/ng-rollout/openclaw-smoke-a2/v1" + ) + closed = client.post("/v1/agent_sessions/close", json=close_body(session_id)) + assert closed.status_code == 200, closed.text + observations = closed.json()["agent_observations"] + assert observations["source"] == "openclaw" + assert "no_sandbox_runtime" not in [gap["code"] for gap in observations["gaps"]] + assert len(observations["records"][0]["model_calls"]) == 2 + assert not agent._sandbox_sessions + assert agent._local_setup_task is None + sandbox.disconnect.assert_awaited_once() + sandbox.stop.assert_not_awaited() + agent.server_client.post.assert_not_called() + + +def test_direct_run_without_resources_rejected_before_execution(setup): + agent, sandbox = setup + with TestClient(agent.setup_webserver()) as client: + response = client.post("/run", json={"responses_create_params": {"input": "task"}}) + assert response.status_code == 422 + assert "use EnvironmentServer /run" in response.json()["detail"] + sandbox.exec.assert_not_awaited() + agent.server_client.post.assert_not_called() + + +@pytest.mark.parametrize("option", ["no-sandbox", "worker", "required-tool", "no-model", "unpinned"]) +def test_unsupported_seed_rejected_before_connection(setup, option): + agent, sandbox = setup + body = seed().model_dump(mode="json") + if option == "no-sandbox": + body["sandbox_access"] = None + elif option == "worker": + agent.config.num_workers = 2 + elif option == "no-model": + agent.config.model_server = None + elif option == "unpinned": + agent.config.openclaw_version = "latest" + else: + body["tool_accesses"] = [ + {"kind": "direct_http", "name": "tools", "base_url": "http://resources", "required": True} + ] + with TestClient(agent.setup_webserver()) as client: + assert client.post("/v1/agent_sessions", json=body).status_code == 422 + sandbox.exec.assert_not_awaited() + + +def test_cookie_identity_and_single_activation(setup): + agent, sandbox = setup + with TestClient(agent.setup_webserver()) as client: + session_id = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).json()["agent_session_id"] + assert client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).status_code == 409 + assert client.post("/v1/responses", json={"input": "task"}).status_code == 409 + assert client.post("/ng-rollout/wrong-a2/v1/responses", json={"input": "task"}).status_code == 409 + bad = close_body(session_id) + bad["episode_id"]["attempt"] = 99 + assert client.post("/v1/agent_sessions/close", json=bad).status_code == 409 + assert client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}).status_code == 200 + assert client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}).status_code == 409 + assert client.post("/v1/agent_sessions/close", json=close_body(session_id)).status_code == 200 + sandbox.pty.create.assert_awaited_once() + + +@pytest.mark.parametrize("reason,expected", [("error", "failed"), ("aborted", "failed"), ("length", "incomplete")]) +def test_failed_or_partial_openclaw_output_is_preserved(setup, reason, expected): + agent, sandbox = setup + sandbox.events = events(stop_reason=reason) + with TestClient(agent.setup_webserver()) as client: + session_id = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).json()["agent_session_id"] + result = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}) + assert result.status_code == 200 + assert result.json()["status"] == expected + assert result.json()["output"][-1]["content"][0]["text"] == "Fixed" + assert client.post("/v1/agent_sessions/close", json=close_body(session_id)).status_code == 200 + + +@pytest.mark.parametrize( + "override", + [ + {"temperature": 0.2}, + {"max_output_tokens": 123}, + {"top_k": 3}, + {"input": ""}, + {"top_p": 0.9}, + {"tools": [{"type": "function", "name": "foo"}]}, + {"input": [{"role": "assistant", "content": "old turn"}]}, + ], +) +def test_unsupported_request_is_not_silently_ignored(setup, override): + agent, sandbox = setup + with TestClient(agent.setup_webserver()) as client: + session_id = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).json()["agent_session_id"] + result = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task", **override}) + assert result.status_code == 422, result.text + assert client.post("/v1/agent_sessions/close", json=close_body(session_id)).status_code == 200 + sandbox.pty.create.assert_not_awaited() + + +def test_rejected_request_does_not_consume_activation(setup): + agent, sandbox = setup + with TestClient(agent.setup_webserver()) as client: + client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).raise_for_status() + path = "/ng-rollout/openclaw-smoke-a2/v1/responses" + assert client.post(path, json={"input": "task", "temperature": 0.2}).status_code == 422 + sandbox.pty.create.assert_not_awaited() + accepted = client.post(path, json={"input": "task"}) + assert accepted.status_code == 200, accepted.text + assert client.post(path, json={"input": "task"}).status_code == 409 + sandbox.pty.create.assert_awaited_once() + + +def test_no_session_keeps_existing_local_path(setup): + agent, sandbox = setup + with patch.object(agent, "_create_response", AsyncMock(side_effect=RuntimeError("legacy path reached"))) as legacy: + with TestClient(agent.setup_webserver()) as client: + with pytest.raises(RuntimeError, match="legacy path reached"): + client.post("/v1/responses", json={"input": "task"}) + legacy.assert_awaited_once() + sandbox.pty.create.assert_not_awaited() + + +async def activate(agent, sandbox): + request = Request( + {"type": "http", "headers": [], "session": {}, "path_params": {"rollout_id": "openclaw-smoke-a2"}} + ) + seeded = await agent.seed_agent_session(request, seed()) + task = asyncio.create_task(agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="task"))) + await asyncio.wait_for(sandbox.started.wait(), 2) + return request, seeded.agent_session_id, task + + +async def test_close_cancels_active_openclaw_before_detaching(setup): + agent, sandbox = setup + sandbox.blocked = True + request, session_id, task = await activate(agent, sandbox) + await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) + with pytest.raises(asyncio.CancelledError): + await task + sandbox.runner.send_signal.assert_awaited_once_with("SIGTERM") + sandbox.disconnect.assert_awaited_once() + sandbox.stop.assert_not_awaited() + + +async def test_failed_cleanup_keeps_handles_and_prevents_close(setup): + agent, sandbox = setup + sandbox.result["cleanup_confirmed"] = False + request, session_id, task = await activate(agent, sandbox) + with pytest.raises(RuntimeError, match="cleanup was not confirmed"): + await task + with pytest.raises(RuntimeError, match="cleanup was not confirmed"): + await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) + assert session_id in agent._sandbox_sessions + sandbox.disconnect.assert_not_awaited() + sandbox.runner.close.assert_not_awaited() + + +async def test_disconnect_failure_retains_session_for_retry(setup): + agent, sandbox = setup + request, session_id, task = await activate(agent, sandbox) + await task + sandbox.disconnect.side_effect = [RuntimeError("provider unavailable"), None] + with pytest.raises(RuntimeError, match="provider unavailable"): + await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) + assert session_id in agent._sandbox_sessions + await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) + assert session_id not in agent._sandbox_sessions + sandbox.runner.close.assert_awaited_once() + + +def test_cleanup_receipt_is_required(): + with pytest.raises(ValueError): + OpenClawSandboxResult.model_validate({"return_code": 0, "error": None}) + + +def test_instructions_and_text_parts_reach_openclaw_without_other_provider_credentials(setup): + agent, sandbox = setup + agent.config.system_prompt = "config instruction" + with TestClient(agent.setup_webserver()) as client: + session_id = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).json()["agent_session_id"] + response = client.post( + "/ng-rollout/openclaw-smoke-a2/v1/responses", + json={ + "instructions": "request instruction", + "input": [ + {"role": "system", "content": "input instruction"}, + {"role": "user", "content": [{"type": "input_text", "text": "task"}]}, + ], + }, + ) + assert response.status_code == 200, response.text + payload = json.loads(sandbox.files[f"{sandbox.directory}/input.json"]) + assert payload["prompt"] == "config instruction\n\nrequest instruction\n\ninput instruction\n\ntask" + models = json.loads(sandbox.files[f"{sandbox.directory}/home/.openclaw/openclaw.json"]) + assert list(models["models"]["providers"]) == ["nemo"] + closed = client.post("/v1/agent_sessions/close", json=close_body(session_id)) + conversation = closed.json()["agent_observations"]["records"][0]["conversation"] + assert conversation[0]["content"] == "config instruction\n\nrequest instruction\n\ninput instruction" + + +def test_observation_parse_failure_preserves_response(setup): + agent, _ = setup + with TestClient(agent.setup_webserver()) as client: + session_id = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).json()["agent_session_id"] + with patch( + "responses_api_agents.openclaw_agent.app.build_openclaw_observations", + side_effect=ValueError("bad event"), + ): + response = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}) + assert response.status_code == 200 + assert response.json()["status"] == "completed" + closed = client.post("/v1/agent_sessions/close", json=close_body(session_id)) + assert closed.status_code == 200 + assert closed.json()["agent_observations"]["gaps"][0]["code"] == "observation_capture_failed" + + +def test_close_retry_and_stale_activation_do_not_run_host_openclaw(setup): + agent, sandbox = setup + with TestClient(agent.setup_webserver()) as client: + session_id = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).json()["agent_session_id"] + client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}).raise_for_status() + closed = client.post("/v1/agent_sessions/close", json=close_body(session_id)) + repeated = client.post("/v1/agent_sessions/close", json=close_body(session_id)) + assert repeated.status_code == 200 + assert repeated.json() == closed.json() + bad = close_body(session_id) + bad["episode_id"]["attempt"] = 99 + assert client.post("/v1/agent_sessions/close", json=bad).status_code == 409 + with patch.object(agent, "_run_openclaw", AsyncMock(side_effect=AssertionError("host OpenClaw must not run"))): + assert client.post("/v1/responses", json={"input": "task"}).status_code == 409 + assert client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).status_code == 200 + sandbox.disconnect.assert_awaited_once() + + +def test_http_close_retry_survives_other_session_closes(setup, monkeypatch): + agent, sandbox = setup + monkeypatch.setattr("responses_api_agents.openclaw_agent.app.monotonic", lambda: 100.0) + with TestClient(agent.setup_webserver()) as client: + + def seed_and_close(index): + client.cookies.clear() + body = seed().model_dump(mode="json") + body["episode_id"] = {"rollout_id": f"episode-{index}"} + created = client.post("/v1/agent_sessions", json=body) + assert created.status_code == 200 + cookies = dict(client.cookies) + close = {"agent_session_id": created.json()["agent_session_id"], "episode_id": body["episode_id"]} + result = client.post("/v1/agent_sessions/close", json=close) + assert result.status_code == 200 + return cookies, close, result.json() + + cookies, close, first = seed_and_close(0) + for index in range(1, 66): + seed_and_close(index) + client.cookies.clear() + client.cookies.update(cookies) + retry = client.post("/v1/agent_sessions/close", json=close) + assert retry.status_code == 200 + assert retry.json() == first + assert sandbox.disconnect.await_count == 66 + + +async def test_close_receipt_expires_without_extending_on_retry(setup, monkeypatch): + agent, sandbox = setup + clock = [100.0] + monkeypatch.setattr("responses_api_agents.openclaw_agent.app.monotonic", lambda: clock[0]) + agent.config.session_close_retry_window_seconds = 10 + request = Request({"type": "http", "session": {}}) + session_id = (await agent.seed_agent_session(request, seed())).agent_session_id + close = AgentCloseSessionRequest(**close_body(session_id)) + first = await agent.close_agent_session(request, close) + clock[0] = 109.0 + assert await agent.close_agent_session(request, close) == first + clock[0] = 110.0 + with pytest.raises(HTTPException) as error: + await agent.close_agent_session(request, close) + assert error.value.status_code == 409 + assert not agent._closed_sandbox_sessions + with patch.object(agent, "_create_episode", AsyncMock(side_effect=AssertionError("host fallback"))): + with pytest.raises(HTTPException) as error: + await agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="task")) + assert error.value.status_code == 409 + sandbox.disconnect.assert_awaited_once() + + +async def test_close_retry_window_starts_after_cleanup(setup, monkeypatch): + agent, sandbox = setup + clock = [100.0] + monkeypatch.setattr("responses_api_agents.openclaw_agent.app.monotonic", lambda: clock[0]) + agent.config.session_close_retry_window_seconds = 10 + request = Request({"type": "http", "session": {}}) + session_id = (await agent.seed_agent_session(request, seed())).agent_session_id + + async def disconnect(): + clock[0] = 200.0 + + sandbox.disconnect.side_effect = disconnect + close = AgentCloseSessionRequest(**close_body(session_id)) + first = await agent.close_agent_session(request, close) + clock[0] = 209.0 + assert await agent.close_agent_session(request, close) == first + sandbox.disconnect.assert_awaited_once() + + +async def test_concurrent_closes_share_receipt(setup): + agent, sandbox = setup + request = Request({"type": "http", "session": {}}) + session_id = (await agent.seed_agent_session(request, seed())).agent_session_id + entered, release = asyncio.Event(), asyncio.Event() + + async def disconnect(): + entered.set() + await release.wait() + + sandbox.disconnect.side_effect = disconnect + close = AgentCloseSessionRequest(**close_body(session_id)) + first = asyncio.create_task(agent.close_agent_session(request, close)) + await asyncio.wait_for(entered.wait(), 2) + second = asyncio.create_task(agent.close_agent_session(request, close)) + await asyncio.sleep(0) + release.set() + first_result, second_result = await asyncio.wait_for(asyncio.gather(first, second), 2) + assert first_result is second_result + sandbox.disconnect.assert_awaited_once() + + +async def test_seed_prunes_expired_close_receipts(setup, monkeypatch): + agent, _ = setup + clock = [100.0] + monkeypatch.setattr("responses_api_agents.openclaw_agent.app.monotonic", lambda: clock[0]) + request = Request({"type": "http", "session": {}}) + session_id = (await agent.seed_agent_session(request, seed())).agent_session_id + await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) + clock[0] += agent.config.session_close_retry_window_seconds + await agent.seed_agent_session(request, seed()) + assert not agent._closed_sandbox_sessions + + +@pytest.mark.parametrize("window", [0, -1, float("inf")]) +def test_close_retry_window_must_be_positive_and_finite(setup, window): + agent, _ = setup + with pytest.raises(ValidationError): + OpenClawAgentConfig(**(agent.config.model_dump() | {"session_close_retry_window_seconds": window})) + + +async def test_unknown_launch_outcome_fails_closed(setup): + agent, sandbox = setup + request = Request({"type": "http", "session": {}, "path_params": {"rollout_id": "openclaw-smoke-a2"}}) + session_id = (await agent.seed_agent_session(request, seed())).agent_session_id + sandbox.pty.create.side_effect = TimeoutError("lost launch response") + with pytest.raises(TimeoutError): + await agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="task")) + with pytest.raises(RuntimeError, match="launch outcome is unknown"): + await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) + assert session_id in agent._sandbox_sessions + sandbox.disconnect.assert_not_awaited() + sandbox.stop.assert_not_awaited() + + +async def test_install_failure_disconnects_without_stopping_owner(setup): + agent, sandbox = setup + sandbox.exec.side_effect = [SimpleNamespace(return_code=0), SimpleNamespace(return_code=1, stderr="npm failed")] + request = Request({"type": "http", "session": {}}) + with pytest.raises(RuntimeError, match="npm failed"): + await agent.seed_agent_session(request, seed()) + assert not agent._sandbox_sessions + assert not request.session + sandbox.disconnect.assert_awaited_once() + sandbox.stop.assert_not_awaited() + + +async def test_cancelled_install_never_publishes_session_or_launches_openclaw(setup): + agent, sandbox = setup + sandbox.exec.side_effect = [SimpleNamespace(return_code=0), asyncio.CancelledError()] + request = Request({"type": "http", "session": {}}) + with pytest.raises(asyncio.CancelledError): + await agent.seed_agent_session(request, seed()) + assert not agent._sandbox_sessions + assert not request.session + sandbox.pty.create.assert_not_awaited() + sandbox.disconnect.assert_awaited_once() + sandbox.stop.assert_not_awaited() + + +@pytest.mark.parametrize( + "override", + [{"model": "different-model"}, {"store": False}, {"include": []}, {"service_tier": "priority"}], +) +def test_model_and_other_request_controls_are_explicitly_rejected(setup, override): + agent, sandbox = setup + with TestClient(agent.setup_webserver()) as client: + client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).raise_for_status() + result = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task", **override}) + assert result.status_code == 422, result.text + sandbox.pty.create.assert_not_awaited() + + +async def test_cleanup_command_provider_error_blocks_close_and_can_retry(setup): + agent, sandbox = setup + request, session_id, task = await activate(agent, sandbox) + await task + sandbox.exec.side_effect = [ + SimpleNamespace(return_code=0, error_type="timeout", stderr="cleanup timed out"), + SimpleNamespace(return_code=0, error_type=None, stderr=""), + ] + with pytest.raises(RuntimeError, match="cleanup timed out"): + await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) + sandbox.disconnect.assert_not_awaited() + await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) + sandbox.disconnect.assert_awaited_once() + + +async def test_simultaneous_activation_is_rejected_before_launch(setup): + agent, sandbox = setup + sandbox.blocked = True + request, session_id, task = await activate(agent, sandbox) + with pytest.raises(HTTPException) as error: + await agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="second task")) + assert error.value.status_code == 409 + await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) + with pytest.raises(asyncio.CancelledError): + await task + sandbox.pty.create.assert_awaited_once() + + +async def test_close_start_prevents_activation(setup): + agent, sandbox = setup + request = Request({"type": "http", "session": {}, "path_params": {"rollout_id": "openclaw-smoke-a2"}}) + session_id = (await agent.seed_agent_session(request, seed())).agent_session_id + entered, release = asyncio.Event(), asyncio.Event() + + async def disconnect(): + entered.set() + await release.wait() + + sandbox.disconnect.side_effect = disconnect + close = asyncio.create_task(agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id)))) + await entered.wait() + with pytest.raises(HTTPException) as error: + await agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="task")) + assert error.value.status_code == 409 + release.set() + await close + sandbox.pty.create.assert_not_awaited() + + +async def test_cancelled_activation_keeps_partial_observations_for_close(setup): + agent, sandbox = setup + sandbox.blocked = True + request, session_id, task = await activate(agent, sandbox) + closed = await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) + with pytest.raises(asyncio.CancelledError): + await task + invocation = closed.agent_observations.records[0] + assert invocation.status == "incomplete" + assert invocation.conversation[-1].content[0].text == "Fixed" + assert len(invocation.model_calls) == 2 + + +def test_usage_sums_cache_writes_and_failed_calls(setup): + agent, sandbox = setup + transcript = [json.loads(line) for line in events(stop_reason="error").splitlines()] + transcript[0]["message"]["usage"]["cacheWrite"] = 4 + sandbox.events = "\n".join(json.dumps(event) for event in transcript) + with TestClient(agent.setup_webserver()) as client: + client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).raise_for_status() + response = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}) + assert response.json()["status"] == "failed" + assert response.json()["usage"]["input_tokens"] == 21 + assert response.json()["usage"]["output_tokens"] == 5 + assert response.json()["usage"]["total_tokens"] == 26 + + +async def test_two_sessions_keep_workspaces_model_routes_and_observations_separate(setup): + agent, first = setup + second = Sandbox() + second.events = second.events.replace("Fixed", "Second result") + request1 = Request({"type": "http", "session": {}, "path_params": {"rollout_id": "openclaw-smoke-a2"}}) + request2 = Request({"type": "http", "session": {}, "path_params": {"rollout_id": "second-a2"}}) + seed2 = seed() + seed2.episode_id = EpisodeId(rollout_id="second", attempt=2) + with patch("responses_api_agents.openclaw_agent.app.AsyncSandbox.connect", AsyncMock(side_effect=[first, second])): + created1 = await agent.seed_agent_session(request1, seed()) + created2 = await agent.seed_agent_session(request2, seed2) + result1, result2 = await asyncio.gather( + agent.responses(request1, NeMoGymResponseCreateParamsNonStreaming(input="first task")), + agent.responses(request2, NeMoGymResponseCreateParamsNonStreaming(input="second task")), + ) + assert result1.output[-1].content[0].text == "Fixed" + assert result2.output[-1].content[0].text == "Second result" + assert first.directory != second.directory + assert created1.agent_session_id != created2.agent_session_id + for sandbox, route in [(first, "openclaw-smoke-a2"), (second, "second-a2")]: + config = json.loads(sandbox.files[f"{sandbox.directory}/home/.openclaw/openclaw.json"]) + assert config["models"]["providers"]["nemo"]["baseUrl"].endswith(f"/ng-rollout/{route}/v1") + await agent.close_agent_session(request1, AgentCloseSessionRequest(**close_body(created1.agent_session_id))) + await agent.close_agent_session( + request2, AgentCloseSessionRequest(agent_session_id=created2.agent_session_id, episode_id=seed2.episode_id) + ) + first.disconnect.assert_awaited_once() + second.disconnect.assert_awaited_once() + + +@pytest.mark.parametrize("marker", [None, 7, "expired-session"]) +async def test_invalid_or_expired_native_cookie_never_uses_local_execution(setup, marker): + agent, sandbox = setup + request = Request( + { + "type": "http", + "session": {"nemo_gym_openclaw_sandbox_session": marker}, + "path_params": {"rollout_id": "openclaw-smoke-a2"}, + } + ) + with patch.object(agent, "_create_response", AsyncMock(side_effect=AssertionError("host fallback"))): + with pytest.raises(HTTPException) as error: + await agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="task")) + assert error.value.status_code == 409 + sandbox.pty.create.assert_not_awaited() + + +def test_native_close_cookie_blocks_legacy_run_with_configured_resources(setup): + from nemo_gym.config_types import ResourcesServerRef + + agent, sandbox = setup + agent.config.resources_server = ResourcesServerRef(type="resources_servers", name="legacy-resources") + with TestClient(agent.setup_webserver()) as client: + session_id = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).json()["agent_session_id"] + closed = client.post("/v1/agent_sessions/close", json=close_body(session_id)) + assert closed.status_code == 200 + assert client.cookies # Exercise the actual signed cookie returned by close. + response = client.post("/run", json={"responses_create_params": {"input": "task"}}) + assert response.status_code == 409 + agent.server_client.post.assert_not_called() + sandbox.pty.create.assert_not_awaited() + + +def test_zero_harness_counters_are_reported_as_missing_usage(setup): + agent, sandbox = setup + transcript = [json.loads(line) for line in events().splitlines()] + for event in transcript: + if event.get("message", {}).get("role") == "assistant": + event["message"]["usage"] = {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0} + sandbox.events = "\n".join(json.dumps(event) for event in transcript) + with TestClient(agent.setup_webserver()) as client: + session_id = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).json()["agent_session_id"] + response = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}) + assert response.status_code == 200 + assert response.json()["usage"]["total_tokens"] == 0 + closed = client.post("/v1/agent_sessions/close", json=close_body(session_id)) + gaps = closed.json()["agent_observations"]["gaps"] + assert any(gap["code"] == "model_call_usage_unavailable" and "zero" in gap["detail"] for gap in gaps) + + +async def test_invalid_envelope_usage_does_not_discard_valid_transcript(setup): + agent, sandbox = setup + request, session_id, task = await activate(agent, sandbox) + await task + state = agent._sandbox_sessions[session_id] + response = await agent._collect_sandbox_response( + state, + NeMoGymResponseCreateParamsNonStreaming(input="task"), + prompt="task", + system="", + stdout=json.dumps({"meta": {"agentMeta": {"usage": {"input": "not-a-number"}}}}), + ) + assert response.status == "completed" + assert response.output[-1].content[0].text == "Fixed" + assert response.usage.total_tokens == 22 + assert "agent_stdout_unparseable" in {gap.code for gap in state.observations.gaps} + + +@pytest.mark.parametrize("owned_root", ["sessions", "runtime"]) +@pytest.mark.parametrize("overlap", ["same", "ancestor", "descendant", "symlink", "dotdot"]) +def test_workdir_cannot_overlap_adapter_storage(tmp_path, owned_root, overlap): + import subprocess + import sys + + from responses_api_agents.openclaw_agent.sandbox import _SANDBOX_PATH_CHECK + + sessions, runtime = tmp_path / "sessions", tmp_path / "runtime" + for directory in (sessions, runtime): + directory.mkdir() + (directory / "repo").mkdir() + owned = tmp_path / owned_root + if overlap == "same": + workdir = owned + elif overlap == "ancestor": + workdir = tmp_path + elif overlap == "descendant": + workdir = owned / "repo" + elif overlap == "symlink": + workdir = tmp_path / "task-alias" + workdir.symlink_to(owned) + else: + workdir = owned / "repo" / ".." + result = subprocess.run( + [sys.executable, "-c", _SANDBOX_PATH_CHECK, str(workdir), str(sessions), str(runtime)], + capture_output=True, + text=True, + errors="replace", + timeout=5, + ) + assert result.returncode != 0 + assert "overlaps adapter-owned storage" in result.stderr + + +def test_workdir_validation_accepts_separate_repository_and_rejects_missing(tmp_path): + import subprocess + import sys + + from responses_api_agents.openclaw_agent.sandbox import _SANDBOX_PATH_CHECK + + workdir = tmp_path / "task" + command = [ + sys.executable, + "-c", + _SANDBOX_PATH_CHECK, + str(workdir), + str(tmp_path / "sessions"), + str(tmp_path / "runtime"), + ] + missing = subprocess.run(command, capture_output=True, text=True, errors="replace", timeout=5) + assert missing.returncode != 0 + assert "task workdir is missing" in missing.stderr + workdir.mkdir() + valid = subprocess.run(command, capture_output=True, text=True, errors="replace", timeout=5) + assert valid.returncode == 0, valid.stderr + assert not (tmp_path / "sessions").exists() + assert not (tmp_path / "runtime").exists() + + +async def test_path_check_failure_never_uploads_or_installs(setup): + agent, sandbox = setup + sandbox.exec.return_value = SimpleNamespace(return_code=1, stderr="task workdir overlaps adapter-owned storage") + request = Request({"type": "http", "session": {}}) + with pytest.raises(RuntimeError, match="overlaps adapter-owned storage"): + await agent.seed_agent_session(request, seed()) + assert not sandbox.files + assert not request.session + sandbox.exec.assert_awaited_once() + sandbox.disconnect.assert_awaited_once() diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py b/responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py new file mode 100644 index 00000000000..a3a8744438e --- /dev/null +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Exercise the real Linux supervisor, not a mocked cleanup acknowledgement.""" + +import json +import os +import signal +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from responses_api_agents.openclaw_agent import sandbox_runner + + +pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Linux subreaper and /proc are required") + + +def launch(tmp_path, code, timeout=3): + request = { + "directory": str(tmp_path), + "command": [sys.executable, "-c", code], + "cwd": str(tmp_path), + "env": {}, + "timeout": timeout, + "cleanup_timeout": 2, + "prompt": "task input", + } + path = tmp_path / "input.json" + path.write_text(json.dumps(request)) + process = subprocess.Popen( + [sys.executable, "-I", str(Path(sandbox_runner.__file__)), str(path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return process + + +def result(tmp_path, process): + try: + stdout, stderr = process.communicate(timeout=10) + assert process.returncode == 0, (stdout, stderr) + return json.loads((tmp_path / "result.json").read_text()) + finally: + if process.poll() is None: + process.kill() + process.wait() + + +def test_capture_and_prompt_file(tmp_path): + process = launch( + tmp_path, "import json,pathlib; print(json.dumps({'prompt':pathlib.Path('prompt.txt').read_text()}))" + ) + summary = result(tmp_path, process) + assert summary["cleanup_confirmed"] is True + assert summary["return_code"] == 0 + event = json.loads((tmp_path / "stdout.log").read_text()) + assert event["prompt"] == "task input" + + +@pytest.mark.parametrize("ending", ["natural", "timeout", "cancel"]) +def test_detached_descendants_are_gone_before_receipt(tmp_path, ending): + # The grandchild starts a new session, escaping process-group-only cleanup. + code = ( + "import subprocess,sys,time,pathlib; " + "p=subprocess.Popen([sys.executable,'-c','import time; time.sleep(60)'],start_new_session=True); " + "pathlib.Path('child.pid').write_text(str(p.pid)); " + ("time.sleep(60)" if ending != "natural" else "pass") + ) + process = launch(tmp_path, code, timeout=0.3 if ending == "timeout" else 3) + if ending == "cancel": + for _ in range(200): + if (tmp_path / "child.pid").exists(): + break + time.sleep(0.01) + process.send_signal(signal.SIGTERM) + summary = result(tmp_path, process) + assert summary["cleanup_confirmed"] is True + assert summary["timed_out"] is (ending != "natural") + pid = int((tmp_path / "child.pid").read_text()) + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + + +def test_spawn_error_is_not_success(tmp_path): + process = launch(tmp_path, "raise RuntimeError('failed')") + summary = result(tmp_path, process) + assert summary["cleanup_confirmed"] is True + assert summary["return_code"] != 0 + + +def test_sigterm_during_spawn_does_not_lose_child_handle(tmp_path): + # Deliver SIGTERM after the real child exists but before Popen returns to run(). + # Isolate signal handlers/subreaper state from pytest, and always reap the test child. + driver = """ +import json, os, runpy, signal, subprocess, sys +runner = runpy.run_path(sys.argv[1]) +spawn = subprocess.Popen +children = [] +def interrupted_spawn(*args, **kwargs): + child = spawn(*args, **kwargs) + children.append(child) + os.kill(os.getpid(), signal.SIGTERM) + return child +subprocess.Popen = interrupted_spawn +try: + summary = runner['run']({ + 'directory': sys.argv[2], 'cwd': sys.argv[2], 'env': {}, 'prompt': 'task', + 'command': [sys.executable, '-c', 'import time; time.sleep(60)'], + 'timeout': 5, 'cleanup_timeout': 2, + }) + summary['child_alive'] = any(child.poll() is None for child in children) + print(json.dumps(summary)) +finally: + for child in children: + if child.poll() is None: + child.kill() + child.wait() +""" + completed = subprocess.run( + [sys.executable, "-c", driver, sandbox_runner.__file__, str(tmp_path)], + capture_output=True, + text=True, + errors="replace", + timeout=10, + check=True, + ) + summary = json.loads(completed.stdout) + assert summary["timed_out"] is True + assert summary["cleanup_confirmed"] is True + assert summary["child_alive"] is False From f4bef5b8043bbda76a0fdb6475aab337f0938fb8 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 22 Sep 2026 16:43:56 -0700 Subject: [PATCH 02/44] fix(openclaw): isolate cached runtime version checks Signed-off-by: yaoyu-33 --- .../openclaw_agent/install_openclaw_runtime.sh | 8 ++++---- .../openclaw_agent/tests/test_install_runtime.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh b/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh index 311ca8b2d18..d4a2b000dc7 100644 --- a/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh +++ b/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh @@ -42,6 +42,10 @@ fi mkdir -p "$runtime" exec 9>"$runtime/.install.lock" flock -x 9 +mkdir -p "$runtime/home" "$runtime/cache" +export HOME="$runtime/home" +export npm_config_cache="$runtime/cache/npm" +export XDG_CACHE_HOME="$runtime/cache" verify_version() { "$runtime/node/bin/node" -e \ 'const p=require(process.argv[1]); if(p.version!==process.argv[2])throw Error("OpenClaw version mismatch")' \ @@ -68,10 +72,6 @@ if [ "${#missing[@]}" -gt 0 ]; then apt-get update DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends curl ca-certificates tar xz-utils coreutils gawk fi -mkdir -p "$runtime/home" "$runtime/cache" -export HOME="$runtime/home" -export npm_config_cache="$runtime/cache/npm" -export XDG_CACHE_HOME="$runtime/cache" cd "$runtime" archive="node-v${node_version}-linux-${arch}.tar.xz" curl -fsSL --retry 3 "https://nodejs.org/dist/v${node_version}/${archive}" -o "$archive" diff --git a/responses_api_agents/openclaw_agent/tests/test_install_runtime.py b/responses_api_agents/openclaw_agent/tests/test_install_runtime.py index 9d0ee1e4524..e191641a169 100644 --- a/responses_api_agents/openclaw_agent/tests/test_install_runtime.py +++ b/responses_api_agents/openclaw_agent/tests/test_install_runtime.py @@ -107,10 +107,11 @@ def test_cached_runtime_needs_no_package_installation(sandbox: tuple[Path, dict[ (root / "runtime/ready").write_text("2026.6.11") (root / "runtime/node/bin").mkdir(parents=True) node = root / "runtime/node/bin/node" - node.write_text('#!/bin/bash\necho "2026.6.11"\n') + node.write_text('#!/bin/bash\nprintf "%s\\n" "$HOME" > "$TEST_ROOT/runtime-home"\necho "2026.6.11"\n') node.chmod(0o755) result = run_installer(root, env) assert result.returncode == 0, result.stderr + assert (root / "runtime-home").read_text().strip() == str(root / "runtime/home") assert not (root / "packages.log").exists() assert not (root / "download.log").exists() From e3766aaefb3b00b429ec7d925209d698b4f4bd61 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 22 Sep 2026 17:45:22 -0700 Subject: [PATCH 03/44] fix(openclaw): account for rewritten transcript usage Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/README.md | 11 +- responses_api_agents/openclaw_agent/app.py | 53 ++++++- .../openclaw_agent/tests/test_sandbox.py | 133 +++++++++++++++++- 3 files changed, 191 insertions(+), 6 deletions(-) diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index c946c7adcf1..b5e46e1c3db 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -138,8 +138,15 @@ Native request support is deliberately explicit: local CLI behavior and configuration. Responses retain reasoning, tool calls/results, final or partial text, and terminal failure status. -Usage sums all retained assistant model calls, including cache reads/writes and failed final calls; -missing per-call usage and unavailable reasoning-token details are reported as observation gaps. +Usage sums observed assistant model calls, including cache reads/writes and failed final calls. +Transcript rewrites retaining the same response ID and message count once; synthetic CLI summary +messages carrying cumulative usage do not count again or override the model's terminal status. +Conflicting messages with the same response ID are excluded with an accounting gap. Records without +response IDs count separately with an identity gap, since their duplication cannot be established. +Auxiliary model calls, such as compaction, are absent from the transcript. A coverage gap is always +reported because interrupted compaction need not leave an event. Totals therefore need not equal +all backend calls. Missing per-call usage and +unavailable reasoning-token details are also reported as observation gaps; the full branch history is retained. A timeout returns partial output as `incomplete`; model errors remain `failed` even when a useful patch exists. Do not infer rollout success from reward alone. Close returns the captured observations, including salvaged transcript evidence after cancellation. diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index b7964a924aa..2e7989e969d 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -151,6 +151,39 @@ def parse_openclaw_output(stdout: str) -> tuple[list[Any], dict[str, int]]: } +def _unique_usage_messages( + assistants: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], list[ObservationGap]]: + # OpenClaw rewrites transcript branches by appending the original messages + # again. Entry IDs change, but the provider response ID and message do not. + identified: dict[str, dict[str, Any]] = {} + ambiguous: set[str] = set() + unidentified = [] + gaps = [] + for message in assistants: + response_id = message.get("responseId") + if not isinstance(response_id, str) or not response_id.strip(): + unidentified.append(message) + gaps.append( + ObservationGap( + code="model_call_usage_identity_unavailable", + detail="Usage counted per transcript record; missing response ID prevents rewrite deduplication", + ) + ) + elif response_id not in identified: + identified[response_id] = message + elif message != identified[response_id]: + ambiguous.add(response_id) + for response_id in sorted(ambiguous): + gaps.append( + ObservationGap( + code="model_call_usage_identity_ambiguous", + detail=f"Conflicting transcript messages for response ID {response_id}; usage excluded from totals", + ) + ) + return [message for key, message in identified.items() if key not in ambiguous] + unidentified, gaps + + def parse_openclaw_session_items(events: list[dict[str, Any]], *, include_input: bool = False) -> list[Any]: """Convert OpenClaw session events into Gym conversation items.""" output_items: list[Any] = [] @@ -736,9 +769,23 @@ async def _collect_sandbox_response( if event.get("type") == "message" and isinstance(event.get("message"), dict) and event["message"].get("role") == "assistant" + # The pinned CLI can append a synthetic final-text mirror with + # aggregate run usage and stopReason="stop". It is neither another + # model call nor evidence that the underlying call succeeded. + and event["message"].get("api") != "cli" ] + usage_messages, identity_gaps = _unique_usage_messages(assistants) + gaps.extend(identity_gaps) + # Auxiliary summarization can consume tokens and then abort before a + # compaction event is written, so absent events do not prove coverage. + gaps.append( + ObservationGap( + code="auxiliary_model_usage_unavailable", + detail="Transcript has no auxiliary-call usage counters; totals cover observed assistant calls only", + ) + ) input_tokens = output_tokens = cached_tokens = 0 - for message in assistants: + for message in usage_messages: usage = message.get("usage") if not isinstance(usage, dict): gaps.append(ObservationGap(code="model_call_usage_unavailable")) @@ -756,8 +803,8 @@ def count(name: str) -> int: value = usage.get(name) return value if type(value) is int and value >= 0 else 0 - # OpenClaw/Pi input excludes cache reads and writes. Sum every - # assistant call, including failed final calls, rather than envelope last-call totals. + # OpenClaw/Pi input excludes cache reads and writes. Count distinct + # observed calls, including failed calls, without adding CLI aggregate mirrors. input_tokens += count("input") + count("cacheRead") + count("cacheWrite") output_tokens += count("output") cached_tokens += count("cacheRead") diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index fbfad379868..ec91a869fcb 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -3,6 +3,7 @@ import asyncio import json +from copy import deepcopy from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch @@ -17,7 +18,7 @@ from nemo_gym.episode_types import EpisodeId, TaskId from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming from nemo_gym.server_utils import ServerClient -from responses_api_agents.openclaw_agent.app import OpenClawAgent, OpenClawAgentConfig +from responses_api_agents.openclaw_agent.app import OpenClawAgent, OpenClawAgentConfig, _unique_usage_messages from responses_api_agents.openclaw_agent.sandbox import OpenClawSandboxResult @@ -223,6 +224,9 @@ def test_http_native_flow_runs_openclaw_in_borrowed_sandbox(setup): observations = closed.json()["agent_observations"] assert observations["source"] == "openclaw" assert "no_sandbox_runtime" not in [gap["code"] for gap in observations["gaps"]] + # Interrupted auxiliary calls need not persist a compaction event, + # so a plain transcript cannot establish full usage coverage either. + assert "auxiliary_model_usage_unavailable" in [gap["code"] for gap in observations["gaps"]] assert len(observations["records"][0]["model_calls"]) == 2 assert not agent._sandbox_sessions assert agent._local_setup_task is None @@ -687,6 +691,133 @@ def test_usage_sums_cache_writes_and_failed_calls(setup): assert response.json()["usage"]["total_tokens"] == 26 +def test_rewritten_transcript_and_cli_mirror_count_each_model_call_once(setup): + agent, sandbox = setup + # Scalar usage from the real Ansible run: eight main calls, three rewritten + # copies, then a CLI mirror of all main-call usage. Auxiliary compaction + # calls (2505 input / 839 output) are not represented in this transcript. + counters = [(4817, 27), (8072, 26), (10442, 27), (13697, 27), (9838, 26), (12208, 27), (15463, 27), (18718, 27)] + transcript = [ + { + "type": "message", + "id": f"entry-{index}", + "parentId": f"entry-{index - 1}" if index else None, + "message": { + "role": "assistant", + "api": "openai-completions", + "responseId": f"call-{index}", + "content": [ + {"type": "toolCall", "id": f"tool-{index}", "name": "read", "arguments": {"path": "/app/a"}} + ], + "usage": {"input": input_tokens, "output": output_tokens, "cacheRead": 0, "cacheWrite": 0}, + "stopReason": "toolUse", + }, + } + for index, (input_tokens, output_tokens) in enumerate(counters) + ] + for index in (1, 2, 3): + rewritten = deepcopy(transcript[index]) + rewritten["id"] = f"rewritten-{index}" + transcript.append(rewritten) + transcript.extend( + [ + { + "type": "compaction", + "tokensBefore": 22737, + "summary": "Partial task history", + "firstKeptEntryId": "entry-0", + }, + { + "type": "message", + "message": { + "role": "assistant", + "api": "openai-completions", + "content": [], + "stopReason": "error", + "errorMessage": "Context overflow recovery exhausted", + "usage": {"input": 0, "output": 0}, + }, + }, + { + "type": "message", + "message": { + "role": "assistant", + "api": "cli", + "content": [{"type": "text", "text": "Context overflow recovery exhausted"}], + "stopReason": "stop", + "usage": {"input": 93255, "output": 214, "cacheRead": 0, "cacheWrite": 0}, + }, + }, + ] + ) + sandbox.events = "\n".join(json.dumps(event) for event in transcript) + with TestClient(agent.setup_webserver()) as client: + session_id = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).json()["agent_session_id"] + response = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}) + assert response.status_code == 200, response.text + body = response.json() + assert body["status"] == "failed" + assert body["error"]["message"] == "Context overflow recovery exhausted" + assert body["usage"]["input_tokens"] == 93255 + assert body["usage"]["output_tokens"] == 214 + assert body["usage"]["total_tokens"] == 93469 + assert sum(item["type"] == "function_call" for item in body["output"]) == 11 + assert body["output"][-1]["content"][0]["text"] == "Context overflow recovery exhausted" + closed = client.post("/v1/agent_sessions/close", json=close_body(session_id)) + assert closed.status_code == 200, closed.text + observations = closed.json()["agent_observations"] + invocation = observations["records"][0] + assert invocation["status"] == "failed" + assert len(invocation["model_calls"]) == 8 + assert sum(item["type"] == "function_call" for item in invocation["conversation"]) == 11 + assert any(record["kind"] == "context_compaction" for record in observations["records"]) + gaps = {gap["code"] for gap in observations["gaps"]} + assert "auxiliary_model_usage_unavailable" in gaps + assert "agent_conversation_branching_unavailable" in gaps + assert "model_call_usage_identity_unavailable" in gaps + + +@pytest.mark.parametrize("conflict", ["content", "usage"]) +def test_conflicting_response_id_usage_is_excluded_and_reported(setup, conflict): + agent, sandbox = setup + transcript = [json.loads(line) for line in events().splitlines()] + collision = deepcopy(transcript[0]) + if conflict == "content": + collision["message"]["content"][0]["thinking"] = "Different call, same ID and usage" + else: + collision["message"]["usage"]["input"] += 1 + transcript.insert(1, collision) + sandbox.events = "\n".join(json.dumps(event) for event in transcript) + with TestClient(agent.setup_webserver()) as client: + session_id = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).json()["agent_session_id"] + response = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}) + assert response.status_code == 200, response.text + body = response.json() + assert body["usage"]["input_tokens"] == 5 + assert body["usage"]["output_tokens"] == 2 + assert sum(item["type"] == "function_call" for item in body["output"]) == 2 + closed = client.post("/v1/agent_sessions/close", json=close_body(session_id)) + assert closed.status_code == 200, closed.text + gaps = closed.json()["agent_observations"]["gaps"] + assert any(gap["code"] == "model_call_usage_identity_ambiguous" and "call-1" in gap["detail"] for gap in gaps) + + +@pytest.mark.parametrize("missing_id", [None, "", " ", 42]) +def test_missing_response_ids_do_not_deduplicate_distinct_calls(missing_id): + message = {"role": "assistant", "content": "Repeated output", "usage": {"input": 7, "output": 3}} + if missing_id is not None: + message["responseId"] = missing_id + messages = [message, deepcopy(message)] + before = deepcopy(messages) + selected, gaps = _unique_usage_messages(messages) + assert selected == messages + assert len(selected) == 2 + assert sum(message["usage"]["input"] for message in selected) == 14 + assert len(gaps) == 2 + assert {gap.code for gap in gaps} == {"model_call_usage_identity_unavailable"} + assert messages == before + + async def test_two_sessions_keep_workspaces_model_routes_and_observations_separate(setup): agent, first = setup second = Sandbox() From 6c81819f4c57be8ead2e325cbc46ed9641d1023e Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 22 Sep 2026 21:25:14 -0700 Subject: [PATCH 04/44] fix(openclaw): preserve unknown optional usage details Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/app.py | 25 ++++- .../openclaw_agent/tests/test_sandbox.py | 95 ++++++++++++++++++- 2 files changed, 114 insertions(+), 6 deletions(-) diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index 2e7989e969d..03f3e75f0c3 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -784,11 +784,14 @@ async def _collect_sandbox_response( detail="Transcript has no auxiliary-call usage counters; totals cover observed assistant calls only", ) ) - input_tokens = output_tokens = cached_tokens = 0 + input_tokens = output_tokens = 0 + cached_tokens: int | None = 0 if usage_messages else None for message in usage_messages: usage = message.get("usage") if not isinstance(usage, dict): gaps.append(ObservationGap(code="model_call_usage_unavailable")) + gaps.append(ObservationGap(code="cached_token_usage_unavailable")) + cached_tokens = None continue if any(type(usage.get(key)) is not int or usage[key] < 0 for key in ("input", "output")): gaps.append(ObservationGap(code="model_call_usage_unavailable", detail="Incomplete usage counters")) @@ -796,8 +799,12 @@ async def _collect_sandbox_response( gaps.append( ObservationGap(code="model_call_usage_unavailable", detail="Harness reported only zero counters") ) - if "cacheRead" not in usage: + cache_read = usage.get("cacheRead") + if type(cache_read) is not int or cache_read < 0: gaps.append(ObservationGap(code="cached_token_usage_unavailable")) + cached_tokens = None + elif cached_tokens is not None: + cached_tokens += cache_read def count(name: str) -> int: value = usage.get(name) @@ -807,11 +814,19 @@ def count(name: str) -> int: # observed calls, including failed calls, without adding CLI aggregate mirrors. input_tokens += count("input") + count("cacheRead") + count("cacheWrite") output_tokens += count("output") - cached_tokens += count("cacheRead") if not assistants: input_tokens = envelope_usage["input_tokens"] output_tokens = envelope_usage["output_tokens"] - cached_tokens = envelope_usage.get("cached_tokens", 0) + # The legacy envelope parser defaults missing counters to zero. + # Keep native optional details unknown unless the artifact supplies them. + envelope = _decode_last_json_dict_suffix(stdout) or {} + meta = envelope.get("meta") + agent_meta = meta.get("agentMeta") if isinstance(meta, dict) else None + raw_usage = agent_meta.get("usage") if isinstance(agent_meta, dict) else None + cache_read = raw_usage.get("cacheRead") if isinstance(raw_usage, dict) else None + cached_tokens = cache_read if type(cache_read) is int and cache_read >= 0 else None + if cached_tokens is None: + gaps.append(ObservationGap(code="cached_token_usage_unavailable")) gaps.append( ObservationGap(code="model_call_usage_unavailable", detail="Only CLI envelope totals available") ) @@ -847,7 +862,7 @@ def count(name: str) -> int: output_tokens=output_tokens, total_tokens=input_tokens + output_tokens, input_tokens_details=NeMoGymResponseInputTokensDetails(cached_tokens=cached_tokens), - output_tokens_details=NeMoGymResponseOutputTokensDetails(reasoning_tokens=0), + output_tokens_details=NeMoGymResponseOutputTokensDetails(reasoning_tokens=None), ), metadata={ "harness_execution": "sandbox", diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index ec91a869fcb..c25e16476cc 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -205,7 +205,8 @@ def test_http_native_flow_runs_openclaw_in_borrowed_sandbox(setup): "message", ] assert body["usage"]["total_tokens"] == 22 - assert body["usage"]["input_tokens_details"]["cached_tokens"] == 2 + assert body["usage"]["input_tokens_details"]["cached_tokens"] is None + assert body["usage"]["output_tokens_details"]["reasoning_tokens"] is None assert body["metadata"]["harness_execution"] == "sandbox" assert "_ng_agent_observations" not in body payload = json.loads(sandbox.files[f"{sandbox.directory}/input.json"]) @@ -691,6 +692,74 @@ def test_usage_sums_cache_writes_and_failed_calls(setup): assert response.json()["usage"]["total_tokens"] == 26 +@pytest.mark.parametrize( + "first_cache,second_cache,expected_cache,expected_input", + [ + (2, {}, None, 17), + (2, {"cacheRead": None}, None, 17), + (2, {"cacheRead": -1}, None, 17), + (2, {"cacheRead": True}, None, 17), + (2, {"cacheRead": "3"}, None, 17), + (2, {"cacheRead": 3.0}, None, 17), + (None, {"cacheRead": 2}, None, 17), + (0, {"cacheRead": 0}, 0, 15), + (2, {"cacheRead": 3}, 5, 20), + ], +) +def test_native_usage_details_preserve_unknown_and_known_counts( + setup, first_cache, second_cache, expected_cache, expected_input +): + agent, sandbox = setup + transcript = [json.loads(line) for line in events().splitlines()] + transcript[0]["message"]["usage"]["cacheRead"] = first_cache + transcript[2]["message"]["usage"].update(second_cache) + sandbox.events = "\n".join(json.dumps(event) for event in transcript) + with TestClient(agent.setup_webserver()) as client: + created = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")) + created.raise_for_status() + response = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}) + response.raise_for_status() + body = response.json() + assert body["status"] == "completed" + assert body["output"][0]["type"] == "reasoning" + assert body["usage"] == { + "input_tokens": expected_input, + "output_tokens": 5, + "total_tokens": expected_input + 5, + "input_tokens_details": {"cached_tokens": expected_cache}, + "output_tokens_details": {"reasoning_tokens": None}, + } + closed = client.post("/v1/agent_sessions/close", json=close_body(created.json()["agent_session_id"])) + closed.raise_for_status() + gaps = {gap["code"] for gap in closed.json()["agent_observations"]["gaps"]} + assert ("cached_token_usage_unavailable" in gaps) == (expected_cache is None) + assert "reasoning_token_usage_unavailable" in gaps + + +@pytest.mark.parametrize("second_usage", [None, "unavailable"]) +def test_missing_call_usage_keeps_native_cache_aggregate_unknown(setup, second_usage): + agent, sandbox = setup + transcript = [json.loads(line) for line in events().splitlines()] + transcript[2]["message"]["usage"] = second_usage + sandbox.events = "\n".join(json.dumps(event) for event in transcript) + with TestClient(agent.setup_webserver()) as client: + created = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")) + created.raise_for_status() + response = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}) + response.raise_for_status() + assert response.json()["usage"] == { + "input_tokens": 12, + "output_tokens": 3, + "total_tokens": 15, + "input_tokens_details": {"cached_tokens": None}, + "output_tokens_details": {"reasoning_tokens": None}, + } + closed = client.post("/v1/agent_sessions/close", json=close_body(created.json()["agent_session_id"])) + closed.raise_for_status() + gaps = {gap["code"] for gap in closed.json()["agent_observations"]["gaps"]} + assert {"model_call_usage_unavailable", "cached_token_usage_unavailable"} <= gaps + + def test_rewritten_transcript_and_cli_mirror_count_each_model_call_once(setup): agent, sandbox = setup # Scalar usage from the real Ansible run: eight main calls, three rewritten @@ -916,6 +985,30 @@ async def test_invalid_envelope_usage_does_not_discard_valid_transcript(setup): assert "agent_stdout_unparseable" in {gap.code for gap in state.observations.gaps} +@pytest.mark.parametrize("cache", [{}, {"cacheRead": None}, {"cacheRead": 0}, {"cacheRead": 5}]) +async def test_native_envelope_fallback_preserves_known_and_unknown_cache_details(setup, cache): + agent, sandbox = setup + request, session_id, task = await activate(agent, sandbox) + await task + state = agent._sandbox_sessions[session_id] + sandbox.files[f"{state.directory}/home/.openclaw/agents/main/sessions/{session_id}.jsonl"] = "" + response = await agent._collect_sandbox_response( + state, + NeMoGymResponseCreateParamsNonStreaming(input="task"), + prompt="task", + system="", + stdout=json.dumps({"meta": {"agentMeta": {"usage": {"input": 10, "output": 3, **cache}}}}), + ) + assert response.usage.input_tokens == 10 + (cache.get("cacheRead") or 0) + assert response.usage.output_tokens == 3 + assert response.usage.input_tokens_details.cached_tokens == cache.get("cacheRead") + assert response.usage.output_tokens_details.reasoning_tokens is None + gaps = {gap.code for gap in state.observations.gaps} + assert ("cached_token_usage_unavailable" in gaps) == (cache.get("cacheRead") is None) + assert "model_call_usage_unavailable" in gaps + await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) + + @pytest.mark.parametrize("owned_root", ["sessions", "runtime"]) @pytest.mark.parametrize("overlap", ["same", "ancestor", "descendant", "symlink", "dotdot"]) def test_workdir_cannot_overlap_adapter_storage(tmp_path, owned_root, overlap): From 44bdd9744dd7e30140e36272000dd1dd7fca961a Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 22 Sep 2026 21:31:46 -0700 Subject: [PATCH 05/44] fix(openclaw): retain valid native envelope usage subtotals Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/app.py | 15 +++++++----- .../openclaw_agent/tests/test_sandbox.py | 23 +++++++++++++++---- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index 03f3e75f0c3..8b775bbb7f9 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -757,9 +757,9 @@ async def _collect_sandbox_response( if getattr(item, "role", None) not in {"user", "system", "developer"} ] try: - fallback, envelope_usage = parse_openclaw_output(stdout) + fallback, _ = parse_openclaw_output(stdout) except (TypeError, ValueError, AttributeError): - fallback, envelope_usage = [], {"input_tokens": 0, "output_tokens": 0} + fallback = [] gaps.append(ObservationGap(code="agent_stdout_unparseable")) if not output: output = fallback @@ -815,16 +815,19 @@ def count(name: str) -> int: input_tokens += count("input") + count("cacheRead") + count("cacheWrite") output_tokens += count("output") if not assistants: - input_tokens = envelope_usage["input_tokens"] - output_tokens = envelope_usage["output_tokens"] # The legacy envelope parser defaults missing counters to zero. - # Keep native optional details unknown unless the artifact supplies them. + # Native totals retain valid subtotals without coercing malformed cache values. envelope = _decode_last_json_dict_suffix(stdout) or {} meta = envelope.get("meta") agent_meta = meta.get("agentMeta") if isinstance(meta, dict) else None raw_usage = agent_meta.get("usage") if isinstance(agent_meta, dict) else None - cache_read = raw_usage.get("cacheRead") if isinstance(raw_usage, dict) else None + raw_usage = raw_usage if isinstance(raw_usage, dict) else {} + cache_read = raw_usage.get("cacheRead") cached_tokens = cache_read if type(cache_read) is int and cache_read >= 0 else None + input_count, output_count = raw_usage.get("input"), raw_usage.get("output") + input_tokens = input_count if type(input_count) is int and input_count >= 0 else 0 + output_tokens = output_count if type(output_count) is int and output_count >= 0 else 0 + input_tokens += cached_tokens or 0 if cached_tokens is None: gaps.append(ObservationGap(code="cached_token_usage_unavailable")) gaps.append( diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index c25e16476cc..b19f34d1a9c 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -985,8 +985,21 @@ async def test_invalid_envelope_usage_does_not_discard_valid_transcript(setup): assert "agent_stdout_unparseable" in {gap.code for gap in state.observations.gaps} -@pytest.mark.parametrize("cache", [{}, {"cacheRead": None}, {"cacheRead": 0}, {"cacheRead": 5}]) -async def test_native_envelope_fallback_preserves_known_and_unknown_cache_details(setup, cache): +@pytest.mark.parametrize( + "cache,expected_cache", + [ + ({}, None), + ({"cacheRead": None}, None), + ({"cacheRead": -1}, None), + ({"cacheRead": True}, None), + ({"cacheRead": "5"}, None), + ({"cacheRead": "bad"}, None), + ({"cacheRead": 3.0}, None), + ({"cacheRead": 0}, 0), + ({"cacheRead": 5}, 5), + ], +) +async def test_native_envelope_fallback_preserves_known_and_unknown_cache_details(setup, cache, expected_cache): agent, sandbox = setup request, session_id, task = await activate(agent, sandbox) await task @@ -999,12 +1012,12 @@ async def test_native_envelope_fallback_preserves_known_and_unknown_cache_detail system="", stdout=json.dumps({"meta": {"agentMeta": {"usage": {"input": 10, "output": 3, **cache}}}}), ) - assert response.usage.input_tokens == 10 + (cache.get("cacheRead") or 0) + assert response.usage.input_tokens == 10 + (expected_cache or 0) assert response.usage.output_tokens == 3 - assert response.usage.input_tokens_details.cached_tokens == cache.get("cacheRead") + assert response.usage.input_tokens_details.cached_tokens == expected_cache assert response.usage.output_tokens_details.reasoning_tokens is None gaps = {gap.code for gap in state.observations.gaps} - assert ("cached_token_usage_unavailable" in gaps) == (cache.get("cacheRead") is None) + assert ("cached_token_usage_unavailable" in gaps) == (expected_cache is None) assert "model_call_usage_unavailable" in gaps await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) From 14b5fbb2ca8e43df9e42a8e7e1b8777b55f5b9ae Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 22 Sep 2026 21:35:09 -0700 Subject: [PATCH 06/44] fix(openclaw): retain fallback text with malformed usage Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/app.py | 33 ++++++++++++------- .../openclaw_agent/tests/test_sandbox.py | 8 ++++- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index 8b775bbb7f9..dffb46de857 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -120,11 +120,8 @@ def _text_from_openclaw_payloads(envelope: dict[str, Any]) -> str: return final.strip() if isinstance(final, str) else "" -def parse_openclaw_output(stdout: str) -> tuple[list[Any], dict[str, int]]: - envelope = _decode_last_json_dict_suffix(stdout) - if not envelope: - return [], {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0} - +def _openclaw_output_items(envelope: dict[str, Any]) -> list[Any]: + """Read envelope text independently of optional numeric usage metadata.""" text = _text_from_openclaw_payloads(envelope) output_items: list[Any] = [] if text: @@ -137,6 +134,14 @@ def parse_openclaw_output(stdout: str) -> tuple[list[Any], dict[str, int]]: type="message", ) ) + return output_items + + +def parse_openclaw_output(stdout: str) -> tuple[list[Any], dict[str, int]]: + envelope = _decode_last_json_dict_suffix(stdout) + if not envelope: + return [], {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0} + output_items = _openclaw_output_items(envelope) meta = envelope.get("meta") if isinstance(envelope.get("meta"), dict) else {} agent_meta = meta.get("agentMeta") if isinstance(meta.get("agentMeta"), dict) else {} @@ -757,9 +762,20 @@ async def _collect_sandbox_response( if getattr(item, "role", None) not in {"user", "system", "developer"} ] try: - fallback, _ = parse_openclaw_output(stdout) + envelope = _decode_last_json_dict_suffix(stdout) or {} + fallback = _openclaw_output_items(envelope) except (TypeError, ValueError, AttributeError): fallback = [] + envelope = {} + gaps.append(ObservationGap(code="agent_stdout_unparseable")) + meta = envelope.get("meta") + agent_meta = meta.get("agentMeta") if isinstance(meta, dict) else None + raw_usage = agent_meta.get("usage") if isinstance(agent_meta, dict) else None + raw_usage = raw_usage if isinstance(raw_usage, dict) else {} + if any( + value is not None and (type(value) is not int or value < 0) + for value in (raw_usage.get(name) for name in ("input", "output", "cacheRead")) + ): gaps.append(ObservationGap(code="agent_stdout_unparseable")) if not output: output = fallback @@ -817,11 +833,6 @@ def count(name: str) -> int: if not assistants: # The legacy envelope parser defaults missing counters to zero. # Native totals retain valid subtotals without coercing malformed cache values. - envelope = _decode_last_json_dict_suffix(stdout) or {} - meta = envelope.get("meta") - agent_meta = meta.get("agentMeta") if isinstance(meta, dict) else None - raw_usage = agent_meta.get("usage") if isinstance(agent_meta, dict) else None - raw_usage = raw_usage if isinstance(raw_usage, dict) else {} cache_read = raw_usage.get("cacheRead") cached_tokens = cache_read if type(cache_read) is int and cache_read >= 0 else None input_count, output_count = raw_usage.get("input"), raw_usage.get("output") diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index b19f34d1a9c..0968ffe52e9 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -1010,8 +1010,14 @@ async def test_native_envelope_fallback_preserves_known_and_unknown_cache_detail NeMoGymResponseCreateParamsNonStreaming(input="task"), prompt="task", system="", - stdout=json.dumps({"meta": {"agentMeta": {"usage": {"input": 10, "output": 3, **cache}}}}), + stdout=json.dumps( + { + "payloads": [{"text": "Retained final output."}], + "meta": {"agentMeta": {"usage": {"input": 10, "output": 3, **cache}}}, + } + ), ) + assert response.output[0].content[0].text == "Retained final output." assert response.usage.input_tokens == 10 + (expected_cache or 0) assert response.usage.output_tokens == 3 assert response.usage.input_tokens_details.cached_tokens == expected_cache From bcab742619ade3d92f063f33f42e689e407d2169 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 22 Sep 2026 21:44:42 -0700 Subject: [PATCH 07/44] fix(openclaw): retain unknown cache detail provenance Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/app.py | 6 ++++-- responses_api_agents/openclaw_agent/tests/test_sandbox.py | 8 +++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index dffb46de857..ea1389ec71f 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -816,7 +816,9 @@ async def _collect_sandbox_response( ObservationGap(code="model_call_usage_unavailable", detail="Harness reported only zero counters") ) cache_read = usage.get("cacheRead") - if type(cache_read) is not int or cache_read < 0: + # The pinned provider defaults unavailable backend cache details to + # zero, so only positive counts retain measurement provenance. + if type(cache_read) is not int or cache_read <= 0: gaps.append(ObservationGap(code="cached_token_usage_unavailable")) cached_tokens = None elif cached_tokens is not None: @@ -834,7 +836,7 @@ def count(name: str) -> int: # The legacy envelope parser defaults missing counters to zero. # Native totals retain valid subtotals without coercing malformed cache values. cache_read = raw_usage.get("cacheRead") - cached_tokens = cache_read if type(cache_read) is int and cache_read >= 0 else None + cached_tokens = cache_read if type(cache_read) is int and cache_read > 0 else None input_count, output_count = raw_usage.get("input"), raw_usage.get("output") input_tokens = input_count if type(input_count) is int and input_count >= 0 else 0 output_tokens = output_count if type(output_count) is int and output_count >= 0 else 0 diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index 0968ffe52e9..0d689028df9 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -702,11 +702,13 @@ def test_usage_sums_cache_writes_and_failed_calls(setup): (2, {"cacheRead": "3"}, None, 17), (2, {"cacheRead": 3.0}, None, 17), (None, {"cacheRead": 2}, None, 17), - (0, {"cacheRead": 0}, 0, 15), + (0, {"cacheRead": 0}, None, 15), + (2, {"cacheRead": 0}, None, 17), + (0, {"cacheRead": 2}, None, 17), (2, {"cacheRead": 3}, 5, 20), ], ) -def test_native_usage_details_preserve_unknown_and_known_counts( +def test_native_usage_details_preserve_positive_counts_and_treat_defaulted_zero_as_unknown( setup, first_cache, second_cache, expected_cache, expected_input ): agent, sandbox = setup @@ -995,7 +997,7 @@ async def test_invalid_envelope_usage_does_not_discard_valid_transcript(setup): ({"cacheRead": "5"}, None), ({"cacheRead": "bad"}, None), ({"cacheRead": 3.0}, None), - ({"cacheRead": 0}, 0), + ({"cacheRead": 0}, None), ({"cacheRead": 5}, 5), ], ) From f4f64fcf3c5542a2fa4238b9a3c1c30fad89d64a Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 23 Sep 2026 13:34:15 -0700 Subject: [PATCH 08/44] fix(openclaw): route native evaluation through environment sessions Signed-off-by: yaoyu-33 --- benchmarks/swebench/pro/openclaw_native.yaml | 24 ++ responses_api_agents/openclaw_agent/README.md | 28 +- responses_api_agents/openclaw_agent/app.py | 180 +++++++--- .../configs/openclaw_agent_native.yaml | 13 + .../openclaw_agent/tests/test_sandbox.py | 310 +++++++++++++++++- 5 files changed, 508 insertions(+), 47 deletions(-) create mode 100644 benchmarks/swebench/pro/openclaw_native.yaml create mode 100644 responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml diff --git a/benchmarks/swebench/pro/openclaw_native.yaml b/benchmarks/swebench/pro/openclaw_native.yaml new file mode 100644 index 00000000000..77a7f1bffbc --- /dev/null +++ b/benchmarks/swebench/pro/openclaw_native.yaml @@ -0,0 +1,24 @@ +config_paths: + - resources_servers/swebench_pro/configs/swebench_pro_resources_server.yaml + - responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml + - environment_servers/single_agent_turn/configs/single_agent_turn.yaml + +environment_routing_mode: taskset +environment_server_routes: + "swebench_pro:smoke": swebench_pro_openclaw + +swebench_pro_openclaw_resources_server: + _inherit_from: swebench_pro_resources_server + resources_servers: + swebench_pro: + allowed_agents: [openclaw_agent] + +swebench_pro_openclaw_agent: + _inherit_from: openclaw_agent_native + +swebench_pro_openclaw: + _inherit_from: single_agent_turn + environment_servers: + single_agent_turn: + resources_server: {type: resources_servers, name: swebench_pro_openclaw_resources_server} + agent_server: {type: responses_api_agents, name: swebench_pro_openclaw_agent} diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index b5e46e1c3db..9397901dd88 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -93,12 +93,30 @@ openclaw_native: native_environment: environment_servers: - single_agent: + single_agent_turn: entrypoint: app.py resources_server: {type: resources_servers, name: task_resources} agent_server: {type: responses_api_agents, name: openclaw_native} ``` +The runnable SWE-Pro composition is `benchmarks/swebench/pro/openclaw_native.yaml`. It selects +`environment_routing_mode: taskset` and binds `swebench_pro:smoke` to the native `single_agent_turn` +EnvironmentServer. The separate native agent config has no legacy Resources reference or local +provider overrides. With sandbox and policy model configuration ready: + +```bash +python benchmarks/swebench/pro/materialize_single_agent_tasks.py \ + resources_servers/swebench_pro/data/example.jsonl /tmp/swe-pro-native.jsonl + +gym eval run --config benchmarks/swebench/pro/openclaw_native.yaml \ + --model-type openai_model --input /tmp/swe-pro-native.jsonl \ + --output outputs/openclaw-native.jsonl --limit 1 +``` + +Supply the sandbox provider config through `env.yaml` or an additional `--config` file, and ensure +`policy_model_name`, `policy_base_url`, and `policy_api_key` match the reachable model endpoint. +Do not add `--agent`: this recipe routes materialized tasks by taskset. + Submit episodes to **EnvironmentServer `/run`**. It seeds Resources, opens the agent session, calls the rollout-scoped Responses endpoint, closes the agent, verifies, and closes Resources. The model server address must be reachable from inside the task sandbox. Its `/ng-rollout/.../v1` @@ -124,6 +142,10 @@ exec host refers to the embedded CLI process inside the borrowed sandbox. Native request support is deliberately explicit: - One activation per session; one worker per agent server. Concurrent sessions are independent. +- EnvironmentServer assigns the session ID. Repeating the full seed request is idempotent, even + without the original cookie; reusing its ID for different inputs is rejected. Close accepts the + explicit ID and episode without a cookie, including after a lost seed response. Closing an + unknown ID prevents a delayed seed from creating it. Filesystem paths use independent random IDs. - A text user prompt, optionally preceded by one system message. Text-part arrays are accepted. Configured `system_prompt`, request `instructions`, and the optional system message are joined and prepended to OpenClaw's user prompt. The harness retains its own system prompt. @@ -159,6 +181,10 @@ retryable for `session_close_retry_window_seconds` after cleanup (300 seconds by and retries do not shorten or extend the window. Stale cookies continue to reject activations after receipt expiry and never enter the host CLI path. +Abandoned sessions expire after `session_lifetime_seconds` (21600 seconds by default) through the +same cleanup path. Expiry failures retain closing state and block activation. Closed-ID tombstones +remain for the longer of the session lifetime and close retry window, then are pruned. + Session state is process-local. Keep requests on one worker. Failed cleanup sessions are retained until owner/provider recovery and agent restart; they are never relabeled as successfully closed. Cached runtime files remain until Resources destroys the sandbox. Supervisor cleanup prevents diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index ea1389ec71f..ac59140c64e 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -25,7 +25,8 @@ import signal from asyncio import Semaphore from collections import OrderedDict -from collections.abc import Mapping +from collections.abc import AsyncIterator, Mapping +from contextlib import asynccontextmanager from pathlib import Path from time import monotonic, time from typing import Any, Callable, ClassVar, Optional @@ -413,6 +414,7 @@ class OpenClawAgentConfig(BaseResponsesAPIAgentConfig): openclaw_version: str sandbox_install_timeout_seconds: float = Field(default=600, gt=0) session_close_timeout_seconds: float = Field(default=60, gt=0) + session_lifetime_seconds: float = Field(default=21600, gt=0, allow_inf_nan=False) session_close_retry_window_seconds: float = Field( default=300.0, gt=0, @@ -458,17 +460,82 @@ class OpenClawAgent(SimpleResponsesAPIAgent): default_factory=OrderedDict ) _local_setup_task: asyncio.Task[None] | None = PrivateAttr(default=None) + _session_locks: dict[str, asyncio.Lock] = PrivateAttr(default_factory=dict) + _session_lock_users: dict[str, int] = PrivateAttr(default_factory=dict) + _session_expiry_tasks: dict[str, asyncio.Task[None]] = PrivateAttr(default_factory=dict) + _closed_session_ids: dict[str, tuple[EpisodeId, float]] = PrivateAttr(default_factory=dict) + + @asynccontextmanager + async def _session_lock(self, session_id: str) -> AsyncIterator[None]: + """Count holders and waiters so pruning cannot replace an in-flight ID lock.""" + lock = self._session_locks.setdefault(session_id, asyncio.Lock()) + self._session_lock_users[session_id] = self._session_lock_users.get(session_id, 0) + 1 + try: + async with lock: + yield + finally: + remaining = self._session_lock_users[session_id] - 1 + if remaining: + self._session_lock_users[session_id] = remaining + else: + self._session_lock_users.pop(session_id) + if session_id not in self._sandbox_sessions and session_id not in self._closed_session_ids: + self._session_locks.pop(session_id, None) async def seed_agent_session(self, request: Request, body: AgentSeedSessionRequest) -> AgentSeedSessionResponse: """Borrow the Resources-owned task sandbox and install a pinned OpenClaw runtime inside it.""" self._expire_closed_agent_sessions() - if request.session.get(_SANDBOX_SESSION_KEY) in self._sandbox_sessions: - raise HTTPException(409, "OpenClaw session already exists") - session_id = f"openclaw-{uuid4().hex}" - state = await self._initialize_agent_session_state(session_id, body) - self._sandbox_sessions[session_id] = state - request.session[_SANDBOX_SESSION_KEY] = session_id - return AgentSeedSessionResponse(agent_session_id=session_id) + session_id = body.agent_session_id + previous = request.session.get(_SANDBOX_SESSION_KEY) + if previous != session_id and previous in self._sandbox_sessions: + raise HTTPException(409, "OpenClaw cookie belongs to another active session") + async with self._session_lock(session_id): + if session_id in self._closed_session_ids: + raise HTTPException(409, "OpenClaw session is already closed") + state = self._sandbox_sessions.get(session_id) + if state is not None: + if state.seed != body: + raise HTTPException(409, "OpenClaw session ID is bound to a different seed request") + if state.closing: + raise HTTPException(409, "OpenClaw session is closing") + else: + try: + state = await self._initialize_agent_session_state(session_id, body) + except BaseException: + if session_id not in self._sandbox_sessions: + # Initialization either never connected or confirmed its cleanup. + self._closed_sandbox_sessions[session_id] = ( + body.episode_id, + AgentCloseSessionResponse(agent_session_id=session_id), + monotonic() + self.config.session_close_retry_window_seconds, + ) + self._remember_closed_session(session_id, body.episode_id) + else: + self._session_expiry_tasks[session_id] = asyncio.create_task( + self._expire_agent_session(session_id, body.episode_id) + ) + raise + self._sandbox_sessions[session_id] = state + self._session_expiry_tasks[session_id] = asyncio.create_task( + self._expire_agent_session(session_id, body.episode_id) + ) + request.session[_SANDBOX_SESSION_KEY] = session_id + return AgentSeedSessionResponse(agent_session_id=session_id) + + async def _expire_agent_session(self, session_id: str, episode_id: EpisodeId) -> None: + """Bound abandoned sessions through the same fail-closed teardown as explicit close.""" + try: + await asyncio.sleep(self.config.session_lifetime_seconds) + await self.close_agent_session( + Request({"type": "http", "session": {}}), + AgentCloseSessionRequest(agent_session_id=session_id, episode_id=episode_id), + ) + except asyncio.CancelledError: + raise + except Exception: + LOG.exception("Could not clean up expired OpenClaw session %s; retaining failed state", session_id) + finally: + self._session_expiry_tasks.pop(session_id, None) async def _initialize_agent_session_state( self, agent_session_id: str, body: AgentSeedSessionRequest @@ -505,8 +572,10 @@ async def _initialize_agent_session_state( except BaseException: await provider.aclose() raise - directory = f"/tmp/nemo-gym-openclaw-sessions/{agent_session_id}" + directory = f"/tmp/nemo-gym-openclaw-sessions/{uuid4().hex}" runtime = f"/tmp/nemo-gym-openclaw-node-22.19.0-{self.config.openclaw_version}" + state = OpenClawSandboxSession(body, sandbox, directory, runtime) + prepared_directory = False try: check_paths = shlex.join( [ @@ -526,6 +595,7 @@ async def _initialize_agent_session_state( ) if prepared.return_code != 0 or getattr(prepared, "error_type", None): raise RuntimeError(prepared.stderr or "Cannot create OpenClaw sandbox session directory") + prepared_directory = True # Install only the agent runtime in the existing task sandbox. # Resources has already prepared the task repository and its dependencies. installer = "install_openclaw_runtime.sh" @@ -544,50 +614,78 @@ async def _initialize_agent_session_state( ) await sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), f"{directory}/sandbox_runner.py") except BaseException: - await sandbox.disconnect() + try: + if prepared_directory: + await state.close(self.config.session_close_timeout_seconds) + else: + # The path check may have rejected overlap with task-owned storage. + state.closing = True + await sandbox.disconnect() + state.closed = True + except BaseException: + self._sandbox_sessions[agent_session_id] = state + LOG.exception("OpenClaw seed cleanup failed; retaining session %s", agent_session_id) raise - return OpenClawSandboxSession(body, sandbox, directory, runtime) + return state async def close_agent_session(self, request: Request, body: AgentCloseSessionRequest) -> AgentCloseSessionResponse: """Confirm OpenClaw teardown before allowing verification; never destroy the borrowed sandbox.""" self._expire_closed_agent_sessions() - session_id = request.session.get(_SANDBOX_SESSION_KEY) - closed = self._closed_sandbox_sessions.get(session_id) - if closed is not None and body.agent_session_id == session_id and body.episode_id == closed[0]: - return closed[1] - state = self._sandbox_sessions.get(session_id) - if state is None or body.agent_session_id != session_id or body.episode_id != state.seed.episode_id: - raise HTTPException(409, "OpenClaw close does not match the seeded session and episode") - await state.close(self.config.session_close_timeout_seconds) - # Another close may have completed while this caller waited for the - # session's cleanup lock. Reuse its receipt without renewing its expiry. - self._expire_closed_agent_sessions() - closed = self._closed_sandbox_sessions.get(session_id) - if closed is not None: - return closed[1] - if self._sandbox_sessions.get(session_id) is not state: - raise HTTPException(409, "OpenClaw close receipt has expired") - observations = state.observations or AgentObservationBundle( - source="openclaw", gaps=[ObservationGap(code="agent_activation_interrupted")] - ) - self._sandbox_sessions.pop(session_id, None) - result = AgentCloseSessionResponse(agent_session_id=session_id, agent_observations=observations) - # Keep the cookie as a tombstone so later activations cannot fall back - # to host execution. Other sessions must not shorten the retry window. - self._closed_sandbox_sessions[session_id] = ( - body.episode_id, - result, - monotonic() + self.config.session_close_retry_window_seconds, - ) - return result + session_id = body.agent_session_id + cookie = request.session.get(_SANDBOX_SESSION_KEY) + if cookie is not None and cookie != session_id: + raise HTTPException(409, "OpenClaw close cookie does not match the requested session") + async with self._session_lock(session_id): + closed = self._closed_sandbox_sessions.get(session_id) + if closed is not None: + if body.episode_id != closed[0]: + raise HTTPException(409, "OpenClaw close does not match the seeded episode") + request.session[_SANDBOX_SESSION_KEY] = session_id + return closed[1] + if session_id in self._closed_session_ids: + raise HTTPException(409, "OpenClaw close receipt has expired") + state = self._sandbox_sessions.get(session_id) + if state is None: + result = AgentCloseSessionResponse(agent_session_id=session_id) + else: + if body.episode_id != state.seed.episode_id: + raise HTTPException(409, "OpenClaw close does not match the seeded episode") + await state.close(self.config.session_close_timeout_seconds) + observations = state.observations or AgentObservationBundle( + source="openclaw", gaps=[ObservationGap(code="agent_activation_interrupted")] + ) + result = AgentCloseSessionResponse(agent_session_id=session_id, agent_observations=observations) + self._sandbox_sessions.pop(session_id, None) + expiry = self._session_expiry_tasks.pop(session_id, None) + if expiry is not None and expiry is not asyncio.current_task(): + expiry.cancel() + request.session[_SANDBOX_SESSION_KEY] = session_id + self._closed_sandbox_sessions[session_id] = ( + body.episode_id, + result, + monotonic() + self.config.session_close_retry_window_seconds, + ) + self._remember_closed_session(session_id, body.episode_id) + return result + + def _remember_closed_session(self, session_id: str, episode_id: EpisodeId) -> None: + """Prevent delayed seeds through the session lifetime and close retry horizon.""" + retention = max(self.config.session_lifetime_seconds, self.config.session_close_retry_window_seconds) + self._closed_session_ids[session_id] = (episode_id, monotonic() + retention) + asyncio.get_running_loop().call_later(retention, self._expire_closed_agent_sessions) def _expire_closed_agent_sessions(self) -> None: - """Prune receipts by close time, never by traffic or retry access order.""" + """Prune receipts and tombstones by elapsed time, never by traffic or retry order.""" now = monotonic() while self._closed_sandbox_sessions: if next(iter(self._closed_sandbox_sessions.values()))[2] > now: break self._closed_sandbox_sessions.popitem(last=False) + for session_id, (_, deadline) in tuple(self._closed_session_ids.items()): + if deadline <= now: + self._closed_session_ids.pop(session_id) + if not self._session_lock_users.get(session_id, 0): + self._session_locks.pop(session_id, None) def _sandbox_input(self, body: NeMoGymResponseCreateParamsNonStreaming) -> tuple[str, str]: """Validate and normalize input before consuming the session's activation.""" diff --git a/responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml b/responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml new file mode 100644 index 00000000000..a82a6bcf4b6 --- /dev/null +++ b/responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml @@ -0,0 +1,13 @@ +openclaw_agent_native: + responses_api_agents: + openclaw_agent: + entrypoint: app.py + num_workers: 1 + model_server: {type: responses_api_models, name: policy_model} + model: ${policy_model_name} + openclaw_version: 2026.6.11 + timeout: 900 + sandbox_install_timeout_seconds: 900 + session_close_timeout_seconds: 60 + session_close_retry_window_seconds: 300 + session_lifetime_seconds: 21600 diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index 0d689028df9..7591e5855ae 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -7,6 +7,7 @@ from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch +from uuid import uuid4 import pytest from fastapi import HTTPException, Request @@ -22,8 +23,9 @@ from responses_api_agents.openclaw_agent.sandbox import OpenClawSandboxResult -def seed() -> AgentSeedSessionRequest: +def seed(*, session_id: str | None = None) -> AgentSeedSessionRequest: return AgentSeedSessionRequest( + agent_session_id=session_id or f"openclaw-{uuid4().hex}", episode_id=EpisodeId(rollout_id="openclaw-smoke", attempt=2), task_id=TaskId(taskset="swe-pro", task_id="task"), sandbox_access={ @@ -183,7 +185,7 @@ def test_http_native_flow_runs_openclaw_in_borrowed_sandbox(setup): created = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")) assert created.status_code == 200, created.text session_id = created.json()["agent_session_id"] - directory = f"/tmp/nemo-gym-openclaw-sessions/{session_id}" + directory = agent._sandbox_sessions[session_id].directory installer = f"{directory}/install_openclaw_runtime.sh" assert installer in sandbox.files assert agent.config.resources_server is None @@ -580,7 +582,11 @@ async def test_unknown_launch_outcome_fails_closed(setup): async def test_install_failure_disconnects_without_stopping_owner(setup): agent, sandbox = setup - sandbox.exec.side_effect = [SimpleNamespace(return_code=0), SimpleNamespace(return_code=1, stderr="npm failed")] + sandbox.exec.side_effect = [ + SimpleNamespace(return_code=0), + SimpleNamespace(return_code=1, stderr="npm failed"), + SimpleNamespace(return_code=0), + ] request = Request({"type": "http", "session": {}}) with pytest.raises(RuntimeError, match="npm failed"): await agent.seed_agent_session(request, seed()) @@ -592,7 +598,11 @@ async def test_install_failure_disconnects_without_stopping_owner(setup): async def test_cancelled_install_never_publishes_session_or_launches_openclaw(setup): agent, sandbox = setup - sandbox.exec.side_effect = [SimpleNamespace(return_code=0), asyncio.CancelledError()] + sandbox.exec.side_effect = [ + SimpleNamespace(return_code=0), + asyncio.CancelledError(), + SimpleNamespace(return_code=0), + ] request = Request({"type": "http", "session": {}}) with pytest.raises(asyncio.CancelledError): await agent.seed_agent_session(request, seed()) @@ -1006,7 +1016,7 @@ async def test_native_envelope_fallback_preserves_known_and_unknown_cache_detail request, session_id, task = await activate(agent, sandbox) await task state = agent._sandbox_sessions[session_id] - sandbox.files[f"{state.directory}/home/.openclaw/agents/main/sessions/{session_id}.jsonl"] = "" + sandbox.files[f"{state.directory}/home/.openclaw/agents/main/sessions/{Path(state.directory).name}.jsonl"] = "" response = await agent._collect_sandbox_response( state, NeMoGymResponseCreateParamsNonStreaming(input="task"), @@ -1100,3 +1110,293 @@ async def test_path_check_failure_never_uploads_or_installs(setup): assert not request.session sandbox.exec.assert_awaited_once() sandbox.disconnect.assert_awaited_once() + + +async def test_caller_id_seed_retries_share_one_session_without_cookies(setup): + agent, sandbox = setup + body = seed(session_id="caller-assigned-id") + requests = [Request({"type": "http", "session": {}}) for _ in range(2)] + first, second = await asyncio.gather(*(agent.seed_agent_session(request, body) for request in requests)) + assert first.agent_session_id == second.agent_session_id == body.agent_session_id + assert len(agent._sandbox_sessions) == 1 + assert sandbox.exec.await_count == 2 # One path check and one installation. + assert requests[0].session == requests[1].session + await agent.close_agent_session( + Request({"type": "http", "session": {}}), + AgentCloseSessionRequest(agent_session_id=body.agent_session_id, episode_id=body.episode_id), + ) + sandbox.disconnect.assert_awaited_once() + + +@pytest.mark.parametrize("field", ["episode", "task", "workdir"]) +async def test_caller_id_rejects_changed_seed_binding(setup, field): + agent, _ = setup + body = seed() + request = Request({"type": "http", "session": {}}) + await agent.seed_agent_session(request, body) + changed = body.model_copy(deep=True) + if field == "episode": + changed.episode_id = changed.episode_id.model_copy(update={"attempt": changed.episode_id.attempt + 1}) + elif field == "task": + changed.task_id = changed.task_id.model_copy(update={"task_id": "another-task"}) + else: + changed.sandbox_access.workdir = "/another-repository" + with pytest.raises(HTTPException, match="different seed"): + await agent.seed_agent_session(Request({"type": "http", "session": {}}), changed) + await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(body.agent_session_id))) + + +async def test_close_before_seed_prevents_late_creation(setup): + agent, sandbox = setup + body = seed() + request = Request({"type": "http", "session": {}}) + close = AgentCloseSessionRequest(agent_session_id=body.agent_session_id, episode_id=body.episode_id) + first = await agent.close_agent_session(request, close) + assert await agent.close_agent_session(Request({"type": "http", "session": {}}), close) is first + with pytest.raises(HTTPException, match="already closed"): + await agent.seed_agent_session(Request({"type": "http", "session": {}}), body) + sandbox.exec.assert_not_awaited() + + +async def test_close_serializes_with_inflight_seed(setup): + agent, sandbox = setup + body = seed() + entered, release = asyncio.Event(), asyncio.Event() + original = agent._initialize_agent_session_state + + async def blocked_initialize(*args): + entered.set() + await release.wait() + return await original(*args) + + with patch.object(agent, "_initialize_agent_session_state", side_effect=blocked_initialize): + pending = asyncio.create_task(agent.seed_agent_session(Request({"type": "http", "session": {}}), body)) + await entered.wait() + close = asyncio.create_task( + agent.close_agent_session( + Request({"type": "http", "session": {}}), + AgentCloseSessionRequest(agent_session_id=body.agent_session_id, episode_id=body.episode_id), + ) + ) + await asyncio.sleep(0) + assert not close.done() + release.set() + await asyncio.wait_for(asyncio.gather(pending, close), 2) + assert not agent._sandbox_sessions + sandbox.disconnect.assert_awaited_once() + + +async def test_caller_id_never_controls_filesystem_path(setup): + agent, _ = setup + body = seed(session_id="../../task-repository\nunsafe") + request = Request({"type": "http", "session": {}}) + response = await agent.seed_agent_session(request, body) + assert response.agent_session_id == body.agent_session_id + directory = agent._sandbox_sessions[body.agent_session_id].directory + assert Path(directory).parent == Path("/tmp/nemo-gym-openclaw-sessions") + assert len(Path(directory).name) == 32 + await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(body.agent_session_id))) + + +async def test_abandoned_session_expires_through_normal_cleanup(setup): + agent, sandbox = setup + agent.config.session_lifetime_seconds = 0.01 + body = seed() + await agent.seed_agent_session(Request({"type": "http", "session": {}}), body) + expiry = agent._session_expiry_tasks[body.agent_session_id] + await asyncio.wait_for(asyncio.shield(expiry), 2) + assert not agent._sandbox_sessions + assert body.agent_session_id in agent._closed_sandbox_sessions + sandbox.disconnect.assert_awaited_once() + sandbox.stop.assert_not_awaited() + + +async def test_failed_expiry_retains_state_and_rejects_activation(setup, caplog): + agent, sandbox = setup + agent.config.session_lifetime_seconds = 0.01 + body = seed() + request = Request({"type": "http", "session": {}, "path_params": {"rollout_id": body.episode_id.capture_key}}) + await agent.seed_agent_session(request, body) + sandbox.disconnect.side_effect = RuntimeError("disconnect unavailable") + await asyncio.wait_for(asyncio.shield(agent._session_expiry_tasks[body.agent_session_id]), 2) + assert agent._sandbox_sessions[body.agent_session_id].closing + assert "retaining failed state" in caplog.text + with pytest.raises(HTTPException): + await agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="task")) + sandbox.pty.create.assert_not_awaited() + sandbox.disconnect.side_effect = None + await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(body.agent_session_id))) + + +@pytest.mark.parametrize("lifetime", [0, -1, float("inf"), float("nan")]) +def test_session_lifetime_is_positive_and_finite(setup, lifetime): + agent, _ = setup + with pytest.raises(ValidationError): + OpenClawAgentConfig(**(agent.config.model_dump() | {"session_lifetime_seconds": lifetime})) + + +async def test_native_recipe_routes_collector_through_environment_run(setup, monkeypatch): + from http.cookies import SimpleCookie + + import orjson + + import nemo_gym.rollout_collection as collection + from benchmarks.swebench.pro.materialize_single_agent_tasks import materialize_row + from environment_servers.single_agent_turn.app import ( + SingleAgentTurnEnvironmentServer, + SingleAgentTurnEnvironmentServerConfig, + ) + from nemo_gym.global_config import GlobalConfigDictParser, GlobalConfigDictParserConfig + + agent, sandbox = setup + global_config = GlobalConfigDictParser().parse( + GlobalConfigDictParserConfig( + initial_global_config_dict=OmegaConf.create( + { + "config_paths": ["benchmarks/swebench/pro/openclaw_native.yaml"], + "policy_model_name": "test-model", + "policy_model": {"responses_api_models": {"openai_model": {"entrypoint": "app.py"}}}, + } + ), + skip_load_from_cli=True, + skip_load_from_dotenv=True, + offline=True, + ) + ) + env_name = global_config.environment_server_routes["swebench_pro:smoke"] + env_config = global_config[env_name].environment_servers.single_agent_turn + agent_name = env_config.agent_server.name + resources_name = env_config.resources_server.name + agent.config = OpenClawAgentConfig( + name=agent_name, + **OmegaConf.to_container(global_config[agent_name].responses_api_agents.openclaw_agent, resolve=True), + ) + assert agent.config.resources_server is None + assert not agent.config.openclaw_config + agent.server_client.global_config_dict = global_config + agent.server_client._resolve_base_url.return_value = "http://resources.example:8000" + environment = SingleAgentTurnEnvironmentServer( + config=SingleAgentTurnEnvironmentServerConfig(name=env_name, **OmegaConf.to_container(env_config)), + server_client=agent.server_client, + ) + calls = [] + resources_id = None + + class Reply: + def __init__(self, payload, *, status=200, cookies=None): + self.status = status + self.ok = status < 400 + self.cookies = SimpleCookie(cookies or {}) + self.payload = payload + + async def read(self): + return orjson.dumps(self.payload) + + def raise_for_status(self): + assert self.ok, self.payload + + async def post(*, server_name, url_path, json, cookies=None, **kwargs): + nonlocal resources_id + payload = json.model_dump(mode="json") if hasattr(json, "model_dump") else json + calls.append((server_name, url_path, payload)) + if server_name == resources_name: + if url_path == "/seed_session": + resources_id = payload["resources_session_id"] + assert payload["task_data"]["instance_id"] == "smoke-instance" + return Reply( + {"resources_session_id": resources_id, "sandbox_access": seed().sandbox_access.model_dump()}, + cookies={"resources": resources_id}, + ) + assert cookies == {"resources": resources_id} + if url_path == "/verify": + sandbox.disconnect.assert_awaited_once() + assert payload["verification_input"]["response"]["status"] == "completed" + return Reply({"reward": 1.0, **payload["verification_input"]}) + assert url_path == "/close_session" + assert payload["resources_session_id"] == resources_id + return Reply({"resources_session_id": resources_id}) + assert server_name in {env_name, agent_name} + assert not (server_name == agent_name and url_path == "/run") + client = environment_http if server_name == env_name else agent_http + client.cookies.clear() + client.cookies.update(cookies or {}) + response = await asyncio.to_thread(client.post, url_path, json=payload) + return Reply(response.json(), status=response.status_code, cookies=dict(response.cookies)) + + agent.server_client.post = AsyncMock(side_effect=post) + monkeypatch.setattr(collection, "setup_server_client_utils", lambda *args, **kwargs: agent.server_client) + materialized = materialize_row( + {"responses_create_params": {"input": "Fix the code"}, "instance_id": "smoke-instance"}, + taskset="swebench_pro:smoke", + ) + config = collection.RolloutCollectionConfig( + input_jsonl_fpath="unused.jsonl", + output_jsonl_fpath="unused-rollouts.jsonl", + environment_routing_mode=global_config.environment_routing_mode, + environment_server_routes=OmegaConf.to_container(global_config.environment_server_routes), + num_repeats=1, + ) + rows = collection.RolloutCollectionHelper._preprocess_raw_rows( + [(0, orjson.dumps(materialized).decode(), materialized)], config + ) + with ( + TestClient(agent.setup_webserver()) as agent_http, + TestClient(environment.setup_webserver()) as environment_http, + ): + _, result = await next(collection.RolloutCollectionHelper().run_examples(rows)) + assert result.get("result") is not None, result + assert result["result"]["verification"]["reward"] == 1.0 + assert result["result"]["agent_observations"]["source"] == "openclaw" + assert [(server_name, path) for server_name, path, _ in calls] == [ + (env_name, "/run"), + (resources_name, "/seed_session"), + (agent_name, "/v1/agent_sessions"), + (agent_name, "/ng-rollout/0-0/v1/responses"), + (agent_name, "/v1/agent_sessions/close"), + (resources_name, "/verify"), + (resources_name, "/close_session"), + ] + assert calls[2][2]["agent_session_id"] == calls[4][2]["agent_session_id"] + assert calls[0][2]["task"]["task_input"] == materialized["task_input"] + assert not agent._sandbox_sessions + sandbox.stop.assert_not_awaited() + + +async def test_failed_seed_with_confirmed_cleanup_accepts_cookie_less_close(setup): + agent, sandbox = setup + body = seed() + sandbox.exec.side_effect = [ + SimpleNamespace(return_code=0), + SimpleNamespace(return_code=1, stderr="npm unavailable"), + SimpleNamespace(return_code=0), + ] + with pytest.raises(RuntimeError, match="npm unavailable"): + await agent.seed_agent_session(Request({"type": "http", "session": {}}), body) + close = AgentCloseSessionRequest(agent_session_id=body.agent_session_id, episode_id=body.episode_id) + result = await agent.close_agent_session(Request({"type": "http", "session": {}}), close) + assert result.agent_session_id == body.agent_session_id + sandbox.disconnect.assert_awaited_once() + assert not agent._sandbox_sessions + + +async def test_failed_seed_cleanup_retains_state_until_explicit_retry(setup): + agent, sandbox = setup + body = seed() + sandbox.exec.side_effect = [ + SimpleNamespace(return_code=0), + SimpleNamespace(return_code=1, stderr="npm unavailable"), + SimpleNamespace(return_code=0), + ] + sandbox.disconnect.side_effect = RuntimeError("disconnect unavailable") + with pytest.raises(RuntimeError, match="npm unavailable"): + await agent.seed_agent_session(Request({"type": "http", "session": {}}), body) + assert agent._sandbox_sessions[body.agent_session_id].closing + assert body.agent_session_id not in agent._closed_sandbox_sessions + with pytest.raises(HTTPException, match="closing"): + await agent.seed_agent_session(Request({"type": "http", "session": {}}), body) + sandbox.exec.side_effect = None + sandbox.disconnect.side_effect = None + close = AgentCloseSessionRequest(agent_session_id=body.agent_session_id, episode_id=body.episode_id) + await agent.close_agent_session(Request({"type": "http", "session": {}}), close) + assert not agent._sandbox_sessions + sandbox.stop.assert_not_awaited() From 41675d80524794bc2425cef9f8875473ef2f1857 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 23 Sep 2026 13:43:37 -0700 Subject: [PATCH 09/44] fix(openclaw): reject expired and malformed session cookies Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/app.py | 6 ++++ .../openclaw_agent/tests/test_sandbox.py | 31 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index ac59140c64e..7e9a450fe67 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -487,6 +487,8 @@ async def seed_agent_session(self, request: Request, body: AgentSeedSessionReque self._expire_closed_agent_sessions() session_id = body.agent_session_id previous = request.session.get(_SANDBOX_SESSION_KEY) + if _SANDBOX_SESSION_KEY in request.session and not isinstance(previous, str): + raise HTTPException(409, "Invalid OpenClaw session marker") if previous != session_id and previous in self._sandbox_sessions: raise HTTPException(409, "OpenClaw cookie belongs to another active session") async with self._session_lock(session_id): @@ -499,6 +501,8 @@ async def seed_agent_session(self, request: Request, body: AgentSeedSessionReque if state.closing: raise HTTPException(409, "OpenClaw session is closing") else: + if previous == session_id: + raise HTTPException(409, "OpenClaw session cookie has expired") try: state = await self._initialize_agent_session_state(session_id, body) except BaseException: @@ -646,6 +650,8 @@ async def close_agent_session(self, request: Request, body: AgentCloseSessionReq raise HTTPException(409, "OpenClaw close receipt has expired") state = self._sandbox_sessions.get(session_id) if state is None: + if cookie is not None: + raise HTTPException(409, "OpenClaw close receipt has expired") result = AgentCloseSessionResponse(agent_session_id=session_id) else: if body.episode_id != state.seed.episode_id: diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index 7591e5855ae..69006862a75 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -1400,3 +1400,34 @@ async def test_failed_seed_cleanup_retains_state_until_explicit_retry(setup): await agent.close_agent_session(Request({"type": "http", "session": {}}), close) assert not agent._sandbox_sessions sandbox.stop.assert_not_awaited() + + +@pytest.mark.parametrize("marker", [None, [], {}, 7]) +async def test_seed_rejects_malformed_native_cookie_before_connecting(setup, marker): + agent, sandbox = setup + request = Request({"type": "http", "session": {"nemo_gym_openclaw_sandbox_session": marker}}) + with pytest.raises(HTTPException, match="Invalid OpenClaw session marker"): + await agent.seed_agent_session(request, seed()) + sandbox.exec.assert_not_awaited() + + +async def test_stale_cookie_cannot_recreate_session_or_close_receipt_after_full_retention(setup, monkeypatch): + agent, sandbox = setup + clock = [100.0] + monkeypatch.setattr("responses_api_agents.openclaw_agent.app.monotonic", lambda: clock[0]) + agent.config.session_lifetime_seconds = 10 + agent.config.session_close_retry_window_seconds = 5 + request = Request({"type": "http", "session": {}}) + body = seed() + await agent.seed_agent_session(request, body) + close = AgentCloseSessionRequest(agent_session_id=body.agent_session_id, episode_id=body.episode_id) + await agent.close_agent_session(request, close) + clock[0] = 111.0 + with pytest.raises(HTTPException, match="expired"): + await agent.close_agent_session(request, close) + with pytest.raises(HTTPException, match="expired"): + await agent.seed_agent_session(request, body) + assert not agent._closed_sandbox_sessions + assert not agent._sandbox_sessions + assert not agent._session_locks + sandbox.disconnect.assert_awaited_once() From 6637a9b27216c78f6e3b27aab0c152ae97ee7cff Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 23 Sep 2026 13:46:25 -0700 Subject: [PATCH 10/44] docs(openclaw): separate server startup from native collection Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index 9397901dd88..88a33806c46 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -108,7 +108,9 @@ provider overrides. With sandbox and policy model configuration ready: python benchmarks/swebench/pro/materialize_single_agent_tasks.py \ resources_servers/swebench_pro/data/example.jsonl /tmp/swe-pro-native.jsonl -gym eval run --config benchmarks/swebench/pro/openclaw_native.yaml \ +gym env start --config benchmarks/swebench/pro/openclaw_native.yaml --model-type openai_model + +gym eval run --no-serve --config benchmarks/swebench/pro/openclaw_native.yaml \ --model-type openai_model --input /tmp/swe-pro-native.jsonl \ --output outputs/openclaw-native.jsonl --limit 1 ``` From 42d203d78c598ff0aa51499f425795eb2cf6190f Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Wed, 23 Sep 2026 13:51:13 -0700 Subject: [PATCH 11/44] docs(openclaw): remove trailing whitespace Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index 88a33806c46..35f2c271df8 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -9,7 +9,7 @@ Minimal, meant to be extended, and currently eval-only. ## Quick start OpenClaw is auto-installed on the first local CLI invocation. Starting an agent server for native -sessions does not install or execute OpenClaw on the agent-server host. +sessions does not install or execute OpenClaw on the agent-server host. Make sure `env.yaml` is also set. ```bash From eb95e8d789ab498e20d7910c5ca9a4ee8a806c45 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Thu, 1 Oct 2026 13:13:17 -0700 Subject: [PATCH 12/44] fix(openclaw): run native sandboxes without PTY sessions Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/README.md | 10 +- responses_api_agents/openclaw_agent/app.py | 2 +- .../install_openclaw_runtime.sh | 83 +++++--- .../openclaw_agent/sandbox.py | 132 +++++++++---- .../openclaw_agent/sandbox_runner.py | 7 +- .../tests/test_exec_transport.py | 186 ++++++++++++++++++ .../tests/test_install_runtime.py | 122 +++++++++++- .../openclaw_agent/tests/test_sandbox.py | 70 ++++--- .../tests/test_sandbox_runner.py | 44 +++++ 9 files changed, 560 insertions(+), 96 deletions(-) create mode 100644 responses_api_agents/openclaw_agent/tests/test_exec_transport.py diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index 35f2c271df8..64e2506de12 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -124,16 +124,22 @@ calls the rollout-scoped Responses endpoint, closes the agent, verifies, and clo The model server address must be reachable from inside the task sandbox. Its `/ng-rollout/.../v1` route is embedded in OpenClaw's Chat Completions provider configuration to retain model-call linkage. -The installer supports Linux glibc on x86_64/aarch64 and requires Python 3.9 or later for the +The installer supports Linux glibc on x86_64/aarch64 and musl on x86_64, with Python 3.8 or later for the supervisor. Node 22.19.0 and OpenClaw are installed under `/tmp/nemo-gym-openclaw-node-22.19.0-`. Node archives are checked against upstream SHA-256 sums; the installed package version is verified before caching and when reusing a cached runtime. A version-scoped `flock` serializes runtime installation and cache validation across session setup -attempts. Missing `flock`, `curl`, CA certificates, `tar`, `xz`, `sha256sum`, and `awk` are installed with `apt-get` only when +attempts. Missing `flock`, `curl`, CA certificates, `tar`, `gzip`, `sha256sum`, and `awk` are installed with `apt-get` or `apk` only when running as root. Other images receive an actionable prerequisite error. Installation errors include the failed command, exit status, and stderr. Supporting these architectures does not establish validation on every task image; record the actual image digest and runtime versions with each run. +The runner uses ordinary sandbox `exec`; no PTY/session API is required. Its internal deadline +leaves time for descendant cleanup before the provider deadline. Cancellation requests a stop and +waits for a cleanup receipt before detaching. A missing or failed receipt blocks verification. +For old musl images, a checksum-pinned C++ runtime is extracted privately and used only by the +private Node binary; task Node/Python and the global library search path are not replaced. + Per-session HOME, config, caches, prompt, transcript, and supervisor output are isolated under `/tmp/nemo-gym-openclaw-sessions/`. OpenClaw reads the generated config directly without onboarding or creating bootstrap files in the task repository. The task's PATH and dependencies are preserved. diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index 7e9a450fe67..4aae4740787 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -593,7 +593,7 @@ async def _initialize_agent_session_state( ) prepared = await sandbox.exec( "command -v python3 >/dev/null 2>&1 || " - "{ echo 'Native OpenClaw requires Python >=3.9 in the task image' >&2; exit 1; }; " + "{ echo 'Native OpenClaw requires Python >=3.8 in the task image' >&2; exit 1; }; " f"{check_paths} && mkdir -p {shlex.quote(directory + '/home/.openclaw')}", timeout_s=30, ) diff --git a/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh b/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh index d4a2b000dc7..3c183a89cd5 100644 --- a/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh +++ b/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh @@ -7,20 +7,51 @@ trap 'status=$?; echo "OpenClaw installer failed (exit $status) at line $LINENO: runtime=$1 openclaw_version=$2 node_version=22.19.0 -if [ "$(uname -s)" != Linux ] || ! getconf GNU_LIBC_VERSION >/dev/null 2>&1; then - echo 'Native OpenClaw requires a Linux glibc sandbox; Alpine/musl is not supported.' >&2 +if [ "$(uname -s)" != Linux ]; then + echo 'Native OpenClaw requires a Linux sandbox.' >&2 exit 1 fi if ! command -v python3 >/dev/null 2>&1; then - echo 'Native OpenClaw requires Python >=3.9 in the task image for descendant supervision.' >&2 + echo 'Native OpenClaw requires Python >=3.8 in the task image for descendant supervision.' >&2 exit 1 fi -python3 -c 'import sys; assert sys.version_info >= (3, 9), "Native OpenClaw requires Python >=3.9"' +python3 -c 'import sys; assert sys.version_info >= (3, 8), "Native OpenClaw requires Python >=3.8"' case "$(uname -m)" in x86_64) arch=x64 ;; aarch64) arch=arm64 ;; - *) echo 'Native OpenClaw supports Linux x86_64/aarch64 glibc only.' >&2; exit 1 ;; + *) echo 'Native OpenClaw supports Linux x86_64/aarch64 only.' >&2; exit 1 ;; esac +platform="linux-${arch}" +node_dist="https://nodejs.org/dist/v${node_version}" +if getconf GNU_LIBC_VERSION >/dev/null 2>&1; then + : +elif [[ "$(ldd --version 2>&1 || true)" == *musl* ]]; then + if [ "$arch" != x64 ]; then + echo "Native OpenClaw's pinned Node ${node_version} musl build supports x86_64 only" >&2 + exit 1 + fi + platform=linux-x64-musl + node_dist="https://unofficial-builds.nodejs.org/download/release/v${node_version}" +else + echo 'Native OpenClaw requires glibc or musl; could not identify the sandbox libc' >&2 + exit 1 +fi + +install_packages() { + if [ "$(id -u)" -ne 0 ]; then + echo "Native OpenClaw requires $*: preinstall these packages (automatic installation requires root)." >&2 + exit 1 + fi + if command -v apk >/dev/null 2>&1; then + apk add --no-cache "$@" + elif command -v apt-get >/dev/null 2>&1; then + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "$@" + else + echo "Native OpenClaw requires $*: preinstall these packages (automatic installation requires apt-get or apk)." >&2 + exit 1 + fi +} # Never change the task repository or its dependency environment. python3 - "$runtime" "$PWD" <<'PY' @@ -32,12 +63,7 @@ PY # One sandbox can receive overlapping session setup attempts. Keep the ready # check and all cache writes under a version-scoped advisory lock. if ! command -v flock >/dev/null 2>&1; then - if [ "$(id -u)" -ne 0 ] || ! command -v apt-get >/dev/null 2>&1; then - echo 'Native OpenClaw requires flock: preinstall util-linux in the task image (automatic installation requires root and apt-get).' >&2 - exit 1 - fi - apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends util-linux + install_packages util-linux fi mkdir -p "$runtime" exec 9>"$runtime/.install.lock" @@ -58,27 +84,38 @@ if [ -f "$runtime/ready" ]; then fi missing=() -for prerequisite in curl tar xz sha256sum awk; do +for prerequisite in curl tar gzip sha256sum awk; do command -v "$prerequisite" >/dev/null 2>&1 || missing+=("$prerequisite") done if [ ! -s /etc/ssl/certs/ca-certificates.crt ] && [ ! -s /etc/pki/tls/certs/ca-bundle.crt ]; then missing+=(ca-certificates) fi if [ "${#missing[@]}" -gt 0 ]; then - if [ "$(id -u)" -ne 0 ] || ! command -v apt-get >/dev/null 2>&1; then - echo "Native OpenClaw prerequisites missing: ${missing[*]}. Preinstall curl, ca-certificates, tar, xz-utils, coreutils, gawk (automatic installation requires root and apt-get)." >&2 - exit 1 - fi - apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends curl ca-certificates tar xz-utils coreutils gawk + echo "Native OpenClaw prerequisites missing: ${missing[*]}. Preinstall curl, ca-certificates, tar, gzip, coreutils, gawk." >&2 + install_packages curl ca-certificates tar gzip coreutils gawk +fi +if [ "$platform" = linux-x64-musl ] && ! python3 -c 'import ctypes; ctypes.CDLL("libstdc++.so.6")' 2>/dev/null; then + install_packages libstdc++ fi cd "$runtime" -archive="node-v${node_version}-linux-${arch}.tar.xz" -curl -fsSL --retry 3 "https://nodejs.org/dist/v${node_version}/${archive}" -o "$archive" -curl -fsSL --retry 3 "https://nodejs.org/dist/v${node_version}/SHASUMS256.txt" -o SHASUMS256.txt -awk -v archive="$archive" '$2 == archive' SHASUMS256.txt | sha256sum --check --strict - +archive="node-v${node_version}-${platform}.tar.gz" +curl -fsSL --retry 3 "${node_dist}/${archive}" -o "$archive" +curl -fsSL --retry 3 "${node_dist}/SHASUMS256.txt" -o SHASUMS256.txt +awk -v archive="$archive" '$2 == archive' SHASUMS256.txt | sha256sum -c - mkdir -p node -tar -xJf "$archive" --strip-components=1 -C node +tar -xzf "$archive" --strip-components=1 -C node +if [ "$platform" = linux-x64-musl ] && ! "$runtime/node/bin/node" --version; then + # Confine the newer C++ runtime to our Node binary, preserving task libraries. + command -v patchelf >/dev/null 2>&1 || install_packages patchelf + curl -fsSL --retry 3 \ + 'https://dl-cdn.alpinelinux.org/alpine/v3.19/main/x86_64/libstdc++-13.2.1_git20231014-r0.apk' \ + -o libstdc++.apk + echo '3cf66a7164240ef590106496d3c75f486bac46cba9cf2198c0c3b318c53ad027 libstdc++.apk' | sha256sum -c - + mkdir -p libstdcpp + tar -xzf libstdc++.apk -C libstdcpp usr/lib + patchelf --set-rpath '$ORIGIN/../../libstdcpp/usr/lib' "$runtime/node/bin/node" +fi +"$runtime/node/bin/node" --version PATH="$runtime/node/bin:$PATH" "$runtime/node/bin/node" \ "$runtime/node/lib/node_modules/npm/bin/npm-cli.js" install \ --prefix "$runtime/openclaw" --no-audit --no-fund "openclaw@${openclaw_version}" diff --git a/responses_api_agents/openclaw_agent/sandbox.py b/responses_api_agents/openclaw_agent/sandbox.py index 35a5869dcfa..a441fd58045 100644 --- a/responses_api_agents/openclaw_agent/sandbox.py +++ b/responses_api_agents/openclaw_agent/sandbox.py @@ -4,10 +4,12 @@ import asyncio import json +import logging import tempfile from dataclasses import dataclass, field from pathlib import Path from shlex import quote +from uuid import uuid4 from pydantic import BaseModel, ConfigDict, JsonValue @@ -15,7 +17,7 @@ from nemo_gym.openai_utils import NeMoGymResponse from nemo_gym.rollout_observability import AgentObservationBundle from nemo_gym.sandbox import AsyncSandbox -from nemo_gym.sandbox.providers.base import SandboxPtySession +from nemo_gym.sandbox.providers.base import SandboxExecResult # Executed by sandbox Python before uploading any adapter-owned files. Resolve @@ -53,14 +55,12 @@ class OpenClawSandboxSession: directory: str runtime: str task: asyncio.Task[NeMoGymResponse] | None = None - runner: SandboxPtySession | None = None - exit_task: asyncio.Task[int] | None = None + exec_task: asyncio.Task[SandboxExecResult] | None = None result: OpenClawSandboxResult | None = None observations: AgentObservationBundle | None = None activated: bool = False closing: bool = False launch_started: bool = False - runner_closed: bool = False closed: bool = False close_lock: asyncio.Lock = field(default_factory=asyncio.Lock) @@ -79,22 +79,50 @@ async def read_text(self, name: str) -> str: return path.read_text(errors="replace") async def stop_runner(self, timeout: float) -> None: - """Wait for the supervisor's cleanup receipt; retain handles on any failure.""" - if not self.launch_started: + """Fence delayed launches and require a receipt before allowing verification.""" + if not self.launch_started or (self.result is not None and self.result.cleanup_confirmed): return - if self.runner is None or self.exit_task is None: - raise RuntimeError("OpenClaw launch outcome is unknown; cannot authorize verification") - if not self.exit_task.done(): - await self.runner.send_signal("SIGTERM") - await asyncio.wait_for(asyncio.shield(self.exit_task), timeout=timeout) - if self.result is None: - self.result = OpenClawSandboxResult.model_validate_json(await self.read_text("result.json")) - if not self.result.cleanup_confirmed: - raise RuntimeError(f"OpenClaw sandbox cleanup was not confirmed: {self.result.error}") - if not self.runner_closed: - await self.runner.close() - self.runner_closed = True - # Keep the successful receipt for retries if file cleanup/disconnect fails. + receipt_path = f"{self.directory}/result.json" + try: + result = OpenClawSandboxResult.model_validate_json(await self.read_text("result.json")) + except Exception: + result = None + if result is None: + # The atomic claim decides whether launch or close won. A stopped + # claim can never start a harness, even if exec arrives much later. + stop_path = quote(f"{self.directory}/runner.stop") + claim_path = quote(f"{self.directory}/launch.claim") + temporary = quote(f"{receipt_path}.{uuid4().hex}.tmp") + stopped = quote( + json.dumps( + { + "return_code": 1, + "timed_out": False, + "cleanup_confirmed": True, + "error": "Closed before runner launch", + "hostname": "", + "pid": 0, + } + ) + ) + script = ( + f"[ -f {quote(receipt_path)} ] && exit 0; " + f"touch {stop_path} || exit 1; " + f"ln -s stop {claim_path} 2>/dev/null || true; " + f'if [ "$(readlink {claim_path})" = stop ]; then ' + f"printf '%s' {stopped} > {temporary} && mv {temporary} {quote(receipt_path)}; exit $?; fi; " + f"for _ in $(seq 1 {max(1, int(timeout))}); do " + f"[ -f {quote(receipt_path)} ] && exit 0; sleep 1; done; exit 1" + ) + await self.sandbox.exec(script, cwd=self.seed.sandbox_access.workdir, timeout_s=timeout + 5) + try: + result = OpenClawSandboxResult.model_validate_json(await self.read_text("result.json")) + except Exception as error: + raise RuntimeError("OpenClaw launch outcome is unknown; cannot authorize verification") from error + if not result.cleanup_confirmed: + raise RuntimeError(f"OpenClaw sandbox cleanup was not confirmed: {result.error}") + # Cache only a successful receipt; failed cleanup remains retryable. + self.result = result async def close(self, timeout: float) -> None: """Stop only OpenClaw-owned work and detach; never call sandbox.stop().""" @@ -102,6 +130,9 @@ async def close(self, timeout: float) -> None: if self.closed: return self.closing = True + # Cancelling provider exec can kill the supervisor. Obtain its + # descendant-cleanup receipt before cancelling the response task. + await self.stop_runner(timeout) if self.task is not None: if not self.task.done() and not self.task.cancelling(): self.task.cancel() @@ -113,28 +144,61 @@ async def close(self, timeout: float) -> None: except Exception: if not self.task.done(): raise - # A response error does not establish cleanup; stop_runner below must. - await self.stop_runner(timeout) - result = await self.sandbox.exec(f"rm -rf -- {quote(self.directory)}", timeout_s=timeout) + if self.exec_task is not None: + # A confirmed receipt makes it safe to cancel a stuck provider + # response; transport completion is not another cleanup gate. + if not self.exec_task.done(): + self.exec_task.cancel() + try: + await asyncio.wait_for(asyncio.shield(self.exec_task), timeout=timeout) + except asyncio.CancelledError: + if not self.exec_task.cancelled(): + raise + except Exception: + if not self.exec_task.done(): + raise + # Transport failure is not cleanup failure once the receipt is confirmed. + retired = f"{self.directory}.closed" + result = await self.sandbox.exec( + f"if [ -d {quote(self.directory)} ]; then " + f"mv {quote(self.directory)} {quote(retired)} || exit 1; fi; rm -rf -- {quote(retired)}", + timeout_s=timeout, + ) if result.return_code != 0 or getattr(result, "error_type", None): raise RuntimeError(f"Could not remove OpenClaw session files: {result.stderr}") await self.sandbox.disconnect() self.closed = True async def execute(self, payload: dict[str, JsonValue], *, timeout: float, close_timeout: float) -> str: - """Start the supervisor and OpenClaw inside the borrowed task sandbox.""" + """Run the supervisor through provider-neutral exec, without a PTY.""" await self.upload_json("input.json", payload) - self.launch_started = True - self.runner = await self.sandbox.pty.create( - command=f"exec python3 -I {quote(self.directory + '/sandbox_runner.py')} {quote(self.directory + '/input.json')}", - cwd=self.seed.sandbox_access.workdir, - pty=False, + command = ( + f"trap '' TERM; ln -s launch {quote(self.directory + '/launch.claim')} 2>/dev/null || exit 0; " + f"exec python3 -I {quote(self.directory + '/sandbox_runner.py')} {quote(self.directory + '/input.json')} " + f">{quote(self.directory + '/runner.log')} 2>&1" ) - self.exit_task = asyncio.create_task(self.runner.wait_exit()) + self.launch_started = True try: - # The supervisor owns the execution timeout; this bounds missing receipts/provider failures too. - await asyncio.wait_for(asyncio.shield(self.exit_task), timeout=timeout + close_timeout * 3) - self.result = OpenClawSandboxResult.model_validate_json(await self.read_text("result.json")) - return await self.read_text("stdout.log") - finally: + # The runner enforces its own deadline and reaps descendants. + # Leave extra time for cleanup and transport before provider timeout. + self.exec_task = asyncio.create_task( + self.sandbox.exec( + command, cwd=self.seed.sandbox_access.workdir, timeout_s=timeout + close_timeout * 3 + 30 + ) + ) + # HTTP cancellation must not propagate into provider exec before + # the supervisor has stopped and reaped the harness descendants. + launched = await asyncio.shield(self.exec_task) + if getattr(launched, "error_type", None) == "timeout": + raise TimeoutError("OpenClaw sandbox supervisor exceeded its execution deadline") + if launched.return_code != 0 or getattr(launched, "error_type", None): + raise RuntimeError(f"OpenClaw sandbox supervisor failed: {launched.stderr}") + except BaseException: + try: + await self.stop_runner(close_timeout) + except Exception: + logging.getLogger(__name__).exception("OpenClaw cleanup remains unconfirmed; close must retry") + raise + else: await self.stop_runner(close_timeout) + return await self.read_text("stdout.log") diff --git a/responses_api_agents/openclaw_agent/sandbox_runner.py b/responses_api_agents/openclaw_agent/sandbox_runner.py index b4ea0b086c5..483d8f33669 100644 --- a/responses_api_agents/openclaw_agent/sandbox_runner.py +++ b/responses_api_agents/openclaw_agent/sandbox_runner.py @@ -81,14 +81,17 @@ def interrupt(_signum: int, _frame: object) -> None: stopping = True signal.signal(signal.SIGTERM, interrupt) + stop_file = directory / "runner.stop" (directory / "prompt.txt").write_text(params["prompt"]) with (directory / "stdout.log").open("wb") as stdout, (directory / "stderr.log").open("wb") as stderr: try: enable_subreaper() + if stopping or stop_file.exists(): + raise RuntimeError("OpenClaw closed before process launch") process = subprocess.Popen( params["command"], cwd=params["cwd"], - env=dict(os.environ) | params["env"], + env={**os.environ, **params["env"]}, stdin=subprocess.DEVNULL, stdout=stdout, stderr=stderr, @@ -96,7 +99,7 @@ def interrupt(_signum: int, _frame: object) -> None: ) deadline = monotonic() + params["timeout"] while process.poll() is None: - if stopping or monotonic() >= deadline: + if stopping or stop_file.exists() or monotonic() >= deadline: timed_out = True break sleep(0.05) diff --git a/responses_api_agents/openclaw_agent/tests/test_exec_transport.py b/responses_api_agents/openclaw_agent/tests/test_exec_transport.py new file mode 100644 index 00000000000..93eea7d1493 --- /dev/null +++ b/responses_api_agents/openclaw_agent/tests/test_exec_transport.py @@ -0,0 +1,186 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Exercise the real supervisor through a provider with no PTY interface.""" + +import asyncio +import json +import os +import shutil +import signal +import sys +from pathlib import Path + +import pytest + +from nemo_gym.sandbox.providers.base import SandboxExecResult +from responses_api_agents.openclaw_agent import sandbox_runner +from responses_api_agents.openclaw_agent.sandbox import OpenClawSandboxSession +from responses_api_agents.openclaw_agent.tests.test_sandbox import seed + + +pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Linux child-subreaper and /proc required") + + +class ExecOnlySandbox: + def __init__(self): + self.cancelled_launch = False + self.lost_launch = False + self.delayed_command = None + self.disconnected = False + self.deadline = None + + async def upload(self, source, destination): + shutil.copyfile(source, destination) + + async def download(self, source, destination): + shutil.copyfile(source, destination) + + async def disconnect(self): + self.disconnected = True + + async def exec(self, command, *, cwd=None, timeout_s=30): + launch = command.startswith("trap '' TERM;") + if launch: + self.deadline = timeout_s + self.delayed_command = command + if self.lost_launch: + raise TimeoutError("lost launch response") + process = await asyncio.create_subprocess_shell( + command, cwd=cwd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, start_new_session=True + ) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout_s) + except BaseException: + self.cancelled_launch |= launch + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + await process.wait() + raise + return SandboxExecResult(stdout.decode(errors="replace"), stderr.decode(errors="replace"), process.returncode) + + +def make_session(tmp_path): + directory = tmp_path / "session" + directory.mkdir() + workdir = tmp_path / "task" + workdir.mkdir() + request = seed() + request.sandbox_access.workdir = str(workdir) + provider = ExecOnlySandbox() + state = OpenClawSandboxSession(request, provider, str(directory), str(tmp_path / "runtime")) + shutil.copyfile(sandbox_runner.__file__, directory / "sandbox_runner.py") + return state, provider, workdir + + +def payload(state, code, timeout=0.5): + return { + "directory": state.directory, + "prompt": "task", + "cwd": state.seed.sandbox_access.workdir, + "command": [sys.executable, "-c", code], + "env": {}, + "timeout": timeout, + "cleanup_timeout": 1, + } + + +@pytest.mark.parametrize("ending", ["natural", "timeout", "cancel", "close"]) +async def test_exec_only_supervision_reaps_detached_child(tmp_path, ending): + state, provider, workdir = make_session(tmp_path) + code = ( + "import subprocess,sys,time,pathlib; " + "p=subprocess.Popen([sys.executable,'-c','import time; time.sleep(60)'],start_new_session=True); " + "pathlib.Path('child.pid').write_text(str(p.pid)); " + + ("print('{}')" if ending == "natural" else "time.sleep(60)") + ) + deadline = 0.5 if ending == "timeout" else 60 + task = state.task = asyncio.create_task( + state.execute(payload(state, code, timeout=deadline), timeout=deadline, close_timeout=3) + ) + try: + async with asyncio.timeout(5): + while not (workdir / "child.pid").exists(): + await asyncio.sleep(0.01) + if ending == "cancel": + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + elif ending == "close": + await state.close(3) + await asyncio.gather(task, return_exceptions=True) + else: + await task + assert state.result.cleanup_confirmed is True + if ending == "timeout": + assert state.result.timed_out is True + with pytest.raises(ProcessLookupError): + os.kill(int((workdir / "child.pid").read_text()), 0) + assert provider.cancelled_launch is False + assert provider.deadline > deadline + 3 * 1 + await state.close(3) + await state.close(3) + assert provider.disconnected + assert not Path(state.directory).exists() + finally: + if not state.closed: + await state.close(3) + + +async def test_lost_launch_is_fenced_even_after_directory_retirement(tmp_path): + state, provider, workdir = make_session(tmp_path) + provider.lost_launch = True + with pytest.raises(TimeoutError, match="lost launch response"): + await state.execute(payload(state, "open('started','w').close()"), timeout=0.5, close_timeout=3) + assert state.result.cleanup_confirmed is True + assert state.result.return_code != 0 + assert state.result.error == "Closed before runner launch" + await state.close(3) + provider.lost_launch = False + await provider.exec(provider.delayed_command, cwd=str(workdir)) + assert not (workdir / "started").exists() + assert not Path(state.directory).exists() + + +async def test_failed_receipt_keeps_files_and_can_retry(tmp_path): + state, provider, _ = make_session(tmp_path) + state.launch_started = True + receipt = { + "return_code": 1, + "timed_out": False, + "cleanup_confirmed": False, + "error": "descendants remain", + "hostname": "sandbox", + "pid": 1, + } + path = Path(state.directory) / "result.json" + path.write_text(json.dumps(receipt)) + with pytest.raises(RuntimeError, match="cleanup was not confirmed"): + await state.close(3) + assert not provider.disconnected + assert path.exists() + receipt.update(cleanup_confirmed=True, error=None) + path.write_text(json.dumps(receipt)) + await state.close(3) + assert provider.disconnected + + +async def test_confirmed_cleanup_cancels_stuck_transport(tmp_path): + state, provider, _ = make_session(tmp_path) + state.launch_started = True + receipt = { + "return_code": 0, + "timed_out": False, + "cleanup_confirmed": True, + "error": None, + "hostname": "sandbox", + "pid": 1, + } + (Path(state.directory) / "result.json").write_text(json.dumps(receipt)) + state.exec_task = asyncio.create_task(asyncio.Event().wait()) + await asyncio.sleep(0) + await state.close(0.5) + assert state.exec_task.cancelled() + assert state.closed and provider.disconnected + assert not Path(state.directory).exists() diff --git a/responses_api_agents/openclaw_agent/tests/test_install_runtime.py b/responses_api_agents/openclaw_agent/tests/test_install_runtime.py index e191641a169..5b7a1610c5a 100644 --- a/responses_api_agents/openclaw_agent/tests/test_install_runtime.py +++ b/responses_api_agents/openclaw_agent/tests/test_install_runtime.py @@ -11,7 +11,7 @@ INSTALLER = Path(__file__).parents[1] / "install_openclaw_runtime.sh" -pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Native OpenClaw requires Linux/glibc") +pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Native OpenClaw requires Linux") @pytest.fixture @@ -19,7 +19,7 @@ def sandbox(tmp_path: Path) -> tuple[Path, dict[str, str]]: """Run the real installer with an isolated PATH and no access to the host package manager.""" bindir = tmp_path / "bin" bindir.mkdir() - for name in ("bash", "uname", "getconf", "mkdir", "cp", "tar", "xz", "sha256sum", "flock", "awk", "sleep"): + for name in ("bash", "uname", "getconf", "mkdir", "cp", "tar", "gzip", "sha256sum", "flock", "awk", "sleep"): executable = shutil.which(name) if executable is None: pytest.skip(f"Installer test requires {name}") @@ -62,7 +62,7 @@ def test_missing_curl_is_installed_before_downloading_node(sandbox: tuple[Path, assert result.returncode == 19, result.stderr assert (root / "packages.log").read_text().splitlines() == [ "update", - "install -y --no-install-recommends curl ca-certificates tar xz-utils coreutils gawk", + "install -y --no-install-recommends curl ca-certificates tar gzip coreutils gawk", ] assert "https://nodejs.org/dist/v22.19.0/node-v22.19.0-linux-" in (root / "download.log").read_text() assert not (root / "runtime/ready").exists() @@ -121,7 +121,7 @@ def test_missing_supervisor_python_explains_task_image_requirement(sandbox): (root / "bin/python3").unlink() result = run_installer(root, env) assert result.returncode == 1 - assert "requires Python >=3.9 in the task image" in result.stderr + assert "requires Python >=3.8 in the task image" in result.stderr assert not (root / "download.log").exists() @@ -140,7 +140,7 @@ def test_missing_install_lock_explains_prerequisite(sandbox): (root / "bin/flock").unlink() result = run_installer(root, env | {"TEST_UID": "1000"}) assert result.returncode == 1 - assert "preinstall util-linux" in result.stderr + assert "requires util-linux" in result.stderr assert not (root / "download.log").exists() @@ -194,3 +194,115 @@ def test_missing_awk_explains_prerequisite_before_download(sandbox): assert "prerequisites missing: awk" in result.stderr assert "gawk" in result.stderr assert not (root / "download.log").exists() + + +def _replace_command(root: Path, name: str, script: str) -> None: + path = root / "bin" / name + path.unlink(missing_ok=True) + path.write_text("#!/bin/bash\n" + script) + path.chmod(0o755) + + +def _use_musl_platform(root: Path, *, architecture: str = "x86_64") -> None: + """Simulate libc detection without using host package managers or downloads.""" + _replace_command(root, "getconf", "exit 1\n") + _replace_command(root, "ldd", "echo 'musl libc' >&2\nexit 1\n") + _replace_command(root, "uname", f'case "$1" in -s) echo Linux;; -m) echo {architecture};; esac\n') + # The task image has its existing C++ library, but our private Node may need + # a newer version. Other Python calls still exercise the real interpreter. + _replace_command( + root, + "python3", + 'if [[ "$*" == *ctypes.CDLL* ]]; then exit 0; fi\n' + f'exec "{sys.executable}" "$@"\n', + ) + _replace_command( + root, + "apk", + 'echo "$*" >> "$TEST_ROOT/packages.log"\ncp "$TEST_ROOT/curl" "$TEST_ROOT/bin/curl"\n', + ) + + +@pytest.mark.parametrize("bootstrap", [False, True]) +def test_musl_uses_pinned_x64_archive_and_apk(sandbox: tuple[Path, dict[str, str]], bootstrap: bool) -> None: + root, env = sandbox + _use_musl_platform(root) + if not bootstrap: + shutil.copy(root / "curl", root / "bin/curl") + result = run_installer(root, env) + assert result.returncode == 19, result.stderr + assert ( + "https://unofficial-builds.nodejs.org/download/release/v22.19.0/node-v22.19.0-linux-x64-musl.tar.gz" + in (root / "download.log").read_text() + ) + if bootstrap: + assert (root / "packages.log").read_text().splitlines() == [ + "add --no-cache curl ca-certificates tar gzip coreutils gawk" + ] + else: + assert not (root / "packages.log").exists() + assert not (root / "runtime/ready").exists() + + +def test_musl_arm64_is_rejected_before_installation(sandbox: tuple[Path, dict[str, str]]) -> None: + root, env = sandbox + _use_musl_platform(root, architecture="aarch64") + result = run_installer(root, env) + assert result.returncode == 1, result.stderr + assert "musl build supports x86_64 only" in result.stderr + assert not (root / "packages.log").exists() + assert not (root / "download.log").exists() + + +@pytest.mark.parametrize("checksum_ok", [True, False]) +def test_old_musl_cpp_repair_is_private_and_checksum_gated( + sandbox: tuple[Path, dict[str, str]], checksum_ok: bool +) -> None: + root, env = sandbox + _use_musl_platform(root) + commands = { + "curl": """for last; do :; done +echo "$*" >> "$TEST_ROOT/download.log" +if [[ "$*" == *SHASUMS256* ]]; then + echo 'stub node-v22.19.0-linux-x64-musl.tar.gz' > "$last" +else + echo fixture > "$last" +fi +""", + "sha256sum": """while IFS= read -r line; do + printf '%s\\n' "$line" >> "$TEST_ROOT/checksums.log" + if [[ "$line" == *libstdc++.apk ]]; then exit "$TEST_APK_CHECKSUM_STATUS"; fi +done +""", + "tar": 'echo "$*" >> "$TEST_ROOT/extract.log"\n', + "patchelf": 'echo "$*" >> "$TEST_ROOT/patchelf.log"\n: > "$TEST_ROOT/patched"\n', + } + for name, script in commands.items(): + _replace_command(root, name, script) + node = root / "runtime/node/bin/node" + node.parent.mkdir(parents=True) + node.write_text("""#!/bin/bash +case "$1" in + --version) [ -f "$TEST_ROOT/patched" ] || exit 1; echo v22.19.0;; + */npm-cli.js) echo install >> "$TEST_ROOT/npm.log";; + *) echo '2026.6.11';; +esac +""") + node.chmod(0o755) + result = run_installer(root, env | {"TEST_APK_CHECKSUM_STATUS": "0" if checksum_ok else "1"}) + assert ( + "3cf66a7164240ef590106496d3c75f486bac46cba9cf2198c0c3b318c53ad027 libstdc++.apk" + in (root / "checksums.log").read_text() + ) + if not checksum_ok: + assert result.returncode != 0 + assert not (root / "patchelf.log").exists() + assert not (root / "npm.log").exists() + assert not (root / "runtime/ready").exists() + assert "libstdc++.apk" not in (root / "extract.log").read_text() + return + assert result.returncode == 0, result.stderr + assert (root / "patchelf.log").read_text().splitlines() == [f"--set-rpath $ORIGIN/../../libstdcpp/usr/lib {node}"] + assert "-xzf libstdc++.apk -C libstdcpp usr/lib" in (root / "extract.log").read_text() + assert (root / "npm.log").read_text().splitlines() == ["install"] + assert (root / "runtime/ready").exists() + assert not (root / "packages.log").exists() diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index 69006862a75..b1c7e09c479 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -98,15 +98,25 @@ def __init__(self): self.blocked = False self.started = asyncio.Event() self.exited = asyncio.Event() - self.exec = AsyncMock(return_value=SimpleNamespace(return_code=0, stdout="", stderr="")) + self.exec = AsyncMock( + side_effect=self.run_exec, + return_value=SimpleNamespace(error_type=None, return_code=0, stdout="", stderr=""), + ) self.stop = AsyncMock() self.disconnect = AsyncMock() - self.runner = SimpleNamespace( - wait_exit=AsyncMock(side_effect=self.wait_exit), - send_signal=AsyncMock(side_effect=self.signal), - close=AsyncMock(), - ) - self.pty = SimpleNamespace(create=AsyncMock(side_effect=self.create)) + self.launch = AsyncMock(side_effect=self.create) + self.request_stop = AsyncMock(side_effect=self.signal) + + @property + def pty(self): + raise AssertionError("This provider supports exec only; no PTY API") + + async def run_exec(self, command, **kwargs): + if command.startswith("trap '' TERM;"): + return await self.launch(command=command, **kwargs) + if "runner.stop" in command: + await self.request_stop() + return self.exec.return_value async def upload(self, source, destination): self.files[destination] = Path(source).read_text() @@ -124,12 +134,14 @@ async def create(self, **kwargs): self.started.set() if not self.blocked: self.exited.set() - return self.runner + await self.wait_exit() + return SimpleNamespace(error_type=None, return_code=0, stdout="", stderr="") - async def signal(self, name): - assert name == "SIGTERM" - self.result["timed_out"] = True - self.exited.set() + async def signal(self): + if hasattr(self, "directory"): + self.result["timed_out"] = True + self.exited.set() + await asyncio.sleep(0) async def wait_exit(self): await self.exited.wait() @@ -189,7 +201,7 @@ def test_http_native_flow_runs_openclaw_in_borrowed_sandbox(setup): installer = f"{directory}/install_openclaw_runtime.sh" assert installer in sandbox.files assert agent.config.resources_server is None - assert not sandbox.pty.create.called + assert not sandbox.launch.called assert not any(path.startswith("/app/") for path in sandbox.files) install_call = sandbox.exec.await_args_list[1] assert install_call.args[0] == ( @@ -282,7 +294,7 @@ def test_cookie_identity_and_single_activation(setup): assert client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}).status_code == 200 assert client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}).status_code == 409 assert client.post("/v1/agent_sessions/close", json=close_body(session_id)).status_code == 200 - sandbox.pty.create.assert_awaited_once() + sandbox.launch.assert_awaited_once() @pytest.mark.parametrize("reason,expected", [("error", "failed"), ("aborted", "failed"), ("length", "incomplete")]) @@ -317,7 +329,7 @@ def test_unsupported_request_is_not_silently_ignored(setup, override): result = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task", **override}) assert result.status_code == 422, result.text assert client.post("/v1/agent_sessions/close", json=close_body(session_id)).status_code == 200 - sandbox.pty.create.assert_not_awaited() + sandbox.launch.assert_not_awaited() def test_rejected_request_does_not_consume_activation(setup): @@ -326,11 +338,11 @@ def test_rejected_request_does_not_consume_activation(setup): client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).raise_for_status() path = "/ng-rollout/openclaw-smoke-a2/v1/responses" assert client.post(path, json={"input": "task", "temperature": 0.2}).status_code == 422 - sandbox.pty.create.assert_not_awaited() + sandbox.launch.assert_not_awaited() accepted = client.post(path, json={"input": "task"}) assert accepted.status_code == 200, accepted.text assert client.post(path, json={"input": "task"}).status_code == 409 - sandbox.pty.create.assert_awaited_once() + sandbox.launch.assert_awaited_once() def test_no_session_keeps_existing_local_path(setup): @@ -340,7 +352,7 @@ def test_no_session_keeps_existing_local_path(setup): with pytest.raises(RuntimeError, match="legacy path reached"): client.post("/v1/responses", json={"input": "task"}) legacy.assert_awaited_once() - sandbox.pty.create.assert_not_awaited() + sandbox.launch.assert_not_awaited() async def activate(agent, sandbox): @@ -360,7 +372,7 @@ async def test_close_cancels_active_openclaw_before_detaching(setup): await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) with pytest.raises(asyncio.CancelledError): await task - sandbox.runner.send_signal.assert_awaited_once_with("SIGTERM") + sandbox.request_stop.assert_awaited_once() sandbox.disconnect.assert_awaited_once() sandbox.stop.assert_not_awaited() @@ -375,7 +387,7 @@ async def test_failed_cleanup_keeps_handles_and_prevents_close(setup): await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) assert session_id in agent._sandbox_sessions sandbox.disconnect.assert_not_awaited() - sandbox.runner.close.assert_not_awaited() + assert not any("rm -rf" in call.args[0] for call in sandbox.exec.await_args_list) async def test_disconnect_failure_retains_session_for_retry(setup): @@ -388,7 +400,7 @@ async def test_disconnect_failure_retains_session_for_retry(setup): assert session_id in agent._sandbox_sessions await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) assert session_id not in agent._sandbox_sessions - sandbox.runner.close.assert_awaited_once() + sandbox.launch.assert_awaited_once() def test_cleanup_receipt_is_required(): @@ -570,7 +582,7 @@ async def test_unknown_launch_outcome_fails_closed(setup): agent, sandbox = setup request = Request({"type": "http", "session": {}, "path_params": {"rollout_id": "openclaw-smoke-a2"}}) session_id = (await agent.seed_agent_session(request, seed())).agent_session_id - sandbox.pty.create.side_effect = TimeoutError("lost launch response") + sandbox.launch.side_effect = TimeoutError("lost launch response") with pytest.raises(TimeoutError): await agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="task")) with pytest.raises(RuntimeError, match="launch outcome is unknown"): @@ -608,7 +620,7 @@ async def test_cancelled_install_never_publishes_session_or_launches_openclaw(se await agent.seed_agent_session(request, seed()) assert not agent._sandbox_sessions assert not request.session - sandbox.pty.create.assert_not_awaited() + sandbox.launch.assert_not_awaited() sandbox.disconnect.assert_awaited_once() sandbox.stop.assert_not_awaited() @@ -623,7 +635,7 @@ def test_model_and_other_request_controls_are_explicitly_rejected(setup, overrid client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).raise_for_status() result = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task", **override}) assert result.status_code == 422, result.text - sandbox.pty.create.assert_not_awaited() + sandbox.launch.assert_not_awaited() async def test_cleanup_command_provider_error_blocks_close_and_can_retry(setup): @@ -651,7 +663,7 @@ async def test_simultaneous_activation_is_rejected_before_launch(setup): await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) with pytest.raises(asyncio.CancelledError): await task - sandbox.pty.create.assert_awaited_once() + sandbox.launch.assert_awaited_once() async def test_close_start_prevents_activation(setup): @@ -672,7 +684,7 @@ async def disconnect(): assert error.value.status_code == 409 release.set() await close - sandbox.pty.create.assert_not_awaited() + sandbox.launch.assert_not_awaited() async def test_cancelled_activation_keeps_partial_observations_for_close(setup): @@ -943,7 +955,7 @@ async def test_invalid_or_expired_native_cookie_never_uses_local_execution(setup with pytest.raises(HTTPException) as error: await agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="task")) assert error.value.status_code == 409 - sandbox.pty.create.assert_not_awaited() + sandbox.launch.assert_not_awaited() def test_native_close_cookie_blocks_legacy_run_with_configured_resources(setup): @@ -959,7 +971,7 @@ def test_native_close_cookie_blocks_legacy_run_with_configured_resources(setup): response = client.post("/run", json={"responses_create_params": {"input": "task"}}) assert response.status_code == 409 agent.server_client.post.assert_not_called() - sandbox.pty.create.assert_not_awaited() + sandbox.launch.assert_not_awaited() def test_zero_harness_counters_are_reported_as_missing_usage(setup): @@ -1223,7 +1235,7 @@ async def test_failed_expiry_retains_state_and_rejects_activation(setup, caplog) assert "retaining failed state" in caplog.text with pytest.raises(HTTPException): await agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="task")) - sandbox.pty.create.assert_not_awaited() + sandbox.launch.assert_not_awaited() sandbox.disconnect.side_effect = None await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(body.agent_session_id))) diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py b/responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py index a3a8744438e..7cca5b5f1af 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py @@ -4,6 +4,7 @@ import json import os +import shutil import signal import subprocess import sys @@ -90,6 +91,49 @@ def test_spawn_error_is_not_success(tmp_path): assert summary["return_code"] != 0 +@pytest.mark.skipif(shutil.which("python3.8") is None, reason="Python 3.8 is not installed") +def test_python38_preserves_environment_and_reaps_detached_child(tmp_path: Path) -> None: + python = shutil.which("python3.8") + code = ( + "import json,os,pathlib,subprocess,sys; " + "p=subprocess.Popen([sys.executable,'-c','import time; time.sleep(60)'],start_new_session=True); " + "pathlib.Path('child.pid').write_text(str(p.pid)); " + "print(json.dumps([os.environ['OPENCLAW_TEST_INHERITED'],os.environ['OPENCLAW_TEST_OVERRIDE']]))" + ) + request = { + "directory": str(tmp_path), + "command": [python, "-c", code], + "cwd": str(tmp_path), + "env": {"OPENCLAW_TEST_OVERRIDE": "new"}, + "timeout": 3, + "cleanup_timeout": 2, + "prompt": "task input", + } + path = tmp_path / "input.json" + path.write_text(json.dumps(request)) + process = subprocess.Popen( + [python, "-I", str(Path(sandbox_runner.__file__)), str(path)], + env={**os.environ, "OPENCLAW_TEST_INHERITED": "kept", "OPENCLAW_TEST_OVERRIDE": "old"}, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + summary = result(tmp_path, process) + assert summary["cleanup_confirmed"] is True + assert summary["return_code"] == 0 + assert json.loads((tmp_path / "stdout.log").read_text()) == ["kept", "new"] + with pytest.raises(ProcessLookupError): + os.kill(int((tmp_path / "child.pid").read_text()), 0) + + +def test_stop_marker_prevents_process_launch(tmp_path): + (tmp_path / "runner.stop").touch() + process = launch(tmp_path, "open('should-not-exist', 'w').close()") + summary = result(tmp_path, process) + assert summary["cleanup_confirmed"] is True + assert summary["return_code"] != 0 + assert not (tmp_path / "should-not-exist").exists() + + def test_sigterm_during_spawn_does_not_lose_child_handle(tmp_path): # Deliver SIGTERM after the real child exists but before Popen returns to run(). # Isolate signal handlers/subreaper state from pytest, and always reap the test child. From 2ec87a2e422dbfb152c182182077aa0c19ac021b Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Thu, 1 Oct 2026 17:51:59 -0700 Subject: [PATCH 13/44] fix(openclaw): align native sessions with the shared contract Use shared lifecycle hooks, join and replay identical activations, isolate waiter cancellation, and propagate runtime errors before verification. Compose prepared rows through independent definitions; retain the upstream SWE-Pro scoring behavior. Signed-off-by: yaoyu-33 --- benchmarks/swebench/pro/openclaw_native.yaml | 24 -- responses_api_agents/openclaw_agent/README.md | 89 ++--- responses_api_agents/openclaw_agent/app.py | 267 ++++---------- .../configs/openclaw_agent_native.yaml | 1 - .../openclaw_agent/sandbox.py | 13 +- .../tests/test_exec_transport.py | 2 +- .../openclaw_agent/tests/test_sandbox.py | 346 +++++++++++------- 7 files changed, 342 insertions(+), 400 deletions(-) delete mode 100644 benchmarks/swebench/pro/openclaw_native.yaml diff --git a/benchmarks/swebench/pro/openclaw_native.yaml b/benchmarks/swebench/pro/openclaw_native.yaml deleted file mode 100644 index 77a7f1bffbc..00000000000 --- a/benchmarks/swebench/pro/openclaw_native.yaml +++ /dev/null @@ -1,24 +0,0 @@ -config_paths: - - resources_servers/swebench_pro/configs/swebench_pro_resources_server.yaml - - responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml - - environment_servers/single_agent_turn/configs/single_agent_turn.yaml - -environment_routing_mode: taskset -environment_server_routes: - "swebench_pro:smoke": swebench_pro_openclaw - -swebench_pro_openclaw_resources_server: - _inherit_from: swebench_pro_resources_server - resources_servers: - swebench_pro: - allowed_agents: [openclaw_agent] - -swebench_pro_openclaw_agent: - _inherit_from: openclaw_agent_native - -swebench_pro_openclaw: - _inherit_from: single_agent_turn - environment_servers: - single_agent_turn: - resources_server: {type: resources_servers, name: swebench_pro_openclaw_resources_server} - agent_server: {type: responses_api_agents, name: swebench_pro_openclaw_agent} diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index 64e2506de12..71322feaa74 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -73,51 +73,48 @@ Resources. The agent server borrows `SandboxAccess`, installs the pinned runtime invocation in `SandboxAccess.workdir`, and disconnects after confirmed cleanup. Resources retains ownership of task preparation, verification, and sandbox destruction. -Bind the agent, Resources, model server, and sandbox provider in your environment config. For -example, the following agent/environment blocks refer to an existing Resources server named -`task_resources` and a Gym model server named `policy`: +Load the independent benchmark, harness, and generic EnvironmentServer definitions. For example, +with an already prepared SWE-Pro JSONL and a reachable policy model, create a run config: ```yaml -openclaw_native: - responses_api_agents: - openclaw_agent: - entrypoint: app.py - num_workers: 1 - openclaw_version: 2026.6.11 - model_server: {type: responses_api_models, name: policy} - model: your-served-model-id - timeout: 900 - sandbox_install_timeout_seconds: 900 - session_close_timeout_seconds: 60 - session_close_retry_window_seconds: 300 - -native_environment: +config_paths: + - resources_servers/swebench_pro/configs/swebench_pro.yaml + - responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml + - environment_servers/single_agent_turn_legacy/configs/single_agent_turn_legacy.yaml + +# Route existing prepared flat rows through native session orchestration. +environment_routing_mode: legacy +environment_server_name: single_agent_turn_legacy + +single_agent_turn_legacy: environment_servers: - single_agent_turn: - entrypoint: app.py - resources_server: {type: resources_servers, name: task_resources} - agent_server: {type: responses_api_agents, name: openclaw_native} + single_agent_turn_legacy: + resources_server: {type: resources_servers, name: swebench_pro_resources_server} + agent_server: {type: responses_api_agents, name: openclaw_agent_native} ``` -The runnable SWE-Pro composition is `benchmarks/swebench/pro/openclaw_native.yaml`. It selects -`environment_routing_mode: taskset` and binds `swebench_pro:smoke` to the native `single_agent_turn` -EnvironmentServer. The separate native agent config has no legacy Resources reference or local -provider overrides. With sandbox and policy model configuration ready: +Save it as `native-run.yaml`, then run: ```bash -python benchmarks/swebench/pro/materialize_single_agent_tasks.py \ - resources_servers/swebench_pro/data/example.jsonl /tmp/swe-pro-native.jsonl +gym env start --config native-run.yaml --model-type openai_model -gym env start --config benchmarks/swebench/pro/openclaw_native.yaml --model-type openai_model - -gym eval run --no-serve --config benchmarks/swebench/pro/openclaw_native.yaml \ - --model-type openai_model --input /tmp/swe-pro-native.jsonl \ - --output outputs/openclaw-native.jsonl --limit 1 +gym eval run --no-serve --config native-run.yaml --model-type openai_model \ + --agent openclaw_agent_native --allow-unsupported-pairing \ + --input /path/to/prepared-swe-pro.jsonl --output outputs/openclaw-native.jsonl --limit 1 ``` Supply the sandbox provider config through `env.yaml` or an additional `--config` file, and ensure `policy_model_name`, `policy_base_url`, and `policy_api_key` match the reachable model endpoint. -Do not add `--agent`: this recipe routes materialized tasks by taskset. +Prepared rows keep benchmark task data and prompts; `task_source`, when present, must name the +selected Resources server. `--agent` selects the harness for existing rows. No extra standalone +materialization script is required. + +SWE-Pro currently has a legacy `allowed_agents` list that excludes OpenClaw. The explicit +`--allow-unsupported-pairing` opt-in permits this smoke without replacing benchmark or verifier +settings. It does not establish compatibility with other benchmarks. Switch compatible components +by changing their independent config paths and the two EnvironmentServer references; keep +benchmark data, preparation, and verifier settings with Resources. There is no combined +benchmark/harness preset. Submit episodes to **EnvironmentServer `/run`**. It seeds Resources, opens the agent session, calls the rollout-scoped Responses endpoint, closes the agent, verifies, and closes Resources. @@ -135,8 +132,8 @@ the failed command, exit status, and stderr. Supporting these architectures does validation on every task image; record the actual image digest and runtime versions with each run. The runner uses ordinary sandbox `exec`; no PTY/session API is required. Its internal deadline -leaves time for descendant cleanup before the provider deadline. Cancellation requests a stop and -waits for a cleanup receipt before detaching. A missing or failed receipt blocks verification. +leaves time for descendant cleanup before the provider deadline. Session close requests a stop and +waits for a cleanup receipt before detaching. A disconnected HTTP waiter leaves the shared invocation running. A missing or failed receipt blocks verification. For old musl images, a checksum-pinned C++ runtime is extracted privately and used only by the private Node binary; task Node/Python and the global library search path are not replaced. @@ -150,6 +147,8 @@ exec host refers to the embedded CLI process inside the borrowed sandbox. Native request support is deliberately explicit: - One activation per session; one worker per agent server. Concurrent sessions are independent. + Identical activation requests join the running task or replay its result/error. A different request + is rejected with HTTP 409. Session close owns cancellation; waiter cancellation does not stop the harness. - EnvironmentServer assigns the session ID. Repeating the full seed request is idempotent, even without the original cookie; reusing its ID for different inputs is rejected. Close accepts the explicit ID and episode without a cookie, including after a lost seed response. Closing an @@ -165,9 +164,12 @@ Native request support is deliberately explicit: effective captured requests. An episode-wide token budget is not implemented. - Custom command, environment, OpenClaw config, Node path, extra arguments, and agent-ID overrides are rejected for native sessions. Existing callers without an agent-session cookie retain the - local CLI behavior and configuration. + local CLI behavior and configuration. Both paths now include request `instructions` in the + same configured-system, request-instructions, input-system order. -Responses retain reasoning, tool calls/results, final or partial text, and terminal failure status. +Successful and limit-truncated Responses retain reasoning, tool calls/results, and final or partial text. +Runtime, provider, and upstream model failures raise an error before verification, with transcript +evidence and failed status retained in the observations returned by close. Usage sums observed assistant model calls, including cache reads/writes and failed final calls. Transcript rewrites retaining the same response ID and message count once; synthetic CLI summary messages carrying cumulative usage do not count again or override the model's terminal status. @@ -177,8 +179,8 @@ Auxiliary model calls, such as compaction, are absent from the transcript. A cov reported because interrupted compaction need not leave an event. Totals therefore need not equal all backend calls. Missing per-call usage and unavailable reasoning-token details are also reported as observation gaps; the full branch history is retained. -A timeout returns partial output as `incomplete`; model errors remain `failed` even when a useful -patch exists. Do not infer rollout success from reward alone. Close returns the captured observations, +An explicit wall or model-output limit returns partial output as `incomplete`; upstream/model errors +raise HTTP 502 even when a useful patch exists. Do not infer rollout success from reward alone. Close returns the captured observations, including salvaged transcript evidence after cancellation. A Linux child subreaper supervises each activation and reaps detached/background descendants before @@ -189,9 +191,12 @@ retryable for `session_close_retry_window_seconds` after cleanup (300 seconds by and retries do not shorten or extend the window. Stale cookies continue to reject activations after receipt expiry and never enter the host CLI path. -Abandoned sessions expire after `session_lifetime_seconds` (21600 seconds by default) through the -same cleanup path. Expiry failures retain closing state and block activation. Closed-ID tombstones -remain for the longer of the session lifetime and close retry window, then are pruned. +The shared `SimpleResponsesAPIAgent` lifecycle owns session identity, serialization, and close +receipts. `session_lifetime_seconds` has been removed: the episode owner and provider TTL govern +abandoned-session recovery; there is no adapter timer cancelling active sessions. Setup failures +whose cleanup fails retain cleanup-only state through `AgentSessionSetupError`. +`timeout`, `sandbox_install_timeout_seconds`, and `session_close_timeout_seconds` must be positive +and finite. The shared `session_close_retry_window_seconds` bounds close receipts and tombstones. Session state is process-local. Keep requests on one worker. Failed cleanup sessions are retained until owner/provider recovery and agent restart; they are never relabeled as successfully closed. diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index 4aae4740787..6693a5554a4 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -24,11 +24,9 @@ import shutil import signal from asyncio import Semaphore -from collections import OrderedDict -from collections.abc import AsyncIterator, Mapping -from contextlib import asynccontextmanager +from collections.abc import Mapping from pathlib import Path -from time import monotonic, time +from time import time from typing import Any, Callable, ClassVar, Optional from uuid import uuid4 @@ -38,16 +36,15 @@ from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import ( - AgentCloseSessionRequest, AgentCloseSessionResponse, AgentSeedSessionRequest, - AgentSeedSessionResponse, + AgentSessionSetupError, + AgentSessionState, BaseResponsesAPIAgentConfig, Body, SimpleResponsesAPIAgent, ) from nemo_gym.config_types import ModelServerRef, ResourcesServerRef -from nemo_gym.episode_types import EpisodeId from nemo_gym.openai_utils import ( NeMoGymEasyInputMessage, NeMoGymFunctionCallOutput, @@ -405,22 +402,15 @@ class OpenClawAgentConfig(BaseResponsesAPIAgentConfig): thinking: str = "off" system_prompt: Optional[str] = None setup_timeout: int = 900 - timeout: int = 900 + timeout: int = Field(default=900, gt=0) extra_args: list[str] = [] openclaw_config: dict[str, Any] = Field(default_factory=dict) context_window: Optional[int] = None max_output_tokens: Optional[int] = None # required: every config must pin an explicit version so runs are reproducible and cannot silently drift openclaw_version: str - sandbox_install_timeout_seconds: float = Field(default=600, gt=0) - session_close_timeout_seconds: float = Field(default=60, gt=0) - session_lifetime_seconds: float = Field(default=21600, gt=0, allow_inf_nan=False) - session_close_retry_window_seconds: float = Field( - default=300.0, - gt=0, - allow_inf_nan=False, - description="Keep successful close receipts for this many seconds; cover the caller's retry horizon.", - ) + sandbox_install_timeout_seconds: float = Field(default=600, gt=0, allow_inf_nan=False) + session_close_timeout_seconds: float = Field(default=60, gt=0, allow_inf_nan=False) @property def command_parts(self) -> list[str]: @@ -455,99 +445,10 @@ class OpenClawAgent(SimpleResponsesAPIAgent): # deny the interactive "message" channel so the headless agent finishes _HEADLESS_TOOL_DENY: ClassVar[tuple[str, ...]] = ("message",) - _sandbox_sessions: dict[str, OpenClawSandboxSession] = PrivateAttr(default_factory=dict) - _closed_sandbox_sessions: OrderedDict[str, tuple[EpisodeId, AgentCloseSessionResponse, float]] = PrivateAttr( - default_factory=OrderedDict - ) _local_setup_task: asyncio.Task[None] | None = PrivateAttr(default=None) - _session_locks: dict[str, asyncio.Lock] = PrivateAttr(default_factory=dict) - _session_lock_users: dict[str, int] = PrivateAttr(default_factory=dict) - _session_expiry_tasks: dict[str, asyncio.Task[None]] = PrivateAttr(default_factory=dict) - _closed_session_ids: dict[str, tuple[EpisodeId, float]] = PrivateAttr(default_factory=dict) - - @asynccontextmanager - async def _session_lock(self, session_id: str) -> AsyncIterator[None]: - """Count holders and waiters so pruning cannot replace an in-flight ID lock.""" - lock = self._session_locks.setdefault(session_id, asyncio.Lock()) - self._session_lock_users[session_id] = self._session_lock_users.get(session_id, 0) + 1 - try: - async with lock: - yield - finally: - remaining = self._session_lock_users[session_id] - 1 - if remaining: - self._session_lock_users[session_id] = remaining - else: - self._session_lock_users.pop(session_id) - if session_id not in self._sandbox_sessions and session_id not in self._closed_session_ids: - self._session_locks.pop(session_id, None) - - async def seed_agent_session(self, request: Request, body: AgentSeedSessionRequest) -> AgentSeedSessionResponse: - """Borrow the Resources-owned task sandbox and install a pinned OpenClaw runtime inside it.""" - self._expire_closed_agent_sessions() - session_id = body.agent_session_id - previous = request.session.get(_SANDBOX_SESSION_KEY) - if _SANDBOX_SESSION_KEY in request.session and not isinstance(previous, str): - raise HTTPException(409, "Invalid OpenClaw session marker") - if previous != session_id and previous in self._sandbox_sessions: - raise HTTPException(409, "OpenClaw cookie belongs to another active session") - async with self._session_lock(session_id): - if session_id in self._closed_session_ids: - raise HTTPException(409, "OpenClaw session is already closed") - state = self._sandbox_sessions.get(session_id) - if state is not None: - if state.seed != body: - raise HTTPException(409, "OpenClaw session ID is bound to a different seed request") - if state.closing: - raise HTTPException(409, "OpenClaw session is closing") - else: - if previous == session_id: - raise HTTPException(409, "OpenClaw session cookie has expired") - try: - state = await self._initialize_agent_session_state(session_id, body) - except BaseException: - if session_id not in self._sandbox_sessions: - # Initialization either never connected or confirmed its cleanup. - self._closed_sandbox_sessions[session_id] = ( - body.episode_id, - AgentCloseSessionResponse(agent_session_id=session_id), - monotonic() + self.config.session_close_retry_window_seconds, - ) - self._remember_closed_session(session_id, body.episode_id) - else: - self._session_expiry_tasks[session_id] = asyncio.create_task( - self._expire_agent_session(session_id, body.episode_id) - ) - raise - self._sandbox_sessions[session_id] = state - self._session_expiry_tasks[session_id] = asyncio.create_task( - self._expire_agent_session(session_id, body.episode_id) - ) - request.session[_SANDBOX_SESSION_KEY] = session_id - return AgentSeedSessionResponse(agent_session_id=session_id) - async def _expire_agent_session(self, session_id: str, episode_id: EpisodeId) -> None: - """Bound abandoned sessions through the same fail-closed teardown as explicit close.""" - try: - await asyncio.sleep(self.config.session_lifetime_seconds) - await self.close_agent_session( - Request({"type": "http", "session": {}}), - AgentCloseSessionRequest(agent_session_id=session_id, episode_id=episode_id), - ) - except asyncio.CancelledError: - raise - except Exception: - LOG.exception("Could not clean up expired OpenClaw session %s; retaining failed state", session_id) - finally: - self._session_expiry_tasks.pop(session_id, None) - - async def _initialize_agent_session_state( - self, agent_session_id: str, body: AgentSeedSessionRequest - ) -> OpenClawSandboxSession: - # Match Hermes: session initialization owns the sandbox runtime setup, - # with the same AgentSeedSessionRequest/SandboxAccess wire contracts. - if self.config.num_workers not in (None, 1): - raise HTTPException(422, "Native OpenClaw sessions require num_workers=1") + async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> OpenClawSandboxSession: + """Install a pinned runtime in the Resources-owned sandbox using the shared session lifecycle.""" if body.sandbox_access is None or not isinstance(body.sandbox_access.connection, DirectSandboxConnection): raise HTTPException(422, "Native OpenClaw requires direct, Resources-owned SandboxAccess") if not body.sandbox_access.workdir.startswith("/") or body.sandbox_access.workdir in ("/", "/tmp"): @@ -617,7 +518,7 @@ async def _initialize_agent_session_state( f"{installed.stderr or installed.stdout}" ) await sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), f"{directory}/sandbox_runner.py") - except BaseException: + except BaseException as error: try: if prepared_directory: await state.close(self.config.session_close_timeout_seconds) @@ -627,71 +528,30 @@ async def _initialize_agent_session_state( await sandbox.disconnect() state.closed = True except BaseException: - self._sandbox_sessions[agent_session_id] = state - LOG.exception("OpenClaw seed cleanup failed; retaining session %s", agent_session_id) + LOG.exception("OpenClaw seed cleanup failed; retaining session %s", body.agent_session_id) + raise AgentSessionSetupError(state, error=error) from error raise return state - async def close_agent_session(self, request: Request, body: AgentCloseSessionRequest) -> AgentCloseSessionResponse: - """Confirm OpenClaw teardown before allowing verification; never destroy the borrowed sandbox.""" - self._expire_closed_agent_sessions() - session_id = body.agent_session_id - cookie = request.session.get(_SANDBOX_SESSION_KEY) - if cookie is not None and cookie != session_id: - raise HTTPException(409, "OpenClaw close cookie does not match the requested session") - async with self._session_lock(session_id): - closed = self._closed_sandbox_sessions.get(session_id) - if closed is not None: - if body.episode_id != closed[0]: - raise HTTPException(409, "OpenClaw close does not match the seeded episode") - request.session[_SANDBOX_SESSION_KEY] = session_id - return closed[1] - if session_id in self._closed_session_ids: - raise HTTPException(409, "OpenClaw close receipt has expired") - state = self._sandbox_sessions.get(session_id) - if state is None: - if cookie is not None: - raise HTTPException(409, "OpenClaw close receipt has expired") - result = AgentCloseSessionResponse(agent_session_id=session_id) - else: - if body.episode_id != state.seed.episode_id: - raise HTTPException(409, "OpenClaw close does not match the seeded episode") - await state.close(self.config.session_close_timeout_seconds) - observations = state.observations or AgentObservationBundle( - source="openclaw", gaps=[ObservationGap(code="agent_activation_interrupted")] - ) - result = AgentCloseSessionResponse(agent_session_id=session_id, agent_observations=observations) - self._sandbox_sessions.pop(session_id, None) - expiry = self._session_expiry_tasks.pop(session_id, None) - if expiry is not None and expiry is not asyncio.current_task(): - expiry.cancel() - request.session[_SANDBOX_SESSION_KEY] = session_id - self._closed_sandbox_sessions[session_id] = ( - body.episode_id, - result, - monotonic() + self.config.session_close_retry_window_seconds, - ) - self._remember_closed_session(session_id, body.episode_id) - return result - - def _remember_closed_session(self, session_id: str, episode_id: EpisodeId) -> None: - """Prevent delayed seeds through the session lifetime and close retry horizon.""" - retention = max(self.config.session_lifetime_seconds, self.config.session_close_retry_window_seconds) - self._closed_session_ids[session_id] = (episode_id, monotonic() + retention) - asyncio.get_running_loop().call_later(retention, self._expire_closed_agent_sessions) - - def _expire_closed_agent_sessions(self) -> None: - """Prune receipts and tombstones by elapsed time, never by traffic or retry order.""" - now = monotonic() - while self._closed_sandbox_sessions: - if next(iter(self._closed_sandbox_sessions.values()))[2] > now: - break - self._closed_sandbox_sessions.popitem(last=False) - for session_id, (_, deadline) in tuple(self._closed_session_ids.items()): - if deadline <= now: - self._closed_session_ids.pop(session_id) - if not self._session_lock_users.get(session_id, 0): - self._session_locks.pop(session_id, None) + async def _close_agent_session_state(self, state: AgentSessionState) -> AgentCloseSessionResponse: + """Confirm teardown and return observations without destroying the borrowed sandbox.""" + assert isinstance(state, OpenClawSandboxSession) + await state.close(self.config.session_close_timeout_seconds) + observations = state.observations or AgentObservationBundle( + source="openclaw", gaps=[ObservationGap(code="agent_activation_interrupted")] + ) + return AgentCloseSessionResponse( + agent_session_id=state.request.agent_session_id, agent_observations=observations + ) + + def _prompt_input(self, body: NeMoGymResponseCreateParamsNonStreaming) -> tuple[str, str]: + """Normalize prompts identically for local and borrowed-sandbox execution.""" + items = ( + [NeMoGymEasyInputMessage(role="user", content=body.input)] if isinstance(body.input, str) else body.input + ) + prompt, input_system = _extract_instruction(items) + system = "\n\n".join(part for part in (self.config.system_prompt, body.instructions, input_system) if part) + return prompt, system def _sandbox_input(self, body: NeMoGymResponseCreateParamsNonStreaming) -> tuple[str, str]: """Validate and normalize input before consuming the session's activation.""" @@ -743,8 +603,7 @@ def _sandbox_input(self, body: NeMoGymResponseCreateParamsNonStreaming) -> tuple for part in item.content ): raise HTTPException(422, "Native OpenClaw only supports text input") - prompt, input_system = _extract_instruction(items) - system = "\n\n".join(part for part in (self.config.system_prompt, body.instructions, input_system) if part) + prompt, system = self._prompt_input(body) if not prompt.strip(): raise HTTPException(422, "Native OpenClaw requires a nonempty user prompt") return prompt, system @@ -759,7 +618,7 @@ async def _sandbox_response( ) -> NeMoGymResponse: # The pinned CLI reads configuration directly; onboarding would create # workspace bootstrap files and unrelated provider/channel state. - native_config = self._build_openclaw_config({}, state.seed.episode_id.capture_key) + native_config = self._build_openclaw_config({}, state.request.episode_id.capture_key) # OpenClaw conservatively disables stream usage for custom origins. Gym # supports it; request the final usage chunk instead of retaining zero counters. native_config["models"]["providers"]["nemo"]["models"][0]["compat"] = { @@ -769,7 +628,7 @@ async def _sandbox_response( { "agents": { "defaults": { - "workspace": state.seed.sandbox_access.workdir, + "workspace": state.request.sandbox_access.workdir, "skipBootstrap": True, "skills": [], "model": {"primary": self._effective_model()}, @@ -808,7 +667,7 @@ async def _sandbox_response( "directory": state.directory, "command": command, "prompt": f"{system}\n\n{prompt}" if system else prompt, - "cwd": state.seed.sandbox_access.workdir, + "cwd": state.request.sandbox_access.workdir, "env": { "HOME": f"{state.directory}/home", "XDG_CACHE_HOME": f"{state.directory}/home/.cache", @@ -834,7 +693,12 @@ async def _sandbox_response( except Exception: LOG.exception("Could not salvage interrupted OpenClaw observations") raise - return await self._collect_sandbox_response(state, body, prompt=prompt, system=system, stdout=stdout) + response = await self._collect_sandbox_response(state, body, prompt=prompt, system=system, stdout=stdout) + if response.error is not None: + # Preserve the transcript in session observations, but let the environment + # classify runtime/provider failures before it invokes the verifier. + raise HTTPException(502, response.error.message) + return response async def _collect_sandbox_response( self, @@ -954,7 +818,9 @@ def count(name: str) -> int: result = state.result error = result.error if result else "OpenClaw activation ended without a cleanup receipt" last = assistants[-1] if assistants else {} - if last.get("stopReason") in {"error", "aborted"}: + if last.get("stopReason") == "error" or ( + last.get("stopReason") == "aborted" and not (result and result.timed_out) + ): error = error or last.get("errorMessage") or f"OpenClaw model call {last['stopReason']}" if result and result.return_code and not result.timed_out: try: @@ -1325,9 +1191,7 @@ async def _create_response( if isinstance(body.input, str): body.input = [NeMoGymEasyInputMessage(role="user", content=body.input)] - user_message, input_system = _extract_instruction(body.input) - system_parts = [p for p in [self.config.system_prompt, input_system] if p] - system_prompt = "\n\n".join(system_parts) if system_parts else None + user_message, system_prompt = self._prompt_input(body) try: output_items, usage, model_name = await self._run_openclaw( @@ -1381,29 +1245,30 @@ async def responses( request: Request, body: NeMoGymResponseCreateParamsNonStreaming = Body(), ) -> NeMoGymResponse: + session_id = self._agent_session_id_from_request(request) try: - session_id = request.session.get(_SANDBOX_SESSION_KEY) - has_session = isinstance(request.session, Mapping) and _SANDBOX_SESSION_KEY in request.session + legacy_marker = _SANDBOX_SESSION_KEY in request.session except (AssertionError, AttributeError): - session_id, has_session = None, False - if has_session: - if not isinstance(session_id, str): - raise HTTPException(409, "Invalid OpenClaw session marker") - state = self._sandbox_sessions.get(session_id) - rollout_id = request.path_params.get("rollout_id") - if state is None or state.seed.episode_id.capture_key != rollout_id: + legacy_marker = False + if legacy_marker: + raise HTTPException(409, "Obsolete OpenClaw session cookie; seed a new agent session") + if session_id is not None: + state = self._require_agent_session(session_id) + assert isinstance(state, OpenClawSandboxSession) + if state.request.episode_id.capture_key != request.path_params.get("rollout_id"): raise HTTPException(409, "OpenClaw activation does not match the seeded session and rollout route") - if state.activated or state.closing: - raise HTTPException(409, "OpenClaw sandbox sessions support one activation") prompt, system = self._sandbox_input(body) - state.activated = True - state.task = asyncio.create_task(self._sandbox_response(state, body, prompt=prompt, system=system)) - try: - return await asyncio.shield(state.task) - except asyncio.CancelledError: - if not state.task.done() and not state.task.cancelling(): - state.task.cancel() - raise + if state.task is None: + # Bind the immutable request before yielding, so identical retries + # share one invocation and different requests cannot replace it. + state.activation_request = body.model_copy(deep=True) + state.task = asyncio.create_task( + self._sandbox_response(state, state.activation_request, prompt=prompt, system=system) + ) + elif body != state.activation_request: + raise HTTPException(409, "OpenClaw sandbox sessions support one activation; retry the same request") + # Session close owns cancellation, not an individual HTTP waiter. + return (await asyncio.shield(state.task)).model_copy(deep=True) path_params = getattr(request, "path_params", None) rollout_id = path_params.get("rollout_id") if isinstance(path_params, Mapping) else None if not isinstance(rollout_id, str): @@ -1493,6 +1358,8 @@ def collect_output(value: list[Any]) -> None: return AgentEpisode(response=response, observations=observations) async def run(self, request: Request, body: OpenClawAgentRunRequest) -> OpenClawAgentVerifyResponse: + if self._agent_session_id_from_request(request) is not None: + raise HTTPException(409, "Native OpenClaw sessions must use EnvironmentServer /run") try: if isinstance(request.session, Mapping) and _SANDBOX_SESSION_KEY in request.session: raise HTTPException(409, "Native OpenClaw sessions must use EnvironmentServer /run") diff --git a/responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml b/responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml index a82a6bcf4b6..08a9521b4f6 100644 --- a/responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml +++ b/responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml @@ -10,4 +10,3 @@ openclaw_agent_native: sandbox_install_timeout_seconds: 900 session_close_timeout_seconds: 60 session_close_retry_window_seconds: 300 - session_lifetime_seconds: 21600 diff --git a/responses_api_agents/openclaw_agent/sandbox.py b/responses_api_agents/openclaw_agent/sandbox.py index a441fd58045..f4001f7c46a 100644 --- a/responses_api_agents/openclaw_agent/sandbox.py +++ b/responses_api_agents/openclaw_agent/sandbox.py @@ -13,8 +13,8 @@ from pydantic import BaseModel, ConfigDict, JsonValue -from nemo_gym.base_responses_api_agent import AgentSeedSessionRequest -from nemo_gym.openai_utils import NeMoGymResponse +from nemo_gym.base_responses_api_agent import AgentSessionState +from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming from nemo_gym.rollout_observability import AgentObservationBundle from nemo_gym.sandbox import AsyncSandbox from nemo_gym.sandbox.providers.base import SandboxExecResult @@ -47,10 +47,9 @@ class OpenClawSandboxResult(BaseModel): @dataclass -class OpenClawSandboxSession: +class OpenClawSandboxSession(AgentSessionState): """Worker-local session with retryable, fail-closed runner teardown.""" - seed: AgentSeedSessionRequest sandbox: AsyncSandbox directory: str runtime: str @@ -58,7 +57,7 @@ class OpenClawSandboxSession: exec_task: asyncio.Task[SandboxExecResult] | None = None result: OpenClawSandboxResult | None = None observations: AgentObservationBundle | None = None - activated: bool = False + activation_request: NeMoGymResponseCreateParamsNonStreaming | None = None closing: bool = False launch_started: bool = False closed: bool = False @@ -114,7 +113,7 @@ async def stop_runner(self, timeout: float) -> None: f"for _ in $(seq 1 {max(1, int(timeout))}); do " f"[ -f {quote(receipt_path)} ] && exit 0; sleep 1; done; exit 1" ) - await self.sandbox.exec(script, cwd=self.seed.sandbox_access.workdir, timeout_s=timeout + 5) + await self.sandbox.exec(script, cwd=self.request.sandbox_access.workdir, timeout_s=timeout + 5) try: result = OpenClawSandboxResult.model_validate_json(await self.read_text("result.json")) except Exception as error: @@ -183,7 +182,7 @@ async def execute(self, payload: dict[str, JsonValue], *, timeout: float, close_ # Leave extra time for cleanup and transport before provider timeout. self.exec_task = asyncio.create_task( self.sandbox.exec( - command, cwd=self.seed.sandbox_access.workdir, timeout_s=timeout + close_timeout * 3 + 30 + command, cwd=self.request.sandbox_access.workdir, timeout_s=timeout + close_timeout * 3 + 30 ) ) # HTTP cancellation must not propagate into provider exec before diff --git a/responses_api_agents/openclaw_agent/tests/test_exec_transport.py b/responses_api_agents/openclaw_agent/tests/test_exec_transport.py index 93eea7d1493..7e911e2c60a 100644 --- a/responses_api_agents/openclaw_agent/tests/test_exec_transport.py +++ b/responses_api_agents/openclaw_agent/tests/test_exec_transport.py @@ -78,7 +78,7 @@ def payload(state, code, timeout=0.5): return { "directory": state.directory, "prompt": "task", - "cwd": state.seed.sandbox_access.workdir, + "cwd": state.request.sandbox_access.workdir, "command": [sys.executable, "-c", code], "env": {}, "timeout": timeout, diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index b1c7e09c479..00fcc6b7617 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -197,7 +197,7 @@ def test_http_native_flow_runs_openclaw_in_borrowed_sandbox(setup): created = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")) assert created.status_code == 200, created.text session_id = created.json()["agent_session_id"] - directory = agent._sandbox_sessions[session_id].directory + directory = agent._session_records[session_id].state.directory installer = f"{directory}/install_openclaw_runtime.sh" assert installer in sandbox.files assert agent.config.resources_server is None @@ -243,7 +243,7 @@ def test_http_native_flow_runs_openclaw_in_borrowed_sandbox(setup): # so a plain transcript cannot establish full usage coverage either. assert "auxiliary_model_usage_unavailable" in [gap["code"] for gap in observations["gaps"]] assert len(observations["records"][0]["model_calls"]) == 2 - assert not agent._sandbox_sessions + assert all(record.state is None for record in agent._session_records.values()) assert agent._local_setup_task is None sandbox.disconnect.assert_awaited_once() sandbox.stop.assert_not_awaited() @@ -277,7 +277,11 @@ def test_unsupported_seed_rejected_before_connection(setup, option): {"kind": "direct_http", "name": "tools", "base_url": "http://resources", "required": True} ] with TestClient(agent.setup_webserver()) as client: - assert client.post("/v1/agent_sessions", json=body).status_code == 422 + if option == "worker": + with pytest.raises(ValueError, match="num_workers=1"): + client.post("/v1/agent_sessions", json=body) + else: + assert client.post("/v1/agent_sessions", json=body).status_code == 422 sandbox.exec.assert_not_awaited() @@ -291,23 +295,41 @@ def test_cookie_identity_and_single_activation(setup): bad = close_body(session_id) bad["episode_id"]["attempt"] = 99 assert client.post("/v1/agent_sessions/close", json=bad).status_code == 409 - assert client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}).status_code == 200 - assert client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}).status_code == 409 + first = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}) + retry = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}) + assert first.status_code == retry.status_code == 200 + assert first.json() == retry.json() + assert ( + client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "different"}).status_code == 409 + ) assert client.post("/v1/agent_sessions/close", json=close_body(session_id)).status_code == 200 sandbox.launch.assert_awaited_once() -@pytest.mark.parametrize("reason,expected", [("error", "failed"), ("aborted", "failed"), ("length", "incomplete")]) -def test_failed_or_partial_openclaw_output_is_preserved(setup, reason, expected): +@pytest.mark.parametrize("reason", ["error", "aborted", "length"]) +def test_failed_or_partial_openclaw_output_is_preserved(setup, reason): agent, sandbox = setup sandbox.events = events(stop_reason=reason) with TestClient(agent.setup_webserver()) as client: session_id = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).json()["agent_session_id"] - result = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}) - assert result.status_code == 200 - assert result.json()["status"] == expected - assert result.json()["output"][-1]["content"][0]["text"] == "Fixed" - assert client.post("/v1/agent_sessions/close", json=close_body(session_id)).status_code == 200 + path = "/ng-rollout/openclaw-smoke-a2/v1/responses" + result = client.post(path, json={"input": "task"}) + if reason == "length": + assert result.status_code == 200 + assert result.json()["status"] == "incomplete" + assert result.json()["output"][-1]["content"][0]["text"] == "Fixed" + else: + assert result.status_code == 502 + assert "model" in result.json()["detail"] + retry = client.post(path, json={"input": "task"}) + assert retry.status_code == 502 + assert retry.json() == result.json() + closed = client.post("/v1/agent_sessions/close", json=close_body(session_id)) + assert closed.status_code == 200 + invocation = closed.json()["agent_observations"]["records"][0] + assert invocation["status"] == ("incomplete" if reason == "length" else "failed") + assert invocation["conversation"][-1]["content"][0]["text"] == "Fixed" + sandbox.launch.assert_awaited_once() @pytest.mark.parametrize( @@ -341,7 +363,7 @@ def test_rejected_request_does_not_consume_activation(setup): sandbox.launch.assert_not_awaited() accepted = client.post(path, json={"input": "task"}) assert accepted.status_code == 200, accepted.text - assert client.post(path, json={"input": "task"}).status_code == 409 + assert client.post(path, json={"input": "task"}).json() == accepted.json() sandbox.launch.assert_awaited_once() @@ -377,6 +399,83 @@ async def test_close_cancels_active_openclaw_before_detaching(setup): sandbox.stop.assert_not_awaited() +async def test_disconnected_waiter_and_identical_retry_share_one_activation(setup): + agent, sandbox = setup + sandbox.blocked = True + request, session_id, waiter = await activate(agent, sandbox) + retry = asyncio.create_task(agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="task"))) + await asyncio.sleep(0) + waiter.cancel() + with pytest.raises(asyncio.CancelledError): + await waiter + state = agent._session_records[session_id].state + assert not state.task.done() + sandbox.request_stop.assert_not_awaited() + with pytest.raises(HTTPException, match="retry the same request"): + await agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="different task")) + sandbox.exited.set() + response = await asyncio.wait_for(retry, 2) + original = response.model_copy(deep=True) + response.output.clear() + replay = await agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="task")) + assert replay == original + sandbox.launch.assert_awaited_once() + await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) + + +@pytest.mark.parametrize("stop_reason", ["stop", "aborted"]) +def test_wall_limit_returns_gradable_partial_output(setup, stop_reason): + agent, sandbox = setup + sandbox.result.update(timed_out=True, return_code=-15) + sandbox.events = events(stop_reason=stop_reason) + with TestClient(agent.setup_webserver()) as client: + session_id = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).json()["agent_session_id"] + response = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}) + assert response.status_code == 200 + assert response.json()["status"] == "incomplete" + assert response.json()["output"][-1]["content"][0]["text"] == "Fixed" + assert client.post("/v1/agent_sessions/close", json=close_body(session_id)).status_code == 200 + + +@pytest.mark.parametrize("failure", ["runner", "exit"]) +def test_runtime_failures_cannot_become_successful_responses(setup, failure): + agent, sandbox = setup + sandbox.result.update(error="runner failed" if failure == "runner" else None, return_code=1) + with TestClient(agent.setup_webserver()) as client: + session_id = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).json()["agent_session_id"] + response = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}) + assert response.status_code == 502 + assert ("runner failed" if failure == "runner" else "exited 1") in response.json()["detail"] + closed = client.post("/v1/agent_sessions/close", json=close_body(session_id)) + invocation = closed.json()["agent_observations"]["records"][0] + assert invocation["status"] == "failed" + assert invocation["conversation"][-1]["content"][0]["text"] == "Fixed" + + +async def test_local_and_native_prompts_include_the_same_instructions(setup): + agent, _ = setup + agent.config.system_prompt = "configured system" + body = NeMoGymResponseCreateParamsNonStreaming( + instructions="request instructions", + input=[{"role": "system", "content": "input system"}, {"role": "user", "content": "task"}], + ) + expected = ("task", "configured system\n\nrequest instructions\n\ninput system") + assert agent._sandbox_input(body) == expected + with patch.object(agent, "_run_openclaw", AsyncMock(return_value=([], {}, "test-model"))) as run: + await agent._create_response(body) + assert run.await_args.args == expected + + +@pytest.mark.parametrize("marker", [None, "expired"]) +async def test_obsolete_native_cookie_never_falls_back_to_host(setup, marker): + agent, sandbox = setup + request = Request({"type": "http", "session": {"nemo_gym_openclaw_sandbox_session": marker}}) + with patch.object(agent, "_create_response", AsyncMock(side_effect=AssertionError("host fallback"))): + with pytest.raises(HTTPException, match="Obsolete"): + await agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="task")) + sandbox.exec.assert_not_awaited() + + async def test_failed_cleanup_keeps_handles_and_prevents_close(setup): agent, sandbox = setup sandbox.result["cleanup_confirmed"] = False @@ -385,7 +484,7 @@ async def test_failed_cleanup_keeps_handles_and_prevents_close(setup): await task with pytest.raises(RuntimeError, match="cleanup was not confirmed"): await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) - assert session_id in agent._sandbox_sessions + assert agent._session_records[session_id].state is not None sandbox.disconnect.assert_not_awaited() assert not any("rm -rf" in call.args[0] for call in sandbox.exec.await_args_list) @@ -397,9 +496,9 @@ async def test_disconnect_failure_retains_session_for_retry(setup): sandbox.disconnect.side_effect = [RuntimeError("provider unavailable"), None] with pytest.raises(RuntimeError, match="provider unavailable"): await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) - assert session_id in agent._sandbox_sessions + assert agent._session_records[session_id].state is not None await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) - assert session_id not in agent._sandbox_sessions + assert agent._session_records[session_id].state is None sandbox.launch.assert_awaited_once() @@ -463,13 +562,14 @@ def test_close_retry_and_stale_activation_do_not_run_host_openclaw(setup): assert client.post("/v1/agent_sessions/close", json=bad).status_code == 409 with patch.object(agent, "_run_openclaw", AsyncMock(side_effect=AssertionError("host OpenClaw must not run"))): assert client.post("/v1/responses", json={"input": "task"}).status_code == 409 + client.cookies.clear() assert client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).status_code == 200 sandbox.disconnect.assert_awaited_once() def test_http_close_retry_survives_other_session_closes(setup, monkeypatch): agent, sandbox = setup - monkeypatch.setattr("responses_api_agents.openclaw_agent.app.monotonic", lambda: 100.0) + monkeypatch.setattr("nemo_gym.base_responses_api_agent.monotonic", lambda: 100.0) with TestClient(agent.setup_webserver()) as client: def seed_and_close(index): @@ -498,7 +598,7 @@ def seed_and_close(index): async def test_close_receipt_expires_without_extending_on_retry(setup, monkeypatch): agent, sandbox = setup clock = [100.0] - monkeypatch.setattr("responses_api_agents.openclaw_agent.app.monotonic", lambda: clock[0]) + monkeypatch.setattr("nemo_gym.base_responses_api_agent.monotonic", lambda: clock[0]) agent.config.session_close_retry_window_seconds = 10 request = Request({"type": "http", "session": {}}) session_id = (await agent.seed_agent_session(request, seed())).agent_session_id @@ -510,7 +610,7 @@ async def test_close_receipt_expires_without_extending_on_retry(setup, monkeypat with pytest.raises(HTTPException) as error: await agent.close_agent_session(request, close) assert error.value.status_code == 409 - assert not agent._closed_sandbox_sessions + assert not agent._closed_session_records with patch.object(agent, "_create_episode", AsyncMock(side_effect=AssertionError("host fallback"))): with pytest.raises(HTTPException) as error: await agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="task")) @@ -521,7 +621,7 @@ async def test_close_receipt_expires_without_extending_on_retry(setup, monkeypat async def test_close_retry_window_starts_after_cleanup(setup, monkeypatch): agent, sandbox = setup clock = [100.0] - monkeypatch.setattr("responses_api_agents.openclaw_agent.app.monotonic", lambda: clock[0]) + monkeypatch.setattr("nemo_gym.base_responses_api_agent.monotonic", lambda: clock[0]) agent.config.session_close_retry_window_seconds = 10 request = Request({"type": "http", "session": {}}) session_id = (await agent.seed_agent_session(request, seed())).agent_session_id @@ -555,20 +655,21 @@ async def disconnect(): await asyncio.sleep(0) release.set() first_result, second_result = await asyncio.wait_for(asyncio.gather(first, second), 2) - assert first_result is second_result + assert first_result == second_result + assert first_result is not second_result sandbox.disconnect.assert_awaited_once() async def test_seed_prunes_expired_close_receipts(setup, monkeypatch): agent, _ = setup clock = [100.0] - monkeypatch.setattr("responses_api_agents.openclaw_agent.app.monotonic", lambda: clock[0]) + monkeypatch.setattr("nemo_gym.base_responses_api_agent.monotonic", lambda: clock[0]) request = Request({"type": "http", "session": {}}) session_id = (await agent.seed_agent_session(request, seed())).agent_session_id await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) clock[0] += agent.config.session_close_retry_window_seconds - await agent.seed_agent_session(request, seed()) - assert not agent._closed_sandbox_sessions + await agent.seed_agent_session(Request({"type": "http", "session": {}}), seed()) + assert not agent._closed_session_records @pytest.mark.parametrize("window", [0, -1, float("inf")]) @@ -587,7 +688,7 @@ async def test_unknown_launch_outcome_fails_closed(setup): await agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="task")) with pytest.raises(RuntimeError, match="launch outcome is unknown"): await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) - assert session_id in agent._sandbox_sessions + assert agent._session_records[session_id].state is not None sandbox.disconnect.assert_not_awaited() sandbox.stop.assert_not_awaited() @@ -602,7 +703,7 @@ async def test_install_failure_disconnects_without_stopping_owner(setup): request = Request({"type": "http", "session": {}}) with pytest.raises(RuntimeError, match="npm failed"): await agent.seed_agent_session(request, seed()) - assert not agent._sandbox_sessions + assert all(record.state is None for record in agent._session_records.values()) assert not request.session sandbox.disconnect.assert_awaited_once() sandbox.stop.assert_not_awaited() @@ -618,7 +719,7 @@ async def test_cancelled_install_never_publishes_session_or_launches_openclaw(se request = Request({"type": "http", "session": {}}) with pytest.raises(asyncio.CancelledError): await agent.seed_agent_session(request, seed()) - assert not agent._sandbox_sessions + assert all(record.state is None for record in agent._session_records.values()) assert not request.session sandbox.launch.assert_not_awaited() sandbox.disconnect.assert_awaited_once() @@ -700,18 +801,23 @@ async def test_cancelled_activation_keeps_partial_observations_for_close(setup): assert len(invocation.model_calls) == 2 -def test_usage_sums_cache_writes_and_failed_calls(setup): +async def test_usage_sums_cache_writes_and_failed_calls(setup): agent, sandbox = setup transcript = [json.loads(line) for line in events(stop_reason="error").splitlines()] transcript[0]["message"]["usage"]["cacheWrite"] = 4 sandbox.events = "\n".join(json.dumps(event) for event in transcript) - with TestClient(agent.setup_webserver()) as client: - client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).raise_for_status() - response = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}) - assert response.json()["status"] == "failed" - assert response.json()["usage"]["input_tokens"] == 21 - assert response.json()["usage"]["output_tokens"] == 5 - assert response.json()["usage"]["total_tokens"] == 26 + request, session_id, task = await activate(agent, sandbox) + with pytest.raises(HTTPException, match="model error"): + await task + state = agent._session_records[session_id].state + response = await agent._collect_sandbox_response( + state, NeMoGymResponseCreateParamsNonStreaming(input="task"), prompt="task", system="" + ) + assert response.status == "failed" + assert response.usage.input_tokens == 21 + assert response.usage.output_tokens == 5 + assert response.usage.total_tokens == 26 + await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) @pytest.mark.parametrize( @@ -847,15 +953,8 @@ def test_rewritten_transcript_and_cli_mirror_count_each_model_call_once(setup): with TestClient(agent.setup_webserver()) as client: session_id = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).json()["agent_session_id"] response = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}) - assert response.status_code == 200, response.text - body = response.json() - assert body["status"] == "failed" - assert body["error"]["message"] == "Context overflow recovery exhausted" - assert body["usage"]["input_tokens"] == 93255 - assert body["usage"]["output_tokens"] == 214 - assert body["usage"]["total_tokens"] == 93469 - assert sum(item["type"] == "function_call" for item in body["output"]) == 11 - assert body["output"][-1]["content"][0]["text"] == "Context overflow recovery exhausted" + assert response.status_code == 502, response.text + assert response.json()["detail"] == "Context overflow recovery exhausted" closed = client.post("/v1/agent_sessions/close", json=close_body(session_id)) assert closed.status_code == 200, closed.text observations = closed.json()["agent_observations"] @@ -947,7 +1046,7 @@ async def test_invalid_or_expired_native_cookie_never_uses_local_execution(setup request = Request( { "type": "http", - "session": {"nemo_gym_openclaw_sandbox_session": marker}, + "session": {"agent_session_id": marker}, "path_params": {"rollout_id": "openclaw-smoke-a2"}, } ) @@ -995,7 +1094,7 @@ async def test_invalid_envelope_usage_does_not_discard_valid_transcript(setup): agent, sandbox = setup request, session_id, task = await activate(agent, sandbox) await task - state = agent._sandbox_sessions[session_id] + state = agent._session_records[session_id].state response = await agent._collect_sandbox_response( state, NeMoGymResponseCreateParamsNonStreaming(input="task"), @@ -1027,7 +1126,7 @@ async def test_native_envelope_fallback_preserves_known_and_unknown_cache_detail agent, sandbox = setup request, session_id, task = await activate(agent, sandbox) await task - state = agent._sandbox_sessions[session_id] + state = agent._session_records[session_id].state sandbox.files[f"{state.directory}/home/.openclaw/agents/main/sessions/{Path(state.directory).name}.jsonl"] = "" response = await agent._collect_sandbox_response( state, @@ -1130,7 +1229,7 @@ async def test_caller_id_seed_retries_share_one_session_without_cookies(setup): requests = [Request({"type": "http", "session": {}}) for _ in range(2)] first, second = await asyncio.gather(*(agent.seed_agent_session(request, body) for request in requests)) assert first.agent_session_id == second.agent_session_id == body.agent_session_id - assert len(agent._sandbox_sessions) == 1 + assert len(agent._session_records) == 1 assert sandbox.exec.await_count == 2 # One path check and one installation. assert requests[0].session == requests[1].session await agent.close_agent_session( @@ -1153,7 +1252,7 @@ async def test_caller_id_rejects_changed_seed_binding(setup, field): changed.task_id = changed.task_id.model_copy(update={"task_id": "another-task"}) else: changed.sandbox_access.workdir = "/another-repository" - with pytest.raises(HTTPException, match="different seed"): + with pytest.raises(HTTPException, match="another seed"): await agent.seed_agent_session(Request({"type": "http", "session": {}}), changed) await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(body.agent_session_id))) @@ -1164,7 +1263,7 @@ async def test_close_before_seed_prevents_late_creation(setup): request = Request({"type": "http", "session": {}}) close = AgentCloseSessionRequest(agent_session_id=body.agent_session_id, episode_id=body.episode_id) first = await agent.close_agent_session(request, close) - assert await agent.close_agent_session(Request({"type": "http", "session": {}}), close) is first + assert await agent.close_agent_session(Request({"type": "http", "session": {}}), close) == first with pytest.raises(HTTPException, match="already closed"): await agent.seed_agent_session(Request({"type": "http", "session": {}}), body) sandbox.exec.assert_not_awaited() @@ -1174,14 +1273,14 @@ async def test_close_serializes_with_inflight_seed(setup): agent, sandbox = setup body = seed() entered, release = asyncio.Event(), asyncio.Event() - original = agent._initialize_agent_session_state + original = agent._seed_agent_session_state async def blocked_initialize(*args): entered.set() await release.wait() return await original(*args) - with patch.object(agent, "_initialize_agent_session_state", side_effect=blocked_initialize): + with patch.object(agent, "_seed_agent_session_state", side_effect=blocked_initialize): pending = asyncio.create_task(agent.seed_agent_session(Request({"type": "http", "session": {}}), body)) await entered.wait() close = asyncio.create_task( @@ -1194,7 +1293,7 @@ async def blocked_initialize(*args): assert not close.done() release.set() await asyncio.wait_for(asyncio.gather(pending, close), 2) - assert not agent._sandbox_sessions + assert all(record.state is None for record in agent._session_records.values()) sandbox.disconnect.assert_awaited_once() @@ -1204,60 +1303,29 @@ async def test_caller_id_never_controls_filesystem_path(setup): request = Request({"type": "http", "session": {}}) response = await agent.seed_agent_session(request, body) assert response.agent_session_id == body.agent_session_id - directory = agent._sandbox_sessions[body.agent_session_id].directory + directory = agent._session_records[body.agent_session_id].state.directory assert Path(directory).parent == Path("/tmp/nemo-gym-openclaw-sessions") assert len(Path(directory).name) == 32 await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(body.agent_session_id))) -async def test_abandoned_session_expires_through_normal_cleanup(setup): - agent, sandbox = setup - agent.config.session_lifetime_seconds = 0.01 - body = seed() - await agent.seed_agent_session(Request({"type": "http", "session": {}}), body) - expiry = agent._session_expiry_tasks[body.agent_session_id] - await asyncio.wait_for(asyncio.shield(expiry), 2) - assert not agent._sandbox_sessions - assert body.agent_session_id in agent._closed_sandbox_sessions - sandbox.disconnect.assert_awaited_once() - sandbox.stop.assert_not_awaited() - - -async def test_failed_expiry_retains_state_and_rejects_activation(setup, caplog): - agent, sandbox = setup - agent.config.session_lifetime_seconds = 0.01 - body = seed() - request = Request({"type": "http", "session": {}, "path_params": {"rollout_id": body.episode_id.capture_key}}) - await agent.seed_agent_session(request, body) - sandbox.disconnect.side_effect = RuntimeError("disconnect unavailable") - await asyncio.wait_for(asyncio.shield(agent._session_expiry_tasks[body.agent_session_id]), 2) - assert agent._sandbox_sessions[body.agent_session_id].closing - assert "retaining failed state" in caplog.text - with pytest.raises(HTTPException): - await agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="task")) - sandbox.launch.assert_not_awaited() - sandbox.disconnect.side_effect = None - await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(body.agent_session_id))) - - -@pytest.mark.parametrize("lifetime", [0, -1, float("inf"), float("nan")]) -def test_session_lifetime_is_positive_and_finite(setup, lifetime): +@pytest.mark.parametrize("field", ["timeout", "sandbox_install_timeout_seconds", "session_close_timeout_seconds"]) +@pytest.mark.parametrize("value", [0, -1, float("inf"), float("nan")]) +def test_deadlines_are_positive_and_finite(setup, field, value): agent, _ = setup with pytest.raises(ValidationError): - OpenClawAgentConfig(**(agent.config.model_dump() | {"session_lifetime_seconds": lifetime})) + OpenClawAgentConfig(**(agent.config.model_dump() | {field: value})) -async def test_native_recipe_routes_collector_through_environment_run(setup, monkeypatch): +@pytest.mark.parametrize("failed", [False, True]) +async def test_independent_configs_route_prepared_rows_through_environment_run(setup, monkeypatch, failed): from http.cookies import SimpleCookie import orjson import nemo_gym.rollout_collection as collection - from benchmarks.swebench.pro.materialize_single_agent_tasks import materialize_row - from environment_servers.single_agent_turn.app import ( - SingleAgentTurnEnvironmentServer, - SingleAgentTurnEnvironmentServerConfig, - ) + from environment_servers.single_agent_turn.app import SingleAgentTurnEnvironmentServerConfig + from environment_servers.single_agent_turn_legacy.app import SingleAgentTurnLegacyEnvironmentServer from nemo_gym.global_config import GlobalConfigDictParser, GlobalConfigDictParserConfig agent, sandbox = setup @@ -1265,7 +1333,22 @@ async def test_native_recipe_routes_collector_through_environment_run(setup, mon GlobalConfigDictParserConfig( initial_global_config_dict=OmegaConf.create( { - "config_paths": ["benchmarks/swebench/pro/openclaw_native.yaml"], + "config_paths": [ + "resources_servers/swebench_pro/configs/swebench_pro.yaml", + "responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml", + "environment_servers/single_agent_turn_legacy/configs/single_agent_turn_legacy.yaml", + ], + "single_agent_turn_legacy": { + "environment_servers": { + "single_agent_turn_legacy": { + "resources_server": { + "type": "resources_servers", + "name": "swebench_pro_resources_server", + }, + "agent_server": {"type": "responses_api_agents", "name": "openclaw_agent_native"}, + } + } + }, "policy_model_name": "test-model", "policy_model": {"responses_api_models": {"openai_model": {"entrypoint": "app.py"}}}, } @@ -1275,8 +1358,13 @@ async def test_native_recipe_routes_collector_through_environment_run(setup, mon offline=True, ) ) - env_name = global_config.environment_server_routes["swebench_pro:smoke"] - env_config = global_config[env_name].environment_servers.single_agent_turn + # The benchmark's legacy allowlist predates this native harness. Keep its + # definition untouched and explicitly opt in, as the CLI smoke does. + monkeypatch.setenv("NEMO_GYM_ALLOW_UNSUPPORTED_PAIRING", "1") + if failed: + sandbox.events = events(stop_reason="error") + env_name = "single_agent_turn_legacy" + env_config = global_config[env_name].environment_servers.single_agent_turn_legacy agent_name = env_config.agent_server.name resources_name = env_config.resources_server.name agent.config = OpenClawAgentConfig( @@ -1287,7 +1375,7 @@ async def test_native_recipe_routes_collector_through_environment_run(setup, mon assert not agent.config.openclaw_config agent.server_client.global_config_dict = global_config agent.server_client._resolve_base_url.return_value = "http://resources.example:8000" - environment = SingleAgentTurnEnvironmentServer( + environment = SingleAgentTurnLegacyEnvironmentServer( config=SingleAgentTurnEnvironmentServerConfig(name=env_name, **OmegaConf.to_container(env_config)), server_client=agent.server_client, ) @@ -1322,8 +1410,8 @@ async def post(*, server_name, url_path, json, cookies=None, **kwargs): assert cookies == {"resources": resources_id} if url_path == "/verify": sandbox.disconnect.assert_awaited_once() - assert payload["verification_input"]["response"]["status"] == "completed" - return Reply({"reward": 1.0, **payload["verification_input"]}) + assert payload["response"]["status"] == "completed" + return Reply({"reward": 1.0, **payload}) assert url_path == "/close_session" assert payload["resources_session_id"] == resources_id return Reply({"resources_session_id": resources_id}) @@ -1337,40 +1425,49 @@ async def post(*, server_name, url_path, json, cookies=None, **kwargs): agent.server_client.post = AsyncMock(side_effect=post) monkeypatch.setattr(collection, "setup_server_client_utils", lambda *args, **kwargs: agent.server_client) - materialized = materialize_row( - {"responses_create_params": {"input": "Fix the code"}, "instance_id": "smoke-instance"}, - taskset="swebench_pro:smoke", - ) + prepared = { + "responses_create_params": {"input": "Fix the code"}, + "instance_id": "smoke-instance", + "task_source": resources_name, + } config = collection.RolloutCollectionConfig( input_jsonl_fpath="unused.jsonl", output_jsonl_fpath="unused-rollouts.jsonl", - environment_routing_mode=global_config.environment_routing_mode, - environment_server_routes=OmegaConf.to_container(global_config.environment_server_routes), + environment_routing_mode="legacy", + environment_server_name=env_name, num_repeats=1, ) rows = collection.RolloutCollectionHelper._preprocess_raw_rows( - [(0, orjson.dumps(materialized).decode(), materialized)], config + [(0, orjson.dumps(prepared).decode(), prepared)], config ) with ( TestClient(agent.setup_webserver()) as agent_http, TestClient(environment.setup_webserver()) as environment_http, ): _, result = await next(collection.RolloutCollectionHelper().run_examples(rows)) - assert result.get("result") is not None, result - assert result["result"]["verification"]["reward"] == 1.0 - assert result["result"]["agent_observations"]["source"] == "openclaw" - assert [(server_name, path) for server_name, path, _ in calls] == [ + if failed: + assert result["_ng_failure_class"] == "environment_server_failed", result + assert result["_ng_failure_stage"] == "agent" + assert "reward" not in result + else: + assert "verification" not in result + assert result.get("reward") == 1.0, result + assert result["ng_agent_observations"]["source"] == "openclaw" + expected = [ (env_name, "/run"), (resources_name, "/seed_session"), (agent_name, "/v1/agent_sessions"), (agent_name, "/ng-rollout/0-0/v1/responses"), (agent_name, "/v1/agent_sessions/close"), - (resources_name, "/verify"), - (resources_name, "/close_session"), ] + if not failed: + expected.append((resources_name, "/verify")) + expected.append((resources_name, "/close_session")) + assert [(server_name, path) for server_name, path, _ in calls] == expected assert calls[2][2]["agent_session_id"] == calls[4][2]["agent_session_id"] - assert calls[0][2]["task"]["task_input"] == materialized["task_input"] - assert not agent._sandbox_sessions + assert calls[0][2]["instance_id"] == prepared["instance_id"] + assert calls[0][2]["responses_create_params"]["input"] == prepared["responses_create_params"]["input"] + assert all(record.state is None for record in agent._session_records.values()) sandbox.stop.assert_not_awaited() @@ -1388,7 +1485,7 @@ async def test_failed_seed_with_confirmed_cleanup_accepts_cookie_less_close(setu result = await agent.close_agent_session(Request({"type": "http", "session": {}}), close) assert result.agent_session_id == body.agent_session_id sandbox.disconnect.assert_awaited_once() - assert not agent._sandbox_sessions + assert all(record.state is None for record in agent._session_records.values()) async def test_failed_seed_cleanup_retains_state_until_explicit_retry(setup): @@ -1402,23 +1499,23 @@ async def test_failed_seed_cleanup_retains_state_until_explicit_retry(setup): sandbox.disconnect.side_effect = RuntimeError("disconnect unavailable") with pytest.raises(RuntimeError, match="npm unavailable"): await agent.seed_agent_session(Request({"type": "http", "session": {}}), body) - assert agent._sandbox_sessions[body.agent_session_id].closing - assert body.agent_session_id not in agent._closed_sandbox_sessions + assert agent._session_records[body.agent_session_id].state.closing + assert body.agent_session_id not in agent._closed_session_records with pytest.raises(HTTPException, match="closing"): await agent.seed_agent_session(Request({"type": "http", "session": {}}), body) sandbox.exec.side_effect = None sandbox.disconnect.side_effect = None close = AgentCloseSessionRequest(agent_session_id=body.agent_session_id, episode_id=body.episode_id) await agent.close_agent_session(Request({"type": "http", "session": {}}), close) - assert not agent._sandbox_sessions + assert all(record.state is None for record in agent._session_records.values()) sandbox.stop.assert_not_awaited() @pytest.mark.parametrize("marker", [None, [], {}, 7]) async def test_seed_rejects_malformed_native_cookie_before_connecting(setup, marker): agent, sandbox = setup - request = Request({"type": "http", "session": {"nemo_gym_openclaw_sandbox_session": marker}}) - with pytest.raises(HTTPException, match="Invalid OpenClaw session marker"): + request = Request({"type": "http", "session": {"agent_session_id": marker}}) + with pytest.raises(HTTPException, match="Invalid agent session marker"): await agent.seed_agent_session(request, seed()) sandbox.exec.assert_not_awaited() @@ -1426,8 +1523,7 @@ async def test_seed_rejects_malformed_native_cookie_before_connecting(setup, mar async def test_stale_cookie_cannot_recreate_session_or_close_receipt_after_full_retention(setup, monkeypatch): agent, sandbox = setup clock = [100.0] - monkeypatch.setattr("responses_api_agents.openclaw_agent.app.monotonic", lambda: clock[0]) - agent.config.session_lifetime_seconds = 10 + monkeypatch.setattr("nemo_gym.base_responses_api_agent.monotonic", lambda: clock[0]) agent.config.session_close_retry_window_seconds = 5 request = Request({"type": "http", "session": {}}) body = seed() @@ -1439,7 +1535,7 @@ async def test_stale_cookie_cannot_recreate_session_or_close_receipt_after_full_ await agent.close_agent_session(request, close) with pytest.raises(HTTPException, match="expired"): await agent.seed_agent_session(request, body) - assert not agent._closed_sandbox_sessions - assert not agent._sandbox_sessions - assert not agent._session_locks + assert not agent._closed_session_records + assert all(record.state is None for record in agent._session_records.values()) + assert not agent._session_records sandbox.disconnect.assert_awaited_once() From 47d0439602b848bf88b061772c8f6dd4842affa3 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Thu, 1 Oct 2026 18:08:25 -0700 Subject: [PATCH 14/44] docs(openclaw): preserve pairing opt-in across head discovery Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index 71322feaa74..5441640a19e 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -98,8 +98,9 @@ Save it as `native-run.yaml`, then run: ```bash gym env start --config native-run.yaml --model-type openai_model -gym eval run --no-serve --config native-run.yaml --model-type openai_model \ - --agent openclaw_agent_native --allow-unsupported-pairing \ +NEMO_GYM_ALLOW_UNSUPPORTED_PAIRING=1 \ + gym eval run --no-serve --config native-run.yaml --model-type openai_model \ + --agent openclaw_agent_native \ --input /path/to/prepared-swe-pro.jsonl --output outputs/openclaw-native.jsonl --limit 1 ``` @@ -110,8 +111,9 @@ selected Resources server. `--agent` selects the harness for existing rows. No e materialization script is required. SWE-Pro currently has a legacy `allowed_agents` list that excludes OpenClaw. The explicit -`--allow-unsupported-pairing` opt-in permits this smoke without replacing benchmark or verifier -settings. It does not establish compatibility with other benchmarks. Switch compatible components +`NEMO_GYM_ALLOW_UNSUPPORTED_PAIRING=1` opt-in permits this smoke without replacing benchmark or +verifier settings. Use the environment form with `--no-serve`: the current collector reloads its +server config from the head server and can lose the CLI-only `--allow-unsupported-pairing` flag. It does not establish compatibility with other benchmarks. Switch compatible components by changing their independent config paths and the two EnvironmentServer references; keep benchmark data, preparation, and verifier settings with Resources. There is no combined benchmark/harness preset. From 75cbc6596d0c80f11a9d554a07a406c72806fc56 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Fri, 2 Oct 2026 10:48:25 -0700 Subject: [PATCH 15/44] fix(openclaw): allow longer model requests before idle timeout Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/README.md | 11 ++++++++++- responses_api_agents/openclaw_agent/app.py | 5 +++++ .../openclaw_agent/configs/openclaw_agent_native.yaml | 1 + responses_api_agents/openclaw_agent/tests/test_app.py | 11 +++++++++++ .../openclaw_agent/tests/test_sandbox.py | 10 ++++++++-- 5 files changed, 35 insertions(+), 3 deletions(-) diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index 5441640a19e..e6df30ca5fa 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -58,6 +58,9 @@ provider configuration is unchanged. - `system_prompt`: prepended to the user message - `setup_timeout`: seconds for `openclaw setup` - `timeout`: seconds for the `openclaw agent` run +- `model_timeout_seconds`: request and stream-idle timeout for the generated Gym model provider + (default: 600 seconds). OpenClaw still caps this by the overall agent deadline. Explicit provider + `timeoutSeconds` settings in local-mode `openclaw_config` take precedence. - `extra_args`: extra flags appended to `openclaw agent` - `env`: extra env vars for the subprocess (e.g. provider API keys) - `openclaw_config`: deep-merged into the generated `openclaw.json` @@ -123,6 +126,12 @@ calls the rollout-scoped Responses endpoint, closes the agent, verifies, and clo The model server address must be reachable from inside the task sandbox. Its `/ng-rollout/.../v1` route is embedded in OpenClaw's Chat Completions provider configuration to retain model-call linkage. +The native adapter sets OpenClaw's provider `timeoutSeconds` to `model_timeout_seconds` (600 seconds +by default). This raises the upstream 120-second idle watchdog so buffered reasoning responses can +finish. For example, set `openclaw_agent_native.responses_api_agents.openclaw_agent.model_timeout_seconds` +to `1200` for a 20-minute model-call limit. Increase the separate `timeout` and EnvironmentServer episode +deadline when a task needs more total time; increasing those alone does not raise the model-idle limit. + The installer supports Linux glibc on x86_64/aarch64 and musl on x86_64, with Python 3.8 or later for the supervisor. Node 22.19.0 and OpenClaw are installed under `/tmp/nemo-gym-openclaw-node-22.19.0-`. Node archives are checked against upstream SHA-256 @@ -197,7 +206,7 @@ The shared `SimpleResponsesAPIAgent` lifecycle owns session identity, serializat receipts. `session_lifetime_seconds` has been removed: the episode owner and provider TTL govern abandoned-session recovery; there is no adapter timer cancelling active sessions. Setup failures whose cleanup fails retain cleanup-only state through `AgentSessionSetupError`. -`timeout`, `sandbox_install_timeout_seconds`, and `session_close_timeout_seconds` must be positive +`timeout`, `model_timeout_seconds`, `sandbox_install_timeout_seconds`, and `session_close_timeout_seconds` must be positive and finite. The shared `session_close_retry_window_seconds` bounds close receipts and tombstones. Session state is process-local. Keep requests on one worker. Failed cleanup sessions are retained diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index 6693a5554a4..ef66b812b8f 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -403,6 +403,7 @@ class OpenClawAgentConfig(BaseResponsesAPIAgentConfig): system_prompt: Optional[str] = None setup_timeout: int = 900 timeout: int = Field(default=900, gt=0) + model_timeout_seconds: int = Field(default=600, gt=0) extra_args: list[str] = [] openclaw_config: dict[str, Any] = Field(default_factory=dict) context_window: Optional[int] = None @@ -927,6 +928,10 @@ def _build_openclaw_config(self, base: dict[str, Any], rollout_id: Optional[str] "models": [model_entry], } ) + # OpenClaw uses this provider request deadline for its stream-idle + # watchdog too. Gym may buffer a long reasoning call before SSE delivery. + # Preserve explicit provider settings on the legacy local CLI path. + nemo.setdefault("timeoutSeconds", self.config.model_timeout_seconds) self._merge_headless_tool_denies(cfg) return cfg diff --git a/responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml b/responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml index 08a9521b4f6..2807560aef4 100644 --- a/responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml +++ b/responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml @@ -7,6 +7,7 @@ openclaw_agent_native: model: ${policy_model_name} openclaw_version: 2026.6.11 timeout: 900 + model_timeout_seconds: 600 sandbox_install_timeout_seconds: 900 session_close_timeout_seconds: 60 session_close_retry_window_seconds: 300 diff --git a/responses_api_agents/openclaw_agent/tests/test_app.py b/responses_api_agents/openclaw_agent/tests/test_app.py index 02c519b7d2f..3b0aafcfc37 100644 --- a/responses_api_agents/openclaw_agent/tests/test_app.py +++ b/responses_api_agents/openclaw_agent/tests/test_app.py @@ -364,6 +364,17 @@ def test_model_server_builds_local_provider(self) -> None: assert agent._effective_model() == "nemo/Qwen3.6-35B-A3B" assert provider["baseUrl"] == "http://model/v1" assert provider["models"][0]["id"] == "Qwen3.6-35B-A3B" + assert provider["timeoutSeconds"] == 600 + + def test_explicit_local_provider_timeout_is_preserved(self) -> None: + agent = _make_agent( + model_server=ModelServerRef(type="responses_api_models", name="policy_model"), + openclaw_config={"models": {"providers": {"nemo": {"timeoutSeconds": 1800}}}}, + ) + with patch.object(agent, "_resolve_model_base_url", return_value="http://model/v1"): + cfg = agent._build_openclaw_config({}) + + assert cfg["models"]["providers"]["nemo"]["timeoutSeconds"] == 1800 def test_context_window_and_max_tokens_omitted_by_default(self) -> None: # Regression: a static default here previously caused every request to fail unconditionally diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index 00fcc6b7617..7842f6c847b 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -190,8 +190,10 @@ def close_body(session_id): return {"agent_session_id": session_id, "episode_id": seed().episode_id.model_dump()} -def test_http_native_flow_runs_openclaw_in_borrowed_sandbox(setup): +@pytest.mark.parametrize("model_timeout_seconds", [600, 1200]) +def test_http_native_flow_runs_openclaw_in_borrowed_sandbox(setup, model_timeout_seconds): agent, sandbox = setup + agent.config.model_timeout_seconds = model_timeout_seconds with patch.object(agent, "_run_openclaw", AsyncMock(side_effect=AssertionError("host OpenClaw must not run"))): with TestClient(agent.setup_webserver()) as client: created = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")) @@ -226,9 +228,11 @@ def test_http_native_flow_runs_openclaw_in_borrowed_sandbox(setup): payload = json.loads(sandbox.files[f"{sandbox.directory}/input.json"]) assert payload["prompt"] == "Fix the code" assert payload["command"][0].endswith("/node/bin/node") + assert payload["timeout"] == 900 assert "PATH" not in payload["env"] models = json.loads(sandbox.files[f"{sandbox.directory}/home/.openclaw/openclaw.json"]) assert models["agents"]["defaults"]["workspace"] == "/app" + assert models["models"]["providers"]["nemo"]["timeoutSeconds"] == model_timeout_seconds assert models["models"]["providers"]["nemo"]["models"][0]["compat"]["supportsUsageInStreaming"] is True assert ( models["models"]["providers"]["nemo"]["baseUrl"] @@ -1309,7 +1313,9 @@ async def test_caller_id_never_controls_filesystem_path(setup): await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(body.agent_session_id))) -@pytest.mark.parametrize("field", ["timeout", "sandbox_install_timeout_seconds", "session_close_timeout_seconds"]) +@pytest.mark.parametrize( + "field", ["timeout", "model_timeout_seconds", "sandbox_install_timeout_seconds", "session_close_timeout_seconds"] +) @pytest.mark.parametrize("value", [0, -1, float("inf"), float("nan")]) def test_deadlines_are_positive_and_finite(setup, field, value): agent, _ = setup From 04121eac4849b463880dbd5a5d111810cc84fd42 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Fri, 2 Oct 2026 15:01:44 -0700 Subject: [PATCH 16/44] fix(openclaw): support agent-owned native sandboxes Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/README.md | 8 + responses_api_agents/openclaw_agent/app.py | 58 +++++-- .../openclaw_agent/sandbox.py | 31 +++- .../tests/test_exec_transport.py | 4 +- .../openclaw_agent/tests/test_sandbox.py | 164 +++++++++++++++++- 5 files changed, 236 insertions(+), 29 deletions(-) diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index e6df30ca5fa..869197673ce 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -218,3 +218,11 @@ This path is eval-only. Working inference does not establish training token-ID/l A tool-and-verifier smoke is required for the actual image/model configuration before review; unit tests alone do not establish runtime compatibility or benchmark accuracy. Process/memory/setup and close-latency overhead have not been benchmarked at scale. + +Without `sandbox_access`, configure `sandbox_provider` and `sandbox_config` +(`SandboxSpec` fields such as `image`, `workdir`, and `ttl_s`) on the agent. +It creates a sandbox, runs there, and destroys it on close; the default workdir is +`/app`. A supplied access always wins, including its workdir; connection failure +never triggers a replacement or host execution. With neither access nor a usable +provider, setup fails. Verifiers that inspect task files must keep using a +Resources-owned sandbox, since agent-owned sandboxes are gone before verification. diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index ef66b812b8f..f0a17cad9cf 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -64,7 +64,7 @@ AgentObservationBundle, ObservationGap, ) -from nemo_gym.sandbox import AsyncSandbox, create_provider +from nemo_gym.sandbox import AsyncSandbox, SandboxSpec, create_provider from nemo_gym.sandbox.access import DirectSandboxConnection from nemo_gym.sandbox.config import resolve_provider_config from nemo_gym.server_utils import get_global_config_dict, get_response_json, raise_for_status @@ -410,6 +410,8 @@ class OpenClawAgentConfig(BaseResponsesAPIAgentConfig): max_output_tokens: Optional[int] = None # required: every config must pin an explicit version so runs are reproducible and cannot silently drift openclaw_version: str + sandbox_provider: str | None = None + sandbox_config: dict[str, Any] = Field(default_factory=dict) sandbox_install_timeout_seconds: float = Field(default=600, gt=0, allow_inf_nan=False) session_close_timeout_seconds: float = Field(default=60, gt=0, allow_inf_nan=False) @@ -449,10 +451,20 @@ class OpenClawAgent(SimpleResponsesAPIAgent): _local_setup_task: asyncio.Task[None] | None = PrivateAttr(default=None) async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> OpenClawSandboxSession: - """Install a pinned runtime in the Resources-owned sandbox using the shared session lifecycle.""" - if body.sandbox_access is None or not isinstance(body.sandbox_access.connection, DirectSandboxConnection): - raise HTTPException(422, "Native OpenClaw requires direct, Resources-owned SandboxAccess") - if not body.sandbox_access.workdir.startswith("/") or body.sandbox_access.workdir in ("/", "/tmp"): + """Install a pinned runtime in the borrowed or agent-owned session sandbox.""" + owns_sandbox = body.sandbox_access is None + if owns_sandbox: + if not self.config.sandbox_provider: + raise HTTPException(422, "OpenClaw requires sandbox_access or a configured sandbox_provider") + spec = SandboxSpec(**{"workdir": "/app", **self.config.sandbox_config}) + workdir = spec.workdir + provider_ref = self.config.sandbox_provider + else: + if not isinstance(body.sandbox_access.connection, DirectSandboxConnection): + raise HTTPException(422, "OpenClaw requires direct SandboxAccess") + workdir = body.sandbox_access.workdir + provider_ref = body.sandbox_access.connection.provider_config_ref + if not isinstance(workdir, str) or not workdir.startswith("/") or workdir in ("/", "/tmp"): raise HTTPException(422, "OpenClaw workdir must be absolute and separate from /tmp runtime storage") if any(access.required for access in self.effective_tool_accesses(body)): raise HTTPException(422, "Native OpenClaw supports its own sandbox tools, not required HTTP/MCP tools") @@ -471,24 +483,34 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Open if self.config.command != "openclaw" or self.config.extra_args or self.config.env: raise HTTPException(422, "Native OpenClaw does not support command, extra_args, or env overrides") - connection = body.sandbox_access.connection - provider = create_provider(resolve_provider_config(connection.provider_config_ref, get_global_config_dict())) - try: - sandbox = await AsyncSandbox.connect(connection.descriptor, provider=provider) - except BaseException: - await provider.aclose() - raise + provider = create_provider(resolve_provider_config(provider_ref, get_global_config_dict())) + if owns_sandbox: + sandbox = AsyncSandbox(provider) + else: + try: + sandbox = await AsyncSandbox.connect(body.sandbox_access.connection.descriptor, provider=provider) + except BaseException: + await provider.aclose() + raise directory = f"/tmp/nemo-gym-openclaw-sessions/{uuid4().hex}" runtime = f"/tmp/nemo-gym-openclaw-node-22.19.0-{self.config.openclaw_version}" - state = OpenClawSandboxSession(body, sandbox, directory, runtime) + state = OpenClawSandboxSession(body, sandbox, directory, runtime, workdir=workdir, owns_sandbox=owns_sandbox) prepared_directory = False try: + if owns_sandbox: + await sandbox.start(spec) + # Providers need not create SandboxSpec.workdir. Never prepare a borrowed task here. + workspace = await sandbox.exec(f"mkdir -p -- {shlex.quote(workdir)}", cwd="/", timeout_s=30) + if workspace.return_code != 0 or workspace.error_type: + raise RuntimeError( + f"Cannot create OpenClaw sandbox workdir {workdir}: {workspace.stderr or workspace.stdout}" + ) check_paths = shlex.join( [ "python3", "-c", _SANDBOX_PATH_CHECK, - body.sandbox_access.workdir, + workdir, "/tmp/nemo-gym-openclaw-sessions", runtime, ] @@ -509,7 +531,7 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Open installed = await sandbox.exec( f"bash {shlex.quote(directory + '/' + installer)} {shlex.quote(runtime)} " f"{shlex.quote(self.config.openclaw_version)}", - cwd=body.sandbox_access.workdir, + cwd=workdir, timeout_s=self.config.sandbox_install_timeout_seconds, ) if installed.return_code != 0 or getattr(installed, "error_type", None): @@ -521,7 +543,7 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Open await sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), f"{directory}/sandbox_runner.py") except BaseException as error: try: - if prepared_directory: + if prepared_directory or owns_sandbox: await state.close(self.config.session_close_timeout_seconds) else: # The path check may have rejected overlap with task-owned storage. @@ -629,7 +651,7 @@ async def _sandbox_response( { "agents": { "defaults": { - "workspace": state.request.sandbox_access.workdir, + "workspace": state.workdir, "skipBootstrap": True, "skills": [], "model": {"primary": self._effective_model()}, @@ -668,7 +690,7 @@ async def _sandbox_response( "directory": state.directory, "command": command, "prompt": f"{system}\n\n{prompt}" if system else prompt, - "cwd": state.request.sandbox_access.workdir, + "cwd": state.workdir, "env": { "HOME": f"{state.directory}/home", "XDG_CACHE_HOME": f"{state.directory}/home/.cache", diff --git a/responses_api_agents/openclaw_agent/sandbox.py b/responses_api_agents/openclaw_agent/sandbox.py index f4001f7c46a..6cdfca57128 100644 --- a/responses_api_agents/openclaw_agent/sandbox.py +++ b/responses_api_agents/openclaw_agent/sandbox.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""OpenClaw-specific borrowed-sandbox execution; Resources retains sandbox ownership.""" +"""OpenClaw execution in borrowed or agent-owned sandboxes.""" import asyncio import json @@ -53,6 +53,9 @@ class OpenClawSandboxSession(AgentSessionState): sandbox: AsyncSandbox directory: str runtime: str + workdir: str = field(kw_only=True) + owns_sandbox: bool = field(default=False, kw_only=True) + sandbox_stopped: bool = False task: asyncio.Task[NeMoGymResponse] | None = None exec_task: asyncio.Task[SandboxExecResult] | None = None result: OpenClawSandboxResult | None = None @@ -79,7 +82,11 @@ async def read_text(self, name: str) -> str: async def stop_runner(self, timeout: float) -> None: """Fence delayed launches and require a receipt before allowing verification.""" - if not self.launch_started or (self.result is not None and self.result.cleanup_confirmed): + if ( + self.sandbox_stopped + or not self.launch_started + or (self.result is not None and self.result.cleanup_confirmed) + ): return receipt_path = f"{self.directory}/result.json" try: @@ -113,7 +120,7 @@ async def stop_runner(self, timeout: float) -> None: f"for _ in $(seq 1 {max(1, int(timeout))}); do " f"[ -f {quote(receipt_path)} ] && exit 0; sleep 1; done; exit 1" ) - await self.sandbox.exec(script, cwd=self.request.sandbox_access.workdir, timeout_s=timeout + 5) + await self.sandbox.exec(script, cwd=self.workdir, timeout_s=timeout + 5) try: result = OpenClawSandboxResult.model_validate_json(await self.read_text("result.json")) except Exception as error: @@ -124,14 +131,21 @@ async def stop_runner(self, timeout: float) -> None: self.result = result async def close(self, timeout: float) -> None: - """Stop only OpenClaw-owned work and detach; never call sandbox.stop().""" + """Stop owned sandboxes; only stop harness work and disconnect borrowed ones.""" async with self.close_lock: if self.closed: return self.closing = True # Cancelling provider exec can kill the supervisor. Obtain its # descendant-cleanup receipt before cancelling the response task. - await self.stop_runner(timeout) + if self.owns_sandbox: + # The provider is the cleanup authority for an agent-owned sandbox. + # Keep the handle retryable if stop fails or times out. + if not self.sandbox_stopped: + await asyncio.wait_for(self.sandbox.stop(), timeout=timeout) + self.sandbox_stopped = True + else: + await self.stop_runner(timeout) if self.task is not None: if not self.task.done() and not self.task.cancelling(): self.task.cancel() @@ -157,6 +171,9 @@ async def close(self, timeout: float) -> None: if not self.exec_task.done(): raise # Transport failure is not cleanup failure once the receipt is confirmed. + if self.owns_sandbox: + self.closed = True + return retired = f"{self.directory}.closed" result = await self.sandbox.exec( f"if [ -d {quote(self.directory)} ]; then " @@ -181,9 +198,7 @@ async def execute(self, payload: dict[str, JsonValue], *, timeout: float, close_ # The runner enforces its own deadline and reaps descendants. # Leave extra time for cleanup and transport before provider timeout. self.exec_task = asyncio.create_task( - self.sandbox.exec( - command, cwd=self.request.sandbox_access.workdir, timeout_s=timeout + close_timeout * 3 + 30 - ) + self.sandbox.exec(command, cwd=self.workdir, timeout_s=timeout + close_timeout * 3 + 30) ) # HTTP cancellation must not propagate into provider exec before # the supervisor has stopped and reaped the harness descendants. diff --git a/responses_api_agents/openclaw_agent/tests/test_exec_transport.py b/responses_api_agents/openclaw_agent/tests/test_exec_transport.py index 7e911e2c60a..0dcf7ebd638 100644 --- a/responses_api_agents/openclaw_agent/tests/test_exec_transport.py +++ b/responses_api_agents/openclaw_agent/tests/test_exec_transport.py @@ -69,7 +69,9 @@ def make_session(tmp_path): request = seed() request.sandbox_access.workdir = str(workdir) provider = ExecOnlySandbox() - state = OpenClawSandboxSession(request, provider, str(directory), str(tmp_path / "runtime")) + state = OpenClawSandboxSession( + request, provider, str(directory), str(tmp_path / "runtime"), workdir=request.sandbox_access.workdir + ) shutil.copyfile(sandbox_runner.__file__, directory / "sandbox_runner.py") return state, provider, workdir diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index 7842f6c847b..163de7dd14e 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -127,8 +127,8 @@ async def download(self, source, destination): async def create(self, **kwargs): payload_path = next(path for path in self.files if path.endswith("/input.json")) payload = json.loads(self.files[payload_path]) - assert payload["cwd"] == "/app" - assert kwargs["cwd"] == "/app" + assert payload["cwd"] == getattr(self, "expected_workdir", "/app") + assert kwargs["cwd"] == getattr(self, "expected_workdir", "/app") assert "sandbox_runner.py" in kwargs["command"] self.directory = payload["directory"] self.started.set() @@ -1545,3 +1545,163 @@ async def test_stale_cookie_cannot_recreate_session_or_close_receipt_after_full_ assert all(record.state is None for record in agent._session_records.values()) assert not agent._session_records sandbox.disconnect.assert_awaited_once() + + +@pytest.mark.parametrize("owned", [False, True]) +def test_sandbox_source_controls_ownership_and_native_routing(setup, owned): + agent, sandbox = setup + agent.config.sandbox_provider = "agent-provider" + agent.config.sandbox_config = {"image": "test-image", "workdir": "/agent-workspace"} + sandbox.expected_workdir = "/agent-workspace" if owned else "/app" + body = seed() + if owned: + body.sandbox_access = None + sandbox.start = AsyncMock() + module = "responses_api_agents.openclaw_agent.app" + with ( + patch(f"{module}.AsyncSandbox", return_value=sandbox) as factory, + patch(f"{module}.resolve_provider_config") as resolve, + TestClient(agent.setup_webserver()) as client, + ): + factory.connect = AsyncMock(return_value=sandbox) + created = client.post("/v1/agent_sessions", json=body.model_dump(mode="json")) + assert created.status_code == 200, created.text + state = agent._session_records[body.agent_session_id].state + assert state.owns_sandbox is owned + assert state.workdir == sandbox.expected_workdir + resolve.assert_called_once_with("agent-provider" if owned else "sandbox", {}) + workspace_calls = [call for call in sandbox.exec.await_args_list if call.args[0].startswith("mkdir -p --")] + assert len(workspace_calls) == int(owned) + if owned: + assert workspace_calls[0].args[0] == "mkdir -p -- /agent-workspace" + assert workspace_calls[0].kwargs["cwd"] == "/" + if owned: + factory.connect.assert_not_awaited() + spec = sandbox.start.await_args.args[0] + assert spec.image == "test-image" + assert spec.workdir == "/agent-workspace" + else: + factory.assert_not_called() + factory.connect.assert_awaited_once() + response = client.post( + f"/ng-rollout/{body.episode_id.capture_key}/v1/responses", json={"input": "Fix the code"} + ) + assert response.status_code == 200, response.text + close_request = {"agent_session_id": body.agent_session_id, "episode_id": body.episode_id.model_dump()} + closed = client.post("/v1/agent_sessions/close", json=close_request) + assert closed.status_code == 200, closed.text + assert client.post("/v1/agent_sessions/close", json=close_request).json() == closed.json() + if owned: + sandbox.stop.assert_awaited_once() + sandbox.disconnect.assert_not_awaited() + else: + sandbox.stop.assert_not_awaited() + sandbox.disconnect.assert_awaited_once() + assert ( + client.post(f"/ng-rollout/{body.episode_id.capture_key}/v1/responses", json={"input": "task"}).status_code + == 409 + ) + + +def test_owned_stop_failure_blocks_close_until_retry(setup): + agent, sandbox = setup + agent.config.sandbox_provider = "agent-provider" + agent.config.sandbox_config = {"image": "test-image", "workdir": "/workspace"} + sandbox.start = AsyncMock() + body = seed() + body.sandbox_access = None + with ( + patch("responses_api_agents.openclaw_agent.app.AsyncSandbox", return_value=sandbox), + TestClient(agent.setup_webserver(), raise_server_exceptions=False) as client, + ): + created = client.post("/v1/agent_sessions", json=body.model_dump(mode="json")) + assert created.status_code == 200, created.text + state = agent._session_records[body.agent_session_id].state + assert state.workdir == "/workspace" + # An owned sandbox can be destroyed even when no runner receipt was returned. + state.launch_started = True + sandbox.stop.side_effect = [RuntimeError("provider stop failed"), None] + close_request = {"agent_session_id": body.agent_session_id, "episode_id": body.episode_id.model_dump()} + assert client.post("/v1/agent_sessions/close", json=close_request).status_code == 500 + assert not state.closed + assert ( + client.post(f"/ng-rollout/{body.episode_id.capture_key}/v1/responses", json={"input": "task"}).status_code + == 409 + ) + assert client.post("/v1/agent_sessions/close", json=close_request).status_code == 200 + assert sandbox.stop.await_count == 2 + sandbox.disconnect.assert_not_awaited() + + +@pytest.mark.parametrize("stage", ["start", "workdir", "install"]) +def test_owned_setup_failure_preserves_error_and_retryable_cleanup(setup, stage): + agent, sandbox = setup + agent.config.sandbox_provider = "agent-provider" + agent.config.sandbox_config = {"image": "test-image", "workdir": "/app"} + sandbox.start = AsyncMock() + body = seed() + body.sandbox_access = None + original = RuntimeError(f"{stage} failed") + if stage == "start": + sandbox.start.side_effect = original + else: + execute = sandbox.exec.side_effect + + async def fail_at_stage(command, **kwargs): + if (stage == "workdir" and command.startswith("mkdir -p --")) or ( + stage == "install" and command.startswith("bash ") and "install_openclaw_runtime.sh" in command + ): + raise original + return await execute(command, **kwargs) + + sandbox.exec.side_effect = fail_at_stage + sandbox.stop.side_effect = [RuntimeError("stop failed"), None] + with ( + patch("responses_api_agents.openclaw_agent.app.AsyncSandbox", return_value=sandbox), + TestClient(agent.setup_webserver()) as client, + ): + with pytest.raises(RuntimeError) as error: + client.post("/v1/agent_sessions", json=body.model_dump(mode="json")) + assert error.value is original + assert client.post("/v1/agent_sessions", json=body.model_dump(mode="json")).status_code == 409 + state = agent._session_records[body.agent_session_id].state + assert state.closing and not state.closed + response = client.post( + "/v1/agent_sessions/close", + json={"agent_session_id": body.agent_session_id, "episode_id": body.episode_id.model_dump()}, + ) + assert response.status_code == 200, response.text + assert sandbox.stop.await_count == 2 + sandbox.disconnect.assert_not_awaited() + + +def test_borrow_connection_failure_never_creates_replacement(setup): + agent, sandbox = setup + agent.config.sandbox_provider = "fallback-must-not-be-used" + module = "responses_api_agents.openclaw_agent.app" + with ( + patch(f"{module}.AsyncSandbox") as factory, + patch(f"{module}.create_provider", return_value=SimpleNamespace(aclose=AsyncMock())), + TestClient(agent.setup_webserver()) as client, + ): + factory.connect = AsyncMock(side_effect=RuntimeError("borrow failed")) + with pytest.raises(RuntimeError, match="borrow failed"): + client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")) + factory.assert_not_called() + sandbox.stop.assert_not_awaited() + + +@pytest.mark.parametrize("workdir", [None, "relative"]) +def test_owned_workdir_is_validated_before_creation(setup, workdir): + agent, sandbox = setup + agent.config.sandbox_provider = "agent-provider" + agent.config.sandbox_config = {"workdir": workdir} + body = seed() + body.sandbox_access = None + with ( + patch("responses_api_agents.openclaw_agent.app.AsyncSandbox") as factory, + TestClient(agent.setup_webserver()) as client, + ): + assert client.post("/v1/agent_sessions", json=body.model_dump(mode="json")).status_code == 422 + factory.assert_not_called() + sandbox.exec.assert_not_awaited() From 5b39930a57ed912c5dce5b7a6c245a92c65adf71 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Fri, 2 Oct 2026 20:50:21 -0700 Subject: [PATCH 17/44] refactor(openclaw): share sandbox cleanup and runtime contracts Use the same supervisor transport as Pi and Codex. Keep cleanup receipts, worker identity, and harness output separate while preserving borrowed/owned teardown and retry behavior. Validation: 314 passed, 38 skipped; full pre-commit passed. Linux-specific tests and fresh model rollouts remain pending. Signed-off-by: yaoyu-33 --- nemo_gym/sandbox/runner.py | 110 +++++++++++ responses_api_agents/openclaw_agent/app.py | 22 +-- .../openclaw_agent/sandbox.py | 114 ++++------- .../openclaw_agent/sandbox_runner.py | 134 +++---------- .../tests/test_exec_transport.py | 21 +- .../tests/test_lifecycle_contract.py | 78 ++++++++ .../openclaw_agent/tests/test_sandbox.py | 13 +- .../tests/test_sandbox_runner.py | 182 +++++++----------- .../test_sandbox_runner_transport.py | 166 ++++++++++++++++ 9 files changed, 516 insertions(+), 324 deletions(-) create mode 100644 nemo_gym/sandbox/runner.py create mode 100644 responses_api_agents/openclaw_agent/tests/test_lifecycle_contract.py create mode 100644 tests/unit_tests/test_sandbox_runner_transport.py diff --git a/nemo_gym/sandbox/runner.py b/nemo_gym/sandbox/runner.py new file mode 100644 index 00000000000..a609fe964da --- /dev/null +++ b/nemo_gym/sandbox/runner.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Controller-side transport for the shared sandbox process supervisor. + +Session ownership, cancellation, and harness output parsing stay with the adapter. +Unlike process_supervisor.py, this module is not uploaded to the task sandbox. +""" + +import json +import tempfile +from pathlib import Path +from shlex import join, quote +from uuid import uuid4 + +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from nemo_gym.sandbox import AsyncSandbox +from nemo_gym.sandbox.process_supervisor import CleanupReceipt + + +_CLEANUP_RECEIPT = TypeAdapter(CleanupReceipt) + + +class RunnerRuntimeInfo(BaseModel): + """Worker identity, independent of supervisor cleanup and harness output.""" + + model_config = ConfigDict(extra="forbid", strict=True) + hostname: str + pid: int + python: str | None = None + + +def parse_cleanup_receipt(payload: object) -> CleanupReceipt: + """Validate the supervisor's existing schema without adding worker dependencies. + + Older stop-before-launch receipts contain only acknowledgement and error. + Normalize that form without inventing a successful worker exit code. + """ + if isinstance(payload, dict) and payload.keys() == {"cleanup_confirmed", "error"}: + payload = {**payload, "return_code": None, "timed_out": False} + return _CLEANUP_RECEIPT.validate_python(payload, strict=True, extra="forbid") + + +async def upload_text(sandbox: AsyncSandbox, *, path: str, text: str) -> None: + """Upload adapter-owned text without interpolating its contents into a shell command.""" + with tempfile.TemporaryDirectory(prefix="sandbox-runner-upload-") as directory: + source = Path(directory) / "payload" + source.write_text(text) + await sandbox.upload(source, path) + + +async def read_text(sandbox: AsyncSandbox, *, path: str) -> str: + """Download an adapter-owned artifact, preserving non-UTF8 output with replacement.""" + with tempfile.TemporaryDirectory(prefix="sandbox-runner-download-") as directory: + destination = Path(directory) / "payload" + await sandbox.download(path, destination) + return destination.read_text(errors="replace") + + +def supervisor_command(*, directory: str, command: list[str], timeout: float, cleanup_timeout: float) -> str: + """Fence delayed launches and run a harness command under the shared supervisor.""" + return ( + f"trap '' TERM; ln -s launch {quote(directory + '/launch.claim')} 2>/dev/null || exit 0; " + f"echo $$ > {quote(directory + '/runner.pid')} && " + f"exec python3 -I {quote(directory + '/process_supervisor.py')} " + f"--timeout {timeout} --cleanup-timeout {cleanup_timeout} " + f"--stop-file {quote(directory + '/runner.stop')} " + f"--receipt {quote(directory + '/cleanup.json')} -- {join(command)} " + f">{quote(directory + '/runner.log')} 2>&1" + ) + + +async def confirm_runner_cleanup( + sandbox: AsyncSandbox, *, directory: str, workdir: str, timeout: float, harness: str +) -> CleanupReceipt: + """Fence a pending launch or require explicit supervisor acknowledgement before teardown. + + A stop that wins the launch claim writes a minimal receipt: no worker ran, so + runtime metadata and a return code are intentionally absent. Worker metadata + and harness output are validated separately by the adapter. + """ + receipt_path = f"{directory}/cleanup.json" + try: + receipt = json.loads(await read_text(sandbox, path=receipt_path)) + except Exception: + receipt = {} + if receipt.get("cleanup_confirmed") is not True: + pid_path = quote(f"{directory}/runner.pid") + stop_path = quote(f"{directory}/runner.stop") + claim_path = quote(f"{directory}/launch.claim") + temporary = quote(f"{receipt_path}.{uuid4().hex}.tmp") + stopped = quote(json.dumps({"cleanup_confirmed": True, "error": None})) + script = ( + f"[ -f {quote(receipt_path)} ] && exit 0; " + f"touch {stop_path} || exit 1; " + f"ln -s stop {claim_path} 2>/dev/null || true; " + f'if [ "$(readlink {claim_path})" = stop ]; then ' + f"printf '%s' {stopped} > {temporary} && mv {temporary} {quote(receipt_path)}; exit $?; fi; " + f'if [ -s {pid_path} ]; then kill -TERM "$(cat {pid_path})" 2>/dev/null || true; fi; ' + f"for _ in $(seq 1 {max(1, int(timeout))}); do " + f"[ -f {quote(receipt_path)} ] && exit 0; sleep 1; done; exit 1" + ) + await sandbox.exec(script, cwd=workdir, timeout_s=timeout + 5) + try: + receipt = json.loads(await read_text(sandbox, path=receipt_path)) + except Exception as error: + raise RuntimeError(f"{harness} launch outcome is unknown; cannot confirm termination") from error + if receipt.get("cleanup_confirmed") is not True: + raise RuntimeError(f"{harness} sandbox cleanup was not confirmed: {receipt.get('error')}") + return parse_cleanup_receipt(receipt) diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index f0a17cad9cf..3ce8023d25a 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -64,7 +64,7 @@ AgentObservationBundle, ObservationGap, ) -from nemo_gym.sandbox import AsyncSandbox, SandboxSpec, create_provider +from nemo_gym.sandbox import AsyncSandbox, SandboxSpec, create_provider, process_supervisor from nemo_gym.sandbox.access import DirectSandboxConnection from nemo_gym.sandbox.config import resolve_provider_config from nemo_gym.server_utils import get_global_config_dict, get_response_json, raise_for_status @@ -541,6 +541,7 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Open f"{installed.stderr or installed.stdout}" ) await sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), f"{directory}/sandbox_runner.py") + await sandbox.upload(Path(process_supervisor.__file__), f"{directory}/process_supervisor.py") except BaseException as error: try: if prepared_directory or owns_sandbox: @@ -700,8 +701,6 @@ async def _sandbox_response( "CLAWHUB_DISABLE_TELEMETRY": "1", "OPENCLAW_EXEC_SHELL_SNAPSHOT": "0", }, - "timeout": self.config.timeout, - "cleanup_timeout": self.config.session_close_timeout_seconds / 3, } try: async with self.sem: @@ -838,20 +837,21 @@ def count(name: str) -> int: ObservationGap(code="model_call_usage_unavailable", detail="Only CLI envelope totals available") ) gaps.append(ObservationGap(code="reasoning_token_usage_unavailable")) - result = state.result - error = result.error if result else "OpenClaw activation ended without a cleanup receipt" + result = state.cleanup + runtime = state.runtime_info + error = result["error"] if result else "OpenClaw activation ended without a cleanup receipt" last = assistants[-1] if assistants else {} if last.get("stopReason") == "error" or ( - last.get("stopReason") == "aborted" and not (result and result.timed_out) + last.get("stopReason") == "aborted" and not (result and result["timed_out"]) ): error = error or last.get("errorMessage") or f"OpenClaw model call {last['stopReason']}" - if result and result.return_code and not result.timed_out: + if result and result["return_code"] and not result["timed_out"]: try: stderr = (await state.read_text("stderr.log"))[-4000:] except Exception: stderr = "stderr unavailable" - error = error or f"OpenClaw exited {result.return_code}: {stderr}" - incomplete = bool(result and result.timed_out) or last.get("stopReason") == "length" + error = error or f"OpenClaw exited {result['return_code']}: {stderr}" + incomplete = bool(result and result["timed_out"]) or last.get("stopReason") == "length" if not incomplete and not error and last.get("stopReason") != "stop": error = "OpenClaw produced no terminal assistant result" status = "failed" if error else "incomplete" if incomplete else "completed" @@ -875,8 +875,8 @@ def count(name: str) -> int: ), metadata={ "harness_execution": "sandbox", - "harness_hostname": result.hostname if result else "unknown", - "harness_pid": str(result.pid) if result else "unknown", + "harness_hostname": runtime.hostname if runtime else "unknown", + "harness_pid": str(runtime.pid) if runtime else "unknown", "openclaw_version": self.config.openclaw_version, }, ) diff --git a/responses_api_agents/openclaw_agent/sandbox.py b/responses_api_agents/openclaw_agent/sandbox.py index 6cdfca57128..d30a6832d59 100644 --- a/responses_api_agents/openclaw_agent/sandbox.py +++ b/responses_api_agents/openclaw_agent/sandbox.py @@ -5,19 +5,24 @@ import asyncio import json import logging -import tempfile from dataclasses import dataclass, field -from pathlib import Path from shlex import quote -from uuid import uuid4 -from pydantic import BaseModel, ConfigDict, JsonValue +from pydantic import JsonValue from nemo_gym.base_responses_api_agent import AgentSessionState from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming from nemo_gym.rollout_observability import AgentObservationBundle -from nemo_gym.sandbox import AsyncSandbox +from nemo_gym.sandbox import AsyncSandbox, process_supervisor +from nemo_gym.sandbox.process_supervisor import CleanupReceipt from nemo_gym.sandbox.providers.base import SandboxExecResult +from nemo_gym.sandbox.runner import ( + RunnerRuntimeInfo, + confirm_runner_cleanup, + read_text, + supervisor_command, + upload_text, +) # Executed by sandbox Python before uploading any adapter-owned files. Resolve @@ -34,18 +39,6 @@ """ -class OpenClawSandboxResult(BaseModel): - """Require an explicit cleanup acknowledgement, not just process exit.""" - - model_config = ConfigDict(extra="forbid", strict=True) - return_code: int - timed_out: bool - cleanup_confirmed: bool - error: str | None - hostname: str - pid: int - - @dataclass class OpenClawSandboxSession(AgentSessionState): """Worker-local session with retryable, fail-closed runner teardown.""" @@ -58,7 +51,8 @@ class OpenClawSandboxSession(AgentSessionState): sandbox_stopped: bool = False task: asyncio.Task[NeMoGymResponse] | None = None exec_task: asyncio.Task[SandboxExecResult] | None = None - result: OpenClawSandboxResult | None = None + cleanup: CleanupReceipt | None = None + runtime_info: RunnerRuntimeInfo | None = None observations: AgentObservationBundle | None = None activation_request: NeMoGymResponseCreateParamsNonStreaming | None = None closing: bool = False @@ -68,67 +62,19 @@ class OpenClawSandboxSession(AgentSessionState): async def upload_json(self, name: str, payload: JsonValue) -> None: """Upload adapter-owned data beneath this session's directory.""" - with tempfile.TemporaryDirectory(prefix="openclaw-session-upload-") as directory: - path = Path(directory) / "payload.json" - path.write_text(json.dumps(payload)) - await self.sandbox.upload(path, f"{self.directory}/{name}") + await upload_text(self.sandbox, path=f"{self.directory}/{name}", text=json.dumps(payload)) async def read_text(self, name: str) -> str: - """Read an adapter-owned result without interpreting it as a shell command.""" - with tempfile.TemporaryDirectory(prefix="openclaw-session-download-") as directory: - path = Path(directory) / "payload" - await self.sandbox.download(f"{self.directory}/{name}", path) - return path.read_text(errors="replace") + """Read adapter-owned output independently of cleanup confirmation.""" + return await read_text(self.sandbox, path=f"{self.directory}/{name}") async def stop_runner(self, timeout: float) -> None: - """Fence delayed launches and require a receipt before allowing verification.""" - if ( - self.sandbox_stopped - or not self.launch_started - or (self.result is not None and self.result.cleanup_confirmed) - ): + """Fence a delayed launch or require the shared supervisor's cleanup receipt.""" + if self.sandbox_stopped or not self.launch_started or self.cleanup is not None: return - receipt_path = f"{self.directory}/result.json" - try: - result = OpenClawSandboxResult.model_validate_json(await self.read_text("result.json")) - except Exception: - result = None - if result is None: - # The atomic claim decides whether launch or close won. A stopped - # claim can never start a harness, even if exec arrives much later. - stop_path = quote(f"{self.directory}/runner.stop") - claim_path = quote(f"{self.directory}/launch.claim") - temporary = quote(f"{receipt_path}.{uuid4().hex}.tmp") - stopped = quote( - json.dumps( - { - "return_code": 1, - "timed_out": False, - "cleanup_confirmed": True, - "error": "Closed before runner launch", - "hostname": "", - "pid": 0, - } - ) - ) - script = ( - f"[ -f {quote(receipt_path)} ] && exit 0; " - f"touch {stop_path} || exit 1; " - f"ln -s stop {claim_path} 2>/dev/null || true; " - f'if [ "$(readlink {claim_path})" = stop ]; then ' - f"printf '%s' {stopped} > {temporary} && mv {temporary} {quote(receipt_path)}; exit $?; fi; " - f"for _ in $(seq 1 {max(1, int(timeout))}); do " - f"[ -f {quote(receipt_path)} ] && exit 0; sleep 1; done; exit 1" - ) - await self.sandbox.exec(script, cwd=self.workdir, timeout_s=timeout + 5) - try: - result = OpenClawSandboxResult.model_validate_json(await self.read_text("result.json")) - except Exception as error: - raise RuntimeError("OpenClaw launch outcome is unknown; cannot authorize verification") from error - if not result.cleanup_confirmed: - raise RuntimeError(f"OpenClaw sandbox cleanup was not confirmed: {result.error}") - # Cache only a successful receipt; failed cleanup remains retryable. - self.result = result + self.cleanup = await confirm_runner_cleanup( + self.sandbox, directory=self.directory, workdir=self.workdir, timeout=timeout, harness="OpenClaw" + ) async def close(self, timeout: float) -> None: """Stop owned sandboxes; only stop harness work and disconnect borrowed ones.""" @@ -188,17 +134,23 @@ async def close(self, timeout: float) -> None: async def execute(self, payload: dict[str, JsonValue], *, timeout: float, close_timeout: float) -> str: """Run the supervisor through provider-neutral exec, without a PTY.""" await self.upload_json("input.json", payload) - command = ( - f"trap '' TERM; ln -s launch {quote(self.directory + '/launch.claim')} 2>/dev/null || exit 0; " - f"exec python3 -I {quote(self.directory + '/sandbox_runner.py')} {quote(self.directory + '/input.json')} " - f">{quote(self.directory + '/runner.log')} 2>&1" + cleanup_timeout = close_timeout / 3 + command = supervisor_command( + directory=self.directory, + command=["python3", "-I", f"{self.directory}/sandbox_runner.py", f"{self.directory}/input.json"], + timeout=timeout, + cleanup_timeout=cleanup_timeout, ) self.launch_started = True try: # The runner enforces its own deadline and reaps descendants. # Leave extra time for cleanup and transport before provider timeout. self.exec_task = asyncio.create_task( - self.sandbox.exec(command, cwd=self.workdir, timeout_s=timeout + close_timeout * 3 + 30) + self.sandbox.exec( + command, + cwd=self.workdir, + timeout_s=process_supervisor.exec_timeout(timeout=timeout, cleanup_timeout=cleanup_timeout), + ) ) # HTTP cancellation must not propagate into provider exec before # the supervisor has stopped and reaped the harness descendants. @@ -215,4 +167,8 @@ async def execute(self, payload: dict[str, JsonValue], *, timeout: float, close_ raise else: await self.stop_runner(close_timeout) + # A fenced launch can be safely closed without ever running a worker. + if self.cleanup is None or self.cleanup["return_code"] is None: + raise RuntimeError("OpenClaw sandbox runner has no worker exit code") + self.runtime_info = RunnerRuntimeInfo.model_validate_json(await self.read_text("runtime.json")) return await self.read_text("stdout.log") diff --git a/responses_api_agents/openclaw_agent/sandbox_runner.py b/responses_api_agents/openclaw_agent/sandbox_runner.py index 483d8f33669..40479be64db 100644 --- a/responses_api_agents/openclaw_agent/sandbox_runner.py +++ b/responses_api_agents/openclaw_agent/sandbox_runner.py @@ -1,141 +1,67 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Supervise one OpenClaw invocation and reap all Linux tool descendants.""" +"""Capture OpenClaw output; the shared process supervisor owns deadlines and cleanup.""" from __future__ import annotations -import ctypes import json import os import signal import subprocess import sys +from contextlib import ExitStack from pathlib import Path -from time import monotonic, sleep from typing import TypedDict -class RunnerRequest(TypedDict): - """Invocation data uploaded by the adapter.""" +class RunnerInput(TypedDict): + """Harness input, independent of supervisor cleanup and runtime metadata.""" directory: str prompt: str command: list[str] cwd: str env: dict[str, str] - timeout: float - cleanup_timeout: float -class RunnerResult(TypedDict): - """Receipt proving whether verification can safely inspect the task.""" - - return_code: int - timed_out: bool - cleanup_confirmed: bool - error: str | None - hostname: str - pid: int - - -def enable_subreaper() -> None: - """Adopt detached children that escape the original process group.""" - if sys.platform != "linux": - raise RuntimeError("Native OpenClaw sessions require Linux") - if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0: - raise OSError(ctypes.get_errno(), "Cannot establish OpenClaw child-subreaper boundary") - - -def drain_children(timeout: float) -> None: - """Kill and reap descendants before acknowledging cleanup.""" - children = Path(f"/proc/self/task/{os.getpid()}/children") - deadline = monotonic() + timeout - while True: - for child in children.read_text().split(): - try: - os.kill(int(child), signal.SIGKILL) - except ProcessLookupError: - pass - try: - while os.waitpid(-1, os.WNOHANG)[0]: - pass - except ChildProcessError: - return - if monotonic() >= deadline: - raise TimeoutError("OpenClaw descendants remain alive; verification must not proceed") - sleep(0.01) - - -def run(params: RunnerRequest) -> RunnerResult: - """Write stdout incrementally; the transcript remains available after failures.""" +def run(params: RunnerInput) -> int: + """Run the harness with isolated files and let the supervisor reap descendants.""" directory = Path(params["directory"]) - process = None - error = None - timed_out = False - cleanup_confirmed = False + (directory / "runtime.json").write_text(json.dumps({"hostname": os.uname().nodename, "pid": os.getpid()})) + (directory / "prompt.txt").write_text(params["prompt"]) stopping = False - def interrupt(_signum: int, _frame: object) -> None: + def interrupt(*_: object) -> None: nonlocal stopping - # Signals must not interrupt Popen between child creation and handle assignment. + # Defer TERM until Popen returns so a signal cannot lose the child handle. stopping = True signal.signal(signal.SIGTERM, interrupt) - stop_file = directory / "runner.stop" - (directory / "prompt.txt").write_text(params["prompt"]) - with (directory / "stdout.log").open("wb") as stdout, (directory / "stderr.log").open("wb") as stderr: - try: - enable_subreaper() - if stopping or stop_file.exists(): - raise RuntimeError("OpenClaw closed before process launch") - process = subprocess.Popen( - params["command"], - cwd=params["cwd"], - env={**os.environ, **params["env"]}, - stdin=subprocess.DEVNULL, - stdout=stdout, - stderr=stderr, - start_new_session=True, - ) - deadline = monotonic() + params["timeout"] - while process.poll() is None: - if stopping or stop_file.exists() or monotonic() >= deadline: - timed_out = True - break - sleep(0.05) - except Exception as exc: - error = str(exc) - finally: - signal.signal(signal.SIGTERM, signal.SIG_IGN) + with ExitStack() as stack: + stdout = stack.enter_context((directory / "stdout.log").open("wb")) + stderr = stack.enter_context((directory / "stderr.log").open("wb")) + stdin = subprocess.DEVNULL + process = subprocess.Popen( + params["command"], + cwd=params["cwd"], + env={**os.environ, **params["env"]}, + stdin=stdin, + stdout=stdout, + stderr=stderr, + ) + while True: + if stopping: + process.terminate() + stopping = False try: - if process is not None: - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - process.wait(timeout=params["cleanup_timeout"]) - drain_children(params["cleanup_timeout"]) - cleanup_confirmed = True - except Exception as exc: - error = f"cleanup: {exc}" - return { - "return_code": process.returncode if process is not None and process.returncode is not None else 1, - "timed_out": timed_out, - "cleanup_confirmed": cleanup_confirmed, - "error": error, - "hostname": os.uname().nodename, - "pid": os.getpid(), - } + return process.wait(timeout=0.05) + except subprocess.TimeoutExpired: + pass def main() -> None: - """Atomically publish the cleanup receipt after all descendants have exited.""" params = json.loads(Path(sys.argv[1]).read_text()) - result = run(params) - output = Path(params["directory"]) / "result.json" - temporary = output.with_suffix(".tmp") - temporary.write_text(json.dumps(result)) - temporary.replace(output) + raise SystemExit(run(params)) if __name__ == "__main__": diff --git a/responses_api_agents/openclaw_agent/tests/test_exec_transport.py b/responses_api_agents/openclaw_agent/tests/test_exec_transport.py index 0dcf7ebd638..3d028645fed 100644 --- a/responses_api_agents/openclaw_agent/tests/test_exec_transport.py +++ b/responses_api_agents/openclaw_agent/tests/test_exec_transport.py @@ -12,6 +12,7 @@ import pytest +from nemo_gym.sandbox import process_supervisor from nemo_gym.sandbox.providers.base import SandboxExecResult from responses_api_agents.openclaw_agent import sandbox_runner from responses_api_agents.openclaw_agent.sandbox import OpenClawSandboxSession @@ -73,6 +74,7 @@ def make_session(tmp_path): request, provider, str(directory), str(tmp_path / "runtime"), workdir=request.sandbox_access.workdir ) shutil.copyfile(sandbox_runner.__file__, directory / "sandbox_runner.py") + shutil.copyfile(process_supervisor.__file__, directory / "process_supervisor.py") return state, provider, workdir @@ -114,9 +116,9 @@ async def test_exec_only_supervision_reaps_detached_child(tmp_path, ending): await asyncio.gather(task, return_exceptions=True) else: await task - assert state.result.cleanup_confirmed is True + assert state.cleanup["cleanup_confirmed"] is True if ending == "timeout": - assert state.result.timed_out is True + assert state.cleanup["timed_out"] is True with pytest.raises(ProcessLookupError): os.kill(int((workdir / "child.pid").read_text()), 0) assert provider.cancelled_launch is False @@ -135,9 +137,10 @@ async def test_lost_launch_is_fenced_even_after_directory_retirement(tmp_path): provider.lost_launch = True with pytest.raises(TimeoutError, match="lost launch response"): await state.execute(payload(state, "open('started','w').close()"), timeout=0.5, close_timeout=3) - assert state.result.cleanup_confirmed is True - assert state.result.return_code != 0 - assert state.result.error == "Closed before runner launch" + assert state.cleanup["cleanup_confirmed"] is True + assert state.cleanup["return_code"] is None + assert state.cleanup["error"] is None + assert state.runtime_info is None await state.close(3) provider.lost_launch = False await provider.exec(provider.delayed_command, cwd=str(workdir)) @@ -153,10 +156,8 @@ async def test_failed_receipt_keeps_files_and_can_retry(tmp_path): "timed_out": False, "cleanup_confirmed": False, "error": "descendants remain", - "hostname": "sandbox", - "pid": 1, } - path = Path(state.directory) / "result.json" + path = Path(state.directory) / "cleanup.json" path.write_text(json.dumps(receipt)) with pytest.raises(RuntimeError, match="cleanup was not confirmed"): await state.close(3) @@ -176,10 +177,8 @@ async def test_confirmed_cleanup_cancels_stuck_transport(tmp_path): "timed_out": False, "cleanup_confirmed": True, "error": None, - "hostname": "sandbox", - "pid": 1, } - (Path(state.directory) / "result.json").write_text(json.dumps(receipt)) + (Path(state.directory) / "cleanup.json").write_text(json.dumps(receipt)) state.exec_task = asyncio.create_task(asyncio.Event().wait()) await asyncio.sleep(0) await state.close(0.5) diff --git a/responses_api_agents/openclaw_agent/tests/test_lifecycle_contract.py b/responses_api_agents/openclaw_agent/tests/test_lifecycle_contract.py new file mode 100644 index 00000000000..f4678802c2b --- /dev/null +++ b/responses_api_agents/openclaw_agent/tests/test_lifecycle_contract.py @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Cleanup evidence must not depend on worker metadata or parseable harness output.""" + +import json + +import pytest +from fastapi import Request +from pydantic import ValidationError + +from responses_api_agents.openclaw_agent.tests.test_sandbox import seed, setup # noqa: F401 + + +async def session(setup): + agent, sandbox = setup + body = seed() + await agent.seed_agent_session(Request({"type": "http", "session": {}}), body) + state = agent._session_records[body.agent_session_id].state + payload = {"directory": state.directory, "cwd": state.workdir, "prompt": "task", "command": [], "env": {}} + return state, sandbox, payload + + +@pytest.mark.parametrize( + "runtime", + [ + {}, + {"hostname": "worker", "pid": "123"}, + {"hostname": 1, "pid": 123}, + {"hostname": "worker", "pid": 123, "return_code": 0}, + ], +) +async def test_invalid_runtime_fails_execution_but_not_confirmed_close(setup, runtime): + state, sandbox, payload = await session(setup) + sandbox.runtime_info = runtime + with pytest.raises(ValidationError): + await state.execute(payload, timeout=5, close_timeout=1) + assert state.cleanup == {"cleanup_confirmed": True, "return_code": 0, "error": None, "timed_out": False} + await state.close(1) + assert state.closed + sandbox.disconnect.assert_awaited_once() + sandbox.stop.assert_not_awaited() + + +@pytest.mark.parametrize("invalid", [{"return_code": "0"}, {"cleanup_confirmed": 1}, {"hostname": "mixed"}]) +async def test_bad_cleanup_keeps_state_retryable_without_reading_runtime(setup, invalid): + state, sandbox, payload = await session(setup) + sandbox.result.update(invalid) + with pytest.raises((RuntimeError, ValidationError)): + await state.execute(payload, timeout=5, close_timeout=1) + assert state.cleanup is None + assert state.runtime_info is None + with pytest.raises((RuntimeError, ValidationError)): + await state.close(1) + assert not state.closed + sandbox.disconnect.assert_not_awaited() + sandbox.files[f"{state.directory}/cleanup.json"] = json.dumps({"cleanup_confirmed": True, "error": None}) + # A stop-before-launch receipt suffices for cleanup, not execution success. + await state.close(1) + assert state.closed + assert state.runtime_info is None + + +async def test_output_failure_does_not_erase_cleanup_confirmation(setup): + state, sandbox, payload = await session(setup) + download = sandbox.download + + async def fail_output(source, destination): + if source.endswith("/stdout.log"): + raise RuntimeError("output unreadable") + return await download(source, destination) + + sandbox.download = fail_output + with pytest.raises(RuntimeError, match="output unreadable"): + await state.execute(payload, timeout=5, close_timeout=1) + assert state.cleanup["cleanup_confirmed"] is True + assert state.cleanup["error"] is None + await state.close(1) + sandbox.disconnect.assert_awaited_once() diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index 163de7dd14e..9e386561a49 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -18,9 +18,9 @@ from nemo_gym.base_responses_api_agent import AgentCloseSessionRequest, AgentSeedSessionRequest from nemo_gym.episode_types import EpisodeId, TaskId from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming +from nemo_gym.sandbox.runner import parse_cleanup_receipt from nemo_gym.server_utils import ServerClient from responses_api_agents.openclaw_agent.app import OpenClawAgent, OpenClawAgentConfig, _unique_usage_messages -from responses_api_agents.openclaw_agent.sandbox import OpenClawSandboxResult def seed(*, session_id: str | None = None) -> AgentSeedSessionRequest: @@ -91,9 +91,8 @@ def __init__(self): "timed_out": False, "cleanup_confirmed": True, "error": None, - "hostname": "task-container", - "pid": 123, } + self.runtime_info = {"hostname": "task-container", "pid": 123} self.events = events() self.blocked = False self.started = asyncio.Event() @@ -145,7 +144,8 @@ async def signal(self): async def wait_exit(self): await self.exited.wait() - self.files[f"{self.directory}/result.json"] = json.dumps(self.result) + self.files[f"{self.directory}/cleanup.json"] = json.dumps(self.result) + self.files[f"{self.directory}/runtime.json"] = json.dumps(self.runtime_info) self.files[f"{self.directory}/stdout.log"] = "" self.files[f"{self.directory}/stderr.log"] = "model error" session_id = self.directory.rsplit("/", 1)[-1] @@ -228,7 +228,8 @@ def test_http_native_flow_runs_openclaw_in_borrowed_sandbox(setup, model_timeout payload = json.loads(sandbox.files[f"{sandbox.directory}/input.json"]) assert payload["prompt"] == "Fix the code" assert payload["command"][0].endswith("/node/bin/node") - assert payload["timeout"] == 900 + assert "--timeout 900" in sandbox.launch.await_args.kwargs["command"] + assert "timeout" not in payload assert "PATH" not in payload["env"] models = json.loads(sandbox.files[f"{sandbox.directory}/home/.openclaw/openclaw.json"]) assert models["agents"]["defaults"]["workspace"] == "/app" @@ -508,7 +509,7 @@ async def test_disconnect_failure_retains_session_for_retry(setup): def test_cleanup_receipt_is_required(): with pytest.raises(ValueError): - OpenClawSandboxResult.model_validate({"return_code": 0, "error": None}) + parse_cleanup_receipt({"return_code": 0, "error": None}) def test_instructions_and_text_parts_reach_openclaw_without_other_provider_credentials(setup): diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py b/responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py index 7cca5b5f1af..b8a9243216a 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Exercise the real Linux supervisor, not a mocked cleanup acknowledgement.""" +"""Test worker I/O separately from the shared Linux supervisor's cleanup contract.""" import json import os @@ -9,67 +9,98 @@ import subprocess import sys import time -from pathlib import Path import pytest +from nemo_gym.sandbox import process_supervisor from responses_api_agents.openclaw_agent import sandbox_runner -pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Linux subreaper and /proc are required") +linux_only = pytest.mark.skipif(sys.platform != "linux", reason="Linux subreaper and /proc required") -def launch(tmp_path, code, timeout=3): - request = { +def launch(tmp_path, code, *, timeout=3, python=sys.executable, env=None, supervised=True): + params = { "directory": str(tmp_path), - "command": [sys.executable, "-c", code], + "command": [python, "-c", code], "cwd": str(tmp_path), - "env": {}, - "timeout": timeout, - "cleanup_timeout": 2, + "env": env or {}, "prompt": "task input", } path = tmp_path / "input.json" - path.write_text(json.dumps(request)) - process = subprocess.Popen( - [sys.executable, "-I", str(Path(sandbox_runner.__file__)), str(path)], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - return process + path.write_text(json.dumps(params)) + command = [python, "-I", sandbox_runner.__file__, str(path)] + if supervised: + command = [ + python, + "-I", + process_supervisor.__file__, + "--timeout", + str(timeout), + "--cleanup-timeout", + "0.5", + "--receipt", + str(tmp_path / "cleanup.json"), + "--stop-file", + str(tmp_path / "runner.stop"), + "--", + *command, + ] + return subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) def result(tmp_path, process): try: stdout, stderr = process.communicate(timeout=10) assert process.returncode == 0, (stdout, stderr) - return json.loads((tmp_path / "result.json").read_text()) + return json.loads((tmp_path / "cleanup.json").read_text()) finally: if process.poll() is None: process.kill() process.wait() -def test_capture_and_prompt_file(tmp_path): +@pytest.mark.parametrize( + "python", + [ + sys.executable, + pytest.param( + shutil.which("python3.8"), + marks=pytest.mark.skipif(shutil.which("python3.8") is None, reason="Python 3.8 unavailable"), + id="python38", + ), + ], +) +def test_worker_io_and_runtime_are_independent_of_cleanup(tmp_path, monkeypatch, python): + monkeypatch.setenv("HARNESS_INHERITED", "kept") + monkeypatch.setenv("HARNESS_OVERRIDE", "old") process = launch( - tmp_path, "import json,pathlib; print(json.dumps({'prompt':pathlib.Path('prompt.txt').read_text()}))" + tmp_path, + "import json,os,sys,pathlib; print(json.dumps([" + "pathlib.Path('prompt.txt').read_text(),os.environ['HARNESS_INHERITED'],os.environ['HARNESS_OVERRIDE']]))", + python=python, + env={"HARNESS_OVERRIDE": "new"}, + supervised=False, ) - summary = result(tmp_path, process) - assert summary["cleanup_confirmed"] is True - assert summary["return_code"] == 0 - event = json.loads((tmp_path / "stdout.log").read_text()) - assert event["prompt"] == "task input" + stdout, stderr = process.communicate(timeout=10) + assert process.returncode == 0, (stdout, stderr) + assert json.loads((tmp_path / "stdout.log").read_text()) == ["task input", "kept", "new"] + runtime = json.loads((tmp_path / "runtime.json").read_text()) + assert runtime["hostname"] == os.uname().nodename + assert runtime["pid"] == process.pid + assert not (tmp_path / "cleanup.json").exists() + assert not (tmp_path / "result.json").exists() +@linux_only @pytest.mark.parametrize("ending", ["natural", "timeout", "cancel"]) def test_detached_descendants_are_gone_before_receipt(tmp_path, ending): - # The grandchild starts a new session, escaping process-group-only cleanup. code = ( "import subprocess,sys,time,pathlib; " "p=subprocess.Popen([sys.executable,'-c','import time; time.sleep(60)'],start_new_session=True); " "pathlib.Path('child.pid').write_text(str(p.pid)); " + ("time.sleep(60)" if ending != "natural" else "pass") ) - process = launch(tmp_path, code, timeout=0.3 if ending == "timeout" else 3) + process = launch(tmp_path, code) if ending == "cancel": for _ in range(200): if (tmp_path / "child.pid").exists(): @@ -78,99 +109,24 @@ def test_detached_descendants_are_gone_before_receipt(tmp_path, ending): process.send_signal(signal.SIGTERM) summary = result(tmp_path, process) assert summary["cleanup_confirmed"] is True - assert summary["timed_out"] is (ending != "natural") - pid = int((tmp_path / "child.pid").read_text()) + assert summary["timed_out"] is (ending == "timeout") + assert set(summary) == {"return_code", "timed_out", "cleanup_confirmed", "error"} with pytest.raises(ProcessLookupError): - os.kill(pid, 0) + os.kill(int((tmp_path / "child.pid").read_text()), 0) -def test_spawn_error_is_not_success(tmp_path): - process = launch(tmp_path, "raise RuntimeError('failed')") - summary = result(tmp_path, process) +@linux_only +def test_spawn_error_does_not_prevent_cleanup(tmp_path): + summary = result(tmp_path, launch(tmp_path, "raise RuntimeError('failed')")) assert summary["cleanup_confirmed"] is True assert summary["return_code"] != 0 -@pytest.mark.skipif(shutil.which("python3.8") is None, reason="Python 3.8 is not installed") -def test_python38_preserves_environment_and_reaps_detached_child(tmp_path: Path) -> None: - python = shutil.which("python3.8") - code = ( - "import json,os,pathlib,subprocess,sys; " - "p=subprocess.Popen([sys.executable,'-c','import time; time.sleep(60)'],start_new_session=True); " - "pathlib.Path('child.pid').write_text(str(p.pid)); " - "print(json.dumps([os.environ['OPENCLAW_TEST_INHERITED'],os.environ['OPENCLAW_TEST_OVERRIDE']]))" - ) - request = { - "directory": str(tmp_path), - "command": [python, "-c", code], - "cwd": str(tmp_path), - "env": {"OPENCLAW_TEST_OVERRIDE": "new"}, - "timeout": 3, - "cleanup_timeout": 2, - "prompt": "task input", - } - path = tmp_path / "input.json" - path.write_text(json.dumps(request)) - process = subprocess.Popen( - [python, "-I", str(Path(sandbox_runner.__file__)), str(path)], - env={**os.environ, "OPENCLAW_TEST_INHERITED": "kept", "OPENCLAW_TEST_OVERRIDE": "old"}, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - summary = result(tmp_path, process) - assert summary["cleanup_confirmed"] is True - assert summary["return_code"] == 0 - assert json.loads((tmp_path / "stdout.log").read_text()) == ["kept", "new"] - with pytest.raises(ProcessLookupError): - os.kill(int((tmp_path / "child.pid").read_text()), 0) - - -def test_stop_marker_prevents_process_launch(tmp_path): +@linux_only +def test_stop_marker_fences_launch_without_inventing_worker_identity(tmp_path): (tmp_path / "runner.stop").touch() - process = launch(tmp_path, "open('should-not-exist', 'w').close()") - summary = result(tmp_path, process) - assert summary["cleanup_confirmed"] is True - assert summary["return_code"] != 0 - assert not (tmp_path / "should-not-exist").exists() - - -def test_sigterm_during_spawn_does_not_lose_child_handle(tmp_path): - # Deliver SIGTERM after the real child exists but before Popen returns to run(). - # Isolate signal handlers/subreaper state from pytest, and always reap the test child. - driver = """ -import json, os, runpy, signal, subprocess, sys -runner = runpy.run_path(sys.argv[1]) -spawn = subprocess.Popen -children = [] -def interrupted_spawn(*args, **kwargs): - child = spawn(*args, **kwargs) - children.append(child) - os.kill(os.getpid(), signal.SIGTERM) - return child -subprocess.Popen = interrupted_spawn -try: - summary = runner['run']({ - 'directory': sys.argv[2], 'cwd': sys.argv[2], 'env': {}, 'prompt': 'task', - 'command': [sys.executable, '-c', 'import time; time.sleep(60)'], - 'timeout': 5, 'cleanup_timeout': 2, - }) - summary['child_alive'] = any(child.poll() is None for child in children) - print(json.dumps(summary)) -finally: - for child in children: - if child.poll() is None: - child.kill() - child.wait() -""" - completed = subprocess.run( - [sys.executable, "-c", driver, sandbox_runner.__file__, str(tmp_path)], - capture_output=True, - text=True, - errors="replace", - timeout=10, - check=True, - ) - summary = json.loads(completed.stdout) - assert summary["timed_out"] is True + summary = result(tmp_path, launch(tmp_path, "open('unexpected', 'w').close()")) assert summary["cleanup_confirmed"] is True - assert summary["child_alive"] is False + assert summary["return_code"] is None + assert not (tmp_path / "runtime.json").exists() + assert not (tmp_path / "unexpected").exists() diff --git a/tests/unit_tests/test_sandbox_runner_transport.py b/tests/unit_tests/test_sandbox_runner_transport.py new file mode 100644 index 00000000000..840a5c4945e --- /dev/null +++ b/tests/unit_tests/test_sandbox_runner_transport.py @@ -0,0 +1,166 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Transport contracts shared by sandboxed harness controllers.""" + +import asyncio +import json +import shutil +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest +from pydantic import ValidationError + +from nemo_gym.sandbox.providers.base import SandboxExecResult +from nemo_gym.sandbox.runner import ( + RunnerRuntimeInfo, + confirm_runner_cleanup, + parse_cleanup_receipt, + read_text, + supervisor_command, + upload_text, +) + + +class LocalSandbox: + async def upload(self, source, destination): + shutil.copyfile(source, destination) + + async def download(self, source, destination): + shutil.copyfile(source, destination) + + async def exec(self, command, *, cwd=None, timeout_s=30): + process = await asyncio.create_subprocess_exec( + "sh", "-c", command, cwd=cwd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout_s) + return SandboxExecResult(stdout.decode(errors="replace"), stderr.decode(errors="replace"), process.returncode) + + +@pytest.fixture +def session(tmp_path): + directory = tmp_path / "session's files" + directory.mkdir() + return LocalSandbox(), directory + + +async def test_file_transport_keeps_contents_out_of_shell(session): + sandbox, directory = session + sandbox.exec = AsyncMock(side_effect=AssertionError("file transfer must not invoke a shell")) + path = str(directory / "input.json") + payload = '{"prompt": "$(touch unwanted); `echo surprise`"}\n' + await upload_text(sandbox, path=path, text=payload) + assert Path(path).read_text() == payload + assert await read_text(sandbox, path=path) == payload + Path(path).write_bytes(b"partial output\xff\n") + assert await read_text(sandbox, path=path) == "partial output\ufffd\n" + sandbox.exec.assert_not_awaited() + + +async def test_confirmed_receipt_does_not_signal_stored_pid(session): + sandbox, directory = session + receipt = {"cleanup_confirmed": True, "error": None} + (directory / "cleanup.json").write_text(json.dumps(receipt)) + (directory / "runner.pid").write_text("12345") + sandbox.exec = AsyncMock(side_effect=AssertionError("must not signal a possibly reused PID")) + assert await confirm_runner_cleanup( + sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + ) == parse_cleanup_receipt(receipt) + sandbox.exec.assert_not_awaited() + + +async def test_stop_wins_claim_and_fences_delayed_launch(session): + sandbox, directory = session + receipt = await confirm_runner_cleanup( + sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + ) + assert receipt == {"cleanup_confirmed": True, "error": None, "return_code": None, "timed_out": False} + assert (directory / "launch.claim").readlink() == Path("stop") + assert (directory / "runner.stop").exists() + command = supervisor_command( + directory=str(directory), command=["touch", str(directory / "started")], timeout=1, cleanup_timeout=1 + ) + assert (await sandbox.exec(command)).return_code == 0 + assert not (directory / "runner.pid").exists() + assert not (directory / "runner.log").exists() + assert not (directory / "started").exists() + # A fenced launch is safe to close, without pretending a worker exited successfully. + assert receipt["return_code"] is None + with pytest.raises(ValidationError): + RunnerRuntimeInfo.model_validate(receipt) + + +async def test_missing_receipt_after_launch_is_not_cleanup_confirmation(session): + sandbox, directory = session + (directory / "launch.claim").symlink_to("launch") + with pytest.raises(RuntimeError, match="test launch outcome is unknown"): + await confirm_runner_cleanup( + sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + ) + assert (directory / "launch.claim").readlink() == Path("launch") + assert not (directory / "cleanup.json").exists() + + +@pytest.mark.parametrize("confirmed", [False, "true", 1]) +async def test_unconfirmed_cleanup_preserves_receipt_for_retry(session, confirmed): + sandbox, directory = session + receipt_path = directory / "cleanup.json" + receipt = {"cleanup_confirmed": confirmed, "error": "descendants remain"} + receipt_path.write_text(json.dumps(receipt)) + with pytest.raises(RuntimeError, match="test sandbox cleanup was not confirmed: descendants remain"): + await confirm_runner_cleanup( + sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + ) + assert json.loads(receipt_path.read_text()) == receipt + receipt = {"cleanup_confirmed": True, "error": None} + receipt_path.write_text(json.dumps(receipt)) + assert await confirm_runner_cleanup( + sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + ) == parse_cleanup_receipt(receipt) + + +def test_cleanup_and_runtime_are_independent(): + cleanup = {"return_code": 0, "timed_out": False, "cleanup_confirmed": True, "error": None} + assert parse_cleanup_receipt(cleanup) == cleanup + runtime = RunnerRuntimeInfo.model_validate({"hostname": "sandbox", "pid": 123}) + assert runtime.hostname == "sandbox" and runtime.pid == 123 + assert runtime.python is None + with pytest.raises(ValidationError): + parse_cleanup_receipt(runtime.model_dump()) + with pytest.raises(ValidationError): + RunnerRuntimeInfo.model_validate(cleanup) + for field in cleanup: + with pytest.raises(ValidationError): + parse_cleanup_receipt({key: value for key, value in cleanup.items() if key != field}) + + +@pytest.mark.parametrize( + "invalid", [{"return_code": "0"}, {"timed_out": "false"}, {"cleanup_confirmed": 1}, {"hostname": "worker"}] +) +def test_cleanup_receipt_keeps_strict_validation(invalid): + with pytest.raises(ValidationError): + parse_cleanup_receipt( + {"return_code": 0, "timed_out": False, "cleanup_confirmed": True, "error": None, **invalid} + ) + + +def test_absent_exit_code_is_not_success(): + full = {"return_code": None, "timed_out": False, "cleanup_confirmed": True, "error": None} + assert parse_cleanup_receipt(full) == full + assert parse_cleanup_receipt({"cleanup_confirmed": True, "error": None}) == full + with pytest.raises(ValidationError): + parse_cleanup_receipt({"return_code": 0, "error": None}) + + +@pytest.mark.parametrize("invalid", [{"pid": "123"}, {"hostname": 123}, {"python": 123}, {"return_code": 0}]) +def test_runtime_info_keeps_strict_validation(invalid): + with pytest.raises(ValidationError): + RunnerRuntimeInfo.model_validate({"hostname": "sandbox", "pid": 123, **invalid}) + + +def test_hermes_runtime_uses_the_same_schema(): + payload = {"hostname": "sandbox", "pid": 123, "python": "/opt/hermes/bin/python"} + assert RunnerRuntimeInfo.model_validate(payload).model_dump() == payload + for field in ("hostname", "pid"): + with pytest.raises(ValidationError): + RunnerRuntimeInfo.model_validate({key: value for key, value in payload.items() if key != field}) From 2d92a636dc96162d2ab961bc2c1a8229e794f632 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Fri, 2 Oct 2026 23:13:38 -0700 Subject: [PATCH 18/44] test(openclaw): wait for cancelled descendants to be reaped Signed-off-by: yaoyu-33 --- .../openclaw_agent/tests/test_app.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/responses_api_agents/openclaw_agent/tests/test_app.py b/responses_api_agents/openclaw_agent/tests/test_app.py index 3b0aafcfc37..c4c1bf00f78 100644 --- a/responses_api_agents/openclaw_agent/tests/test_app.py +++ b/responses_api_agents/openclaw_agent/tests/test_app.py @@ -14,7 +14,6 @@ # limitations under the License. import asyncio -import contextlib import json import os import signal @@ -816,16 +815,19 @@ async def _main(): break await asyncio.sleep(0.01) assert children, "descendant process never started" - child_pids = [child.pid for child in children] task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await task - return child_pids + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=5) + return children - child_pids = asyncio.run(_main()) + children = asyncio.run(_main()) assert captured["proc"].returncode is not None - assert all(not psutil.pid_exists(pid) for pid in child_pids) + # Orphaned descendants are reaped asynchronously by init after SIGKILL. + # Require disappearance within a bound, without racing that OS cleanup. + _, alive = psutil.wait_procs(children, timeout=3) + assert not alive, f"Descendants remain after cancellation: {alive}" + assert all(not psutil.pid_exists(child.pid) for child in children) class TestSigtermSalvage: From 8952633c83f5b8274c9d11247b45962a4294a1bb Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Fri, 2 Oct 2026 15:03:07 -0700 Subject: [PATCH 19/44] refactor(hermes): decouple default harness configuration Signed-off-by: yaoyu-33 --- responses_api_agents/hermes_agent/README.md | 129 ++++++++++++------ responses_api_agents/hermes_agent/app.py | 4 +- .../hermes_agent/configs/hermes_agent.yaml | 5 +- .../hermes_agent/tests/test_agent_sessions.py | 28 +++- .../hermes_agent/tests/test_recipe.py | 37 ++++- 5 files changed, 156 insertions(+), 47 deletions(-) diff --git a/responses_api_agents/hermes_agent/README.md b/responses_api_agents/hermes_agent/README.md index 7b53e7bcc6a..e86e11a065e 100644 --- a/responses_api_agents/hermes_agent/README.md +++ b/responses_api_agents/hermes_agent/README.md @@ -1,50 +1,76 @@ # Hermes Agent -# Quick start +Runs [Hermes](https://github.com/NousResearch/hermes-agent) inside a task sandbox through +Gym's agent-session interface. The agent server stays outside as an adapter. -## Create env.yaml in Gym/ +## Configure and run -``` -policy_base_url: https://api.openai.com/v1 -policy_api_key: sk... -policy_model_name: gpt-4o -``` +[configs/hermes_agent.yaml](configs/hermes_agent.yaml) is the default harness definition. +The benchmark owns task data, preparation, verification, and task sandbox settings. +The harness owns its runtime and model/tool loop. The Environment Server binds the two +and closes the agent before verification. -## Launch nemo gym servers - -```bash -gym env start \ - --config environments/hermes_math/config.yaml \ - --model-type openai_model -``` +Run from the Gym repository root with Gym and the benchmark's preparation dependencies +installed. For SWE-bench Pro, save this composition as `run.yaml`: -## Collect rollouts +```yaml +config_paths: + - resources_servers/swebench_pro/configs/swebench_pro.yaml + - responses_api_agents/hermes_agent/configs/hermes_agent.yaml + - environment_servers/single_agent_turn_legacy/configs/single_agent_turn_legacy.yaml + +single_agent_turn_legacy: + environment_servers: + single_agent_turn_legacy: + resources_server: + name: swebench_pro_resources_server + agent_server: + name: hermes_agent + resources_tool_transports: [] -```bash -gym eval run --no-serve \ - --agent hermes_math_agent \ - --input environments/hermes_math/data/example.jsonl \ - --output hermes_agent_rollout.jsonl \ - --limit 1 +hermes_agent: + responses_api_agents: + hermes_agent: + enabled_toolsets: [terminal] ``` -Example math rollouts are in `environments/hermes_math/data/example_rollouts.jsonl`. - -Example training reward for small multi environment test is shown [here](https://github.com/NVIDIA-NeMo/Gym/pull/1033#issuecomment-4399509664). +Supply the `policy_model` Gym Model Server, `policy_model_name`, and `sandbox` provider in +`model-provider.yaml`. The agent's `model` defaults to `${policy_model_name}` and remains +overridable. The sandbox must be able to reach the Model Server. -## Description +```bash +python benchmarks/swebench/pro/prepare.py -Runs [hermes-agent](https://github.com/NousResearch/hermes-agent) in a nemo gym agent server via the `run_agent.AIAgent` entrypoint, which matches the hermes-agent CLI and user experience. Can be used for benchmarks with hermes agent, or training in the harness. +gym env start --config run.yaml --config model-provider.yaml -## Setup +gym eval run --no-serve \ + --config run.yaml --config model-provider.yaml \ + --agent hermes_agent \ + -i benchmarks/swebench/data/swebench_pro_benchmark.jsonl \ + -o rollouts.jsonl --limit 3 --concurrency 3 +``` -`hermes-agent` is pinned in `requirements.txt` to a fork branch with patches for token id tracking, chat template, and sampling parameters needed for training. +The collector calls Environment Server `/run`: seed Resources, seed the agent, call its +rollout-prefixed `/v1/responses`, close the agent, verify, then close Resources. Prepared +flat rows use `single_agent_turn_legacy` with this native session lifecycle; no additional +materialization script is needed. Collection does not call the agent's compatibility `/run`. +Pass the same configuration to startup and `--no-serve` collection; collection does not +inherit routing settings from the running servers. -For agent integrations like this, the agent must point at Gym's model server, it must include prompt and generation token id in requests for Nemo RL and other trainer integration on policy token id correction, it must not override sampling parameters like temperature and top p, and it must not do non-monotonic things like dropping past reasoning content or context compaction. +### Switch harness or benchmark -## Resources server compatibility +To change a compatible harness, replace its config import, harness-specific settings, +the Environment Server's `agent_server.name`, and the collection command's `--agent`. +Keep benchmark data, preparation, and verifier settings unchanged. To change a compatible +benchmark, replace its Resources config/reference and prepared input, keeping the harness +definition unchanged. Check tool grants, task-image/runtime support, model API, and the +benchmark's declared `allowed_agents` before running a new pairing. -Works with any resources server based verifier, but does not work for resources server tools or other endpoints out of the box. Hermes Agent ships its own toolset (terminal, file, code_execution, web, etc.), so it does not rely on tools defined in the dataset. It may work with Gymnasium style resources servers, though. In testing, only the resources server's task data and `verify` are used. This means existing benchmarks (math, code, reasoning_gym, mcqa, instruction_following, ...) can be used as-is by adding a `_hermes_agent` config. +Use this explicit composition for now. `--agent` selects a configured agent; it does not +install or rebind one. The existing `--agent-type` swap and automatic benchmark-data lookup +still depend on legacy Agent-to-Resources bindings; they are not equivalent to this workflow. +Existing [Hermes SWE-Pro recipes](../../benchmarks/swebench/pro/hermes.yaml) remain supported +for compatibility; a new pairing does not need another combined preset. ## Configuration example @@ -53,13 +79,11 @@ hermes_agent: responses_api_agents: hermes_agent: entrypoint: app.py - resources_server: - type: resources_servers - name: my_verifier + resources_server: null model_server: type: responses_api_models name: policy_model - model: served-model-name + model: ${policy_model_name} enabled_toolsets: [terminal, file, code_execution] max_turns: 30 concurrency: 32 @@ -77,7 +101,8 @@ hermes_agent: |-------|---------|-------------| | `enabled_toolsets` | `null` (all) | forwarded to `AIAgent(enabled_toolsets=...)` | | `disabled_toolsets` | `null` | forwarded to `AIAgent(disabled_toolsets=...)` | -| `model` | `null` | served model id; defaults to `model_server.name` for backward compatibility | +| `model` | `${policy_model_name}` in the default YAML | served model ID; configs that omit it retain the legacy `model_server.name` fallback | +| `resources_server` | `null` | required only for the agent's compatibility `/run`; Environment Server binds Resources for native sessions | | `max_turns` | `30` | maps to `AIAgent.max_iterations` | | `concurrency` | `32` | max simultaneous `run()` calls | | `temperature` | `null` | sampling temperature passed to `AIAgent`; request `temperature` overrides it, including `0.0` | @@ -103,11 +128,7 @@ system message and ignored request `temperature`. It now combines the prompts an the request temperature, just like sandbox execution. These changes can affect scores. Remove unsupported fields that older versions silently ignored. -For SWE-bench Pro, use [`hermes.yaml`](../../benchmarks/swebench/pro/hermes.yaml). -See [Evaluate SWE-bench Pro with Hermes](../../fern/versions/latest/pages/evaluation-tutorials/hermes-swe-bench-pro.mdx) -for task preparation, Environment Server configuration, evaluation commands, and session limits. - -## Sandbox-mode requirements +## Runtime and model requirements Sandbox sessions live in the memory of the worker that seeded them, so seeding a session requires `num_workers: 1`. Calling the agent's `/run` directly keeps no session and still supports several workers. @@ -116,3 +137,29 @@ Each sandbox session installs the Hermes version pinned in `requirements.txt`, t Sessions use MCP tool grants and reject other required grants. Each granted MCP server is added to that session's Hermes configuration, so the sandbox must reach it at the granted URL, usually the Resources Server's `/mcp` endpoint. An activation fails before its first model call when a required server does not connect. Hermes names MCP tools `mcp__`; the response reports them as `mcp____`, the form Gym strips before verification, while captured model calls keep Hermes' names. The session token in each grant is readable inside the sandbox and gives access only to that episode's tools. The Hermes runner and the model's terminal tool execute as the same user in the same sandbox. The host reads the final result, including token IDs, from `/tmp/nemo-gym-hermes-sessions//output.json`; commands issued by the model can also write that file. Sandbox mode is suitable for evaluation, but it must not be used to produce RL training data until results are returned through a channel the model cannot modify. + +## Local compatibility + +Calls without an agent session run Hermes on the agent-server host. They do not operate on +a Resources-owned task sandbox. The agent's compatibility `/run` requires an explicit +`resources_server`; native sessions and direct `/v1/responses` do not. + +The default YAML now selects native composition rather than acting as an unbound legacy +swap source. Legacy swap configurations must explicitly restore +`resources_server: {type: resources_servers, name: "???"}` on the agent. Existing bound +recipes, including `hermes_math` and `swebench/pro/hermes`, keep their bindings. + +For the existing host-side math example, put `policy_base_url`, `policy_api_key`, and +`policy_model_name` in `env.yaml`, then run: + +```bash +gym env start --config environments/hermes_math/config.yaml --model-type openai_model + +gym eval run --no-serve \ + --config environments/hermes_math/config.yaml --model-type openai_model \ + --agent hermes_math_agent \ + --input environments/hermes_math/data/example.jsonl \ + --output hermes_agent_rollout.jsonl --limit 1 +``` + +Example math rollouts are in `environments/hermes_math/data/example_rollouts.jsonl`. diff --git a/responses_api_agents/hermes_agent/app.py b/responses_api_agents/hermes_agent/app.py index 45355923db7..666d17e9549 100644 --- a/responses_api_agents/hermes_agent/app.py +++ b/responses_api_agents/hermes_agent/app.py @@ -258,7 +258,7 @@ def _split_input_to_user_and_history(input_items) -> tuple[str, list[dict], Opti class HermesAgentConfig(BaseResponsesAPIAgentConfig): - resources_server: ResourcesServerRef + resources_server: ResourcesServerRef | None = None model_server: ModelServerRef model: Optional[str] = None concurrency: int = 32 @@ -1145,6 +1145,8 @@ def collect(bundle: AgentObservationBundle) -> None: async def run(self, request: Request, body: HermesAgentRunRequest) -> HermesAgentVerifyResponse: if self._agent_session_id_from_request(request) is not None: raise HTTPException(409, "Use the agent session responses and close routes") + if self.config.resources_server is None: + raise HTTPException(422, "Hermes /run requires resources_server; use Environment Server /run for sessions") async with self.sem: cookies = request.cookies diff --git a/responses_api_agents/hermes_agent/configs/hermes_agent.yaml b/responses_api_agents/hermes_agent/configs/hermes_agent.yaml index db4d015fe47..fbf7920a8a2 100644 --- a/responses_api_agents/hermes_agent/configs/hermes_agent.yaml +++ b/responses_api_agents/hermes_agent/configs/hermes_agent.yaml @@ -2,12 +2,11 @@ hermes_agent: responses_api_agents: hermes_agent: entrypoint: app.py - resources_server: - type: resources_servers - name: ??? + resources_server: null model_server: type: responses_api_models name: policy_model + model: ${policy_model_name} concurrency: 32 max_turns: 30 enabled_toolsets: null diff --git a/responses_api_agents/hermes_agent/tests/test_agent_sessions.py b/responses_api_agents/hermes_agent/tests/test_agent_sessions.py index 7166a59aa4d..7126f969b5f 100644 --- a/responses_api_agents/hermes_agent/tests/test_agent_sessions.py +++ b/responses_api_agents/hermes_agent/tests/test_agent_sessions.py @@ -97,7 +97,33 @@ def episode(agent): ) -def test_http_close_retry_and_stale_activation_never_fall_back(agent, state): +async def test_unbound_compatibility_run_fails_before_calling_resources(agent): + agent.config.resources_server = None + agent.server_client.post = AsyncMock() + with pytest.raises(HTTPException, match="/run requires resources_server") as error: + await agent.run( + SimpleNamespace(session={}, cookies={}), + HermesAgentRunRequest(responses_create_params={"input": "task"}), + ) + assert error.value.status_code == 422 + agent.server_client.post.assert_not_awaited() + + +async def test_unbound_direct_responses_remains_available(agent): + agent.config.resources_server = None + result = episode(agent) + agent._create_response = AsyncMock(return_value=result.response) + response = await agent.responses( + SimpleNamespace(session={}, path_params={}), NeMoGymResponseCreateParamsNonStreaming(input="task") + ) + assert response == result.response + agent._create_response.assert_awaited_once() + + +@pytest.mark.parametrize("bind_resources", [True, False]) +def test_http_close_retry_and_stale_activation_never_fall_back(agent, state, bind_resources): + if not bind_resources: + agent.config.resources_server = None agent._initialize_agent_session_state = AsyncMock(return_value=state) agent._run_sandbox_episode = AsyncMock(return_value=episode(agent)) agent._create_response = AsyncMock(side_effect=AssertionError("host fallback")) diff --git a/responses_api_agents/hermes_agent/tests/test_recipe.py b/responses_api_agents/hermes_agent/tests/test_recipe.py index 390b57480c0..9995b2e573a 100644 --- a/responses_api_agents/hermes_agent/tests/test_recipe.py +++ b/responses_api_agents/hermes_agent/tests/test_recipe.py @@ -1,17 +1,52 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import re from pathlib import Path import pytest from omegaconf import OmegaConf from environment_servers.single_agent_turn.app import SingleAgentTurnEnvironmentServerConfig -from nemo_gym.global_config import GlobalConfigDictParser +from nemo_gym.global_config import GlobalConfigDictParser, GlobalConfigDictParserConfig from nemo_gym.rollout_collection import _environment_server_for_agent, _environment_servers_by_agent from responses_api_agents.hermes_agent.app import HermesAgentConfig +@pytest.mark.parametrize("model_override", [None, "other-served-model"]) +def test_readme_composes_independent_harness_and_benchmark(model_override: str | None) -> None: + root = Path(__file__).parents[3] + readme = (root / "responses_api_agents/hermes_agent/README.md").read_text() + composition = re.search(r"```yaml\n(config_paths:.*?)\n```", readme, re.DOTALL) + assert composition is not None + initial = OmegaConf.merge( + OmegaConf.create(composition.group(1)), + GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + {"policy_model_name": "served-policy-model"}, + ) + if model_override is not None: + initial.hermes_agent.responses_api_agents.hermes_agent.model = model_override + config = GlobalConfigDictParser().parse( + GlobalConfigDictParserConfig( + initial_global_config_dict=initial, + skip_load_from_cli=True, + skip_load_from_dotenv=True, + offline=True, + ) + ) + agent = HermesAgentConfig(name="hermes_agent", **config.hermes_agent.responses_api_agents.hermes_agent) + assert agent.resources_server is None + assert agent.model == (model_override or "served-policy-model") + assert agent.enabled_toolsets == ["terminal"] + assert "datasets" not in config.hermes_agent.responses_api_agents.hermes_agent + environment_name = _environment_server_for_agent(agent.name, _environment_servers_by_agent(config)) + assert environment_name == "single_agent_turn_legacy" + environment = config[environment_name].environment_servers.single_agent_turn_legacy + assert environment.resources_server.name == "swebench_pro_resources_server" + assert environment.resources_tool_transports == [] + assert not any(name.startswith("swebench_pro_hermes") for name in config) + + @pytest.mark.parametrize("model_override", [None, "other-served-model"]) def test_hermes_recipe_resolves_to_session_environment(model_override: str | None) -> None: recipe = Path(__file__).parents[3] / "benchmarks/swebench/pro/hermes.yaml" From 6d443b6e1ca0e4ebc64364d2cdb42c1e27595a56 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Fri, 2 Oct 2026 19:53:57 -0700 Subject: [PATCH 20/44] refactor(hermes): share cleanup and runtime validation Integrate Hermes-specific changes from yaoyu-33/Gym#10 and #11 into NVIDIA-NeMo/Gym#3961. Keep harness output separate from cleanup and worker runtime metadata. Signed-off-by: yaoyu-33 --- nemo_gym/sandbox/runner.py | 110 ++++++++++++ responses_api_agents/hermes_agent/app.py | 15 +- .../hermes_agent/tests/test_agent_sessions.py | 58 ++++-- .../hermes_agent/tests/test_app.py | 22 ++- .../test_sandbox_runner_transport.py | 166 ++++++++++++++++++ 5 files changed, 351 insertions(+), 20 deletions(-) create mode 100644 nemo_gym/sandbox/runner.py create mode 100644 tests/unit_tests/test_sandbox_runner_transport.py diff --git a/nemo_gym/sandbox/runner.py b/nemo_gym/sandbox/runner.py new file mode 100644 index 00000000000..a609fe964da --- /dev/null +++ b/nemo_gym/sandbox/runner.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Controller-side transport for the shared sandbox process supervisor. + +Session ownership, cancellation, and harness output parsing stay with the adapter. +Unlike process_supervisor.py, this module is not uploaded to the task sandbox. +""" + +import json +import tempfile +from pathlib import Path +from shlex import join, quote +from uuid import uuid4 + +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from nemo_gym.sandbox import AsyncSandbox +from nemo_gym.sandbox.process_supervisor import CleanupReceipt + + +_CLEANUP_RECEIPT = TypeAdapter(CleanupReceipt) + + +class RunnerRuntimeInfo(BaseModel): + """Worker identity, independent of supervisor cleanup and harness output.""" + + model_config = ConfigDict(extra="forbid", strict=True) + hostname: str + pid: int + python: str | None = None + + +def parse_cleanup_receipt(payload: object) -> CleanupReceipt: + """Validate the supervisor's existing schema without adding worker dependencies. + + Older stop-before-launch receipts contain only acknowledgement and error. + Normalize that form without inventing a successful worker exit code. + """ + if isinstance(payload, dict) and payload.keys() == {"cleanup_confirmed", "error"}: + payload = {**payload, "return_code": None, "timed_out": False} + return _CLEANUP_RECEIPT.validate_python(payload, strict=True, extra="forbid") + + +async def upload_text(sandbox: AsyncSandbox, *, path: str, text: str) -> None: + """Upload adapter-owned text without interpolating its contents into a shell command.""" + with tempfile.TemporaryDirectory(prefix="sandbox-runner-upload-") as directory: + source = Path(directory) / "payload" + source.write_text(text) + await sandbox.upload(source, path) + + +async def read_text(sandbox: AsyncSandbox, *, path: str) -> str: + """Download an adapter-owned artifact, preserving non-UTF8 output with replacement.""" + with tempfile.TemporaryDirectory(prefix="sandbox-runner-download-") as directory: + destination = Path(directory) / "payload" + await sandbox.download(path, destination) + return destination.read_text(errors="replace") + + +def supervisor_command(*, directory: str, command: list[str], timeout: float, cleanup_timeout: float) -> str: + """Fence delayed launches and run a harness command under the shared supervisor.""" + return ( + f"trap '' TERM; ln -s launch {quote(directory + '/launch.claim')} 2>/dev/null || exit 0; " + f"echo $$ > {quote(directory + '/runner.pid')} && " + f"exec python3 -I {quote(directory + '/process_supervisor.py')} " + f"--timeout {timeout} --cleanup-timeout {cleanup_timeout} " + f"--stop-file {quote(directory + '/runner.stop')} " + f"--receipt {quote(directory + '/cleanup.json')} -- {join(command)} " + f">{quote(directory + '/runner.log')} 2>&1" + ) + + +async def confirm_runner_cleanup( + sandbox: AsyncSandbox, *, directory: str, workdir: str, timeout: float, harness: str +) -> CleanupReceipt: + """Fence a pending launch or require explicit supervisor acknowledgement before teardown. + + A stop that wins the launch claim writes a minimal receipt: no worker ran, so + runtime metadata and a return code are intentionally absent. Worker metadata + and harness output are validated separately by the adapter. + """ + receipt_path = f"{directory}/cleanup.json" + try: + receipt = json.loads(await read_text(sandbox, path=receipt_path)) + except Exception: + receipt = {} + if receipt.get("cleanup_confirmed") is not True: + pid_path = quote(f"{directory}/runner.pid") + stop_path = quote(f"{directory}/runner.stop") + claim_path = quote(f"{directory}/launch.claim") + temporary = quote(f"{receipt_path}.{uuid4().hex}.tmp") + stopped = quote(json.dumps({"cleanup_confirmed": True, "error": None})) + script = ( + f"[ -f {quote(receipt_path)} ] && exit 0; " + f"touch {stop_path} || exit 1; " + f"ln -s stop {claim_path} 2>/dev/null || true; " + f'if [ "$(readlink {claim_path})" = stop ]; then ' + f"printf '%s' {stopped} > {temporary} && mv {temporary} {quote(receipt_path)}; exit $?; fi; " + f'if [ -s {pid_path} ]; then kill -TERM "$(cat {pid_path})" 2>/dev/null || true; fi; ' + f"for _ in $(seq 1 {max(1, int(timeout))}); do " + f"[ -f {quote(receipt_path)} ] && exit 0; sleep 1; done; exit 1" + ) + await sandbox.exec(script, cwd=workdir, timeout_s=timeout + 5) + try: + receipt = json.loads(await read_text(sandbox, path=receipt_path)) + except Exception as error: + raise RuntimeError(f"{harness} launch outcome is unknown; cannot confirm termination") from error + if receipt.get("cleanup_confirmed") is not True: + raise RuntimeError(f"{harness} sandbox cleanup was not confirmed: {receipt.get('error')}") + return parse_cleanup_receipt(receipt) diff --git a/responses_api_agents/hermes_agent/app.py b/responses_api_agents/hermes_agent/app.py index 666d17e9549..ed2eca5ccc1 100644 --- a/responses_api_agents/hermes_agent/app.py +++ b/responses_api_agents/hermes_agent/app.py @@ -76,6 +76,7 @@ from nemo_gym.sandbox.access import DirectSandboxConnection from nemo_gym.sandbox.config import resolve_provider_config from nemo_gym.sandbox.providers import create_provider +from nemo_gym.sandbox.runner import RunnerRuntimeInfo, parse_cleanup_receipt from nemo_gym.server_utils import get_response_json, raise_for_status from nemo_gym.tool_access import MCPToolAccess from responses_api_agents.hermes_agent.model_kwargs import _model_api_kwargs @@ -552,6 +553,7 @@ async def _terminate_sandbox_runner(self, state: HermesAgentSessionState) -> Non raise RuntimeError("Hermes launch outcome is unknown; cannot confirm termination") from error if receipt.get("cleanup_confirmed") is not True: raise RuntimeError(f"Hermes descendant cleanup was not confirmed: {receipt.get('error')}") + parse_cleanup_receipt(receipt) state.runner_cleanup = RunnerCleanup.CONFIRMED async def _cleanup_sandbox_session( @@ -785,9 +787,12 @@ async def _run_sandbox_episode( if output.get("error") is not None: raise RuntimeError(f"Hermes sandbox runner failed: {output['error']}\n{output.get('traceback', '')}") result = output.get("result") - runtime = output.get("runtime") - if not isinstance(result, dict) or not isinstance(runtime, dict): + if not isinstance(result, dict): raise RuntimeError("Hermes sandbox runner returned an invalid output") + try: + runtime = RunnerRuntimeInfo.model_validate(output.get("runtime")) + except ValueError as error: + raise RuntimeError("Hermes sandbox runner returned invalid runtime metadata") from error response = self._response_from_result( body=body, result=result, @@ -802,9 +807,9 @@ async def _run_sandbox_episode( response.metadata = { **(response.metadata or {}), "harness_execution": "sandbox", - "harness_hostname": str(runtime.get("hostname") or ""), - "harness_pid": str(runtime.get("pid") or ""), - "harness_python": str(runtime.get("python") or ""), + "harness_hostname": runtime.hostname, + "harness_pid": str(runtime.pid), + "harness_python": runtime.python or "", } return AgentEpisode( response=response, diff --git a/responses_api_agents/hermes_agent/tests/test_agent_sessions.py b/responses_api_agents/hermes_agent/tests/test_agent_sessions.py index 7126f969b5f..8b057cafb61 100644 --- a/responses_api_agents/hermes_agent/tests/test_agent_sessions.py +++ b/responses_api_agents/hermes_agent/tests/test_agent_sessions.py @@ -261,8 +261,11 @@ def test_provider_failure_is_http_500_and_replays_without_rerunning(agent, state agent._upload_json = AsyncMock() agent._download_json = AsyncMock( side_effect=[ - {"cleanup_confirmed": True}, - {"result": {"failed": True, "error": "HTTP 429 Too Many Requests", "messages": []}, "runtime": {}}, + {"cleanup_confirmed": True, "error": None}, + { + "result": {"failed": True, "error": "HTTP 429 Too Many Requests", "messages": []}, + "runtime": {"hostname": "sandbox", "pid": 123}, + }, ] ) with TestClient(agent.setup_webserver(), raise_server_exceptions=False) as client: @@ -288,7 +291,7 @@ async def test_close_failure_keeps_session_for_retry( agent._session_records["session"] = _AgentSessionRecord(state=state, episode_id=state.request.episode_id) if runner_started: state.runner_cleanup = RunnerCleanup.UNCONFIRMED - agent._download_json = AsyncMock(return_value={"cleanup_confirmed": True}) + agent._download_json = AsyncMock(return_value={"cleanup_confirmed": True, "error": None}) state.sandbox.exec.side_effect = [SimpleNamespace(return_code=1), SimpleNamespace(return_code=0)] close = AgentCloseSessionRequest(agent_session_id="session", episode_id=state.request.episode_id) with pytest.raises(RuntimeError, match="session files"): @@ -517,7 +520,7 @@ async def test_receipt_download_failure_never_signals_a_reused_pid(agent, state, assert await child.stdout.readline() == b"ready\n" (directory / "runner.pid").write_text(str(child.pid)) (directory / "launch.claim").symlink_to("launch") - (directory / "cleanup.json").write_text(json.dumps({"cleanup_confirmed": confirmed})) + (directory / "cleanup.json").write_text(json.dumps({"cleanup_confirmed": confirmed, "error": None})) download = agent._download_json attempts = 0 @@ -668,6 +671,36 @@ async def test_stop_during_interpreter_startup_closes_without_starting_a_worker( await asyncio.gather(state.task, return_exceptions=True) +@pytest.mark.parametrize("runtime", [None, {}, {"hostname": "sandbox", "pid": "123"}]) +async def test_invalid_runtime_keeps_cleanup_confirmed(agent, state, runtime): + agent._upload_json = AsyncMock() + agent._download_json = AsyncMock( + side_effect=[ + {"cleanup_confirmed": True, "error": None}, + {"result": {"completed": True, "messages": []}, "runtime": runtime}, + ] + ) + with pytest.raises(RuntimeError, match="invalid runtime metadata"): + await agent._run_sandbox_episode( + body=NeMoGymResponseCreateParamsNonStreaming(input="task"), agent_session_id="session", state=state + ) + assert state.runner_cleanup is RunnerCleanup.CONFIRMED + await agent._close_agent_session_state(state) + state.sandbox.disconnect.assert_awaited_once() + state.sandbox.stop.assert_not_awaited() + + +async def test_invalid_cleanup_schema_does_not_mark_runner_confirmed(agent, state): + state.runner_cleanup = RunnerCleanup.UNCONFIRMED + agent._download_json = AsyncMock( + return_value={"cleanup_confirmed": True, "error": None, "timed_out": False, "return_code": "0"} + ) + with pytest.raises(ValidationError): + await agent._terminate_sandbox_runner(state) + assert state.runner_cleanup is RunnerCleanup.UNCONFIRMED + state.sandbox.disconnect.assert_not_awaited() + + @pytest.mark.parametrize("receipt", [{}, {"cleanup_confirmed": False}, {"cleanup_confirmed": "true"}]) async def test_runner_exit_without_cleanup_receipt_blocks_close(agent, state, receipt): state.runner_cleanup = RunnerCleanup.UNCONFIRMED @@ -676,7 +709,7 @@ async def test_runner_exit_without_cleanup_receipt_blocks_close(agent, state, re await agent._close_agent_session_state(state) assert state.runner_cleanup is RunnerCleanup.UNCONFIRMED state.sandbox.disconnect.assert_not_awaited() - agent._download_json.return_value = {"cleanup_confirmed": True} + agent._download_json.return_value = {"cleanup_confirmed": True, "error": None} await agent._close_agent_session_state(state) assert state.runner_cleanup is RunnerCleanup.CONFIRMED state.sandbox.disconnect.assert_awaited_once() @@ -692,10 +725,10 @@ async def test_session_prompt_and_limits_reach_runner(agent, state, overrides, t agent._upload_json = AsyncMock() agent._download_json = AsyncMock( side_effect=[ - {"cleanup_confirmed": True}, + {"cleanup_confirmed": True, "error": None}, { "result": {"completed": True, "messages": [{"role": "assistant", "content": "done"}]}, - "runtime": {"pid": 123}, + "runtime": {"hostname": "sandbox", "pid": 123}, }, ] ) @@ -742,7 +775,7 @@ async def execute(command, **kwargs): async def download(sandbox, path): if path.endswith("/cleanup.json"): events.append("cleanup") - return {"cleanup_confirmed": True} + return {"cleanup_confirmed": True, "error": None} assert path.endswith("/output.json") events.append("output") if not output_available: @@ -765,7 +798,7 @@ async def download(sandbox, path): } ] }, - "runtime": {"pid": 123}, + "runtime": {"hostname": "sandbox", "pid": 123}, } agent._download_json = AsyncMock(side_effect=download) @@ -867,7 +900,12 @@ def count_validation(self, body): agent._session_records["session"] = _AgentSessionRecord(state=state, episode_id=state.request.episode_id) agent._upload_json = AsyncMock() - agent._download_json = AsyncMock(side_effect=[{"cleanup_confirmed": True}, {"result": result, "runtime": {}}]) + agent._download_json = AsyncMock( + side_effect=[ + {"cleanup_confirmed": True, "error": None}, + {"result": result, "runtime": {"hostname": "sandbox", "pid": 123}}, + ] + ) await agent.responses(request(state), body) payload = agent._upload_json.await_args.args[2] assert payload["user_message"] == user_message == "Follow-up" diff --git a/responses_api_agents/hermes_agent/tests/test_app.py b/responses_api_agents/hermes_agent/tests/test_app.py index be41c6c607a..474217aa474 100644 --- a/responses_api_agents/hermes_agent/tests/test_app.py +++ b/responses_api_agents/hermes_agent/tests/test_app.py @@ -132,7 +132,10 @@ async def test_sandbox_access_selects_runtime_provider(self, monkeypatch, with_m sandbox.exec.return_value = MagicMock(return_code=0, stdout="", stderr="") connect = AsyncMock(return_value=sandbox) monkeypatch.setattr("responses_api_agents.hermes_agent.app.shutil.which", lambda name: "/test/uv") - monkeypatch.setattr("responses_api_agents.hermes_agent.app.get_global_config_dict", lambda: {"runtime": {}}) + monkeypatch.setattr( + "responses_api_agents.hermes_agent.app.get_global_config_dict", + lambda: {"runtime": {"hostname": "sandbox", "pid": 123}}, + ) monkeypatch.setattr("responses_api_agents.hermes_agent.app.resolve_provider_config", resolve) monkeypatch.setattr("responses_api_agents.hermes_agent.app.create_provider", lambda config: provider) monkeypatch.setattr("responses_api_agents.hermes_agent.app.AsyncSandbox.connect", connect) @@ -155,7 +158,7 @@ async def test_sandbox_access_selects_runtime_provider(self, monkeypatch, with_m ), ) - resolve.assert_called_once_with("runtime", {"runtime": {}}) + resolve.assert_called_once_with("runtime", {"runtime": {"hostname": "sandbox", "pid": 123}}) connect.assert_awaited_once_with( {"sandbox_id": "sandbox"}, provider=provider, @@ -183,7 +186,10 @@ async def exec_(command, **_kwargs): return MagicMock(return_code=0, stdout="", stderr="") sandbox.exec.side_effect = exec_ - monkeypatch.setattr("responses_api_agents.hermes_agent.app.get_global_config_dict", lambda: {"runtime": {}}) + monkeypatch.setattr( + "responses_api_agents.hermes_agent.app.get_global_config_dict", + lambda: {"runtime": {"hostname": "sandbox", "pid": 123}}, + ) monkeypatch.setattr("responses_api_agents.hermes_agent.app.resolve_provider_config", MagicMock()) monkeypatch.setattr("responses_api_agents.hermes_agent.app.create_provider", lambda config: AsyncMock()) monkeypatch.setattr( @@ -225,7 +231,10 @@ async def test_missing_sandbox_access_uses_configured_fallback(self, monkeypatch sandbox = AsyncMock() sandbox.exec.return_value = MagicMock(return_code=0, stdout="", stderr="") sandbox_factory = MagicMock(return_value=sandbox) - monkeypatch.setattr("responses_api_agents.hermes_agent.app.get_global_config_dict", lambda: {"runtime": {}}) + monkeypatch.setattr( + "responses_api_agents.hermes_agent.app.get_global_config_dict", + lambda: {"runtime": {"hostname": "sandbox", "pid": 123}}, + ) monkeypatch.setattr( "responses_api_agents.hermes_agent.app.resolve_provider_config", MagicMock(return_value={"local": {}}), @@ -391,7 +400,10 @@ def _sandbox_session( async def download(sandbox, remote_path): if remote_path.endswith("/cleanup.json"): commands = getattr(sandbox, "commands", None) - return {"cleanup_confirmed": commands is None or any("kill -TERM" in cmd for cmd in commands)} + return { + "cleanup_confirmed": commands is None or any("kill -TERM" in cmd for cmd in commands), + "error": None, + } return await original_download(sandbox, remote_path) hermes._download_json = AsyncMock(side_effect=download) diff --git a/tests/unit_tests/test_sandbox_runner_transport.py b/tests/unit_tests/test_sandbox_runner_transport.py new file mode 100644 index 00000000000..840a5c4945e --- /dev/null +++ b/tests/unit_tests/test_sandbox_runner_transport.py @@ -0,0 +1,166 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Transport contracts shared by sandboxed harness controllers.""" + +import asyncio +import json +import shutil +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest +from pydantic import ValidationError + +from nemo_gym.sandbox.providers.base import SandboxExecResult +from nemo_gym.sandbox.runner import ( + RunnerRuntimeInfo, + confirm_runner_cleanup, + parse_cleanup_receipt, + read_text, + supervisor_command, + upload_text, +) + + +class LocalSandbox: + async def upload(self, source, destination): + shutil.copyfile(source, destination) + + async def download(self, source, destination): + shutil.copyfile(source, destination) + + async def exec(self, command, *, cwd=None, timeout_s=30): + process = await asyncio.create_subprocess_exec( + "sh", "-c", command, cwd=cwd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout_s) + return SandboxExecResult(stdout.decode(errors="replace"), stderr.decode(errors="replace"), process.returncode) + + +@pytest.fixture +def session(tmp_path): + directory = tmp_path / "session's files" + directory.mkdir() + return LocalSandbox(), directory + + +async def test_file_transport_keeps_contents_out_of_shell(session): + sandbox, directory = session + sandbox.exec = AsyncMock(side_effect=AssertionError("file transfer must not invoke a shell")) + path = str(directory / "input.json") + payload = '{"prompt": "$(touch unwanted); `echo surprise`"}\n' + await upload_text(sandbox, path=path, text=payload) + assert Path(path).read_text() == payload + assert await read_text(sandbox, path=path) == payload + Path(path).write_bytes(b"partial output\xff\n") + assert await read_text(sandbox, path=path) == "partial output\ufffd\n" + sandbox.exec.assert_not_awaited() + + +async def test_confirmed_receipt_does_not_signal_stored_pid(session): + sandbox, directory = session + receipt = {"cleanup_confirmed": True, "error": None} + (directory / "cleanup.json").write_text(json.dumps(receipt)) + (directory / "runner.pid").write_text("12345") + sandbox.exec = AsyncMock(side_effect=AssertionError("must not signal a possibly reused PID")) + assert await confirm_runner_cleanup( + sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + ) == parse_cleanup_receipt(receipt) + sandbox.exec.assert_not_awaited() + + +async def test_stop_wins_claim_and_fences_delayed_launch(session): + sandbox, directory = session + receipt = await confirm_runner_cleanup( + sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + ) + assert receipt == {"cleanup_confirmed": True, "error": None, "return_code": None, "timed_out": False} + assert (directory / "launch.claim").readlink() == Path("stop") + assert (directory / "runner.stop").exists() + command = supervisor_command( + directory=str(directory), command=["touch", str(directory / "started")], timeout=1, cleanup_timeout=1 + ) + assert (await sandbox.exec(command)).return_code == 0 + assert not (directory / "runner.pid").exists() + assert not (directory / "runner.log").exists() + assert not (directory / "started").exists() + # A fenced launch is safe to close, without pretending a worker exited successfully. + assert receipt["return_code"] is None + with pytest.raises(ValidationError): + RunnerRuntimeInfo.model_validate(receipt) + + +async def test_missing_receipt_after_launch_is_not_cleanup_confirmation(session): + sandbox, directory = session + (directory / "launch.claim").symlink_to("launch") + with pytest.raises(RuntimeError, match="test launch outcome is unknown"): + await confirm_runner_cleanup( + sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + ) + assert (directory / "launch.claim").readlink() == Path("launch") + assert not (directory / "cleanup.json").exists() + + +@pytest.mark.parametrize("confirmed", [False, "true", 1]) +async def test_unconfirmed_cleanup_preserves_receipt_for_retry(session, confirmed): + sandbox, directory = session + receipt_path = directory / "cleanup.json" + receipt = {"cleanup_confirmed": confirmed, "error": "descendants remain"} + receipt_path.write_text(json.dumps(receipt)) + with pytest.raises(RuntimeError, match="test sandbox cleanup was not confirmed: descendants remain"): + await confirm_runner_cleanup( + sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + ) + assert json.loads(receipt_path.read_text()) == receipt + receipt = {"cleanup_confirmed": True, "error": None} + receipt_path.write_text(json.dumps(receipt)) + assert await confirm_runner_cleanup( + sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + ) == parse_cleanup_receipt(receipt) + + +def test_cleanup_and_runtime_are_independent(): + cleanup = {"return_code": 0, "timed_out": False, "cleanup_confirmed": True, "error": None} + assert parse_cleanup_receipt(cleanup) == cleanup + runtime = RunnerRuntimeInfo.model_validate({"hostname": "sandbox", "pid": 123}) + assert runtime.hostname == "sandbox" and runtime.pid == 123 + assert runtime.python is None + with pytest.raises(ValidationError): + parse_cleanup_receipt(runtime.model_dump()) + with pytest.raises(ValidationError): + RunnerRuntimeInfo.model_validate(cleanup) + for field in cleanup: + with pytest.raises(ValidationError): + parse_cleanup_receipt({key: value for key, value in cleanup.items() if key != field}) + + +@pytest.mark.parametrize( + "invalid", [{"return_code": "0"}, {"timed_out": "false"}, {"cleanup_confirmed": 1}, {"hostname": "worker"}] +) +def test_cleanup_receipt_keeps_strict_validation(invalid): + with pytest.raises(ValidationError): + parse_cleanup_receipt( + {"return_code": 0, "timed_out": False, "cleanup_confirmed": True, "error": None, **invalid} + ) + + +def test_absent_exit_code_is_not_success(): + full = {"return_code": None, "timed_out": False, "cleanup_confirmed": True, "error": None} + assert parse_cleanup_receipt(full) == full + assert parse_cleanup_receipt({"cleanup_confirmed": True, "error": None}) == full + with pytest.raises(ValidationError): + parse_cleanup_receipt({"return_code": 0, "error": None}) + + +@pytest.mark.parametrize("invalid", [{"pid": "123"}, {"hostname": 123}, {"python": 123}, {"return_code": 0}]) +def test_runtime_info_keeps_strict_validation(invalid): + with pytest.raises(ValidationError): + RunnerRuntimeInfo.model_validate({"hostname": "sandbox", "pid": 123, **invalid}) + + +def test_hermes_runtime_uses_the_same_schema(): + payload = {"hostname": "sandbox", "pid": 123, "python": "/opt/hermes/bin/python"} + assert RunnerRuntimeInfo.model_validate(payload).model_dump() == payload + for field in ("hostname", "pid"): + with pytest.raises(ValidationError): + RunnerRuntimeInfo.model_validate({key: value for key, value in payload.items() if key != field}) From 7b1b5937b5c3c6b123cd8cddcecfb25cc9572847 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Fri, 2 Oct 2026 20:09:36 -0700 Subject: [PATCH 21/44] fix(hermes): report provider token usage for root model calls Signed-off-by: yaoyu-33 --- responses_api_agents/hermes_agent/README.md | 7 ++ responses_api_agents/hermes_agent/app.py | 17 ++- .../hermes_agent/sandbox_runner.py | 7 +- .../hermes_agent/tests/test_app.py | 1 + .../hermes_agent/tests/test_model_kwargs.py | 2 +- .../hermes_agent/tests/test_sandbox_runner.py | 104 ++++++++++++++++ .../hermes_agent/token_usage.py | 112 ++++++++++++++++++ 7 files changed, 238 insertions(+), 12 deletions(-) create mode 100644 responses_api_agents/hermes_agent/token_usage.py diff --git a/responses_api_agents/hermes_agent/README.md b/responses_api_agents/hermes_agent/README.md index e86e11a065e..4f2aa5d9333 100644 --- a/responses_api_agents/hermes_agent/README.md +++ b/responses_api_agents/hermes_agent/README.md @@ -138,6 +138,13 @@ Sessions use MCP tool grants and reject other required grants. Each granted MCP The Hermes runner and the model's terminal tool execute as the same user in the same sandbox. The host reads the final result, including token IDs, from `/tmp/nemo-gym-hermes-sessions//output.json`; commands issued by the model can also write that file. Sandbox mode is suitable for evaluation, but it must not be used to produce RL training data until results are returned through a channel the model cannot modify. +Response `usage` sums provider-reported tokens for the root agent's model replies, +including retries, truncated replies, and the iteration-limit summary. Cached input +and reasoning tokens remain subsets of input and output tokens. Missing detail counts +stay `null`; if any reply omits token usage, the aggregate is `null`. Counts do not depend +on token IDs or the retained conversation history. Delegated agents and auxiliary model +clients (such as context compression) are outside this root-agent total. + ## Local compatibility Calls without an agent session run Hermes on the agent-server host. They do not operate on diff --git a/responses_api_agents/hermes_agent/app.py b/responses_api_agents/hermes_agent/app.py index ed2eca5ccc1..ac12a83bbc3 100644 --- a/responses_api_agents/hermes_agent/app.py +++ b/responses_api_agents/hermes_agent/app.py @@ -54,10 +54,8 @@ NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming, NeMoGymResponseFunctionToolCall, - NeMoGymResponseInputTokensDetails, NeMoGymResponseOutputMessageForTraining, NeMoGymResponseOutputText, - NeMoGymResponseOutputTokensDetails, NeMoGymResponseReasoningItem, NeMoGymResponseUsage, NeMoGymSummary, @@ -81,6 +79,7 @@ from nemo_gym.tool_access import MCPToolAccess from responses_api_agents.hermes_agent.model_kwargs import _model_api_kwargs from responses_api_agents.hermes_agent.observability import HermesAgentObserver, normalize_hermes_messages +from responses_api_agents.hermes_agent.token_usage import HermesTokenUsage def _trajectory_to_output_items(messages, n_input): @@ -186,6 +185,7 @@ def _sandbox_hermes_install() -> tuple[str, str]: _SANDBOX_SUPERVISOR = f"{_SANDBOX_RUNTIME_DIR}/process_supervisor.py" _SANDBOX_OBSERVER = f"{_SANDBOX_RUNTIME_DIR}/sandbox_observer.py" _SANDBOX_MODEL_KWARGS = f"{_SANDBOX_RUNTIME_DIR}/model_kwargs.py" +_SANDBOX_TOKEN_USAGE = f"{_SANDBOX_RUNTIME_DIR}/token_usage.py" class RunnerCleanup(Enum): @@ -473,6 +473,7 @@ async def _initialize_agent_session_state( await sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), _SANDBOX_RUNNER) await sandbox.upload(Path(__file__).with_name("sandbox_observer.py"), _SANDBOX_OBSERVER) await sandbox.upload(Path(__file__).with_name("model_kwargs.py"), _SANDBOX_MODEL_KWARGS) + await sandbox.upload(Path(__file__).with_name("token_usage.py"), _SANDBOX_TOKEN_USAGE) await sandbox.upload(Path(process_supervisor.__file__), _SANDBOX_SUPERVISOR) except BaseException as error: try: @@ -956,13 +957,7 @@ def _response_from_result( tool_choice=body.tool_choice, tools=body.tools, parallel_tool_calls=body.parallel_tool_calls, - usage=NeMoGymResponseUsage( - input_tokens=0, - input_tokens_details=NeMoGymResponseInputTokensDetails(cached_tokens=0), - output_tokens=0, - output_tokens_details=NeMoGymResponseOutputTokensDetails(reasoning_tokens=0), - total_tokens=0, - ), + usage=NeMoGymResponseUsage.model_validate(result["gym_usage"]) if result.get("gym_usage") else None, ) async def _create_response( @@ -1005,6 +1000,8 @@ def _patched_build_api_kwargs(api_messages: list[dict[str, Any]]) -> dict[str, A ) agent._build_api_kwargs = _patched_build_api_kwargs + usage = HermesTokenUsage() + usage.instrument(agent) observer = None if observation_collector is not None: try: @@ -1060,7 +1057,7 @@ def _patched_build_api_kwargs(api_messages: list[dict[str, Any]]) -> dict[str, A return self._response_from_result( body=body, - result=result, + result={**result, "gym_usage": usage.snapshot()}, model_name=model_name, interrupted_by_dispatch=interrupted_by_dispatch, n_input=len(params["history"]) + 1, diff --git a/responses_api_agents/hermes_agent/sandbox_runner.py b/responses_api_agents/hermes_agent/sandbox_runner.py index d240b045630..5c82cba572a 100644 --- a/responses_api_agents/hermes_agent/sandbox_runner.py +++ b/responses_api_agents/hermes_agent/sandbox_runner.py @@ -19,9 +19,11 @@ try: from .model_kwargs import _model_api_kwargs from .sandbox_observer import SandboxHermesObserver + from .token_usage import HermesTokenUsage except ImportError: from model_kwargs import _model_api_kwargs from sandbox_observer import SandboxHermesObserver + from token_usage import HermesTokenUsage def _write_atomic(path: Path, payload: dict[str, Any]) -> None: @@ -98,6 +100,8 @@ def _run(payload: dict[str, Any], session_dir: Path, *, output_path: Path | None save_trajectories=False, ) observer = SandboxHermesObserver().instrument(agent) + usage = HermesTokenUsage() + usage.instrument(agent) original_build_api_kwargs = agent._build_api_kwargs @@ -118,6 +122,7 @@ def progress() -> dict[str, Any]: "completed": False, "interrupted": True, "stop_reason": stop_reason, + "gym_usage": usage.snapshot(), "messages": getattr(agent, "_session_messages", []) or [*payload["history"], {"role": "user", "content": payload["user_message"]}], } @@ -156,7 +161,7 @@ def interrupt(*_: object) -> None: result = {**result, "completed": False, "interrupted": True, "stop_reason": stop_reason} return { "observations": observer.finish(result, error), - "result": result, + "result": {**result, "gym_usage": usage.snapshot()}, "runtime": runtime, } diff --git a/responses_api_agents/hermes_agent/tests/test_app.py b/responses_api_agents/hermes_agent/tests/test_app.py index 474217aa474..3702185c33f 100644 --- a/responses_api_agents/hermes_agent/tests/test_app.py +++ b/responses_api_agents/hermes_agent/tests/test_app.py @@ -172,6 +172,7 @@ async def test_sandbox_access_selects_runtime_provider(self, monkeypatch, with_m "sandbox_runner.py", "sandbox_observer.py", "model_kwargs.py", + "token_usage.py", "process_supervisor.py", } diff --git a/responses_api_agents/hermes_agent/tests/test_model_kwargs.py b/responses_api_agents/hermes_agent/tests/test_model_kwargs.py index 59819a2ead6..2c7c87c92e3 100644 --- a/responses_api_agents/hermes_agent/tests/test_model_kwargs.py +++ b/responses_api_agents/hermes_agent/tests/test_model_kwargs.py @@ -61,7 +61,7 @@ def test_template_settings_merge_without_mutating_hermes_input(): def test_standalone_runner_loads_shared_helper_without_gym_on_path(tmp_path): source = Path(__file__).parents[1] - for name in ("sandbox_runner.py", "sandbox_observer.py", "model_kwargs.py"): + for name in ("sandbox_runner.py", "sandbox_observer.py", "model_kwargs.py", "token_usage.py"): shutil.copyfile(source / name, tmp_path / name) # -S excludes site-packages (including Gym); only the uploaded runtime files are importable. completed = subprocess.run( diff --git a/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py b/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py index e12350413fd..b79b59f8975 100644 --- a/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py @@ -246,6 +246,14 @@ def with_override(self, messages): ], } answers = [_completion(tool_call), _completion({"content": "summary of the work"})] + for answer, prompt, completion, cached, reasoning in zip(answers, [100, 180], [20, 40], [10, 30], [5, 15]): + answer["usage"] = { + "prompt_tokens": prompt, + "completion_tokens": completion, + "total_tokens": prompt + completion, + "prompt_tokens_details": {"cached_tokens": cached}, + "completion_tokens_details": {"reasoning_tokens": reasoning}, + } with _ModelServer(answers) as model_server: if execution == "sandbox": @@ -254,6 +262,7 @@ def with_override(self, messages): tmp_path, ) assert output["result"]["final_response"] == "summary of the work" + usage = output["result"]["gym_usage"] else: from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming @@ -288,8 +297,17 @@ def with_override(self, messages): monkeypatch.setattr(HermesAgent, "resolve_model_base_url", lambda *_args: model_server.base_url) output = asyncio.run(agent._create_response(NeMoGymResponseCreateParamsNonStreaming(input="fix bug"))) assert output.output[-1].content[0].text == "summary of the work" + usage = output.usage.model_dump() assert len(model_server.requests) == 2 + # The root's summary bypasses Hermes' session counters, and neither reply has token IDs. + assert usage == { + "input_tokens": 280, + "output_tokens": 60, + "total_tokens": 340, + "input_tokens_details": {"cached_tokens": 40}, + "output_tokens_details": {"reasoning_tokens": 20}, + } assert all(not request.get("stream") for request in model_server.requests) first = NeMoGymChatCompletionCreateParamsNonStreaming.model_validate(model_server.requests[0]) assert all("chat_template_kwargs" not in body for body in model_server.requests) @@ -338,6 +356,92 @@ def test_clients_hermes_builds_itself_use_the_model_server(restore_process_globa assert child.use_streaming is False +@pytest.mark.parametrize("execution", ["sandbox", "local"]) +@pytest.mark.parametrize("finish_reason", ["stop", "length"]) +@pytest.mark.parametrize("reported_usage", [None, {"prompt_tokens": 17, "completion_tokens": 7, "total_tokens": 24}]) +def test_provider_usage_survives_truncation_without_fabricating_details( + tmp_path, restore_process_globals, monkeypatch, execution, finish_reason, reported_usage +) -> None: + from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming + + answer = _completion({"content": "42" if finish_reason == "stop" else ""}) + answer["choices"][0]["finish_reason"] = finish_reason + answer["usage"] = reported_usage + body = NeMoGymResponseCreateParamsNonStreaming(input="What is 6 times 7?") + agent = HermesAgent( + config=HermesAgentConfig( + host="127.0.0.1", + port=0, + name="hermes", + entrypoint="app.py", + model_server={"type": "responses_api_models", "name": "model"}, + resources_server={"type": "resources_servers", "name": "resources"}, + enabled_toolsets=[], + max_turns=2, + ), + server_client=MagicMock(spec=ServerClient, global_config_dict={}), + ) + with _ModelServer([answer]) as model_server: + if execution == "sandbox": + output = _run( + _payload(model_server.base_url, user_message=body.input, enabled_toolsets=[], max_turns=2), tmp_path + ) + response = agent._response_from_result(body=body, result=output["result"], model_name="model", n_input=1) + else: + monkeypatch.setattr(HermesAgent, "resolve_model_base_url", lambda *_args: model_server.base_url) + response = asyncio.run(agent._create_response(body)) + + assert len(model_server.requests) == 1 + assert response.status == ("failed" if finish_reason == "length" else "completed") + if reported_usage is None: + assert response.usage is None + else: + assert response.usage.input_tokens == 17 + assert response.usage.output_tokens == 7 + assert response.usage.total_tokens == 24 + assert response.usage.input_tokens_details.cached_tokens is None + assert response.usage.output_tokens_details.reasoning_tokens is None + + +@pytest.mark.parametrize("second_usage", [None, {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}]) +def test_missing_usage_is_not_a_measured_zero(tmp_path, restore_process_globals, second_usage) -> None: + from responses_api_agents.hermes_agent.token_usage import HermesTokenUsage + + answers = [_completion({"content": "first"}), _completion({"content": "second"})] + answers[0]["usage"] = { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + "prompt_tokens_details": {"cached_tokens": 2}, + "completion_tokens_details": {"reasoning_tokens": 1}, + } + answers[1]["usage"] = second_usage + with _ModelServer(answers) as model_server: + agent = AIAgent(base_url=model_server.base_url, api_key="gym", model="model", quiet_mode=True) + usage = HermesTokenUsage() + usage.instrument(agent) + assert usage.snapshot() is None + agent.client.chat.completions.create(model="model", messages=[{"role": "user", "content": "first"}]) + first = usage.snapshot() + assert first["input_tokens_details"]["cached_tokens"] == 2 + # The pinned Hermes creates a separate client for each ordinary model call. + with agent._create_request_openai_client(reason="test") as client: + client.chat.completions.create(model="model", messages=[{"role": "user", "content": "second"}]) + if second_usage is None: + assert usage.snapshot() is None + else: + assert usage.snapshot() == { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "input_tokens_details": {"cached_tokens": None}, + "output_tokens_details": {"reasoning_tokens": None}, + } + # A later response cannot mutate a checkpoint already returned to the runner. + assert first["input_tokens_details"]["cached_tokens"] == 2 + agent.client.close() + + @pytest.mark.skipif(sys.platform != "linux", reason="Linux supervisor contract") @pytest.mark.parametrize("cooperative", [True, False]) @pytest.mark.parametrize("ending", ["deadline", "close"]) diff --git a/responses_api_agents/hermes_agent/token_usage.py b/responses_api_agents/hermes_agent/token_usage.py new file mode 100644 index 00000000000..81ab0689be4 --- /dev/null +++ b/responses_api_agents/hermes_agent/token_usage.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Collect provider usage before Hermes can discard a response or compact its history.""" + +from __future__ import annotations + +from functools import wraps +from threading import RLock +from typing import TYPE_CHECKING, TypedDict + + +if TYPE_CHECKING: + from openai import OpenAI + from openai.types.chat import ChatCompletion + from run_agent import AIAgent + + +class _InputDetails(TypedDict): + cached_tokens: int | None + + +class _OutputDetails(TypedDict): + reasoning_tokens: int | None + + +class ResponseUsage(TypedDict): + """JSON usage shared by the host adapter and the standalone sandbox runner.""" + + input_tokens: int + output_tokens: int + total_tokens: int + input_tokens_details: _InputDetails + output_tokens_details: _OutputDetails + + +class HermesTokenUsage: + """Count the root agent's Chat Completions, including retries and final summaries. + + Hermes' session counters omit summaries and some early returns. Instrument its + existing client and its client factory instead, without modifying global SDK state. + Auxiliary clients and delegated agents have separate usage and are not included. + """ + + def __init__(self) -> None: + self._lock = RLock() + self._calls = 0 + self._complete = True + self._input = self._output = 0 + self._cached: int | None = 0 + self._reasoning: int | None = 0 + + def instrument(self, agent: AIAgent) -> None: + """Track each client the pinned Hermes creates for this agent instance.""" + original = getattr(agent, "_create_openai_client", None) + if not callable(original): + return + self._instrument_client(agent.client) + + @wraps(original) + def create(*args, **kwargs): + client = original(*args, **kwargs) + self._instrument_client(client) + return client + + agent._create_openai_client = create + + def _instrument_client(self, client: OpenAI) -> None: + original = client.chat.completions.create + + @wraps(original) + def create(*args, **kwargs): + response = original(*args, **kwargs) + self._record(response) + return response + + client.chat.completions.create = create + + def _record(self, response: ChatCompletion) -> None: + usage = getattr(response, "usage", None) + prompt = getattr(usage, "prompt_tokens", None) + completion = getattr(usage, "completion_tokens", None) + cached = getattr(getattr(usage, "prompt_tokens_details", None), "cached_tokens", None) + reasoning = getattr(getattr(usage, "completion_tokens_details", None), "reasoning_tokens", None) + with self._lock: + self._calls += 1 + if type(prompt) is not int or prompt < 0 or type(completion) is not int or completion < 0: + self._complete = False + return + self._input += prompt + self._output += completion + self._cached = ( + self._cached + cached if self._cached is not None and type(cached) is int and cached >= 0 else None + ) + self._reasoning = ( + self._reasoning + reasoning + if self._reasoning is not None and type(reasoning) is int and reasoning >= 0 + else None + ) + + def snapshot(self) -> ResponseUsage | None: + """Return reported totals, or unknown if any returned response omitted usage.""" + with self._lock: + if not self._calls or not self._complete: + return None + return { + "input_tokens": self._input, + "output_tokens": self._output, + "total_tokens": self._input + self._output, + "input_tokens_details": {"cached_tokens": self._cached}, + "output_tokens_details": {"reasoning_tokens": self._reasoning}, + } From 7dd777fabdf44b9da9a2edf18f8a72f23fd891d1 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Fri, 2 Oct 2026 20:52:24 -0700 Subject: [PATCH 22/44] fix(agent): derive usage from shared model capture Signed-off-by: yaoyu-33 --- .../single_agent_turn/README.md | 13 ++ environment_servers/single_agent_turn/app.py | 12 +- .../single_agent_turn/tests/test_app.py | 56 +++++++ nemo_gym/base_responses_api_model.py | 22 ++- nemo_gym/model_usage.py | 102 ++++++++++++ responses_api_agents/hermes_agent/README.md | 9 +- responses_api_agents/hermes_agent/app.py | 11 +- .../hermes_agent/sandbox_runner.py | 7 +- .../hermes_agent/tests/test_app.py | 1 - .../hermes_agent/tests/test_model_kwargs.py | 2 +- .../hermes_agent/tests/test_sandbox_runner.py | 104 ------------ .../hermes_agent/token_usage.py | 112 ------------- tests/unit_tests/test_model_usage.py | 154 ++++++++++++++++++ 13 files changed, 363 insertions(+), 242 deletions(-) create mode 100644 nemo_gym/model_usage.py delete mode 100644 responses_api_agents/hermes_agent/token_usage.py create mode 100644 tests/unit_tests/test_model_usage.py diff --git a/environment_servers/single_agent_turn/README.md b/environment_servers/single_agent_turn/README.md index 6f5720b2a64..a0b2344a44a 100644 --- a/environment_servers/single_agent_turn/README.md +++ b/environment_servers/single_agent_turn/README.md @@ -21,3 +21,16 @@ environment_servers: `direct_http` grants the Agent Server scoped HTTP access to the Resources Server. `mcp` requires the Resources seed response to provide HTTP MCP connection metadata. Selecting neither transport sends `tool_accesses=[]` when the agent session is seeded. The server supports `num_workers > 1`. Episode state and cleanup remain local to the worker handling that request. `max_concurrent_episodes` and queue admission are also enforced independently by each worker, so the deployment-wide concurrency limit is approximately `num_workers * max_concurrent_episodes`. + +With `observability_enabled: true`, the Environment Server derives `response.usage` from the shared Model Server capture before verification. This applies to every harness using this lifecycle, including the legacy flat-row adapter. It counts exchanges from Agent session creation through Agent close, including delegated work, compaction and retries that return usage. Resources seeding, earlier calls under the same rollout ID, and later verifier/judge calls are excluded. Native CLI counters and response IDs are not needed. + +Configure an absolute `model_call_capture_dir` shared by the Model and Environment Servers. Submitted evaluations already enable capture and derive this directory unless explicitly disabled. Direct local deployments can set: + +```yaml +observability_enabled: true +model_call_capture_dir: /absolute/path/to/run/model-calls +``` + +Input counts include cached input; output counts include reasoning. Anthropic cache-read and cache-write tokens are folded into input counts. Optional details remain `null` when unavailable. Missing counts on any exchange, truncated streams, or unavailable/damaged capture yield `usage: null`, not zero or a partial sum. The shared count covers only traffic routed through Gym Model Servers with the episode's rollout prefix; it is not a provider billing audit. Custom Environment Servers can use `nemo_gym.model_usage.ModelUsageCapture` around their own agent lifecycle. + +When observability is disabled, or when calling an Agent Server directly, usage remains whatever that harness reports. Direct calls do not get this Environment Server accounting. diff --git a/environment_servers/single_agent_turn/app.py b/environment_servers/single_agent_turn/app.py index 797b573c66f..74131d7ab0b 100644 --- a/environment_servers/single_agent_turn/app.py +++ b/environment_servers/single_agent_turn/app.py @@ -38,6 +38,7 @@ TOKEN_ID_CAPTURE_BLOCK, get_first_server_config_dict, ) +from nemo_gym.model_usage import ModelUsageCapture from nemo_gym.server_utils import get_response_json, is_nemo_gym_fastapi_entrypoint, raise_for_status from nemo_gym.single_agent_turn_types import ( SingleAgentTurnFailure, @@ -166,6 +167,9 @@ async def close_resources() -> None: ), ) ) + usage_capture = await ModelUsageCapture.start( + self.server_client.global_config_dict, rollout_id=request.episode_id.capture_key + ) agent_session_id = f"agent-session-{uuid4().hex}" agent_cookies: dict[str, str] = {} @@ -249,12 +253,17 @@ async def close_agent() -> None: try: await agent_cleanup.close() except Exception as error: + if usage_capture is not None: + # Cleanup has not confirmed the end of model activity. + agent_response = agent_response.model_copy(update={"usage": None}) raise self._failure( stage="cleanup", failure_reason=str(error), terminal=not _is_retryable_dependency_error(error), partial_response=agent_response, ) from error + if usage_capture is not None: + agent_response = agent_response.model_copy(update={"usage": await usage_capture.usage()}) try: verify_http_response = await self.server_client.post( server_name=self.config.resources_server.name, @@ -287,9 +296,10 @@ async def close_agent() -> None: task_id=request.task.task_id, result=verification.model_copy( update={ + "response": verification.response.model_copy(update={"usage": agent_response.usage}), "ng_agent_observations": agent_close_response.agent_observations if agent_close_response is not None - else None + else None, } ), ) diff --git a/environment_servers/single_agent_turn/tests/test_app.py b/environment_servers/single_agent_turn/tests/test_app.py index e40cca4e82a..bb98ffafda5 100644 --- a/environment_servers/single_agent_turn/tests/test_app.py +++ b/environment_servers/single_agent_turn/tests/test_app.py @@ -250,6 +250,62 @@ async def test_single_agent_turn_with_direct_resources_tools() -> None: assert resources_close_body.episode_id == EpisodeId(rollout_id="rollout", attempt=2) +@pytest.mark.parametrize("missing_usage", [False, True]) +async def test_shared_model_usage_covers_agent_lifecycle_but_excludes_resources_and_judge( + tmp_path, monkeypatch, missing_usage +) -> None: + from nemo_gym.base_responses_api_model import CaptureStore + + server, client = _environment_server() + client.global_config_dict.observability_enabled = True + client.global_config_dict.model_call_capture_dir = str(tmp_path) + for index in (2, 4): + payload = orjson.loads(client.responses[index].body) + response_payload = payload if index == 2 else payload["response"] + response_payload["usage"] = { + "input_tokens": 999, + "output_tokens": 1, + "total_tokens": 1000, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + } + client.responses[index] = _Response(payload) + store = CaptureStore(tmp_path) + original_post = _Client.post + + async def post(self, server_name, url_path, **kwargs): + response = await original_post(self, server_name, url_path, **kwargs) + # Capture from setup, root, delegate/compaction and close, regardless of + # harness transcript format or whether it retained a model response ID. + prompt = { + "/seed_session": 1000, + "/v1/agent_sessions": 10, + "/ng-rollout/rollout-a2/v1/responses": 20, + "/v1/agent_sessions/close": 30, + "/verify": 2000, + }.get(url_path) + if prompt is not None: + usage = {"prompt_tokens": prompt, "completion_tokens": 5} + if missing_usage and url_path == "/v1/agent_sessions/close": + usage = None + store.record("rollout-a2", {"response": {"usage": usage}}) + return response + + monkeypatch.setattr(_Client, "post", post) + result = await server.run_request(_request()) + assert result.result.reward == 1.0 + usage = result.result.response.usage + sent_usage = next(kwargs["json"]["response"]["usage"] for _, path, kwargs in client.calls if path == "/verify") + if missing_usage: + assert usage is None + assert sent_usage is None + else: + assert (usage.input_tokens, usage.output_tokens, usage.total_tokens) == (60, 15, 75) + assert usage.input_tokens_details.cached_tokens is None + assert usage.output_tokens_details.reasoning_tokens is None + assert sent_usage == usage.model_dump() + + async def test_single_agent_turn_translates_resources_mcp_metadata_to_canonical_tool_access() -> None: environment_server, client = _environment_server(resources_tool_transports=["direct_http", "mcp"]) client.responses[0] = _Response( diff --git a/nemo_gym/base_responses_api_model.py b/nemo_gym/base_responses_api_model.py index 46e0204bdf9..9c5cda07824 100644 --- a/nemo_gym/base_responses_api_model.py +++ b/nemo_gym/base_responses_api_model.py @@ -632,10 +632,25 @@ def read(self, rollout_id: str) -> list[dict[str, Any]]: fcntl.flock(handle.fileno(), fcntl.LOCK_UN) return exchanges - def read_available(self, rollout_id: str) -> tuple[list[tuple[int, dict[str, Any]]], int]: + def offset(self, rollout_id: str) -> int: + """Return an append boundary while excluding an in-progress writer.""" + try: + handle = self.path_for(rollout_id).open("rb") + except FileNotFoundError: + return 0 + with handle: + fcntl.flock(handle.fileno(), fcntl.LOCK_SH) + try: + return handle.seek(0, os.SEEK_END) + finally: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + + def read_available( + self, rollout_id: str, *, start_offset: int = 0 + ) -> tuple[list[tuple[int, dict[str, Any]]], int]: """Read valid exchanges without letting one damaged line hide the rest.""" path = self.path_for(rollout_id) - if not path.exists(): + if not path.exists() and start_offset == 0: return [], 0 exchanges: list[tuple[int, dict[str, Any]]] = [] invalid_count = 0 @@ -643,6 +658,9 @@ def read_available(self, rollout_id: str) -> tuple[list[tuple[int, dict[str, Any with path.open("rb") as handle: fcntl.flock(handle.fileno(), fcntl.LOCK_SH) try: + if handle.seek(0, os.SEEK_END) < start_offset: + raise OSError("Model-call capture was truncated during agent execution") + handle.seek(start_offset) for line in handle: stripped = line.strip() if not stripped: diff --git a/nemo_gym/model_usage.py b/nemo_gym/model_usage.py new file mode 100644 index 00000000000..dbb3e1563af --- /dev/null +++ b/nemo_gym/model_usage.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Provider-reported usage for an agent execution, independent of its harness.""" + +import asyncio +import logging +from collections.abc import Mapping, Sequence +from dataclasses import dataclass + +from nemo_gym.base_responses_api_model import ( + CaptureStore, + ModelCallRecord, + build_model_call_record, + model_call_capture_dirs_from_config, +) +from nemo_gym.openai_utils import ( + NeMoGymResponseInputTokensDetails, + NeMoGymResponseOutputTokensDetails, + NeMoGymResponseUsage, +) + + +logger = logging.getLogger(__name__) + + +def aggregate_model_usage(calls: Sequence[ModelCallRecord]) -> NeMoGymResponseUsage | None: + """Sum actual model exchanges, preserving unknown totals and optional details. + + Each record is one exchange, including retries, auxiliary calls and replies + ending at a token limit. Response IDs need not be unique across exchanges. + A partial sum is not a complete usage measurement. + """ + + if any(call.error_category in {"stream_truncated", "capture_parse_error"} for call in calls): + return None + + def total(field: str) -> int | None: + values = [getattr(call, field) for call in calls] + if not values or any(type(value) is not int or value < 0 for value in values): + return None + return sum(values) + + prompt, completion, tokens = (total(field) for field in ("tokens_in", "tokens_out", "tokens_total")) + if prompt is None or completion is None or tokens is None: + return None + return NeMoGymResponseUsage( + input_tokens=prompt, + output_tokens=completion, + total_tokens=tokens, + input_tokens_details=NeMoGymResponseInputTokensDetails(cached_tokens=total("cached_tokens")), + output_tokens_details=NeMoGymResponseOutputTokensDetails(reasoning_tokens=total("tokens_reasoning")), + ) + + +@dataclass(frozen=True) +class ModelUsageCapture: + """Read usage appended during one agent lifecycle from the shared capture store. + + Start after Resources seeding and read after Agent close, before verification. + This excludes pre-existing calls and judge usage without depending on native + transcript IDs, which some harnesses omit. The episode must exclusively own + its rollout ID, as required by the Environment Server lifecycle. + """ + + rollout_id: str + store: CaptureStore | None + start_offset: int = 0 + + @classmethod + async def start(cls, config: Mapping[str, object], *, rollout_id: str) -> "ModelUsageCapture | None": + """Checkpoint enabled capture; return None when observability is disabled.""" + directories = model_call_capture_dirs_from_config(config) + if not directories: + return None + + def checkpoint() -> ModelUsageCapture: + try: + store = CaptureStore(directories[0]) + return cls(rollout_id, store, store.offset(rollout_id)) + except Exception: + logger.warning("Could not checkpoint model usage for %s", rollout_id, exc_info=True) + return cls(rollout_id, None) + + return await asyncio.to_thread(checkpoint) + + async def usage(self) -> NeMoGymResponseUsage | None: + """Return usage for this execution, or unknown for missing/damaged capture.""" + return await asyncio.to_thread(self._read) + + def _read(self) -> NeMoGymResponseUsage | None: + try: + if self.store is None or self.store.is_incomplete(self.rollout_id): + return None + exchanges, invalid_count = self.store.read_available(self.rollout_id, start_offset=self.start_offset) + if invalid_count: + return None + calls = [build_model_call_record(exchange, call_index=index) for index, exchange in exchanges] + return aggregate_model_usage(calls) + except Exception: + logger.warning("Could not read model usage for %s", self.rollout_id, exc_info=True) + return None diff --git a/responses_api_agents/hermes_agent/README.md b/responses_api_agents/hermes_agent/README.md index 4f2aa5d9333..58a3ba88e86 100644 --- a/responses_api_agents/hermes_agent/README.md +++ b/responses_api_agents/hermes_agent/README.md @@ -138,15 +138,10 @@ Sessions use MCP tool grants and reject other required grants. Each granted MCP The Hermes runner and the model's terminal tool execute as the same user in the same sandbox. The host reads the final result, including token IDs, from `/tmp/nemo-gym-hermes-sessions//output.json`; commands issued by the model can also write that file. Sandbox mode is suitable for evaluation, but it must not be used to produce RL training data until results are returned through a channel the model cannot modify. -Response `usage` sums provider-reported tokens for the root agent's model replies, -including retries, truncated replies, and the iteration-limit summary. Cached input -and reasoning tokens remain subsets of input and output tokens. Missing detail counts -stay `null`; if any reply omits token usage, the aggregate is `null`. Counts do not depend -on token IDs or the retained conversation history. Delegated agents and auxiliary model -clients (such as context compression) are outside this root-agent total. - ## Local compatibility +Hermes does not report reliable aggregate usage itself. Its response uses `usage: null`; the shared [single-agent-turn Environment Server](../../environment_servers/single_agent_turn/README.md) supplies provider counts when Model Server capture is enabled. This includes delegated and auxiliary model calls routed through the episode's Model Server URL. Direct Agent Server calls leave usage unknown. + Calls without an agent session run Hermes on the agent-server host. They do not operate on a Resources-owned task sandbox. The agent's compatibility `/run` requires an explicit `resources_server`; native sessions and direct `/v1/responses` do not. diff --git a/responses_api_agents/hermes_agent/app.py b/responses_api_agents/hermes_agent/app.py index ac12a83bbc3..5da57fe6ace 100644 --- a/responses_api_agents/hermes_agent/app.py +++ b/responses_api_agents/hermes_agent/app.py @@ -57,7 +57,6 @@ NeMoGymResponseOutputMessageForTraining, NeMoGymResponseOutputText, NeMoGymResponseReasoningItem, - NeMoGymResponseUsage, NeMoGymSummary, ) from nemo_gym.responses_converter import ResponsesConverter @@ -79,7 +78,6 @@ from nemo_gym.tool_access import MCPToolAccess from responses_api_agents.hermes_agent.model_kwargs import _model_api_kwargs from responses_api_agents.hermes_agent.observability import HermesAgentObserver, normalize_hermes_messages -from responses_api_agents.hermes_agent.token_usage import HermesTokenUsage def _trajectory_to_output_items(messages, n_input): @@ -185,7 +183,6 @@ def _sandbox_hermes_install() -> tuple[str, str]: _SANDBOX_SUPERVISOR = f"{_SANDBOX_RUNTIME_DIR}/process_supervisor.py" _SANDBOX_OBSERVER = f"{_SANDBOX_RUNTIME_DIR}/sandbox_observer.py" _SANDBOX_MODEL_KWARGS = f"{_SANDBOX_RUNTIME_DIR}/model_kwargs.py" -_SANDBOX_TOKEN_USAGE = f"{_SANDBOX_RUNTIME_DIR}/token_usage.py" class RunnerCleanup(Enum): @@ -473,7 +470,6 @@ async def _initialize_agent_session_state( await sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), _SANDBOX_RUNNER) await sandbox.upload(Path(__file__).with_name("sandbox_observer.py"), _SANDBOX_OBSERVER) await sandbox.upload(Path(__file__).with_name("model_kwargs.py"), _SANDBOX_MODEL_KWARGS) - await sandbox.upload(Path(__file__).with_name("token_usage.py"), _SANDBOX_TOKEN_USAGE) await sandbox.upload(Path(process_supervisor.__file__), _SANDBOX_SUPERVISOR) except BaseException as error: try: @@ -957,7 +953,8 @@ def _response_from_result( tool_choice=body.tool_choice, tools=body.tools, parallel_tool_calls=body.parallel_tool_calls, - usage=NeMoGymResponseUsage.model_validate(result["gym_usage"]) if result.get("gym_usage") else None, + # The Environment Server derives usage from shared Model Server capture. + usage=None, ) async def _create_response( @@ -1000,8 +997,6 @@ def _patched_build_api_kwargs(api_messages: list[dict[str, Any]]) -> dict[str, A ) agent._build_api_kwargs = _patched_build_api_kwargs - usage = HermesTokenUsage() - usage.instrument(agent) observer = None if observation_collector is not None: try: @@ -1057,7 +1052,7 @@ def _patched_build_api_kwargs(api_messages: list[dict[str, Any]]) -> dict[str, A return self._response_from_result( body=body, - result={**result, "gym_usage": usage.snapshot()}, + result=result, model_name=model_name, interrupted_by_dispatch=interrupted_by_dispatch, n_input=len(params["history"]) + 1, diff --git a/responses_api_agents/hermes_agent/sandbox_runner.py b/responses_api_agents/hermes_agent/sandbox_runner.py index 5c82cba572a..d240b045630 100644 --- a/responses_api_agents/hermes_agent/sandbox_runner.py +++ b/responses_api_agents/hermes_agent/sandbox_runner.py @@ -19,11 +19,9 @@ try: from .model_kwargs import _model_api_kwargs from .sandbox_observer import SandboxHermesObserver - from .token_usage import HermesTokenUsage except ImportError: from model_kwargs import _model_api_kwargs from sandbox_observer import SandboxHermesObserver - from token_usage import HermesTokenUsage def _write_atomic(path: Path, payload: dict[str, Any]) -> None: @@ -100,8 +98,6 @@ def _run(payload: dict[str, Any], session_dir: Path, *, output_path: Path | None save_trajectories=False, ) observer = SandboxHermesObserver().instrument(agent) - usage = HermesTokenUsage() - usage.instrument(agent) original_build_api_kwargs = agent._build_api_kwargs @@ -122,7 +118,6 @@ def progress() -> dict[str, Any]: "completed": False, "interrupted": True, "stop_reason": stop_reason, - "gym_usage": usage.snapshot(), "messages": getattr(agent, "_session_messages", []) or [*payload["history"], {"role": "user", "content": payload["user_message"]}], } @@ -161,7 +156,7 @@ def interrupt(*_: object) -> None: result = {**result, "completed": False, "interrupted": True, "stop_reason": stop_reason} return { "observations": observer.finish(result, error), - "result": {**result, "gym_usage": usage.snapshot()}, + "result": result, "runtime": runtime, } diff --git a/responses_api_agents/hermes_agent/tests/test_app.py b/responses_api_agents/hermes_agent/tests/test_app.py index 3702185c33f..474217aa474 100644 --- a/responses_api_agents/hermes_agent/tests/test_app.py +++ b/responses_api_agents/hermes_agent/tests/test_app.py @@ -172,7 +172,6 @@ async def test_sandbox_access_selects_runtime_provider(self, monkeypatch, with_m "sandbox_runner.py", "sandbox_observer.py", "model_kwargs.py", - "token_usage.py", "process_supervisor.py", } diff --git a/responses_api_agents/hermes_agent/tests/test_model_kwargs.py b/responses_api_agents/hermes_agent/tests/test_model_kwargs.py index 2c7c87c92e3..59819a2ead6 100644 --- a/responses_api_agents/hermes_agent/tests/test_model_kwargs.py +++ b/responses_api_agents/hermes_agent/tests/test_model_kwargs.py @@ -61,7 +61,7 @@ def test_template_settings_merge_without_mutating_hermes_input(): def test_standalone_runner_loads_shared_helper_without_gym_on_path(tmp_path): source = Path(__file__).parents[1] - for name in ("sandbox_runner.py", "sandbox_observer.py", "model_kwargs.py", "token_usage.py"): + for name in ("sandbox_runner.py", "sandbox_observer.py", "model_kwargs.py"): shutil.copyfile(source / name, tmp_path / name) # -S excludes site-packages (including Gym); only the uploaded runtime files are importable. completed = subprocess.run( diff --git a/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py b/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py index b79b59f8975..e12350413fd 100644 --- a/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py @@ -246,14 +246,6 @@ def with_override(self, messages): ], } answers = [_completion(tool_call), _completion({"content": "summary of the work"})] - for answer, prompt, completion, cached, reasoning in zip(answers, [100, 180], [20, 40], [10, 30], [5, 15]): - answer["usage"] = { - "prompt_tokens": prompt, - "completion_tokens": completion, - "total_tokens": prompt + completion, - "prompt_tokens_details": {"cached_tokens": cached}, - "completion_tokens_details": {"reasoning_tokens": reasoning}, - } with _ModelServer(answers) as model_server: if execution == "sandbox": @@ -262,7 +254,6 @@ def with_override(self, messages): tmp_path, ) assert output["result"]["final_response"] == "summary of the work" - usage = output["result"]["gym_usage"] else: from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming @@ -297,17 +288,8 @@ def with_override(self, messages): monkeypatch.setattr(HermesAgent, "resolve_model_base_url", lambda *_args: model_server.base_url) output = asyncio.run(agent._create_response(NeMoGymResponseCreateParamsNonStreaming(input="fix bug"))) assert output.output[-1].content[0].text == "summary of the work" - usage = output.usage.model_dump() assert len(model_server.requests) == 2 - # The root's summary bypasses Hermes' session counters, and neither reply has token IDs. - assert usage == { - "input_tokens": 280, - "output_tokens": 60, - "total_tokens": 340, - "input_tokens_details": {"cached_tokens": 40}, - "output_tokens_details": {"reasoning_tokens": 20}, - } assert all(not request.get("stream") for request in model_server.requests) first = NeMoGymChatCompletionCreateParamsNonStreaming.model_validate(model_server.requests[0]) assert all("chat_template_kwargs" not in body for body in model_server.requests) @@ -356,92 +338,6 @@ def test_clients_hermes_builds_itself_use_the_model_server(restore_process_globa assert child.use_streaming is False -@pytest.mark.parametrize("execution", ["sandbox", "local"]) -@pytest.mark.parametrize("finish_reason", ["stop", "length"]) -@pytest.mark.parametrize("reported_usage", [None, {"prompt_tokens": 17, "completion_tokens": 7, "total_tokens": 24}]) -def test_provider_usage_survives_truncation_without_fabricating_details( - tmp_path, restore_process_globals, monkeypatch, execution, finish_reason, reported_usage -) -> None: - from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming - - answer = _completion({"content": "42" if finish_reason == "stop" else ""}) - answer["choices"][0]["finish_reason"] = finish_reason - answer["usage"] = reported_usage - body = NeMoGymResponseCreateParamsNonStreaming(input="What is 6 times 7?") - agent = HermesAgent( - config=HermesAgentConfig( - host="127.0.0.1", - port=0, - name="hermes", - entrypoint="app.py", - model_server={"type": "responses_api_models", "name": "model"}, - resources_server={"type": "resources_servers", "name": "resources"}, - enabled_toolsets=[], - max_turns=2, - ), - server_client=MagicMock(spec=ServerClient, global_config_dict={}), - ) - with _ModelServer([answer]) as model_server: - if execution == "sandbox": - output = _run( - _payload(model_server.base_url, user_message=body.input, enabled_toolsets=[], max_turns=2), tmp_path - ) - response = agent._response_from_result(body=body, result=output["result"], model_name="model", n_input=1) - else: - monkeypatch.setattr(HermesAgent, "resolve_model_base_url", lambda *_args: model_server.base_url) - response = asyncio.run(agent._create_response(body)) - - assert len(model_server.requests) == 1 - assert response.status == ("failed" if finish_reason == "length" else "completed") - if reported_usage is None: - assert response.usage is None - else: - assert response.usage.input_tokens == 17 - assert response.usage.output_tokens == 7 - assert response.usage.total_tokens == 24 - assert response.usage.input_tokens_details.cached_tokens is None - assert response.usage.output_tokens_details.reasoning_tokens is None - - -@pytest.mark.parametrize("second_usage", [None, {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}]) -def test_missing_usage_is_not_a_measured_zero(tmp_path, restore_process_globals, second_usage) -> None: - from responses_api_agents.hermes_agent.token_usage import HermesTokenUsage - - answers = [_completion({"content": "first"}), _completion({"content": "second"})] - answers[0]["usage"] = { - "prompt_tokens": 10, - "completion_tokens": 5, - "total_tokens": 15, - "prompt_tokens_details": {"cached_tokens": 2}, - "completion_tokens_details": {"reasoning_tokens": 1}, - } - answers[1]["usage"] = second_usage - with _ModelServer(answers) as model_server: - agent = AIAgent(base_url=model_server.base_url, api_key="gym", model="model", quiet_mode=True) - usage = HermesTokenUsage() - usage.instrument(agent) - assert usage.snapshot() is None - agent.client.chat.completions.create(model="model", messages=[{"role": "user", "content": "first"}]) - first = usage.snapshot() - assert first["input_tokens_details"]["cached_tokens"] == 2 - # The pinned Hermes creates a separate client for each ordinary model call. - with agent._create_request_openai_client(reason="test") as client: - client.chat.completions.create(model="model", messages=[{"role": "user", "content": "second"}]) - if second_usage is None: - assert usage.snapshot() is None - else: - assert usage.snapshot() == { - "input_tokens": 10, - "output_tokens": 5, - "total_tokens": 15, - "input_tokens_details": {"cached_tokens": None}, - "output_tokens_details": {"reasoning_tokens": None}, - } - # A later response cannot mutate a checkpoint already returned to the runner. - assert first["input_tokens_details"]["cached_tokens"] == 2 - agent.client.close() - - @pytest.mark.skipif(sys.platform != "linux", reason="Linux supervisor contract") @pytest.mark.parametrize("cooperative", [True, False]) @pytest.mark.parametrize("ending", ["deadline", "close"]) diff --git a/responses_api_agents/hermes_agent/token_usage.py b/responses_api_agents/hermes_agent/token_usage.py deleted file mode 100644 index 81ab0689be4..00000000000 --- a/responses_api_agents/hermes_agent/token_usage.py +++ /dev/null @@ -1,112 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Collect provider usage before Hermes can discard a response or compact its history.""" - -from __future__ import annotations - -from functools import wraps -from threading import RLock -from typing import TYPE_CHECKING, TypedDict - - -if TYPE_CHECKING: - from openai import OpenAI - from openai.types.chat import ChatCompletion - from run_agent import AIAgent - - -class _InputDetails(TypedDict): - cached_tokens: int | None - - -class _OutputDetails(TypedDict): - reasoning_tokens: int | None - - -class ResponseUsage(TypedDict): - """JSON usage shared by the host adapter and the standalone sandbox runner.""" - - input_tokens: int - output_tokens: int - total_tokens: int - input_tokens_details: _InputDetails - output_tokens_details: _OutputDetails - - -class HermesTokenUsage: - """Count the root agent's Chat Completions, including retries and final summaries. - - Hermes' session counters omit summaries and some early returns. Instrument its - existing client and its client factory instead, without modifying global SDK state. - Auxiliary clients and delegated agents have separate usage and are not included. - """ - - def __init__(self) -> None: - self._lock = RLock() - self._calls = 0 - self._complete = True - self._input = self._output = 0 - self._cached: int | None = 0 - self._reasoning: int | None = 0 - - def instrument(self, agent: AIAgent) -> None: - """Track each client the pinned Hermes creates for this agent instance.""" - original = getattr(agent, "_create_openai_client", None) - if not callable(original): - return - self._instrument_client(agent.client) - - @wraps(original) - def create(*args, **kwargs): - client = original(*args, **kwargs) - self._instrument_client(client) - return client - - agent._create_openai_client = create - - def _instrument_client(self, client: OpenAI) -> None: - original = client.chat.completions.create - - @wraps(original) - def create(*args, **kwargs): - response = original(*args, **kwargs) - self._record(response) - return response - - client.chat.completions.create = create - - def _record(self, response: ChatCompletion) -> None: - usage = getattr(response, "usage", None) - prompt = getattr(usage, "prompt_tokens", None) - completion = getattr(usage, "completion_tokens", None) - cached = getattr(getattr(usage, "prompt_tokens_details", None), "cached_tokens", None) - reasoning = getattr(getattr(usage, "completion_tokens_details", None), "reasoning_tokens", None) - with self._lock: - self._calls += 1 - if type(prompt) is not int or prompt < 0 or type(completion) is not int or completion < 0: - self._complete = False - return - self._input += prompt - self._output += completion - self._cached = ( - self._cached + cached if self._cached is not None and type(cached) is int and cached >= 0 else None - ) - self._reasoning = ( - self._reasoning + reasoning - if self._reasoning is not None and type(reasoning) is int and reasoning >= 0 - else None - ) - - def snapshot(self) -> ResponseUsage | None: - """Return reported totals, or unknown if any returned response omitted usage.""" - with self._lock: - if not self._calls or not self._complete: - return None - return { - "input_tokens": self._input, - "output_tokens": self._output, - "total_tokens": self._input + self._output, - "input_tokens_details": {"cached_tokens": self._cached}, - "output_tokens_details": {"reasoning_tokens": self._reasoning}, - } diff --git a/tests/unit_tests/test_model_usage.py b/tests/unit_tests/test_model_usage.py new file mode 100644 index 00000000000..611279ca3dc --- /dev/null +++ b/tests/unit_tests/test_model_usage.py @@ -0,0 +1,154 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import pytest +from fastapi import FastAPI +from fastapi.responses import StreamingResponse +from fastapi.testclient import TestClient + +from nemo_gym.base_responses_api_model import ( + CaptureStore, + ModelCallCaptureConfig, + ModelCallRecord, + install_model_call_capture, +) +from nemo_gym.model_usage import ModelUsageCapture, aggregate_model_usage + + +def _exchange(prompt: int = 10, completion: int = 3) -> dict: + return {"response": {"usage": {"input_tokens": prompt, "output_tokens": completion}}} + + +async def _capture(tmp_path: Path, rollout_id: str = "rollout") -> ModelUsageCapture: + capture = await ModelUsageCapture.start( + {"observability_enabled": True, "model_call_capture_dir": tmp_path}, rollout_id=rollout_id + ) + assert capture is not None + return capture + + +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("dialect", ["chat", "responses", "messages"]) +async def test_counts_model_server_json_and_sse_without_native_harness_usage(tmp_path, dialect, streaming): + from nemo_gym.anthropic_converter import AnthropicConverter + from nemo_gym.chat_streaming import synthesize_chat_completion_sse + from nemo_gym.responses_streaming import synthesize_responses_sse + + usage = {"input_tokens": 10, "output_tokens": 3, "input_tokens_details": {"cached_tokens": 4}} + if dialect == "chat": + path = "/v1/chat/completions" + payload = { + "id": "repeated-provider-id", + "model": "policy", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "answer"}, "finish_reason": "length"} + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 3, "prompt_tokens_details": {"cached_tokens": 4}}, + } + events = synthesize_chat_completion_sse(payload, include_usage=True) + elif dialect == "responses": + path = "/v1/responses" + payload = {"id": "repeated-provider-id", "status": "incomplete", "output": [], "usage": usage} + events = synthesize_responses_sse(payload) + else: + path = "/v1/messages" + payload = { + "id": "repeated-provider-id", + "type": "message", + "role": "assistant", + "model": "policy", + "content": [{"type": "text", "text": "answer"}], + "stop_reason": "max_tokens", + "stop_sequence": None, + "usage": { + "input_tokens": 4, + "output_tokens": 3, + "cache_read_input_tokens": 4, + "cache_creation_input_tokens": 2, + }, + } + events = AnthropicConverter().anthropic_response_to_sse(payload) + + # Materialize once: the provider reuses its response ID on a retry, but both + # physical exchanges consumed tokens and must count. + events = list(events) + app = FastAPI() + + @app.post(path) + async def model(): + return StreamingResponse(iter(events), media_type="text/event-stream") if streaming else payload + + install_model_call_capture( + app, ModelCallCaptureConfig(observability_enabled=True, model_call_capture_dir=tmp_path) + ) + capture = await _capture(tmp_path) + with TestClient(app) as client: + for _ in range(2): + assert client.post(f"/ng-rollout/rollout{path}", json={}).status_code == 200 + assert client.post(f"/ng-rollout/unrelated{path}", json={}).status_code == 200 + result = await capture.usage() + assert result is not None + assert (result.input_tokens, result.output_tokens, result.total_tokens) == (20, 6, 26) + assert result.input_tokens_details.cached_tokens == 8 + assert result.output_tokens_details.reasoning_tokens is None + + +async def test_capture_is_scoped_to_agent_lifecycle_and_snapshot_is_stable(tmp_path): + store = CaptureStore(tmp_path) + store.record("rollout", _exchange(1000)) # Prior attempt or Resources seed. + capture = await _capture(tmp_path) + store.record("rollout", _exchange()) + store.record("rollout", _exchange(20, 7)) # Delegate/compaction/close-time reply. + result = await capture.usage() + store.record("rollout", _exchange(2000)) # Judge runs after usage is finalized. + assert (result.input_tokens, result.output_tokens, result.total_tokens) == (30, 10, 40) + assert result.input_tokens_details.cached_tokens is None + + +@pytest.mark.parametrize("failure", ["missing", "missing_usage", "damaged", "incomplete", "truncated", "deleted"]) +async def test_incomplete_capture_never_turns_into_a_partial_or_zero_total(tmp_path, failure): + store = CaptureStore(tmp_path) + if failure in {"truncated", "deleted"}: + store.record("rollout", _exchange(1000)) + capture = await _capture(tmp_path) + if failure != "missing": + store.record("rollout", _exchange()) + if failure == "missing_usage": + store.record("rollout", {"response": {}}) + elif failure == "damaged": + with store.path_for("rollout").open("ab") as handle: + handle.write(b"broken json\n") + elif failure == "incomplete": + store.mark_incomplete("rollout") + elif failure == "truncated": + store.path_for("rollout").write_bytes(b"") + elif failure == "deleted": + store.path_for("rollout").unlink() + assert await capture.usage() is None + + +async def test_disabled_capture_does_not_create_a_store(tmp_path): + assert await ModelUsageCapture.start({"model_call_capture_dir": tmp_path}, rollout_id="rollout") is None + assert not list(tmp_path.iterdir()) + + +def test_zero_is_measured_but_unknown_details_and_truncated_streams_are_not(): + first = ModelCallRecord( + call_index=0, tokens_in=0, tokens_out=0, tokens_total=0, cached_tokens=0, tokens_reasoning=0 + ) + zero = aggregate_model_usage([first]) + assert ( + zero.total_tokens + == zero.input_tokens_details.cached_tokens + == zero.output_tokens_details.reasoning_tokens + == 0 + ) + second = first.model_copy(update={"call_index": 1, "cached_tokens": None, "tokens_reasoning": None}) + total = aggregate_model_usage([first, second]) + assert total.total_tokens == 0 + assert total.input_tokens_details.cached_tokens is None + assert total.output_tokens_details.reasoning_tokens is None + assert aggregate_model_usage([first, second.model_copy(update={"error_category": "stream_truncated"})]) is None + assert aggregate_model_usage([]) is None From 55d742ffa857656046c353df092e850db9f7f670 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Fri, 2 Oct 2026 21:38:27 -0700 Subject: [PATCH 23/44] refactor(hermes): move token usage to its own PR Token accounting is isolated in #3981. Preserve the configuration and lifecycle changes from before the usage work. Signed-off-by: yaoyu-33 --- .../single_agent_turn/README.md | 13 -- environment_servers/single_agent_turn/app.py | 12 +- .../single_agent_turn/tests/test_app.py | 56 ------- nemo_gym/base_responses_api_model.py | 22 +-- nemo_gym/model_usage.py | 102 ------------ responses_api_agents/hermes_agent/README.md | 2 - responses_api_agents/hermes_agent/app.py | 12 +- tests/unit_tests/test_model_usage.py | 154 ------------------ 8 files changed, 13 insertions(+), 360 deletions(-) delete mode 100644 nemo_gym/model_usage.py delete mode 100644 tests/unit_tests/test_model_usage.py diff --git a/environment_servers/single_agent_turn/README.md b/environment_servers/single_agent_turn/README.md index a0b2344a44a..6f5720b2a64 100644 --- a/environment_servers/single_agent_turn/README.md +++ b/environment_servers/single_agent_turn/README.md @@ -21,16 +21,3 @@ environment_servers: `direct_http` grants the Agent Server scoped HTTP access to the Resources Server. `mcp` requires the Resources seed response to provide HTTP MCP connection metadata. Selecting neither transport sends `tool_accesses=[]` when the agent session is seeded. The server supports `num_workers > 1`. Episode state and cleanup remain local to the worker handling that request. `max_concurrent_episodes` and queue admission are also enforced independently by each worker, so the deployment-wide concurrency limit is approximately `num_workers * max_concurrent_episodes`. - -With `observability_enabled: true`, the Environment Server derives `response.usage` from the shared Model Server capture before verification. This applies to every harness using this lifecycle, including the legacy flat-row adapter. It counts exchanges from Agent session creation through Agent close, including delegated work, compaction and retries that return usage. Resources seeding, earlier calls under the same rollout ID, and later verifier/judge calls are excluded. Native CLI counters and response IDs are not needed. - -Configure an absolute `model_call_capture_dir` shared by the Model and Environment Servers. Submitted evaluations already enable capture and derive this directory unless explicitly disabled. Direct local deployments can set: - -```yaml -observability_enabled: true -model_call_capture_dir: /absolute/path/to/run/model-calls -``` - -Input counts include cached input; output counts include reasoning. Anthropic cache-read and cache-write tokens are folded into input counts. Optional details remain `null` when unavailable. Missing counts on any exchange, truncated streams, or unavailable/damaged capture yield `usage: null`, not zero or a partial sum. The shared count covers only traffic routed through Gym Model Servers with the episode's rollout prefix; it is not a provider billing audit. Custom Environment Servers can use `nemo_gym.model_usage.ModelUsageCapture` around their own agent lifecycle. - -When observability is disabled, or when calling an Agent Server directly, usage remains whatever that harness reports. Direct calls do not get this Environment Server accounting. diff --git a/environment_servers/single_agent_turn/app.py b/environment_servers/single_agent_turn/app.py index 74131d7ab0b..797b573c66f 100644 --- a/environment_servers/single_agent_turn/app.py +++ b/environment_servers/single_agent_turn/app.py @@ -38,7 +38,6 @@ TOKEN_ID_CAPTURE_BLOCK, get_first_server_config_dict, ) -from nemo_gym.model_usage import ModelUsageCapture from nemo_gym.server_utils import get_response_json, is_nemo_gym_fastapi_entrypoint, raise_for_status from nemo_gym.single_agent_turn_types import ( SingleAgentTurnFailure, @@ -167,9 +166,6 @@ async def close_resources() -> None: ), ) ) - usage_capture = await ModelUsageCapture.start( - self.server_client.global_config_dict, rollout_id=request.episode_id.capture_key - ) agent_session_id = f"agent-session-{uuid4().hex}" agent_cookies: dict[str, str] = {} @@ -253,17 +249,12 @@ async def close_agent() -> None: try: await agent_cleanup.close() except Exception as error: - if usage_capture is not None: - # Cleanup has not confirmed the end of model activity. - agent_response = agent_response.model_copy(update={"usage": None}) raise self._failure( stage="cleanup", failure_reason=str(error), terminal=not _is_retryable_dependency_error(error), partial_response=agent_response, ) from error - if usage_capture is not None: - agent_response = agent_response.model_copy(update={"usage": await usage_capture.usage()}) try: verify_http_response = await self.server_client.post( server_name=self.config.resources_server.name, @@ -296,10 +287,9 @@ async def close_agent() -> None: task_id=request.task.task_id, result=verification.model_copy( update={ - "response": verification.response.model_copy(update={"usage": agent_response.usage}), "ng_agent_observations": agent_close_response.agent_observations if agent_close_response is not None - else None, + else None } ), ) diff --git a/environment_servers/single_agent_turn/tests/test_app.py b/environment_servers/single_agent_turn/tests/test_app.py index bb98ffafda5..e40cca4e82a 100644 --- a/environment_servers/single_agent_turn/tests/test_app.py +++ b/environment_servers/single_agent_turn/tests/test_app.py @@ -250,62 +250,6 @@ async def test_single_agent_turn_with_direct_resources_tools() -> None: assert resources_close_body.episode_id == EpisodeId(rollout_id="rollout", attempt=2) -@pytest.mark.parametrize("missing_usage", [False, True]) -async def test_shared_model_usage_covers_agent_lifecycle_but_excludes_resources_and_judge( - tmp_path, monkeypatch, missing_usage -) -> None: - from nemo_gym.base_responses_api_model import CaptureStore - - server, client = _environment_server() - client.global_config_dict.observability_enabled = True - client.global_config_dict.model_call_capture_dir = str(tmp_path) - for index in (2, 4): - payload = orjson.loads(client.responses[index].body) - response_payload = payload if index == 2 else payload["response"] - response_payload["usage"] = { - "input_tokens": 999, - "output_tokens": 1, - "total_tokens": 1000, - "input_tokens_details": {"cached_tokens": 0}, - "output_tokens_details": {"reasoning_tokens": 0}, - } - client.responses[index] = _Response(payload) - store = CaptureStore(tmp_path) - original_post = _Client.post - - async def post(self, server_name, url_path, **kwargs): - response = await original_post(self, server_name, url_path, **kwargs) - # Capture from setup, root, delegate/compaction and close, regardless of - # harness transcript format or whether it retained a model response ID. - prompt = { - "/seed_session": 1000, - "/v1/agent_sessions": 10, - "/ng-rollout/rollout-a2/v1/responses": 20, - "/v1/agent_sessions/close": 30, - "/verify": 2000, - }.get(url_path) - if prompt is not None: - usage = {"prompt_tokens": prompt, "completion_tokens": 5} - if missing_usage and url_path == "/v1/agent_sessions/close": - usage = None - store.record("rollout-a2", {"response": {"usage": usage}}) - return response - - monkeypatch.setattr(_Client, "post", post) - result = await server.run_request(_request()) - assert result.result.reward == 1.0 - usage = result.result.response.usage - sent_usage = next(kwargs["json"]["response"]["usage"] for _, path, kwargs in client.calls if path == "/verify") - if missing_usage: - assert usage is None - assert sent_usage is None - else: - assert (usage.input_tokens, usage.output_tokens, usage.total_tokens) == (60, 15, 75) - assert usage.input_tokens_details.cached_tokens is None - assert usage.output_tokens_details.reasoning_tokens is None - assert sent_usage == usage.model_dump() - - async def test_single_agent_turn_translates_resources_mcp_metadata_to_canonical_tool_access() -> None: environment_server, client = _environment_server(resources_tool_transports=["direct_http", "mcp"]) client.responses[0] = _Response( diff --git a/nemo_gym/base_responses_api_model.py b/nemo_gym/base_responses_api_model.py index 9c5cda07824..46e0204bdf9 100644 --- a/nemo_gym/base_responses_api_model.py +++ b/nemo_gym/base_responses_api_model.py @@ -632,25 +632,10 @@ def read(self, rollout_id: str) -> list[dict[str, Any]]: fcntl.flock(handle.fileno(), fcntl.LOCK_UN) return exchanges - def offset(self, rollout_id: str) -> int: - """Return an append boundary while excluding an in-progress writer.""" - try: - handle = self.path_for(rollout_id).open("rb") - except FileNotFoundError: - return 0 - with handle: - fcntl.flock(handle.fileno(), fcntl.LOCK_SH) - try: - return handle.seek(0, os.SEEK_END) - finally: - fcntl.flock(handle.fileno(), fcntl.LOCK_UN) - - def read_available( - self, rollout_id: str, *, start_offset: int = 0 - ) -> tuple[list[tuple[int, dict[str, Any]]], int]: + def read_available(self, rollout_id: str) -> tuple[list[tuple[int, dict[str, Any]]], int]: """Read valid exchanges without letting one damaged line hide the rest.""" path = self.path_for(rollout_id) - if not path.exists() and start_offset == 0: + if not path.exists(): return [], 0 exchanges: list[tuple[int, dict[str, Any]]] = [] invalid_count = 0 @@ -658,9 +643,6 @@ def read_available( with path.open("rb") as handle: fcntl.flock(handle.fileno(), fcntl.LOCK_SH) try: - if handle.seek(0, os.SEEK_END) < start_offset: - raise OSError("Model-call capture was truncated during agent execution") - handle.seek(start_offset) for line in handle: stripped = line.strip() if not stripped: diff --git a/nemo_gym/model_usage.py b/nemo_gym/model_usage.py deleted file mode 100644 index dbb3e1563af..00000000000 --- a/nemo_gym/model_usage.py +++ /dev/null @@ -1,102 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Provider-reported usage for an agent execution, independent of its harness.""" - -import asyncio -import logging -from collections.abc import Mapping, Sequence -from dataclasses import dataclass - -from nemo_gym.base_responses_api_model import ( - CaptureStore, - ModelCallRecord, - build_model_call_record, - model_call_capture_dirs_from_config, -) -from nemo_gym.openai_utils import ( - NeMoGymResponseInputTokensDetails, - NeMoGymResponseOutputTokensDetails, - NeMoGymResponseUsage, -) - - -logger = logging.getLogger(__name__) - - -def aggregate_model_usage(calls: Sequence[ModelCallRecord]) -> NeMoGymResponseUsage | None: - """Sum actual model exchanges, preserving unknown totals and optional details. - - Each record is one exchange, including retries, auxiliary calls and replies - ending at a token limit. Response IDs need not be unique across exchanges. - A partial sum is not a complete usage measurement. - """ - - if any(call.error_category in {"stream_truncated", "capture_parse_error"} for call in calls): - return None - - def total(field: str) -> int | None: - values = [getattr(call, field) for call in calls] - if not values or any(type(value) is not int or value < 0 for value in values): - return None - return sum(values) - - prompt, completion, tokens = (total(field) for field in ("tokens_in", "tokens_out", "tokens_total")) - if prompt is None or completion is None or tokens is None: - return None - return NeMoGymResponseUsage( - input_tokens=prompt, - output_tokens=completion, - total_tokens=tokens, - input_tokens_details=NeMoGymResponseInputTokensDetails(cached_tokens=total("cached_tokens")), - output_tokens_details=NeMoGymResponseOutputTokensDetails(reasoning_tokens=total("tokens_reasoning")), - ) - - -@dataclass(frozen=True) -class ModelUsageCapture: - """Read usage appended during one agent lifecycle from the shared capture store. - - Start after Resources seeding and read after Agent close, before verification. - This excludes pre-existing calls and judge usage without depending on native - transcript IDs, which some harnesses omit. The episode must exclusively own - its rollout ID, as required by the Environment Server lifecycle. - """ - - rollout_id: str - store: CaptureStore | None - start_offset: int = 0 - - @classmethod - async def start(cls, config: Mapping[str, object], *, rollout_id: str) -> "ModelUsageCapture | None": - """Checkpoint enabled capture; return None when observability is disabled.""" - directories = model_call_capture_dirs_from_config(config) - if not directories: - return None - - def checkpoint() -> ModelUsageCapture: - try: - store = CaptureStore(directories[0]) - return cls(rollout_id, store, store.offset(rollout_id)) - except Exception: - logger.warning("Could not checkpoint model usage for %s", rollout_id, exc_info=True) - return cls(rollout_id, None) - - return await asyncio.to_thread(checkpoint) - - async def usage(self) -> NeMoGymResponseUsage | None: - """Return usage for this execution, or unknown for missing/damaged capture.""" - return await asyncio.to_thread(self._read) - - def _read(self) -> NeMoGymResponseUsage | None: - try: - if self.store is None or self.store.is_incomplete(self.rollout_id): - return None - exchanges, invalid_count = self.store.read_available(self.rollout_id, start_offset=self.start_offset) - if invalid_count: - return None - calls = [build_model_call_record(exchange, call_index=index) for index, exchange in exchanges] - return aggregate_model_usage(calls) - except Exception: - logger.warning("Could not read model usage for %s", self.rollout_id, exc_info=True) - return None diff --git a/responses_api_agents/hermes_agent/README.md b/responses_api_agents/hermes_agent/README.md index 58a3ba88e86..e86e11a065e 100644 --- a/responses_api_agents/hermes_agent/README.md +++ b/responses_api_agents/hermes_agent/README.md @@ -140,8 +140,6 @@ The Hermes runner and the model's terminal tool execute as the same user in the ## Local compatibility -Hermes does not report reliable aggregate usage itself. Its response uses `usage: null`; the shared [single-agent-turn Environment Server](../../environment_servers/single_agent_turn/README.md) supplies provider counts when Model Server capture is enabled. This includes delegated and auxiliary model calls routed through the episode's Model Server URL. Direct Agent Server calls leave usage unknown. - Calls without an agent session run Hermes on the agent-server host. They do not operate on a Resources-owned task sandbox. The agent's compatibility `/run` requires an explicit `resources_server`; native sessions and direct `/v1/responses` do not. diff --git a/responses_api_agents/hermes_agent/app.py b/responses_api_agents/hermes_agent/app.py index 5da57fe6ace..ed2eca5ccc1 100644 --- a/responses_api_agents/hermes_agent/app.py +++ b/responses_api_agents/hermes_agent/app.py @@ -54,9 +54,12 @@ NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming, NeMoGymResponseFunctionToolCall, + NeMoGymResponseInputTokensDetails, NeMoGymResponseOutputMessageForTraining, NeMoGymResponseOutputText, + NeMoGymResponseOutputTokensDetails, NeMoGymResponseReasoningItem, + NeMoGymResponseUsage, NeMoGymSummary, ) from nemo_gym.responses_converter import ResponsesConverter @@ -953,8 +956,13 @@ def _response_from_result( tool_choice=body.tool_choice, tools=body.tools, parallel_tool_calls=body.parallel_tool_calls, - # The Environment Server derives usage from shared Model Server capture. - usage=None, + usage=NeMoGymResponseUsage( + input_tokens=0, + input_tokens_details=NeMoGymResponseInputTokensDetails(cached_tokens=0), + output_tokens=0, + output_tokens_details=NeMoGymResponseOutputTokensDetails(reasoning_tokens=0), + total_tokens=0, + ), ) async def _create_response( diff --git a/tests/unit_tests/test_model_usage.py b/tests/unit_tests/test_model_usage.py deleted file mode 100644 index 611279ca3dc..00000000000 --- a/tests/unit_tests/test_model_usage.py +++ /dev/null @@ -1,154 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from pathlib import Path - -import pytest -from fastapi import FastAPI -from fastapi.responses import StreamingResponse -from fastapi.testclient import TestClient - -from nemo_gym.base_responses_api_model import ( - CaptureStore, - ModelCallCaptureConfig, - ModelCallRecord, - install_model_call_capture, -) -from nemo_gym.model_usage import ModelUsageCapture, aggregate_model_usage - - -def _exchange(prompt: int = 10, completion: int = 3) -> dict: - return {"response": {"usage": {"input_tokens": prompt, "output_tokens": completion}}} - - -async def _capture(tmp_path: Path, rollout_id: str = "rollout") -> ModelUsageCapture: - capture = await ModelUsageCapture.start( - {"observability_enabled": True, "model_call_capture_dir": tmp_path}, rollout_id=rollout_id - ) - assert capture is not None - return capture - - -@pytest.mark.parametrize("streaming", [False, True]) -@pytest.mark.parametrize("dialect", ["chat", "responses", "messages"]) -async def test_counts_model_server_json_and_sse_without_native_harness_usage(tmp_path, dialect, streaming): - from nemo_gym.anthropic_converter import AnthropicConverter - from nemo_gym.chat_streaming import synthesize_chat_completion_sse - from nemo_gym.responses_streaming import synthesize_responses_sse - - usage = {"input_tokens": 10, "output_tokens": 3, "input_tokens_details": {"cached_tokens": 4}} - if dialect == "chat": - path = "/v1/chat/completions" - payload = { - "id": "repeated-provider-id", - "model": "policy", - "choices": [ - {"index": 0, "message": {"role": "assistant", "content": "answer"}, "finish_reason": "length"} - ], - "usage": {"prompt_tokens": 10, "completion_tokens": 3, "prompt_tokens_details": {"cached_tokens": 4}}, - } - events = synthesize_chat_completion_sse(payload, include_usage=True) - elif dialect == "responses": - path = "/v1/responses" - payload = {"id": "repeated-provider-id", "status": "incomplete", "output": [], "usage": usage} - events = synthesize_responses_sse(payload) - else: - path = "/v1/messages" - payload = { - "id": "repeated-provider-id", - "type": "message", - "role": "assistant", - "model": "policy", - "content": [{"type": "text", "text": "answer"}], - "stop_reason": "max_tokens", - "stop_sequence": None, - "usage": { - "input_tokens": 4, - "output_tokens": 3, - "cache_read_input_tokens": 4, - "cache_creation_input_tokens": 2, - }, - } - events = AnthropicConverter().anthropic_response_to_sse(payload) - - # Materialize once: the provider reuses its response ID on a retry, but both - # physical exchanges consumed tokens and must count. - events = list(events) - app = FastAPI() - - @app.post(path) - async def model(): - return StreamingResponse(iter(events), media_type="text/event-stream") if streaming else payload - - install_model_call_capture( - app, ModelCallCaptureConfig(observability_enabled=True, model_call_capture_dir=tmp_path) - ) - capture = await _capture(tmp_path) - with TestClient(app) as client: - for _ in range(2): - assert client.post(f"/ng-rollout/rollout{path}", json={}).status_code == 200 - assert client.post(f"/ng-rollout/unrelated{path}", json={}).status_code == 200 - result = await capture.usage() - assert result is not None - assert (result.input_tokens, result.output_tokens, result.total_tokens) == (20, 6, 26) - assert result.input_tokens_details.cached_tokens == 8 - assert result.output_tokens_details.reasoning_tokens is None - - -async def test_capture_is_scoped_to_agent_lifecycle_and_snapshot_is_stable(tmp_path): - store = CaptureStore(tmp_path) - store.record("rollout", _exchange(1000)) # Prior attempt or Resources seed. - capture = await _capture(tmp_path) - store.record("rollout", _exchange()) - store.record("rollout", _exchange(20, 7)) # Delegate/compaction/close-time reply. - result = await capture.usage() - store.record("rollout", _exchange(2000)) # Judge runs after usage is finalized. - assert (result.input_tokens, result.output_tokens, result.total_tokens) == (30, 10, 40) - assert result.input_tokens_details.cached_tokens is None - - -@pytest.mark.parametrize("failure", ["missing", "missing_usage", "damaged", "incomplete", "truncated", "deleted"]) -async def test_incomplete_capture_never_turns_into_a_partial_or_zero_total(tmp_path, failure): - store = CaptureStore(tmp_path) - if failure in {"truncated", "deleted"}: - store.record("rollout", _exchange(1000)) - capture = await _capture(tmp_path) - if failure != "missing": - store.record("rollout", _exchange()) - if failure == "missing_usage": - store.record("rollout", {"response": {}}) - elif failure == "damaged": - with store.path_for("rollout").open("ab") as handle: - handle.write(b"broken json\n") - elif failure == "incomplete": - store.mark_incomplete("rollout") - elif failure == "truncated": - store.path_for("rollout").write_bytes(b"") - elif failure == "deleted": - store.path_for("rollout").unlink() - assert await capture.usage() is None - - -async def test_disabled_capture_does_not_create_a_store(tmp_path): - assert await ModelUsageCapture.start({"model_call_capture_dir": tmp_path}, rollout_id="rollout") is None - assert not list(tmp_path.iterdir()) - - -def test_zero_is_measured_but_unknown_details_and_truncated_streams_are_not(): - first = ModelCallRecord( - call_index=0, tokens_in=0, tokens_out=0, tokens_total=0, cached_tokens=0, tokens_reasoning=0 - ) - zero = aggregate_model_usage([first]) - assert ( - zero.total_tokens - == zero.input_tokens_details.cached_tokens - == zero.output_tokens_details.reasoning_tokens - == 0 - ) - second = first.model_copy(update={"call_index": 1, "cached_tokens": None, "tokens_reasoning": None}) - total = aggregate_model_usage([first, second]) - assert total.total_tokens == 0 - assert total.input_tokens_details.cached_tokens is None - assert total.output_tokens_details.reasoning_tokens is None - assert aggregate_model_usage([first, second.model_copy(update={"error_category": "stream_truncated"})]) is None - assert aggregate_model_usage([]) is None From 1f95e8ee95940a1a2621ce40651706d5eacee9be Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 5 Oct 2026 16:22:41 -0700 Subject: [PATCH 24/44] refactor(hermes): share sandbox supervision and session transport Signed-off-by: yaoyu-33 --- nemo_gym/sandbox/api.py | 14 + .../{runner.py => supervisor_client.py} | 70 +++-- responses_api_agents/hermes_agent/app.py | 288 ++---------------- responses_api_agents/hermes_agent/sandbox.py | 225 ++++++++++++++ .../hermes_agent/tests/test_app.py | 35 +-- ...test_agent_sessions.py => test_sandbox.py} | 215 +++++++------ ...t.py => test_sandbox_supervisor_client.py} | 79 +++-- 7 files changed, 499 insertions(+), 427 deletions(-) rename nemo_gym/sandbox/{runner.py => supervisor_client.py} (65%) create mode 100644 responses_api_agents/hermes_agent/sandbox.py rename responses_api_agents/hermes_agent/tests/{test_agent_sessions.py => test_sandbox.py} (87%) rename tests/unit_tests/{test_sandbox_runner_transport.py => test_sandbox_supervisor_client.py} (70%) diff --git a/nemo_gym/sandbox/api.py b/nemo_gym/sandbox/api.py index ebe556c69fb..81ae7e55b56 100644 --- a/nemo_gym/sandbox/api.py +++ b/nemo_gym/sandbox/api.py @@ -576,6 +576,20 @@ async def upload(self, local_path: Path | str, remote_path: str) -> None: async def download(self, remote_path: str, local_path: Path | str) -> None: await self._provider.download_file(self._require_handle(), remote_path, Path(local_path)) + async def upload_text(self, path: str, *, text: str) -> None: + """Upload UTF-8 text through file transfer without shell interpolation.""" + with tempfile.TemporaryDirectory(prefix="sandbox-upload-") as directory: + source = Path(directory) / "payload" + source.write_text(text, encoding="utf-8") + await self.upload(source, path) + + async def read_text(self, path: str) -> str: + """Download UTF-8 text, replacing invalid bytes in partial process output.""" + with tempfile.TemporaryDirectory(prefix="sandbox-download-") as directory: + destination = Path(directory) / "payload" + await self.download(path, destination) + return destination.read_text(encoding="utf-8", errors="replace") + async def status(self) -> SandboxStatus: if self._handle is None: return SandboxStatus.UNKNOWN diff --git a/nemo_gym/sandbox/runner.py b/nemo_gym/sandbox/supervisor_client.py similarity index 65% rename from nemo_gym/sandbox/runner.py rename to nemo_gym/sandbox/supervisor_client.py index a609fe964da..65841546503 100644 --- a/nemo_gym/sandbox/runner.py +++ b/nemo_gym/sandbox/supervisor_client.py @@ -7,8 +7,6 @@ """ import json -import tempfile -from pathlib import Path from shlex import join, quote from uuid import uuid4 @@ -21,8 +19,12 @@ _CLEANUP_RECEIPT = TypeAdapter(CleanupReceipt) -class RunnerRuntimeInfo(BaseModel): - """Worker identity, independent of supervisor cleanup and harness output.""" +class HarnessProcessInfo(BaseModel): + """Identity of the supervised worker, which may be a shim that spawns the harness CLI. + + This is independent of supervisor cleanup and harness output. The PID is + diagnostic metadata, not the supervisor PID used to request cleanup. + """ model_config = ConfigDict(extra="forbid", strict=True) hostname: str @@ -41,28 +43,24 @@ def parse_cleanup_receipt(payload: object) -> CleanupReceipt: return _CLEANUP_RECEIPT.validate_python(payload, strict=True, extra="forbid") -async def upload_text(sandbox: AsyncSandbox, *, path: str, text: str) -> None: - """Upload adapter-owned text without interpolating its contents into a shell command.""" - with tempfile.TemporaryDirectory(prefix="sandbox-runner-upload-") as directory: - source = Path(directory) / "payload" - source.write_text(text) - await sandbox.upload(source, path) - +def supervised_launch_command( + *, + directory: str, + command: list[str], + timeout: float, + cleanup_timeout: float, + python: str = "python3", + supervisor_path: str | None = None, +) -> str: + """Fence delayed launches and run a harness command under the shared supervisor. -async def read_text(sandbox: AsyncSandbox, *, path: str) -> str: - """Download an adapter-owned artifact, preserving non-UTF8 output with replacement.""" - with tempfile.TemporaryDirectory(prefix="sandbox-runner-download-") as directory: - destination = Path(directory) / "payload" - await sandbox.download(path, destination) - return destination.read_text(errors="replace") - - -def supervisor_command(*, directory: str, command: list[str], timeout: float, cleanup_timeout: float) -> str: - """Fence delayed launches and run a harness command under the shared supervisor.""" + Use a private interpreter and supervisor path when the harness installs its + own runtime. Supervisor and worker diagnostics are combined in runner.log. + """ return ( f"trap '' TERM; ln -s launch {quote(directory + '/launch.claim')} 2>/dev/null || exit 0; " f"echo $$ > {quote(directory + '/runner.pid')} && " - f"exec python3 -I {quote(directory + '/process_supervisor.py')} " + f"exec {quote(python)} -I {quote(supervisor_path or directory + '/process_supervisor.py')} " f"--timeout {timeout} --cleanup-timeout {cleanup_timeout} " f"--stop-file {quote(directory + '/runner.stop')} " f"--receipt {quote(directory + '/cleanup.json')} -- {join(command)} " @@ -70,8 +68,8 @@ def supervisor_command(*, directory: str, command: list[str], timeout: float, cl ) -async def confirm_runner_cleanup( - sandbox: AsyncSandbox, *, directory: str, workdir: str, timeout: float, harness: str +async def stop_and_confirm_cleanup( + sandbox: AsyncSandbox, *, directory: str, workdir: str | None, timeout: float, harness: str ) -> CleanupReceipt: """Fence a pending launch or require explicit supervisor acknowledgement before teardown. @@ -81,7 +79,7 @@ async def confirm_runner_cleanup( """ receipt_path = f"{directory}/cleanup.json" try: - receipt = json.loads(await read_text(sandbox, path=receipt_path)) + receipt = json.loads(await sandbox.read_text(receipt_path)) except Exception: receipt = {} if receipt.get("cleanup_confirmed") is not True: @@ -102,9 +100,29 @@ async def confirm_runner_cleanup( ) await sandbox.exec(script, cwd=workdir, timeout_s=timeout + 5) try: - receipt = json.loads(await read_text(sandbox, path=receipt_path)) + receipt = json.loads(await sandbox.read_text(receipt_path)) except Exception as error: raise RuntimeError(f"{harness} launch outcome is unknown; cannot confirm termination") from error if receipt.get("cleanup_confirmed") is not True: raise RuntimeError(f"{harness} sandbox cleanup was not confirmed: {receipt.get('error')}") return parse_cleanup_receipt(receipt) + + +async def remove_session_directory( + sandbox: AsyncSandbox, *, directory: str, workdir: str | None, timeout: float, harness: str +) -> None: + """Remove adapter-owned files after cleanup, keeping delayed launches fenced. + + Retire the directory atomically before unlinking its claim. Otherwise a + delayed launch could win the claim while recursive deletion is in progress. + A failed removal leaves the retired path available for a close retry. + """ + retired = f"{directory}.closed" + result = await sandbox.exec( + f"if [ -d {quote(directory)} ]; then " + f"mv {quote(directory)} {quote(retired)} || exit 1; fi; rm -rf -- {quote(retired)}", + cwd=workdir, + timeout_s=timeout, + ) + if result.return_code != 0 or result.error_type: + raise RuntimeError(f"Could not remove {harness} session files") diff --git a/responses_api_agents/hermes_agent/app.py b/responses_api_agents/hermes_agent/app.py index ed2eca5ccc1..97e9f9e1490 100644 --- a/responses_api_agents/hermes_agent/app.py +++ b/responses_api_agents/hermes_agent/app.py @@ -14,8 +14,6 @@ # limitations under the License. import asyncio import atexit -import importlib.metadata -import json import logging import os import shutil @@ -23,10 +21,6 @@ import tempfile from asyncio import Semaphore from collections.abc import Mapping -from dataclasses import dataclass -from enum import Enum, auto -from pathlib import Path -from shlex import quote from time import time from typing import Any, Callable, Optional from uuid import uuid4 @@ -72,15 +66,15 @@ ObservationGap, ToolCallObservation, ) -from nemo_gym.sandbox import AsyncSandbox, SandboxSpec, process_supervisor +from nemo_gym.sandbox import AsyncSandbox, SandboxSpec from nemo_gym.sandbox.access import DirectSandboxConnection from nemo_gym.sandbox.config import resolve_provider_config from nemo_gym.sandbox.providers import create_provider -from nemo_gym.sandbox.runner import RunnerRuntimeInfo, parse_cleanup_receipt from nemo_gym.server_utils import get_response_json, raise_for_status from nemo_gym.tool_access import MCPToolAccess from responses_api_agents.hermes_agent.model_kwargs import _model_api_kwargs from responses_api_agents.hermes_agent.observability import HermesAgentObserver, normalize_hermes_messages +from responses_api_agents.hermes_agent.sandbox import HermesSandboxSession def _trajectory_to_output_items(messages, n_input): @@ -160,54 +154,6 @@ def _gym_mcp_tool_name(name: str, server_names: list[str]) -> str: return name -def _sandbox_hermes_install() -> tuple[str, str]: - """Return the requirement the sandbox installs and the key that names its runtime directory. - - Both come from the Hermes installed with this server, so ``requirements.txt`` is the only version pin and - the sandbox runs the same Hermes as the host. A git install is fetched as a GitHub archive, so the sandbox - does not need git. The ``mcp`` extra carries Hermes' MCP client, which episode tool grants use. - """ - distribution = importlib.metadata.distribution("hermes-agent") - direct_url = json.loads(distribution.read_text("direct_url.json") or "{}") - commit = (direct_url.get("vcs_info") or {}).get("commit_id") - if commit is None: - return f"hermes-agent[mcp]=={distribution.version}", distribution.version - url = str(direct_url.get("url") or "").removesuffix(".git") - if not url.startswith("https://github.com/"): - raise RuntimeError(f"Cannot build a sandbox install URL for hermes-agent installed from {url!r}") - return f"hermes-agent[mcp] @ {url}/archive/{commit}.tar.gz", commit[:12] - - -_HERMES_REQUIREMENT, _HERMES_RUNTIME_KEY = _sandbox_hermes_install() -_SANDBOX_RUNTIME_DIR = f"/tmp/nemo-gym-hermes-runtime-{_HERMES_RUNTIME_KEY}" -_SANDBOX_UV = f"{_SANDBOX_RUNTIME_DIR}/uv" -_SANDBOX_PYTHON = f"{_SANDBOX_RUNTIME_DIR}/venv/bin/python" -_SANDBOX_RUNNER = f"{_SANDBOX_RUNTIME_DIR}/sandbox_runner.py" -_SANDBOX_SUPERVISOR = f"{_SANDBOX_RUNTIME_DIR}/process_supervisor.py" -_SANDBOX_OBSERVER = f"{_SANDBOX_RUNTIME_DIR}/sandbox_observer.py" -_SANDBOX_MODEL_KWARGS = f"{_SANDBOX_RUNTIME_DIR}/model_kwargs.py" - - -class RunnerCleanup(Enum): - """Track remote process cleanup independently of session files and connections.""" - - IDLE = auto() # No remote launch has been attempted. - UNCONFIRMED = auto() # A launch attempt may have succeeded without returning a handle. - CONFIRMED = auto() - - -@dataclass -class HermesAgentSessionState(AgentSessionState): - sandbox: AsyncSandbox - workdir: str | None - session_dir: str - owns_sandbox: bool = False - observations: AgentObservationBundle | None = None - activation_request: NeMoGymResponseCreateParamsNonStreaming | None = None - task: asyncio.Task[NeMoGymResponse] | None = None - runner_cleanup: RunnerCleanup = RunnerCleanup.IDLE - - # if ray close sys.stderr mid-request, write to the original fd class _SafeStderrHandler(logging.Handler): def emit(self, record): @@ -309,7 +255,7 @@ class HermesAgent(SimpleResponsesAPIAgent): sigterm_installed: bool = False model_config = ConfigDict(arbitrary_types_allowed=True) - async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> HermesAgentSessionState: + async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> HermesSandboxSession: tool_accesses = self.effective_tool_accesses(body) unsupported = [ access.name for access in tool_accesses if access.required and not isinstance(access, MCPToolAccess) @@ -328,16 +274,19 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Herm raise ValueError("MCP tool grants collide with Hermes toolsets: " + ", ".join(sorted(colliding))) return await self._initialize_agent_session_state(body.agent_session_id, body) - def _require_agent_session(self, agent_session_id: str) -> HermesAgentSessionState: + def _require_agent_session(self, agent_session_id: str) -> HermesSandboxSession: state = super()._require_agent_session(agent_session_id) - if not isinstance(state, HermesAgentSessionState): + if not isinstance(state, HermesSandboxSession): raise TypeError("Expected Hermes agent session state") return state async def _close_agent_session_state(self, state: AgentSessionState) -> AgentCloseSessionResponse: - if not isinstance(state, HermesAgentSessionState): + if not isinstance(state, HermesSandboxSession): raise TypeError("Expected Hermes agent session state") - observations = await self._cleanup_sandbox_session(state) + await state.close(self.config.session_close_timeout_seconds) + observations = state.observations or AgentObservationBundle( + source="hermes", gaps=[ObservationGap(code="observation_capture_failed")] + ) return AgentCloseSessionResponse( agent_session_id=state.request.agent_session_id, agent_observations=observations ) @@ -424,7 +373,7 @@ async def _initialize_agent_session_state( self, agent_session_id: str, body: AgentSeedSessionRequest, - ) -> HermesAgentSessionState: + ) -> HermesSandboxSession: owns_sandbox = body.sandbox_access is None if owns_sandbox: if self.config.sandbox_provider is None: @@ -453,175 +402,27 @@ async def _initialize_agent_session_state( raise session_dir = f"/tmp/nemo-gym-hermes-sessions/{uuid4().hex}" - state = HermesAgentSessionState( + state = HermesSandboxSession( request=body, sandbox=sandbox, workdir=workdir, - session_dir=session_dir, + directory=session_dir, owns_sandbox=owns_sandbox, ) try: - prepare = await sandbox.exec( - f"mkdir -p {quote(_SANDBOX_RUNTIME_DIR)} {quote(session_dir)}", - cwd=workdir, - timeout_s=30, - ) - if prepare.return_code != 0: - raise RuntimeError(prepare.stderr or prepare.stdout or "Failed to prepare Hermes sandbox paths") - if not await self._sandbox_hermes_installed(sandbox, workdir): - await self._install_sandbox_hermes(sandbox, workdir) - await sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), _SANDBOX_RUNNER) - await sandbox.upload(Path(__file__).with_name("sandbox_observer.py"), _SANDBOX_OBSERVER) - await sandbox.upload(Path(__file__).with_name("model_kwargs.py"), _SANDBOX_MODEL_KWARGS) - await sandbox.upload(Path(process_supervisor.__file__), _SANDBOX_SUPERVISOR) + await state.prepare(install_timeout=self.config.sandbox_install_timeout_seconds) except BaseException as error: try: - await self._cleanup_sandbox_session(state) + await state.close(self.config.session_close_timeout_seconds) except BaseException: LOG.exception("Could not clean failed Hermes setup %s; retaining session for close", agent_session_id) raise AgentSessionSetupError(state, error=error) from error raise return state - @staticmethod - async def _sandbox_hermes_installed(sandbox: AsyncSandbox, workdir: str | None) -> bool: - """Whether the pinned Hermes and its MCP client import from its runtime path. - - The path is keyed by the pinned commit, so a runtime baked into the image or left by an earlier - session in this sandbox is reused. - """ - check = await sandbox.exec( - f"{quote(_SANDBOX_PYTHON)} -c 'import run_agent, mcp'", - cwd=workdir, - timeout_s=120, - ) - return check.return_code == 0 - - async def _install_sandbox_hermes(self, sandbox: AsyncSandbox, workdir: str | None) -> None: - uv_path = shutil.which("uv") - if uv_path is None: - raise RuntimeError("Hermes agent server requires uv to install the sandbox runtime") - await sandbox.upload(uv_path, _SANDBOX_UV) - venv = quote(_SANDBOX_RUNTIME_DIR + "/venv") - # A runtime that failed the import check is incomplete, so rebuild it rather than reuse it. - install = await sandbox.exec( - f"chmod 755 {quote(_SANDBOX_UV)} && rm -rf {venv} && " - f"{quote(_SANDBOX_UV)} venv {venv} --python 3.13 && " - f"{quote(_SANDBOX_UV)} pip install --python {quote(_SANDBOX_PYTHON)} {quote(_HERMES_REQUIREMENT)}", - cwd=workdir, - timeout_s=self.config.sandbox_install_timeout_seconds, - ) - if install.return_code != 0 or not await self._sandbox_hermes_installed(sandbox, workdir): - raise RuntimeError(install.stderr or install.stdout or "Hermes sandbox installation failed") - - async def _terminate_sandbox_runner(self, state: HermesAgentSessionState) -> None: - """Fence an unstarted launch, or require the running subreaper's cleanup receipt.""" - if state.runner_cleanup in (RunnerCleanup.IDLE, RunnerCleanup.CONFIRMED): - return - receipt_path = f"{state.session_dir}/cleanup.json" - try: - receipt = await self._download_json(state.sandbox, receipt_path) - except Exception: - receipt = {} - if receipt.get("cleanup_confirmed") is not True: - pid_path = quote(f"{state.session_dir}/runner.pid") - stop_path = quote(f"{state.session_dir}/runner.stop") - claim_path = quote(f"{state.session_dir}/launch.claim") - temporary_receipt = quote(f"{receipt_path}.{uuid4().hex}.tmp") - # The symlink atomically records who won the launch/stop race. A stop-owned claim - # also lets a retry finish publishing its receipt if the first close was interrupted. - stopped_receipt = quote(json.dumps({"cleanup_confirmed": True, "error": None})) - # A launch whose response was lost must still be stopped. Never kill the - # supervisor with SIGKILL: only it can reap detached tools and acknowledge cleanup. - script = ( - f"touch {stop_path} || exit 1; " - f"ln -s stop {claim_path} 2>/dev/null || true; " - f'if [ "$(readlink {claim_path})" = stop ]; then ' - f"printf '%s' {stopped_receipt} > {temporary_receipt} && " - f"mv {temporary_receipt} {quote(receipt_path)}; exit $?; fi; " - f"[ -f {quote(receipt_path)} ] && exit 0; " - f'if [ -s {pid_path} ]; then kill -TERM "$(cat {pid_path})" 2>/dev/null || true; fi; ' - f"for _ in $(seq 1 {max(1, int(self.config.session_close_timeout_seconds))}); do " - f"[ -f {quote(receipt_path)} ] && exit 0; sleep 1; done; exit 1" - ) - await state.sandbox.exec( - script, cwd=state.workdir, timeout_s=self.config.session_close_timeout_seconds + 5 - ) - try: - receipt = await self._download_json(state.sandbox, receipt_path) - except Exception as error: - raise RuntimeError("Hermes launch outcome is unknown; cannot confirm termination") from error - if receipt.get("cleanup_confirmed") is not True: - raise RuntimeError(f"Hermes descendant cleanup was not confirmed: {receipt.get('error')}") - parse_cleanup_receipt(receipt) - state.runner_cleanup = RunnerCleanup.CONFIRMED - - async def _cleanup_sandbox_session( - self, - state: HermesAgentSessionState, - ) -> AgentObservationBundle: - if not state.owns_sandbox: - # Cancelling provider exec may kill its process group. Confirm remote cleanup first. - # A queued activation has not attempted launch and can be cancelled immediately. - await self._terminate_sandbox_runner(state) - if state.task is not None and not state.task.done() and not state.task.cancelling(): - state.task.cancel() - if state.owns_sandbox: - # Container teardown is the cleanup boundary for an owned sandbox. Do not let a - # missing runner receipt or a stuck activation prevent stopping all its processes. - await state.sandbox.stop() - state.runner_cleanup = RunnerCleanup.CONFIRMED - if state.task is not None: - try: - await asyncio.wait_for(asyncio.shield(state.task), timeout=self.config.session_close_timeout_seconds) - except asyncio.CancelledError: - if not state.task.cancelled(): - raise - except Exception: - if not state.task.done(): - raise - # A borrowed sandbox still needs proof of cleanup after an activation error. - if not state.owns_sandbox: - # Retire the path atomically before deleting its launch fence. Otherwise a delayed - # exec could claim the directory between rm unlinking launch.claim and removing the directory. - retired_dir = f"{state.session_dir}.closed" - removed = await state.sandbox.exec( - f"if [ -d {quote(state.session_dir)} ]; then " - f"mv {quote(state.session_dir)} {quote(retired_dir)} || exit 1; fi; " - f"rm -rf {quote(retired_dir)}", - cwd=state.workdir, - timeout_s=self.config.session_close_timeout_seconds, - ) - if removed.return_code != 0: - raise RuntimeError("Could not remove Hermes session files") - await state.sandbox.disconnect() - observations = state.observations - if observations is None: - observations = AgentObservationBundle( - source="hermes", gaps=[ObservationGap(code="observation_capture_failed")] - ) - return observations - def _model_name(self) -> str: return self.config.model or str(self.config.model_server.name) - @staticmethod - async def _upload_json(sandbox: AsyncSandbox, remote_path: str, payload: dict[str, Any]) -> None: - with tempfile.TemporaryDirectory(prefix="hermes_sandbox_upload_") as directory: - local_path = Path(directory) / "payload.json" - local_path.write_text(json.dumps(payload)) - await sandbox.upload(local_path, remote_path) - - @staticmethod - async def _download_json(sandbox: AsyncSandbox, remote_path: str) -> dict[str, Any]: - with tempfile.TemporaryDirectory(prefix="hermes_sandbox_download_") as directory: - local_path = Path(directory) / "payload.json" - await sandbox.download(remote_path, local_path) - payload = json.loads(local_path.read_text()) - if not isinstance(payload, dict): - raise TypeError(f"Hermes sandbox payload at {remote_path} is not an object") - return payload - def _sandbox_observations( self, result: dict[str, Any], @@ -710,15 +511,9 @@ async def _run_sandbox_episode( *, body: NeMoGymResponseCreateParamsNonStreaming, agent_session_id: str, - state: HermesAgentSessionState, + state: HermesSandboxSession, ) -> AgentEpisode: params = self._conversation_params(body) - input_path = f"{state.session_dir}/input.json" - output_path = f"{state.session_dir}/output.json" - stdout_path = f"{state.session_dir}/stdout.log" - stderr_path = f"{state.session_dir}/stderr.log" - pid_path = f"{state.session_dir}/runner.pid" - claim_path = f"{state.session_dir}/launch.claim" mcp_accesses = [ access for access in self.effective_tool_accesses(state.request) if isinstance(access, MCPToolAccess) ] @@ -743,56 +538,15 @@ async def _run_sandbox_episode( ), "terminal_timeout": self.config.terminal_timeout, } - await self._upload_json(state.sandbox, input_path, payload) - cleanup_timeout = self.config.session_close_timeout_seconds / 3 - # Only the claim winner can launch. Do not recreate the session directory: a delayed exec - # must remain fenced even after close removes it. Ignore TERM across exec until the Python - # supervisor installs its handler; it checks runner.stop before starting the worker. - command = ( - f"trap '' TERM; ln -s launch {quote(claim_path)} 2>/dev/null || exit 0; " - f"echo $$ > {quote(pid_path)} && " - f"exec {quote(_SANDBOX_PYTHON)} -I {quote(_SANDBOX_SUPERVISOR)} " - f"--timeout {self.config.sandbox_runner_timeout_seconds} --cleanup-timeout {cleanup_timeout} " - f"--stop-file {quote(state.session_dir + '/runner.stop')} " - f"--receipt {quote(state.session_dir + '/cleanup.json')} -- " - f"{quote(_SANDBOX_PYTHON)} {quote(_SANDBOX_RUNNER)} {quote(input_path)} {quote(output_path)} " - f">{quote(stdout_path)} 2>{quote(stderr_path)}" + output = await state.execute( + payload, + timeout=self.config.sandbox_runner_timeout_seconds, + close_timeout=self.config.session_close_timeout_seconds, ) - state.runner_cleanup = RunnerCleanup.UNCONFIRMED - try: - await state.sandbox.exec( - command, - cwd=state.workdir, - timeout_s=process_supervisor.exec_timeout( - timeout=self.config.sandbox_runner_timeout_seconds, cleanup_timeout=cleanup_timeout - ), - ) - except BaseException: - try: - await self._terminate_sandbox_runner(state) - except Exception: - LOG.exception("Hermes cleanup remains unconfirmed; close must retry before verification") - raise - else: - await self._terminate_sandbox_runner(state) - try: - output = await self._download_json(state.sandbox, output_path) - except Exception as error: - logs = await state.sandbox.exec( - f"cat {quote(stderr_path)} 2>/dev/null || true", - cwd=state.workdir, - timeout_s=30, - ) - raise RuntimeError(f"Hermes sandbox runner exited without output: {logs.stdout or ''}") from error - if output.get("error") is not None: - raise RuntimeError(f"Hermes sandbox runner failed: {output['error']}\n{output.get('traceback', '')}") result = output.get("result") if not isinstance(result, dict): raise RuntimeError("Hermes sandbox runner returned an invalid output") - try: - runtime = RunnerRuntimeInfo.model_validate(output.get("runtime")) - except ValueError as error: - raise RuntimeError("Hermes sandbox runner returned invalid runtime metadata") from error + runtime = state.runtime_info response = self._response_from_result( body=body, result=result, diff --git a/responses_api_agents/hermes_agent/sandbox.py b/responses_api_agents/hermes_agent/sandbox.py new file mode 100644 index 00000000000..77261bc98f1 --- /dev/null +++ b/responses_api_agents/hermes_agent/sandbox.py @@ -0,0 +1,225 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Agent-server side of Hermes execution in borrowed or owned task sandboxes.""" + +import asyncio +import importlib.metadata +import json +import logging +import shutil +from dataclasses import dataclass +from pathlib import Path +from shlex import quote + +from pydantic import JsonValue + +from nemo_gym.base_responses_api_agent import AgentSessionState +from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming +from nemo_gym.rollout_observability import AgentObservationBundle +from nemo_gym.sandbox import AsyncSandbox, process_supervisor +from nemo_gym.sandbox.process_supervisor import CleanupReceipt +from nemo_gym.sandbox.supervisor_client import ( + HarnessProcessInfo, + remove_session_directory, + stop_and_confirm_cleanup, + supervised_launch_command, +) + + +LOG = logging.getLogger(__name__) + + +def _sandbox_hermes_install() -> tuple[str, str]: + """Return the requirement the sandbox installs and the key that names its runtime directory. + + Both come from the Hermes installed with this server, so ``requirements.txt`` is the only version pin and + the sandbox runs the same Hermes as the host. A git install is fetched as a GitHub archive, so the sandbox + does not need git. The ``mcp`` extra carries Hermes' MCP client, which episode tool grants use. + """ + distribution = importlib.metadata.distribution("hermes-agent") + direct_url = json.loads(distribution.read_text("direct_url.json") or "{}") + commit = (direct_url.get("vcs_info") or {}).get("commit_id") + if commit is None: + return f"hermes-agent[mcp]=={distribution.version}", distribution.version + url = str(direct_url.get("url") or "").removesuffix(".git") + if not url.startswith("https://github.com/"): + raise RuntimeError(f"Cannot build a sandbox install URL for hermes-agent installed from {url!r}") + return f"hermes-agent[mcp] @ {url}/archive/{commit}.tar.gz", commit[:12] + + +_HERMES_REQUIREMENT, _HERMES_RUNTIME_KEY = _sandbox_hermes_install() +_SANDBOX_RUNTIME_DIR = f"/tmp/nemo-gym-hermes-runtime-{_HERMES_RUNTIME_KEY}" +_SANDBOX_UV = f"{_SANDBOX_RUNTIME_DIR}/uv" +_SANDBOX_PYTHON = f"{_SANDBOX_RUNTIME_DIR}/venv/bin/python" +_SANDBOX_RUNNER = f"{_SANDBOX_RUNTIME_DIR}/sandbox_runner.py" +_SANDBOX_SUPERVISOR = f"{_SANDBOX_RUNTIME_DIR}/process_supervisor.py" +_SANDBOX_OBSERVER = f"{_SANDBOX_RUNTIME_DIR}/sandbox_observer.py" +_SANDBOX_MODEL_KWARGS = f"{_SANDBOX_RUNTIME_DIR}/model_kwargs.py" + + +@dataclass +class HermesSandboxSession(AgentSessionState): + """Hermes transport and cleanup; HTTP retry bookkeeping stays in the agent base.""" + + sandbox: AsyncSandbox + workdir: str | None + directory: str + owns_sandbox: bool = False + observations: AgentObservationBundle | None = None + activation_request: NeMoGymResponseCreateParamsNonStreaming | None = None + task: asyncio.Task[NeMoGymResponse] | None = None + runtime_info: HarnessProcessInfo | None = None + launch_started: bool = False + cleanup: CleanupReceipt | None = None + sandbox_stopped: bool = False + closed: bool = False + + async def prepare(self, *, install_timeout: float) -> None: + """Reuse or install the pinned runtime and stage the Hermes worker files.""" + prepared = await self.sandbox.exec( + f"mkdir -p {quote(_SANDBOX_RUNTIME_DIR)} {quote(self.directory)}", + cwd=self.workdir, + timeout_s=30, + ) + if prepared.return_code != 0: + raise RuntimeError(prepared.stderr or prepared.stdout or "Failed to prepare Hermes sandbox paths") + if not await self._runtime_installed(): + await self._install_runtime(install_timeout) + await self.sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), _SANDBOX_RUNNER) + await self.sandbox.upload(Path(__file__).with_name("sandbox_observer.py"), _SANDBOX_OBSERVER) + await self.sandbox.upload(Path(__file__).with_name("model_kwargs.py"), _SANDBOX_MODEL_KWARGS) + await self.sandbox.upload(Path(process_supervisor.__file__), _SANDBOX_SUPERVISOR) + + async def _runtime_installed(self) -> bool: + """Whether the pinned Hermes and its MCP client import from its runtime path. + + The path is keyed by the pinned commit, so a runtime baked into the image or left by an earlier + session in this sandbox is reused. + """ + check = await self.sandbox.exec( + f"{quote(_SANDBOX_PYTHON)} -c 'import run_agent, mcp'", + cwd=self.workdir, + timeout_s=120, + ) + return check.return_code == 0 + + async def _install_runtime(self, timeout: float) -> None: + uv_path = shutil.which("uv") + if uv_path is None: + raise RuntimeError("Hermes agent server requires uv to install the sandbox runtime") + await self.sandbox.upload(uv_path, _SANDBOX_UV) + venv = quote(_SANDBOX_RUNTIME_DIR + "/venv") + # A runtime that failed the import check is incomplete, so rebuild it rather than reuse it. + install = await self.sandbox.exec( + f"chmod 755 {quote(_SANDBOX_UV)} && rm -rf {venv} && " + f"{quote(_SANDBOX_UV)} venv {venv} --python 3.13 && " + f"{quote(_SANDBOX_UV)} pip install --python {quote(_SANDBOX_PYTHON)} {quote(_HERMES_REQUIREMENT)}", + cwd=self.workdir, + timeout_s=timeout, + ) + if install.return_code != 0 or not await self._runtime_installed(): + raise RuntimeError(install.stderr or install.stdout or "Hermes sandbox installation failed") + + async def upload_json(self, name: str, payload: dict[str, JsonValue]) -> None: + """Write a Hermes input under the session directory using file transfer.""" + await self.sandbox.upload_text(f"{self.directory}/{name}", text=json.dumps(payload)) + + async def read_json(self, name: str) -> dict[str, JsonValue]: + """Read a Hermes output object without mixing it with the cleanup contract.""" + path = f"{self.directory}/{name}" + payload = json.loads(await self.sandbox.read_text(path)) + if not isinstance(payload, dict): + raise TypeError(f"Hermes sandbox payload at {path} is not an object") + return payload + + async def stop_runner(self, timeout: float) -> None: + """Fence a delayed launch or require the supervisor's cleanup receipt.""" + if self.sandbox_stopped or not self.launch_started or self.cleanup is not None: + return + self.cleanup = await stop_and_confirm_cleanup( + self.sandbox, directory=self.directory, workdir=self.workdir, timeout=timeout, harness="Hermes" + ) + + async def close(self, timeout: float) -> None: + """Stop owned sandboxes; clean only harness work in borrowed sandboxes.""" + if self.closed: + return + if not self.owns_sandbox: + # Provider cancellation may kill its process group, including the + # supervisor. Require descendant cleanup before cancelling exec. + await self.stop_runner(timeout) + if self.task is not None and not self.task.done() and not self.task.cancelling(): + self.task.cancel() + if self.owns_sandbox and not self.sandbox_stopped: + # Container teardown is the authority, even without a runner receipt. + # Stop before waiting for an activation that might be stuck in cleanup. + await self.sandbox.stop() + self.sandbox_stopped = True + if self.task is not None: + try: + await asyncio.wait_for(asyncio.shield(self.task), timeout=timeout) + except asyncio.CancelledError: + if not self.task.cancelled(): + raise + except Exception: + if not self.task.done(): + raise + # The activation error is replayed by the agent base. Cleanup + # is independent and must still release the borrowed connection. + if not self.owns_sandbox: + await remove_session_directory( + self.sandbox, directory=self.directory, workdir=self.workdir, timeout=timeout, harness="Hermes" + ) + await self.sandbox.disconnect() + self.closed = True + + async def execute( + self, payload: dict[str, JsonValue], *, timeout: float, close_timeout: float + ) -> dict[str, JsonValue]: + """Run Hermes under supervision and read output only after confirmed cleanup.""" + await self.upload_json("input.json", payload) + cleanup_timeout = close_timeout / 3 + command = supervised_launch_command( + directory=self.directory, + command=[ + _SANDBOX_PYTHON, + _SANDBOX_RUNNER, + f"{self.directory}/input.json", + f"{self.directory}/output.json", + ], + timeout=timeout, + cleanup_timeout=cleanup_timeout, + python=_SANDBOX_PYTHON, + supervisor_path=_SANDBOX_SUPERVISOR, + ) + self.launch_started = True + try: + await self.sandbox.exec( + command, + cwd=self.workdir, + timeout_s=process_supervisor.exec_timeout(timeout=timeout, cleanup_timeout=cleanup_timeout), + ) + except BaseException: + try: + await self.stop_runner(close_timeout) + except Exception: + LOG.exception("Hermes cleanup remains unconfirmed; close must retry before verification") + raise + else: + await self.stop_runner(close_timeout) + try: + output = await self.read_json("output.json") + except Exception as error: + logs = await self.sandbox.exec( + f"cat {quote(self.directory + '/runner.log')} 2>/dev/null || true", + cwd=self.workdir, + timeout_s=30, + ) + raise RuntimeError(f"Hermes sandbox runner exited without output: {logs.stdout or ''}") from error + if output.get("error") is not None: + raise RuntimeError(f"Hermes sandbox runner failed: {output['error']}\n{output.get('traceback', '')}") + try: + self.runtime_info = HarnessProcessInfo.model_validate(output.get("runtime")) + except ValueError as error: + raise RuntimeError("Hermes sandbox runner returned invalid runtime metadata") from error + return output diff --git a/responses_api_agents/hermes_agent/tests/test_app.py b/responses_api_agents/hermes_agent/tests/test_app.py index 474217aa474..0fd5a9f009c 100644 --- a/responses_api_agents/hermes_agent/tests/test_app.py +++ b/responses_api_agents/hermes_agent/tests/test_app.py @@ -15,7 +15,7 @@ import asyncio import json from pathlib import Path -from types import SimpleNamespace +from types import MethodType, SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -39,7 +39,7 @@ NeMoGymResponseReasoningItem, ) from nemo_gym.rollout_observability import AgentEpisode, AgentObservationBundle -from nemo_gym.sandbox import SandboxExecResult, SandboxSpec +from nemo_gym.sandbox import AsyncSandbox, SandboxExecResult, SandboxSpec from nemo_gym.sandbox.access import DirectSandboxConnection, SandboxAccess from nemo_gym.server_utils import ServerClient from nemo_gym.tool_access import DirectHTTPToolAccess, MCPStreamableHTTPConnection, MCPToolAccess @@ -47,15 +47,14 @@ HermesAgent, HermesAgentConfig, HermesAgentRunRequest, - HermesAgentSessionState, ModelServerRef, ResourcesServerRef, _gym_mcp_tool_name, - _sandbox_hermes_install, _split_input_to_user_and_history, _trajectory_to_output_items, ) from responses_api_agents.hermes_agent.observability import HermesAgentObserver +from responses_api_agents.hermes_agent.sandbox import HermesSandboxSession, _sandbox_hermes_install class _FakeResponse: @@ -131,7 +130,7 @@ async def test_sandbox_access_selects_runtime_provider(self, monkeypatch, with_m sandbox = AsyncMock() sandbox.exec.return_value = MagicMock(return_code=0, stdout="", stderr="") connect = AsyncMock(return_value=sandbox) - monkeypatch.setattr("responses_api_agents.hermes_agent.app.shutil.which", lambda name: "/test/uv") + monkeypatch.setattr("responses_api_agents.hermes_agent.sandbox.shutil.which", lambda name: "/test/uv") monkeypatch.setattr( "responses_api_agents.hermes_agent.app.get_global_config_dict", lambda: {"runtime": {"hostname": "sandbox", "pid": 123}}, @@ -165,8 +164,8 @@ async def test_sandbox_access_selects_runtime_provider(self, monkeypatch, with_m ) assert state.sandbox is sandbox assert state.workdir == "/app" - assert state.session_dir.startswith("/tmp/nemo-gym-hermes-sessions/") - assert len(state.session_dir.rsplit("/", 1)[-1]) == 32 + assert state.directory.startswith("/tmp/nemo-gym-hermes-sessions/") + assert len(state.directory.rsplit("/", 1)[-1]) == 32 assert sandbox.exec.await_count == 2 assert {call.args[0].name for call in sandbox.upload.await_args_list} == { "sandbox_runner.py", @@ -195,7 +194,7 @@ async def exec_(command, **_kwargs): monkeypatch.setattr( "responses_api_agents.hermes_agent.app.AsyncSandbox.connect", AsyncMock(return_value=sandbox) ) - monkeypatch.setattr("responses_api_agents.hermes_agent.app.shutil.which", lambda _name: "/usr/bin/uv") + monkeypatch.setattr("responses_api_agents.hermes_agent.sandbox.shutil.which", lambda _name: "/usr/bin/uv") hermes._session_records["session"] = _AgentSessionRecord() await hermes._initialize_agent_session_state( @@ -388,25 +387,27 @@ def _sandbox_session( tool_accesses=list(tool_accesses), ) hermes._session_records["session"] = _AgentSessionRecord( - state=HermesAgentSessionState(request=seed, sandbox=sandbox, workdir=None, session_dir="/session"), + state=HermesSandboxSession(request=seed, sandbox=sandbox, workdir=None, directory="/session"), episode_id=seed.episode_id, ) request = SimpleNamespace( session={"agent_session_id": "session"}, path_params={"rollout_id": seed.episode_id.capture_key}, ) - original_download = hermes._download_json + sandbox.upload_text = MethodType(AsyncSandbox.upload_text, sandbox) - async def download(sandbox, remote_path): + async def read_text(remote_path): if remote_path.endswith("/cleanup.json"): commands = getattr(sandbox, "commands", None) - return { - "cleanup_confirmed": commands is None or any("kill -TERM" in cmd for cmd in commands), - "error": None, - } - return await original_download(sandbox, remote_path) + return json.dumps( + { + "cleanup_confirmed": commands is None or any("kill -TERM" in cmd for cmd in commands), + "error": None, + } + ) + return await AsyncSandbox.read_text(sandbox, remote_path) - hermes._download_json = AsyncMock(side_effect=download) + sandbox.read_text = AsyncMock(side_effect=read_text) return hermes, request, seed async def test_sandbox_activation_calls_the_model_server_directly(self, monkeypatch) -> None: diff --git a/responses_api_agents/hermes_agent/tests/test_agent_sessions.py b/responses_api_agents/hermes_agent/tests/test_sandbox.py similarity index 87% rename from responses_api_agents/hermes_agent/tests/test_agent_sessions.py rename to responses_api_agents/hermes_agent/tests/test_sandbox.py index 8b057cafb61..6c1dda2fed0 100644 --- a/responses_api_agents/hermes_agent/tests/test_agent_sessions.py +++ b/responses_api_agents/hermes_agent/tests/test_sandbox.py @@ -9,7 +9,7 @@ import signal import sys from pathlib import Path -from types import SimpleNamespace +from types import MethodType, SimpleNamespace from unittest.mock import AsyncMock, MagicMock import pytest @@ -21,15 +21,15 @@ from nemo_gym.episode_types import EpisodeId, TaskId from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming from nemo_gym.rollout_observability import AgentEpisode, AgentObservationBundle +from nemo_gym.sandbox import AsyncSandbox from nemo_gym.server_utils import ServerClient -from responses_api_agents.hermes_agent import app as hermes_app +from responses_api_agents.hermes_agent import sandbox as hermes_sandbox from responses_api_agents.hermes_agent.app import ( HermesAgent, HermesAgentConfig, HermesAgentRunRequest, - HermesAgentSessionState, - RunnerCleanup, ) +from responses_api_agents.hermes_agent.sandbox import HermesSandboxSession @pytest.fixture @@ -58,8 +58,10 @@ def agent(monkeypatch): @pytest.fixture def state(): sandbox = AsyncMock() + sandbox.read_text = MethodType(AsyncSandbox.read_text, sandbox) + sandbox.upload_text = MethodType(AsyncSandbox.upload_text, sandbox) sandbox.exec.return_value = SimpleNamespace(return_code=0, stdout="", stderr="", error_type=None) - return HermesAgentSessionState( + return HermesSandboxSession( request=AgentSeedSessionRequest( agent_session_id="session", episode_id=EpisodeId(rollout_id="episode", attempt=1), @@ -75,10 +77,21 @@ def state(): ), sandbox=sandbox, workdir="/app", - session_dir="/tmp/nemo-gym-hermes-sessions/session", + directory="/tmp/nemo-gym-hermes-sessions/session", ) +def mock_json_download(sandbox, **kwargs): + """Supply JSON artifacts through the same text API as a real sandbox.""" + download = AsyncMock(**kwargs) + + async def read_text(path): + return json.dumps(await download(sandbox, path)) + + sandbox.read_text = AsyncMock(side_effect=read_text) + return download + + def request(state): return SimpleNamespace( session={"agent_session_id": "session"}, path_params={"rollout_id": state.request.episode_id.capture_key} @@ -132,7 +145,7 @@ def test_http_close_retry_and_stale_activation_never_fall_back(agent, state, bin seed = client.post("/v1/agent_sessions", json=state.request.model_dump(mode="json")) assert seed.status_code == 200 assert state.task is None - assert state.runner_cleanup is RunnerCleanup.IDLE + assert not state.launch_started and state.cleanup is None assert client.post("/v1/agent_sessions", json=state.request.model_dump(mode="json")).status_code == 200 path = f"/ng-rollout/{state.request.episode_id.capture_key}/v1/responses" activation = client.post(path, json={"input": "task"}) @@ -161,7 +174,7 @@ def test_http_close_retry_and_stale_activation_never_fall_back(agent, state, bin state.sandbox.stop.assert_not_awaited() -async def test_invalid_activation_keeps_session_ready(agent: HermesAgent, state: HermesAgentSessionState) -> None: +async def test_invalid_activation_keeps_session_ready(agent: HermesAgent, state: HermesSandboxSession) -> None: agent._session_records["session"] = _AgentSessionRecord(state=state, episode_id=state.request.episode_id) agent._run_sandbox_episode = AsyncMock(return_value=episode(agent)) with pytest.raises(HTTPException) as error: @@ -258,15 +271,16 @@ async def activate(**kwargs): def test_provider_failure_is_http_500_and_replays_without_rerunning(agent, state): agent._initialize_agent_session_state = AsyncMock(return_value=state) - agent._upload_json = AsyncMock() - agent._download_json = AsyncMock( + state.upload_json = AsyncMock() + mock_json_download( + state.sandbox, side_effect=[ {"cleanup_confirmed": True, "error": None}, { "result": {"failed": True, "error": "HTTP 429 Too Many Requests", "messages": []}, "runtime": {"hostname": "sandbox", "pid": 123}, }, - ] + ], ) with TestClient(agent.setup_webserver(), raise_server_exceptions=False) as client: assert client.post("/v1/agent_sessions", json=state.request.model_dump(mode="json")).status_code == 200 @@ -274,8 +288,8 @@ def test_provider_failure_is_http_500_and_replays_without_rerunning(agent, state for _ in range(2): response = client.post(path, json={"input": "task"}) assert response.status_code == 500 - agent._upload_json.assert_awaited_once() - assert state.runner_cleanup is RunnerCleanup.CONFIRMED + state.upload_json.assert_awaited_once() + assert state.cleanup is not None and state.cleanup["cleanup_confirmed"] is True close = client.post( "/v1/agent_sessions/close", json={"agent_session_id": "session", "episode_id": state.request.episode_id.model_dump()}, @@ -286,79 +300,83 @@ def test_provider_failure_is_http_500_and_replays_without_rerunning(agent, state @pytest.mark.parametrize("runner_started", [False, True], ids=["not-launched", "cleanup-confirmed"]) async def test_close_failure_keeps_session_for_retry( - agent: HermesAgent, state: HermesAgentSessionState, runner_started: bool + agent: HermesAgent, state: HermesSandboxSession, runner_started: bool ) -> None: agent._session_records["session"] = _AgentSessionRecord(state=state, episode_id=state.request.episode_id) if runner_started: - state.runner_cleanup = RunnerCleanup.UNCONFIRMED - agent._download_json = AsyncMock(return_value={"cleanup_confirmed": True, "error": None}) - state.sandbox.exec.side_effect = [SimpleNamespace(return_code=1), SimpleNamespace(return_code=0)] + state.launch_started = True + download_json = mock_json_download(state.sandbox, return_value={"cleanup_confirmed": True, "error": None}) + state.sandbox.exec.side_effect = [ + SimpleNamespace(return_code=1, error_type=None), + SimpleNamespace(return_code=0, error_type=None), + ] close = AgentCloseSessionRequest(agent_session_id="session", episode_id=state.request.episode_id) with pytest.raises(RuntimeError, match="session files"): await agent.close_agent_session(request(state), close) assert agent._session_records["session"].state is state assert agent._session_records["session"].closing - assert state.runner_cleanup is (RunnerCleanup.CONFIRMED if runner_started else RunnerCleanup.IDLE) + assert (state.cleanup is not None) == runner_started state.sandbox.disconnect.assert_not_awaited() with pytest.raises(HTTPException): await agent.responses(request(state), NeMoGymResponseCreateParamsNonStreaming(input="task")) await agent.close_agent_session(request(state), close) state.sandbox.disconnect.assert_awaited_once() if runner_started: - agent._download_json.assert_awaited_once() + download_json.assert_awaited_once() @pytest.mark.parametrize("receipt", [None, {"cleanup_confirmed": False, "error": "cleanup failed"}]) -@pytest.mark.parametrize("runner_cleanup", list(RunnerCleanup)) -async def test_owned_close_stops_without_receipt_or_filesystem_cleanup(agent, state, receipt, runner_cleanup): +@pytest.mark.parametrize("launch_started,cleanup", [(False, None), (True, None), (True, {"cleanup_confirmed": True})]) +async def test_owned_close_stops_without_receipt_or_filesystem_cleanup(agent, state, receipt, launch_started, cleanup): state.owns_sandbox = True - state.runner_cleanup = runner_cleanup + state.launch_started = launch_started + state.cleanup = cleanup state.observations = AgentObservationBundle(source="hermes") agent._session_records["session"] = _AgentSessionRecord(state=state, episode_id=state.request.episode_id) - agent._download_json = AsyncMock(return_value=receipt) + download_json = mock_json_download(state.sandbox, return_value=receipt) if receipt is None: - agent._download_json.side_effect = FileNotFoundError("missing cleanup receipt") + download_json.side_effect = FileNotFoundError("missing cleanup receipt") state.sandbox.exec.side_effect = AssertionError("Owned close must not depend on sandbox exec") close = AgentCloseSessionRequest(agent_session_id="session", episode_id=state.request.episode_id) response = await agent.close_agent_session(request(state), close) assert response.agent_observations == state.observations - assert state.runner_cleanup is RunnerCleanup.CONFIRMED + assert state.sandbox_stopped assert agent._session_records["session"].state is None assert await agent.close_agent_session(request(state), close) == response state.sandbox.stop.assert_awaited_once() state.sandbox.disconnect.assert_not_awaited() state.sandbox.exec.assert_not_awaited() - agent._download_json.assert_not_awaited() + download_json.assert_not_awaited() @pytest.mark.parametrize("failure", [RuntimeError, TimeoutError, asyncio.CancelledError]) async def test_owned_stop_failure_keeps_close_retryable(agent, state, failure): state.owns_sandbox = True - state.runner_cleanup = RunnerCleanup.UNCONFIRMED + state.launch_started = True agent._session_records["session"] = _AgentSessionRecord(state=state, episode_id=state.request.episode_id) state.sandbox.stop.side_effect = [failure("stop failed"), None] - agent._download_json = AsyncMock(side_effect=FileNotFoundError("missing cleanup receipt")) + download_json = mock_json_download(state.sandbox, side_effect=FileNotFoundError("missing cleanup receipt")) close = AgentCloseSessionRequest(agent_session_id="session", episode_id=state.request.episode_id) with pytest.raises(failure, match="stop failed"): await agent.close_agent_session(request(state), close) assert agent._session_records["session"].state is state assert agent._session_records["session"].closing - assert state.runner_cleanup is RunnerCleanup.UNCONFIRMED + assert state.launch_started and state.cleanup is None and not state.sandbox_stopped assert "session" not in agent._closed_session_records response = await agent.close_agent_session(request(state), close) assert await agent.close_agent_session(request(state), close) == response assert state.sandbox.stop.await_count == 2 - assert state.runner_cleanup is RunnerCleanup.CONFIRMED + assert state.sandbox_stopped state.sandbox.disconnect.assert_not_awaited() state.sandbox.exec.assert_not_awaited() - agent._download_json.assert_not_awaited() + download_json.assert_not_awaited() async def test_owned_stop_precedes_waiting_for_cancelled_activation(agent, state): state.owns_sandbox = True - state.runner_cleanup = RunnerCleanup.UNCONFIRMED + state.launch_started = True agent.config.session_close_timeout_seconds = 0.1 started, stopped = asyncio.Event(), asyncio.Event() @@ -372,16 +390,16 @@ async def activation(): raise state.sandbox.stop.side_effect = stopped.set - agent._download_json = AsyncMock(side_effect=FileNotFoundError("missing cleanup receipt")) + download_json = mock_json_download(state.sandbox, side_effect=FileNotFoundError("missing cleanup receipt")) state.task = asyncio.create_task(activation()) await started.wait() try: await agent._close_agent_session_state(state) assert state.task.cancelled() - assert state.runner_cleanup is RunnerCleanup.CONFIRMED + assert state.sandbox_stopped state.sandbox.stop.assert_awaited_once() state.sandbox.exec.assert_not_awaited() - agent._download_json.assert_not_awaited() + download_json.assert_not_awaited() finally: stopped.set() if not state.task.done(): @@ -393,15 +411,15 @@ async def activation(): async def test_unavailable_remote_fence_still_blocks_close(agent, state, failure): # Losing contact with the sandbox is not proof that its pending launch is fenced. state.sandbox.exec.side_effect = [failure("launch status unavailable"), SimpleNamespace(return_code=0)] - agent._download_json = AsyncMock(side_effect=FileNotFoundError("missing cleanup receipt")) - agent._upload_json = AsyncMock() + mock_json_download(state.sandbox, side_effect=FileNotFoundError("missing cleanup receipt")) + state.upload_json = AsyncMock() with pytest.raises(failure): await agent._run_sandbox_episode( body=NeMoGymResponseCreateParamsNonStreaming(input="task"), agent_session_id="session", state=state, ) - assert state.runner_cleanup is RunnerCleanup.UNCONFIRMED + assert state.launch_started and state.cleanup is None and not state.sandbox_stopped state.sandbox.exec.side_effect = None with pytest.raises(RuntimeError, match="launch outcome is unknown"): await agent._close_agent_session_state(state) @@ -413,12 +431,14 @@ def local_runner(agent, state, monkeypatch, tmp_path): """Execute the actual launch/close shell commands and exchange files, without a remote provider.""" directory = tmp_path / "session" directory.mkdir() - state.session_dir = str(directory) + state.directory = str(directory) state.workdir = str(tmp_path) agent.config.session_close_timeout_seconds = 2 - monkeypatch.setattr(hermes_app, "_SANDBOX_PYTHON", sys.executable) - monkeypatch.setattr(hermes_app, "_SANDBOX_RUNNER", str(Path(hermes_app.__file__).with_name("sandbox_runner.py"))) - monkeypatch.setattr(hermes_app, "_SANDBOX_SUPERVISOR", hermes_app.process_supervisor.__file__) + monkeypatch.setattr(hermes_sandbox, "_SANDBOX_PYTHON", sys.executable) + monkeypatch.setattr( + hermes_sandbox, "_SANDBOX_RUNNER", str(Path(hermes_sandbox.__file__).with_name("sandbox_runner.py")) + ) + monkeypatch.setattr(hermes_sandbox, "_SANDBOX_SUPERVISOR", hermes_sandbox.process_supervisor.__file__) state.sandbox.upload.side_effect = shutil.copyfile state.sandbox.download.side_effect = shutil.copyfile @@ -475,8 +495,8 @@ async def queued_exec(command, **kwargs): state.task.cancel() with pytest.raises(asyncio.CancelledError if failure == "cancel" else OSError): await state.task - assert state.runner_cleanup is RunnerCleanup.CONFIRMED - directory = Path(state.session_dir) + assert state.cleanup is not None and state.cleanup["cleanup_confirmed"] is True + directory = Path(state.directory) assert (directory / "launch.claim").readlink() == Path("stop") assert json.loads((directory / "cleanup.json").read_text())["cleanup_confirmed"] is True @@ -494,18 +514,18 @@ async def queued_exec(command, **kwargs): async def test_close_can_recover_a_stop_claim_with_no_receipt(agent, state, local_runner): - directory = Path(state.session_dir) + directory = Path(state.directory) # The first close won the claim but was interrupted before publishing its receipt. (directory / "launch.claim").symlink_to("stop") - state.runner_cleanup = RunnerCleanup.UNCONFIRMED - await agent._terminate_sandbox_runner(state) - assert state.runner_cleanup is RunnerCleanup.CONFIRMED + state.launch_started = True + await state.stop_runner(agent.config.session_close_timeout_seconds) + assert state.cleanup is not None and state.cleanup["cleanup_confirmed"] is True assert json.loads((directory / "cleanup.json").read_text())["cleanup_confirmed"] is True @pytest.mark.parametrize("confirmed", [True, False]) async def test_receipt_download_failure_never_signals_a_reused_pid(agent, state, local_runner, tmp_path, confirmed): - directory = Path(state.session_dir) + directory = Path(state.directory) signaled = tmp_path / "unrelated-process-signaled" child = await asyncio.create_subprocess_exec( sys.executable, @@ -521,7 +541,7 @@ async def test_receipt_download_failure_never_signals_a_reused_pid(agent, state, (directory / "runner.pid").write_text(str(child.pid)) (directory / "launch.claim").symlink_to("launch") (directory / "cleanup.json").write_text(json.dumps({"cleanup_confirmed": confirmed, "error": None})) - download = agent._download_json + original_read = state.sandbox.read_text attempts = 0 async def transient_download(*args): @@ -529,15 +549,15 @@ async def transient_download(*args): attempts += 1 if attempts == 1: raise OSError("transient download failure") - return await download(*args) + return await original_read(*args) - agent._download_json = AsyncMock(side_effect=transient_download) - state.runner_cleanup = RunnerCleanup.UNCONFIRMED + state.sandbox.read_text = AsyncMock(side_effect=transient_download) + state.launch_started = True if confirmed: - await agent._terminate_sandbox_runner(state) + await state.stop_runner(agent.config.session_close_timeout_seconds) else: with pytest.raises(RuntimeError, match="cleanup was not confirmed"): - await agent._terminate_sandbox_runner(state) + await state.stop_runner(agent.config.session_close_timeout_seconds) await asyncio.sleep(0.05) assert not signaled.exists() assert child.returncode is None @@ -567,10 +587,10 @@ async def execute(): async def terminate(_state): assert not state.task.cancelling() - state.runner_cleanup = RunnerCleanup.CONFIRMED + state.cleanup = {"cleanup_confirmed": True, "error": None, "return_code": None, "timed_out": False} confirmed.set() - agent._terminate_sandbox_runner = AsyncMock(side_effect=terminate) + state.stop_runner = AsyncMock(side_effect=terminate) state.task = asyncio.create_task(execute()) await entered.wait() try: @@ -586,9 +606,9 @@ async def terminate(_state): async def test_close_retires_the_launch_path_before_removing_its_fence( agent, state, local_runner, monkeypatch, tmp_path ): - directory = Path(state.session_dir) - retired = Path(f"{state.session_dir}.closed") - state.runner_cleanup = RunnerCleanup.UNCONFIRMED + directory = Path(state.directory) + retired = Path(f"{state.directory}.closed") + state.launch_started = True commands = tmp_path / "bin" commands.mkdir() remove = commands / "rm" @@ -608,12 +628,12 @@ async def test_close_retires_the_launch_path_before_removing_its_fence( async def test_launch_claim_without_pid_is_not_proof_of_cleanup(agent, state, local_runner): - directory = Path(state.session_dir) + directory = Path(state.directory) (directory / "launch.claim").symlink_to("launch") - state.runner_cleanup = RunnerCleanup.UNCONFIRMED + state.launch_started = True with pytest.raises(RuntimeError, match="launch outcome is unknown"): await agent._close_agent_session_state(state) - assert state.runner_cleanup is RunnerCleanup.UNCONFIRMED + assert state.launch_started and state.cleanup is None and not state.sandbox_stopped assert not (directory / "cleanup.json").exists() state.sandbox.disconnect.assert_not_awaited() @@ -622,7 +642,7 @@ async def test_launch_claim_without_pid_is_not_proof_of_cleanup(agent, state, lo async def test_stop_during_interpreter_startup_closes_without_starting_a_worker( agent, state, local_runner, monkeypatch, tmp_path, stop_timing ): - directory = Path(state.session_dir) + directory = Path(state.directory) ready = tmp_path / "before-handler" worker_started = tmp_path / "worker-started" wrapper = tmp_path / "delayed_supervisor.py" @@ -632,14 +652,14 @@ async def test_stop_during_interpreter_startup_closes_without_starting_a_worker( f"while not pathlib.Path({str(directory / 'runner.stop')!r}).exists(): time.sleep(0.01)\n" # Keep the interpreter in the pre-handler window while close sends TERM. "time.sleep(0.15)\n" - f"runner=runpy.run_path({hermes_app._SANDBOX_SUPERVISOR!r})\n" + f"runner=runpy.run_path({hermes_sandbox._SANDBOX_SUPERVISOR!r})\n" "def unexpected_worker(*args, **kwargs):\n" f" pathlib.Path({str(worker_started)!r}).touch()\n" " raise AssertionError('Worker must not start after the stop marker')\n" "runner['subprocess'].Popen=unexpected_worker\n" "raise SystemExit(runner['main']())\n" ) - monkeypatch.setattr(hermes_app, "_SANDBOX_SUPERVISOR", str(wrapper)) + monkeypatch.setattr(hermes_sandbox, "_SANDBOX_SUPERVISOR", str(wrapper)) if stop_timing == "before-shell": (directory / "runner.stop").touch() state.task = asyncio.create_task( @@ -652,10 +672,10 @@ async def test_stop_during_interpreter_startup_closes_without_starting_a_worker( async with asyncio.timeout(3): while not ready.exists(): await asyncio.sleep(0.01) - await agent._terminate_sandbox_runner(state) + await state.stop_runner(agent.config.session_close_timeout_seconds) with pytest.raises(RuntimeError, match="exited without output"): await state.task - assert state.runner_cleanup is RunnerCleanup.CONFIRMED + assert state.cleanup is not None and state.cleanup["cleanup_confirmed"] is True assert not worker_started.exists() assert json.loads((directory / "cleanup.json").read_text()) == { "cleanup_confirmed": True, @@ -673,64 +693,66 @@ async def test_stop_during_interpreter_startup_closes_without_starting_a_worker( @pytest.mark.parametrize("runtime", [None, {}, {"hostname": "sandbox", "pid": "123"}]) async def test_invalid_runtime_keeps_cleanup_confirmed(agent, state, runtime): - agent._upload_json = AsyncMock() - agent._download_json = AsyncMock( + state.upload_json = AsyncMock() + mock_json_download( + state.sandbox, side_effect=[ {"cleanup_confirmed": True, "error": None}, {"result": {"completed": True, "messages": []}, "runtime": runtime}, - ] + ], ) with pytest.raises(RuntimeError, match="invalid runtime metadata"): await agent._run_sandbox_episode( body=NeMoGymResponseCreateParamsNonStreaming(input="task"), agent_session_id="session", state=state ) - assert state.runner_cleanup is RunnerCleanup.CONFIRMED + assert state.cleanup is not None and state.cleanup["cleanup_confirmed"] is True await agent._close_agent_session_state(state) state.sandbox.disconnect.assert_awaited_once() state.sandbox.stop.assert_not_awaited() async def test_invalid_cleanup_schema_does_not_mark_runner_confirmed(agent, state): - state.runner_cleanup = RunnerCleanup.UNCONFIRMED - agent._download_json = AsyncMock( - return_value={"cleanup_confirmed": True, "error": None, "timed_out": False, "return_code": "0"} + state.launch_started = True + mock_json_download( + state.sandbox, return_value={"cleanup_confirmed": True, "error": None, "timed_out": False, "return_code": "0"} ) with pytest.raises(ValidationError): - await agent._terminate_sandbox_runner(state) - assert state.runner_cleanup is RunnerCleanup.UNCONFIRMED + await state.stop_runner(agent.config.session_close_timeout_seconds) + assert state.launch_started and state.cleanup is None and not state.sandbox_stopped state.sandbox.disconnect.assert_not_awaited() @pytest.mark.parametrize("receipt", [{}, {"cleanup_confirmed": False}, {"cleanup_confirmed": "true"}]) async def test_runner_exit_without_cleanup_receipt_blocks_close(agent, state, receipt): - state.runner_cleanup = RunnerCleanup.UNCONFIRMED - agent._download_json = AsyncMock(return_value=receipt) + state.launch_started = True + download_json = mock_json_download(state.sandbox, return_value=receipt) with pytest.raises(RuntimeError, match="cleanup was not confirmed"): await agent._close_agent_session_state(state) - assert state.runner_cleanup is RunnerCleanup.UNCONFIRMED + assert state.launch_started and state.cleanup is None and not state.sandbox_stopped state.sandbox.disconnect.assert_not_awaited() - agent._download_json.return_value = {"cleanup_confirmed": True, "error": None} + download_json.return_value = {"cleanup_confirmed": True, "error": None} await agent._close_agent_session_state(state) - assert state.runner_cleanup is RunnerCleanup.CONFIRMED + assert state.cleanup is not None and state.cleanup["cleanup_confirmed"] is True state.sandbox.disconnect.assert_awaited_once() @pytest.mark.parametrize("overrides", [{}, {"temperature": 0.0}]) async def test_session_prompt_and_limits_reach_runner(agent, state, overrides, tmp_path): - state.session_dir = str(tmp_path) + state.directory = str(tmp_path) agent.server_client.global_config_dict = { "model": {"responses_api_models": {"vllm_model": {"chat_template_kwargs": {"enable_thinking": False}}}} } agent.config.system_prompt = "Configured instruction" - agent._upload_json = AsyncMock() - agent._download_json = AsyncMock( + state.upload_json = AsyncMock() + mock_json_download( + state.sandbox, side_effect=[ {"cleanup_confirmed": True, "error": None}, { "result": {"completed": True, "messages": [{"role": "assistant", "content": "done"}]}, "runtime": {"hostname": "sandbox", "pid": 123}, }, - ] + ], ) body = NeMoGymResponseCreateParamsNonStreaming( input="Fix the bug", instructions="Request instruction", **overrides @@ -751,7 +773,7 @@ async def test_session_prompt_and_limits_reach_runner(agent, state, overrides, t process = await asyncio.create_subprocess_exec("sh", "-c", launch_prefix) assert await process.wait() == 0 assert int((tmp_path / "runner.pid").read_text()) == process.pid - payload = agent._upload_json.await_args.args[2] + payload = state.upload_json.await_args.args[1] assert payload["user_message"] == "Fix the bug" assert payload["history"] == [] assert payload["system_message"] == "Configured instruction\n\nRequest instruction" @@ -763,7 +785,7 @@ async def test_session_prompt_and_limits_reach_runner(agent, state, overrides, t @pytest.mark.parametrize("output_available", [True, False]) @pytest.mark.parametrize("hermes_error", [None, "Model generated invalid tool call"]) async def test_exec_reads_final_output_after_confirmed_cleanup(agent, state, output_available, hermes_error): - agent._upload_json = AsyncMock() + state.upload_json = AsyncMock() events = [] async def execute(command, **kwargs): @@ -801,7 +823,7 @@ async def download(sandbox, path): "runtime": {"hostname": "sandbox", "pid": 123}, } - agent._download_json = AsyncMock(side_effect=download) + mock_json_download(state.sandbox, side_effect=download) activation = agent._run_sandbox_episode( body=NeMoGymResponseCreateParamsNonStreaming(input="task"), agent_session_id="session", state=state ) @@ -815,9 +837,9 @@ async def download(sandbox, path): with pytest.raises(RuntimeError, match="runner exited without output: runner stderr"): await activation assert events[:3] == ["exec", "cleanup", "output"] - assert state.runner_cleanup is RunnerCleanup.CONFIRMED + assert state.cleanup is not None and state.cleanup["cleanup_confirmed"] is True agent.server_client.post.assert_not_called() - payload = agent._upload_json.await_args.args[2] + payload = state.upload_json.await_args.args[1] assert payload["model_base_url"] == "http://model:8000/ng-rollout/episode-a1/v1" @@ -899,15 +921,16 @@ def count_validation(self, body): user_message, system_message, history = runner.run_conversation.call_args.args agent._session_records["session"] = _AgentSessionRecord(state=state, episode_id=state.request.episode_id) - agent._upload_json = AsyncMock() - agent._download_json = AsyncMock( + state.upload_json = AsyncMock() + mock_json_download( + state.sandbox, side_effect=[ {"cleanup_confirmed": True, "error": None}, {"result": result, "runtime": {"hostname": "sandbox", "pid": 123}}, - ] + ], ) await agent.responses(request(state), body) - payload = agent._upload_json.await_args.args[2] + payload = state.upload_json.await_args.args[1] assert payload["user_message"] == user_message == "Follow-up" assert ( payload["history"] @@ -1015,7 +1038,7 @@ async def test_failed_setup_retains_handle_until_cleanup_confirmed( monkeypatch.setattr(module, "resolve_provider_config", lambda *args: {}) monkeypatch.setattr(module, "create_provider", lambda config: AsyncMock()) monkeypatch.setattr(module.shutil, "which", lambda name: "/test/uv") - ok = SimpleNamespace(return_code=0, stdout="", stderr="") + ok = SimpleNamespace(return_code=0, stdout="", stderr="", error_type=None) failed = SimpleNamespace(return_code=1, stdout="", stderr="installer failed") # Prepare paths, detect a missing runtime, fail installation, then remove session files. sandbox.exec.side_effect = [ok, failed, failed, ok] diff --git a/tests/unit_tests/test_sandbox_runner_transport.py b/tests/unit_tests/test_sandbox_supervisor_client.py similarity index 70% rename from tests/unit_tests/test_sandbox_runner_transport.py rename to tests/unit_tests/test_sandbox_supervisor_client.py index 840a5c4945e..7208022e40f 100644 --- a/tests/unit_tests/test_sandbox_runner_transport.py +++ b/tests/unit_tests/test_sandbox_supervisor_client.py @@ -5,24 +5,27 @@ import asyncio import json import shutil +import sys from pathlib import Path from unittest.mock import AsyncMock import pytest from pydantic import ValidationError +from nemo_gym.sandbox import AsyncSandbox, process_supervisor from nemo_gym.sandbox.providers.base import SandboxExecResult -from nemo_gym.sandbox.runner import ( - RunnerRuntimeInfo, - confirm_runner_cleanup, +from nemo_gym.sandbox.supervisor_client import ( + HarnessProcessInfo, parse_cleanup_receipt, - read_text, - supervisor_command, - upload_text, + stop_and_confirm_cleanup, + supervised_launch_command, ) class LocalSandbox: + upload_text = AsyncSandbox.upload_text + read_text = AsyncSandbox.read_text + async def upload(self, source, destination): shutil.copyfile(source, destination) @@ -49,11 +52,11 @@ async def test_file_transport_keeps_contents_out_of_shell(session): sandbox.exec = AsyncMock(side_effect=AssertionError("file transfer must not invoke a shell")) path = str(directory / "input.json") payload = '{"prompt": "$(touch unwanted); `echo surprise`"}\n' - await upload_text(sandbox, path=path, text=payload) + await sandbox.upload_text(path, text=payload) assert Path(path).read_text() == payload - assert await read_text(sandbox, path=path) == payload + assert await sandbox.read_text(path) == payload Path(path).write_bytes(b"partial output\xff\n") - assert await read_text(sandbox, path=path) == "partial output\ufffd\n" + assert await sandbox.read_text(path) == "partial output\ufffd\n" sandbox.exec.assert_not_awaited() @@ -63,7 +66,7 @@ async def test_confirmed_receipt_does_not_signal_stored_pid(session): (directory / "cleanup.json").write_text(json.dumps(receipt)) (directory / "runner.pid").write_text("12345") sandbox.exec = AsyncMock(side_effect=AssertionError("must not signal a possibly reused PID")) - assert await confirm_runner_cleanup( + assert await stop_and_confirm_cleanup( sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" ) == parse_cleanup_receipt(receipt) sandbox.exec.assert_not_awaited() @@ -71,13 +74,13 @@ async def test_confirmed_receipt_does_not_signal_stored_pid(session): async def test_stop_wins_claim_and_fences_delayed_launch(session): sandbox, directory = session - receipt = await confirm_runner_cleanup( + receipt = await stop_and_confirm_cleanup( sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" ) assert receipt == {"cleanup_confirmed": True, "error": None, "return_code": None, "timed_out": False} assert (directory / "launch.claim").readlink() == Path("stop") assert (directory / "runner.stop").exists() - command = supervisor_command( + command = supervised_launch_command( directory=str(directory), command=["touch", str(directory / "started")], timeout=1, cleanup_timeout=1 ) assert (await sandbox.exec(command)).return_code == 0 @@ -87,20 +90,54 @@ async def test_stop_wins_claim_and_fences_delayed_launch(session): # A fenced launch is safe to close, without pretending a worker exited successfully. assert receipt["return_code"] is None with pytest.raises(ValidationError): - RunnerRuntimeInfo.model_validate(receipt) + HarnessProcessInfo.model_validate(receipt) async def test_missing_receipt_after_launch_is_not_cleanup_confirmation(session): sandbox, directory = session (directory / "launch.claim").symlink_to("launch") with pytest.raises(RuntimeError, match="test launch outcome is unknown"): - await confirm_runner_cleanup( + await stop_and_confirm_cleanup( sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" ) assert (directory / "launch.claim").readlink() == Path("launch") assert not (directory / "cleanup.json").exists() +@pytest.mark.skipif(sys.platform != "linux", reason="Linux supervisor contract") +@pytest.mark.parametrize("private_runtime", [False, True]) +async def test_launch_supervises_worker_with_default_or_private_runtime(session, tmp_path, private_runtime): + sandbox, directory = session + runtime = tmp_path / "runtime's files" + runtime.mkdir() + supervisor = (runtime if private_runtime else directory) / "process_supervisor.py" + shutil.copyfile(process_supervisor.__file__, supervisor) + options = {} + if private_runtime: + python = runtime / "python interpreter" + python.symlink_to(sys.executable) + options = {"python": str(python), "supervisor_path": str(supervisor)} + command = supervised_launch_command( + directory=str(directory), + command=[ + sys.executable, + "-c", + "import sys; print(sys.argv[1]); print('stderr', file=sys.stderr)", + "$(literal)", + ], + timeout=5, + cleanup_timeout=1, + **options, + ) + launched = await sandbox.exec(command) + assert launched.return_code == 0, launched.stderr + receipt = await stop_and_confirm_cleanup( + sandbox, directory=str(directory), workdir=str(tmp_path), timeout=2, harness="test" + ) + assert receipt == {"cleanup_confirmed": True, "error": None, "return_code": 0, "timed_out": False} + assert set((directory / "runner.log").read_text().splitlines()) == {"$(literal)", "stderr"} + + @pytest.mark.parametrize("confirmed", [False, "true", 1]) async def test_unconfirmed_cleanup_preserves_receipt_for_retry(session, confirmed): sandbox, directory = session @@ -108,13 +145,13 @@ async def test_unconfirmed_cleanup_preserves_receipt_for_retry(session, confirme receipt = {"cleanup_confirmed": confirmed, "error": "descendants remain"} receipt_path.write_text(json.dumps(receipt)) with pytest.raises(RuntimeError, match="test sandbox cleanup was not confirmed: descendants remain"): - await confirm_runner_cleanup( + await stop_and_confirm_cleanup( sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" ) assert json.loads(receipt_path.read_text()) == receipt receipt = {"cleanup_confirmed": True, "error": None} receipt_path.write_text(json.dumps(receipt)) - assert await confirm_runner_cleanup( + assert await stop_and_confirm_cleanup( sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" ) == parse_cleanup_receipt(receipt) @@ -122,13 +159,13 @@ async def test_unconfirmed_cleanup_preserves_receipt_for_retry(session, confirme def test_cleanup_and_runtime_are_independent(): cleanup = {"return_code": 0, "timed_out": False, "cleanup_confirmed": True, "error": None} assert parse_cleanup_receipt(cleanup) == cleanup - runtime = RunnerRuntimeInfo.model_validate({"hostname": "sandbox", "pid": 123}) + runtime = HarnessProcessInfo.model_validate({"hostname": "sandbox", "pid": 123}) assert runtime.hostname == "sandbox" and runtime.pid == 123 assert runtime.python is None with pytest.raises(ValidationError): parse_cleanup_receipt(runtime.model_dump()) with pytest.raises(ValidationError): - RunnerRuntimeInfo.model_validate(cleanup) + HarnessProcessInfo.model_validate(cleanup) for field in cleanup: with pytest.raises(ValidationError): parse_cleanup_receipt({key: value for key, value in cleanup.items() if key != field}) @@ -155,12 +192,12 @@ def test_absent_exit_code_is_not_success(): @pytest.mark.parametrize("invalid", [{"pid": "123"}, {"hostname": 123}, {"python": 123}, {"return_code": 0}]) def test_runtime_info_keeps_strict_validation(invalid): with pytest.raises(ValidationError): - RunnerRuntimeInfo.model_validate({"hostname": "sandbox", "pid": 123, **invalid}) + HarnessProcessInfo.model_validate({"hostname": "sandbox", "pid": 123, **invalid}) def test_hermes_runtime_uses_the_same_schema(): payload = {"hostname": "sandbox", "pid": 123, "python": "/opt/hermes/bin/python"} - assert RunnerRuntimeInfo.model_validate(payload).model_dump() == payload + assert HarnessProcessInfo.model_validate(payload).model_dump() == payload for field in ("hostname", "pid"): with pytest.raises(ValidationError): - RunnerRuntimeInfo.model_validate({key: value for key, value in payload.items() if key != field}) + HarnessProcessInfo.model_validate({key: value for key, value in payload.items() if key != field}) From ba8a2c660536ff2ff7d03ab591cf29a0b6b9247e Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 5 Oct 2026 16:48:11 -0700 Subject: [PATCH 25/44] refactor(sandbox): share harness execution and close lifecycle Signed-off-by: yaoyu-33 --- nemo_gym/sandbox/session.py | 221 ++++++++++++++ nemo_gym/sandbox/supervisor_client.py | 2 +- responses_api_agents/hermes_agent/app.py | 20 +- responses_api_agents/hermes_agent/sandbox.py | 148 ++++----- .../hermes_agent/tests/test_app.py | 22 +- .../hermes_agent/tests/test_sandbox.py | 282 +++++++++++------- tests/unit_tests/test_sandbox_session.py | 259 ++++++++++++++++ 7 files changed, 730 insertions(+), 224 deletions(-) create mode 100644 nemo_gym/sandbox/session.py create mode 100644 tests/unit_tests/test_sandbox_session.py diff --git a/nemo_gym/sandbox/session.py b/nemo_gym/sandbox/session.py new file mode 100644 index 00000000000..a4e4d876856 --- /dev/null +++ b/nemo_gym/sandbox/session.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Agent-server lifecycle for one supervised harness activation.""" + +import asyncio +import logging +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from typing import cast + +from nemo_gym.sandbox.api import AsyncSandbox +from nemo_gym.sandbox.process_supervisor import CleanupReceipt, exec_timeout +from nemo_gym.sandbox.providers.base import SandboxExecResult +from nemo_gym.sandbox.supervisor_client import ( + remove_session_directory, + stop_and_confirm_cleanup, + supervised_launch_command, +) + + +LOG = logging.getLogger(__name__) + + +@dataclass(frozen=True, kw_only=True) +class SandboxCommand: + """Harness argv and the interpreter/path used to launch its supervisor.""" + + argv: list[str] + python: str = "python3" + supervisor_path: str | None = None + + +@dataclass(kw_only=True) +class SandboxSession[Artifacts]: + """Execute, stop, capture, then release an owned or borrowed sandbox. + + Runtime installation belongs in the adapter's seed hook. ``prepare`` stages + activation input and returns a command; ``collect`` copies harness artifacts + to the agent server after cleanup is confirmed. It may run a bounded snapshot + command, but must not restart the harness. Parsing responses stays in the adapter. + + Each stop, collection and release phase is bounded by ``close_timeout`` (or + the timeout passed to ``close``). Failed cleanup/release retains the handle + for retry. Failed capture is recorded separately and does not prevent release. + HTTP request binding and activation retries belong to the agent session layer. + """ + + sandbox: AsyncSandbox + directory: str + workdir: str | None + harness: str + owns_sandbox: bool = False + launch_started: bool = field(default=False, init=False) + cleanup: CleanupReceipt | None = field(default=None, init=False) + artifacts: Artifacts | None = field(default=None, init=False) + capture_error: Exception | None = field(default=None, init=False) + sandbox_stopped: bool = field(default=False, init=False) + closing: bool = field(default=False, init=False) + closed: bool = field(default=False, init=False) + _prepare_task: asyncio.Future[SandboxCommand] | None = field(default=None, init=False) + _exec_task: asyncio.Task[SandboxExecResult] | None = field(default=None, init=False) + _finalize_task: asyncio.Task[None] | None = field(default=None, init=False) + _close_task: asyncio.Task[None] | None = field(default=None, init=False) + _collect: Callable[[], Awaitable[Artifacts]] | None = field(default=None, init=False) + _capture_attempted: bool = field(default=False, init=False) + + async def execute( + self, + *, + prepare: Callable[[], Awaitable[SandboxCommand]], + collect: Callable[[], Awaitable[Artifacts]], + timeout: float, + close_timeout: float, + ) -> Artifacts: + """Run once, preserving captured artifacts even when execution raises. + + Cancellation stops the remote harness before releasing provider exec. + Callers that merely stop waiting (such as disconnected HTTP requests) + must shield their shared activation task. + """ + if self.closing or self._prepare_task is not None: + raise RuntimeError(f"{self.harness} sandbox session is closing or already activated") + # Register both hooks before yielding, so close can fence preparation too. + self._collect = collect + self._prepare_task = asyncio.ensure_future(prepare()) + try: + command = await asyncio.shield(self._prepare_task) + if self.closing or self._finalize_task is not None: + raise RuntimeError(f"{self.harness} sandbox session closed before launch") + cleanup_timeout = close_timeout / 3 + launch = supervised_launch_command( + directory=self.directory, + command=command.argv, + timeout=timeout, + cleanup_timeout=cleanup_timeout, + python=command.python, + supervisor_path=command.supervisor_path, + ) + self.launch_started = True + self._exec_task = asyncio.create_task( + self.sandbox.exec( + launch, + cwd=self.workdir, + timeout_s=exec_timeout(timeout=timeout, cleanup_timeout=cleanup_timeout), + ) + ) + result = await asyncio.shield(self._exec_task) + if result.error_type == "timeout": + raise TimeoutError(f"{self.harness} sandbox supervisor exceeded its execution deadline") + if result.error_type: + raise RuntimeError(f"{self.harness} sandbox execution failed: {result.error_type}: {result.stderr}") + # A worker's nonzero exit can still carry useful, adapter-specific output. + except BaseException: + try: + await self._finish(timeout=close_timeout) + except Exception: + LOG.exception("%s cleanup remains unconfirmed; close must retry", self.harness) + raise + await self._finish(timeout=close_timeout) + if self.capture_error is not None: + raise self.capture_error + # Collection either returned artifacts (including a valid None) or recorded an error. + return cast(Artifacts, self.artifacts) + + async def stop_runner(self, *, timeout: float) -> None: + """Fence a delayed launch or confirm descendant cleanup before transport cancellation.""" + if self.sandbox_stopped or not self.launch_started or self.cleanup is not None: + return + try: + async with asyncio.timeout(timeout): + self.cleanup = await stop_and_confirm_cleanup( + self.sandbox, + directory=self.directory, + workdir=self.workdir, + timeout=timeout, + harness=self.harness, + ) + except TimeoutError as error: + raise RuntimeError(f"{self.harness} launch outcome is unknown; cleanup deadline exceeded") from error + + async def _finish(self, *, timeout: float) -> None: + if self._finalize_task is None or ( + self._finalize_task.done() + and (self._finalize_task.cancelled() or self._finalize_task.exception() is not None) + ): + self._finalize_task = asyncio.create_task(self._finalize(timeout=timeout)) + await asyncio.shield(self._finalize_task) + + async def _cancel_and_wait[T](self, task: asyncio.Future[T], *, timeout: float) -> None: + if not task.done(): + task.cancel() + try: + await asyncio.wait_for(asyncio.shield(task), timeout=timeout) + except asyncio.CancelledError: + if not task.cancelled(): + raise + except Exception: + if not task.done(): + raise + # Execution/preparation errors are propagated by execute, independently of cleanup. + + async def _finalize(self, *, timeout: float) -> None: + if self._prepare_task is not None: + await self._cancel_and_wait(self._prepare_task, timeout=timeout) + await self.stop_runner(timeout=timeout) + if self.launch_started and not self._capture_attempted and self._collect is not None: + if self.sandbox_stopped: + self.capture_error = RuntimeError(f"{self.harness} sandbox stopped before artifact capture") + else: + try: + async with asyncio.timeout(timeout): + self.artifacts = await self._collect() + self.capture_error = None + except Exception as error: + self.capture_error = error + LOG.exception("%s artifact capture failed after confirmed cleanup", self.harness) + self._capture_attempted = True + await self._release_exec(timeout=timeout) + + async def _release_exec(self, *, timeout: float) -> None: + if (self.cleanup is not None or self.sandbox_stopped) and self._exec_task is not None: + await self._cancel_and_wait(self._exec_task, timeout=timeout) + + async def close(self, *, timeout: float) -> None: + """Join finalization before release; concurrent/retried closes never duplicate capture.""" + self.closing = True + if self.closed: + return + if self._close_task is None or ( + self._close_task.done() and (self._close_task.cancelled() or self._close_task.exception() is not None) + ): + self._close_task = asyncio.create_task(self._close(timeout=timeout)) + await asyncio.shield(self._close_task) + + async def _close(self, *, timeout: float) -> None: + try: + await self._finish(timeout=timeout) + except Exception: + if not self.owns_sandbox: + raise + # Container teardown remains the authority if graceful cleanup is unavailable. + LOG.exception("%s graceful cleanup failed; stopping owned sandbox", self.harness) + if not self._capture_attempted: + self.capture_error = RuntimeError(f"{self.harness} artifact capture unavailable during forced stop") + if self.owns_sandbox: + if not self.sandbox_stopped: + async with asyncio.timeout(timeout): + await self.sandbox.stop() + self.sandbox_stopped = True + await self._release_exec(timeout=timeout) + else: + async with asyncio.timeout(timeout): + await remove_session_directory( + self.sandbox, + directory=self.directory, + workdir=self.workdir, + timeout=timeout, + harness=self.harness, + ) + await self.sandbox.disconnect() + self.closed = True diff --git a/nemo_gym/sandbox/supervisor_client.py b/nemo_gym/sandbox/supervisor_client.py index 65841546503..79d1a9481c4 100644 --- a/nemo_gym/sandbox/supervisor_client.py +++ b/nemo_gym/sandbox/supervisor_client.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Controller-side transport for the shared sandbox process supervisor. -Session ownership, cancellation, and harness output parsing stay with the adapter. +Session ownership and cancellation live in session.py; output parsing stays with the adapter. Unlike process_supervisor.py, this module is not uploaded to the task sandbox. """ diff --git a/responses_api_agents/hermes_agent/app.py b/responses_api_agents/hermes_agent/app.py index 97e9f9e1490..726dc2b96aa 100644 --- a/responses_api_agents/hermes_agent/app.py +++ b/responses_api_agents/hermes_agent/app.py @@ -70,6 +70,7 @@ from nemo_gym.sandbox.access import DirectSandboxConnection from nemo_gym.sandbox.config import resolve_provider_config from nemo_gym.sandbox.providers import create_provider +from nemo_gym.sandbox.session import SandboxSession from nemo_gym.server_utils import get_response_json, raise_for_status from nemo_gym.tool_access import MCPToolAccess from responses_api_agents.hermes_agent.model_kwargs import _model_api_kwargs @@ -284,6 +285,14 @@ async def _close_agent_session_state(self, state: AgentSessionState) -> AgentClo if not isinstance(state, HermesSandboxSession): raise TypeError("Expected Hermes agent session state") await state.close(self.config.session_close_timeout_seconds) + # Errors/cancellation can bypass response parsing. The common session + # still captures available output before releasing the sandbox. + output = state.session.artifacts + if state.observations is None and output is not None: + result = output.get("result") + state.observations = self._sandbox_observations( + result if isinstance(result, dict) else {"failed": True}, output.get("observations") + ) observations = state.observations or AgentObservationBundle( source="hermes", gaps=[ObservationGap(code="observation_capture_failed")] ) @@ -404,10 +413,13 @@ async def _initialize_agent_session_state( session_dir = f"/tmp/nemo-gym-hermes-sessions/{uuid4().hex}" state = HermesSandboxSession( request=body, - sandbox=sandbox, - workdir=workdir, - directory=session_dir, - owns_sandbox=owns_sandbox, + session=SandboxSession( + sandbox=sandbox, + workdir=workdir, + directory=session_dir, + owns_sandbox=owns_sandbox, + harness="Hermes", + ), ) try: await state.prepare(install_timeout=self.config.sandbox_install_timeout_seconds) diff --git a/responses_api_agents/hermes_agent/sandbox.py b/responses_api_agents/hermes_agent/sandbox.py index 77261bc98f1..9c8819b5ca9 100644 --- a/responses_api_agents/hermes_agent/sandbox.py +++ b/responses_api_agents/hermes_agent/sandbox.py @@ -5,7 +5,6 @@ import asyncio import importlib.metadata import json -import logging import shutil from dataclasses import dataclass from pathlib import Path @@ -16,17 +15,9 @@ from nemo_gym.base_responses_api_agent import AgentSessionState from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming from nemo_gym.rollout_observability import AgentObservationBundle -from nemo_gym.sandbox import AsyncSandbox, process_supervisor -from nemo_gym.sandbox.process_supervisor import CleanupReceipt -from nemo_gym.sandbox.supervisor_client import ( - HarnessProcessInfo, - remove_session_directory, - stop_and_confirm_cleanup, - supervised_launch_command, -) - - -LOG = logging.getLogger(__name__) +from nemo_gym.sandbox import process_supervisor +from nemo_gym.sandbox.session import SandboxCommand, SandboxSession +from nemo_gym.sandbox.supervisor_client import HarnessProcessInfo def _sandbox_hermes_install() -> tuple[str, str]: @@ -61,34 +52,27 @@ def _sandbox_hermes_install() -> tuple[str, str]: class HermesSandboxSession(AgentSessionState): """Hermes transport and cleanup; HTTP retry bookkeeping stays in the agent base.""" - sandbox: AsyncSandbox - workdir: str | None - directory: str - owns_sandbox: bool = False + session: SandboxSession[dict[str, JsonValue]] observations: AgentObservationBundle | None = None activation_request: NeMoGymResponseCreateParamsNonStreaming | None = None task: asyncio.Task[NeMoGymResponse] | None = None runtime_info: HarnessProcessInfo | None = None - launch_started: bool = False - cleanup: CleanupReceipt | None = None - sandbox_stopped: bool = False - closed: bool = False async def prepare(self, *, install_timeout: float) -> None: """Reuse or install the pinned runtime and stage the Hermes worker files.""" - prepared = await self.sandbox.exec( - f"mkdir -p {quote(_SANDBOX_RUNTIME_DIR)} {quote(self.directory)}", - cwd=self.workdir, + prepared = await self.session.sandbox.exec( + f"mkdir -p {quote(_SANDBOX_RUNTIME_DIR)} {quote(self.session.directory)}", + cwd=self.session.workdir, timeout_s=30, ) if prepared.return_code != 0: raise RuntimeError(prepared.stderr or prepared.stdout or "Failed to prepare Hermes sandbox paths") if not await self._runtime_installed(): await self._install_runtime(install_timeout) - await self.sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), _SANDBOX_RUNNER) - await self.sandbox.upload(Path(__file__).with_name("sandbox_observer.py"), _SANDBOX_OBSERVER) - await self.sandbox.upload(Path(__file__).with_name("model_kwargs.py"), _SANDBOX_MODEL_KWARGS) - await self.sandbox.upload(Path(process_supervisor.__file__), _SANDBOX_SUPERVISOR) + await self.session.sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), _SANDBOX_RUNNER) + await self.session.sandbox.upload(Path(__file__).with_name("sandbox_observer.py"), _SANDBOX_OBSERVER) + await self.session.sandbox.upload(Path(__file__).with_name("model_kwargs.py"), _SANDBOX_MODEL_KWARGS) + await self.session.sandbox.upload(Path(process_supervisor.__file__), _SANDBOX_SUPERVISOR) async def _runtime_installed(self) -> bool: """Whether the pinned Hermes and its MCP client import from its runtime path. @@ -96,9 +80,9 @@ async def _runtime_installed(self) -> bool: The path is keyed by the pinned commit, so a runtime baked into the image or left by an earlier session in this sandbox is reused. """ - check = await self.sandbox.exec( + check = await self.session.sandbox.exec( f"{quote(_SANDBOX_PYTHON)} -c 'import run_agent, mcp'", - cwd=self.workdir, + cwd=self.session.workdir, timeout_s=120, ) return check.return_code == 0 @@ -107,14 +91,14 @@ async def _install_runtime(self, timeout: float) -> None: uv_path = shutil.which("uv") if uv_path is None: raise RuntimeError("Hermes agent server requires uv to install the sandbox runtime") - await self.sandbox.upload(uv_path, _SANDBOX_UV) + await self.session.sandbox.upload(uv_path, _SANDBOX_UV) venv = quote(_SANDBOX_RUNTIME_DIR + "/venv") # A runtime that failed the import check is incomplete, so rebuild it rather than reuse it. - install = await self.sandbox.exec( + install = await self.session.sandbox.exec( f"chmod 755 {quote(_SANDBOX_UV)} && rm -rf {venv} && " f"{quote(_SANDBOX_UV)} venv {venv} --python 3.13 && " f"{quote(_SANDBOX_UV)} pip install --python {quote(_SANDBOX_PYTHON)} {quote(_HERMES_REQUIREMENT)}", - cwd=self.workdir, + cwd=self.session.workdir, timeout_s=timeout, ) if install.return_code != 0 or not await self._runtime_installed(): @@ -122,39 +106,21 @@ async def _install_runtime(self, timeout: float) -> None: async def upload_json(self, name: str, payload: dict[str, JsonValue]) -> None: """Write a Hermes input under the session directory using file transfer.""" - await self.sandbox.upload_text(f"{self.directory}/{name}", text=json.dumps(payload)) + await self.session.sandbox.upload_text(f"{self.session.directory}/{name}", text=json.dumps(payload)) async def read_json(self, name: str) -> dict[str, JsonValue]: """Read a Hermes output object without mixing it with the cleanup contract.""" - path = f"{self.directory}/{name}" - payload = json.loads(await self.sandbox.read_text(path)) + path = f"{self.session.directory}/{name}" + payload = json.loads(await self.session.sandbox.read_text(path)) if not isinstance(payload, dict): raise TypeError(f"Hermes sandbox payload at {path} is not an object") return payload - async def stop_runner(self, timeout: float) -> None: - """Fence a delayed launch or require the supervisor's cleanup receipt.""" - if self.sandbox_stopped or not self.launch_started or self.cleanup is not None: - return - self.cleanup = await stop_and_confirm_cleanup( - self.sandbox, directory=self.directory, workdir=self.workdir, timeout=timeout, harness="Hermes" - ) - async def close(self, timeout: float) -> None: - """Stop owned sandboxes; clean only harness work in borrowed sandboxes.""" - if self.closed: - return - if not self.owns_sandbox: - # Provider cancellation may kill its process group, including the - # supervisor. Require descendant cleanup before cancelling exec. - await self.stop_runner(timeout) + """Finish sandbox capture/release before cancelling the HTTP activation.""" + await self.session.close(timeout=timeout) if self.task is not None and not self.task.done() and not self.task.cancelling(): self.task.cancel() - if self.owns_sandbox and not self.sandbox_stopped: - # Container teardown is the authority, even without a runner receipt. - # Stop before waiting for an activation that might be stuck in cleanup. - await self.sandbox.stop() - self.sandbox_stopped = True if self.task is not None: try: await asyncio.wait_for(asyncio.shield(self.task), timeout=timeout) @@ -164,62 +130,48 @@ async def close(self, timeout: float) -> None: except Exception: if not self.task.done(): raise - # The activation error is replayed by the agent base. Cleanup - # is independent and must still release the borrowed connection. - if not self.owns_sandbox: - await remove_session_directory( - self.sandbox, directory=self.directory, workdir=self.workdir, timeout=timeout, harness="Hermes" - ) - await self.sandbox.disconnect() - self.closed = True + # The agent base replays the activation error independently of close. async def execute( self, payload: dict[str, JsonValue], *, timeout: float, close_timeout: float ) -> dict[str, JsonValue]: - """Run Hermes under supervision and read output only after confirmed cleanup.""" + """Use the common lifecycle, then validate Hermes-specific result metadata.""" + output = await self.session.execute( + prepare=lambda: self.prepare_execution(payload), + collect=self.collect_artifacts, + timeout=timeout, + close_timeout=close_timeout, + ) + if output.get("error") is not None: + raise RuntimeError(f"Hermes sandbox runner failed: {output['error']}\n{output.get('traceback', '')}") + try: + self.runtime_info = HarnessProcessInfo.model_validate(output.get("runtime")) + except ValueError as error: + raise RuntimeError("Hermes sandbox runner returned invalid runtime metadata") from error + return output + + async def prepare_execution(self, payload: dict[str, JsonValue]) -> SandboxCommand: + """Stage this activation's input and describe its worker command.""" await self.upload_json("input.json", payload) - cleanup_timeout = close_timeout / 3 - command = supervised_launch_command( - directory=self.directory, - command=[ + return SandboxCommand( + argv=[ _SANDBOX_PYTHON, _SANDBOX_RUNNER, - f"{self.directory}/input.json", - f"{self.directory}/output.json", + f"{self.session.directory}/input.json", + f"{self.session.directory}/output.json", ], - timeout=timeout, - cleanup_timeout=cleanup_timeout, python=_SANDBOX_PYTHON, supervisor_path=_SANDBOX_SUPERVISOR, ) - self.launch_started = True - try: - await self.sandbox.exec( - command, - cwd=self.workdir, - timeout_s=process_supervisor.exec_timeout(timeout=timeout, cleanup_timeout=cleanup_timeout), - ) - except BaseException: - try: - await self.stop_runner(close_timeout) - except Exception: - LOG.exception("Hermes cleanup remains unconfirmed; close must retry before verification") - raise - else: - await self.stop_runner(close_timeout) + + async def collect_artifacts(self) -> dict[str, JsonValue]: + """Copy Hermes output before close removes files, including interrupted output.""" try: - output = await self.read_json("output.json") + return await self.read_json("output.json") except Exception as error: - logs = await self.sandbox.exec( - f"cat {quote(self.directory + '/runner.log')} 2>/dev/null || true", - cwd=self.workdir, + logs = await self.session.sandbox.exec( + f"cat {quote(self.session.directory + '/runner.log')} 2>/dev/null || true", + cwd=self.session.workdir, timeout_s=30, ) raise RuntimeError(f"Hermes sandbox runner exited without output: {logs.stdout or ''}") from error - if output.get("error") is not None: - raise RuntimeError(f"Hermes sandbox runner failed: {output['error']}\n{output.get('traceback', '')}") - try: - self.runtime_info = HarnessProcessInfo.model_validate(output.get("runtime")) - except ValueError as error: - raise RuntimeError("Hermes sandbox runner returned invalid runtime metadata") from error - return output diff --git a/responses_api_agents/hermes_agent/tests/test_app.py b/responses_api_agents/hermes_agent/tests/test_app.py index 0fd5a9f009c..5640d4392b1 100644 --- a/responses_api_agents/hermes_agent/tests/test_app.py +++ b/responses_api_agents/hermes_agent/tests/test_app.py @@ -41,6 +41,7 @@ from nemo_gym.rollout_observability import AgentEpisode, AgentObservationBundle from nemo_gym.sandbox import AsyncSandbox, SandboxExecResult, SandboxSpec from nemo_gym.sandbox.access import DirectSandboxConnection, SandboxAccess +from nemo_gym.sandbox.session import SandboxSession from nemo_gym.server_utils import ServerClient from nemo_gym.tool_access import DirectHTTPToolAccess, MCPStreamableHTTPConnection, MCPToolAccess from responses_api_agents.hermes_agent.app import ( @@ -162,10 +163,10 @@ async def test_sandbox_access_selects_runtime_provider(self, monkeypatch, with_m {"sandbox_id": "sandbox"}, provider=provider, ) - assert state.sandbox is sandbox - assert state.workdir == "/app" - assert state.directory.startswith("/tmp/nemo-gym-hermes-sessions/") - assert len(state.directory.rsplit("/", 1)[-1]) == 32 + assert state.session.sandbox is sandbox + assert state.session.workdir == "/app" + assert state.session.directory.startswith("/tmp/nemo-gym-hermes-sessions/") + assert len(state.session.directory.rsplit("/", 1)[-1]) == 32 assert sandbox.exec.await_count == 2 assert {call.args[0].name for call in sandbox.upload.await_args_list} == { "sandbox_runner.py", @@ -253,9 +254,9 @@ async def test_missing_sandbox_access_uses_configured_fallback(self, monkeypatch sandbox_factory.assert_called_once_with(provider) sandbox.start.assert_awaited_once_with(SandboxSpec(workdir="/fallback")) - assert state.sandbox is sandbox - assert state.workdir == "/fallback" - assert state.owns_sandbox is True + assert state.session.sandbox is sandbox + assert state.session.workdir == "/fallback" + assert state.session.owns_sandbox is True await hermes._close_agent_session_state(state) sandbox.stop.assert_awaited_once() sandbox.disconnect.assert_not_awaited() @@ -387,7 +388,10 @@ def _sandbox_session( tool_accesses=list(tool_accesses), ) hermes._session_records["session"] = _AgentSessionRecord( - state=HermesSandboxSession(request=seed, sandbox=sandbox, workdir=None, directory="/session"), + state=HermesSandboxSession( + request=seed, + session=SandboxSession(sandbox=sandbox, workdir=None, directory="/session", harness="Hermes"), + ), episode_id=seed.episode_id, ) request = SimpleNamespace( @@ -516,7 +520,7 @@ async def exec( sandbox = _Sandbox() hermes, request, _ = self._sandbox_session(monkeypatch, sandbox) - with pytest.raises(RuntimeError, match="runner exited without output"): + with pytest.raises(TimeoutError, match="exceeded its execution deadline"): await hermes.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="fix bug")) stop = [command for command in sandbox.commands if "runner.stop" in command and "kill -TERM" in command] diff --git a/responses_api_agents/hermes_agent/tests/test_sandbox.py b/responses_api_agents/hermes_agent/tests/test_sandbox.py index 6c1dda2fed0..dc8a1ab7367 100644 --- a/responses_api_agents/hermes_agent/tests/test_sandbox.py +++ b/responses_api_agents/hermes_agent/tests/test_sandbox.py @@ -22,6 +22,7 @@ from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming from nemo_gym.rollout_observability import AgentEpisode, AgentObservationBundle from nemo_gym.sandbox import AsyncSandbox +from nemo_gym.sandbox.session import SandboxSession from nemo_gym.server_utils import ServerClient from responses_api_agents.hermes_agent import sandbox as hermes_sandbox from responses_api_agents.hermes_agent.app import ( @@ -75,9 +76,9 @@ def state(): "workdir": "/app", }, ), - sandbox=sandbox, - workdir="/app", - directory="/tmp/nemo-gym-hermes-sessions/session", + session=SandboxSession( + sandbox=sandbox, workdir="/app", directory="/tmp/nemo-gym-hermes-sessions/session", harness="Hermes" + ), ) @@ -145,7 +146,7 @@ def test_http_close_retry_and_stale_activation_never_fall_back(agent, state, bin seed = client.post("/v1/agent_sessions", json=state.request.model_dump(mode="json")) assert seed.status_code == 200 assert state.task is None - assert not state.launch_started and state.cleanup is None + assert not state.session.launch_started and state.session.cleanup is None assert client.post("/v1/agent_sessions", json=state.request.model_dump(mode="json")).status_code == 200 path = f"/ng-rollout/{state.request.episode_id.capture_key}/v1/responses" activation = client.post(path, json={"input": "task"}) @@ -170,8 +171,8 @@ def test_http_close_retry_and_stale_activation_never_fall_back(agent, state, bin wrong = dict(close, episode_id={"rollout_id": "other"}) assert client.post("/v1/agent_sessions/close", json=wrong).status_code == 409 assert agent._run_sandbox_episode.await_count == 1 - state.sandbox.disconnect.assert_awaited_once() - state.sandbox.stop.assert_not_awaited() + state.session.sandbox.disconnect.assert_awaited_once() + state.session.sandbox.stop.assert_not_awaited() async def test_invalid_activation_keeps_session_ready(agent: HermesAgent, state: HermesSandboxSession) -> None: @@ -222,7 +223,7 @@ async def activate(**kwargs): with pytest.raises(asyncio.CancelledError): await replay agent._run_sandbox_episode.assert_awaited_once() - state.sandbox.disconnect.assert_awaited_once() + state.session.sandbox.disconnect.assert_awaited_once() async def test_activation_replay_survives_disconnected_waiter(agent, state): @@ -273,7 +274,7 @@ def test_provider_failure_is_http_500_and_replays_without_rerunning(agent, state agent._initialize_agent_session_state = AsyncMock(return_value=state) state.upload_json = AsyncMock() mock_json_download( - state.sandbox, + state.session.sandbox, side_effect=[ {"cleanup_confirmed": True, "error": None}, { @@ -289,13 +290,13 @@ def test_provider_failure_is_http_500_and_replays_without_rerunning(agent, state response = client.post(path, json={"input": "task"}) assert response.status_code == 500 state.upload_json.assert_awaited_once() - assert state.cleanup is not None and state.cleanup["cleanup_confirmed"] is True + assert state.session.cleanup is not None and state.session.cleanup["cleanup_confirmed"] is True close = client.post( "/v1/agent_sessions/close", json={"agent_session_id": "session", "episode_id": state.request.episode_id.model_dump()}, ) assert close.status_code == 200 - state.sandbox.disconnect.assert_awaited_once() + state.session.sandbox.disconnect.assert_awaited_once() @pytest.mark.parametrize("runner_started", [False, True], ids=["not-launched", "cleanup-confirmed"]) @@ -304,9 +305,11 @@ async def test_close_failure_keeps_session_for_retry( ) -> None: agent._session_records["session"] = _AgentSessionRecord(state=state, episode_id=state.request.episode_id) if runner_started: - state.launch_started = True - download_json = mock_json_download(state.sandbox, return_value={"cleanup_confirmed": True, "error": None}) - state.sandbox.exec.side_effect = [ + state.session.launch_started = True + download_json = mock_json_download( + state.session.sandbox, return_value={"cleanup_confirmed": True, "error": None} + ) + state.session.sandbox.exec.side_effect = [ SimpleNamespace(return_code=1, error_type=None), SimpleNamespace(return_code=0, error_type=None), ] @@ -315,68 +318,72 @@ async def test_close_failure_keeps_session_for_retry( await agent.close_agent_session(request(state), close) assert agent._session_records["session"].state is state assert agent._session_records["session"].closing - assert (state.cleanup is not None) == runner_started - state.sandbox.disconnect.assert_not_awaited() + assert (state.session.cleanup is not None) == runner_started + state.session.sandbox.disconnect.assert_not_awaited() with pytest.raises(HTTPException): await agent.responses(request(state), NeMoGymResponseCreateParamsNonStreaming(input="task")) await agent.close_agent_session(request(state), close) - state.sandbox.disconnect.assert_awaited_once() + state.session.sandbox.disconnect.assert_awaited_once() if runner_started: download_json.assert_awaited_once() @pytest.mark.parametrize("receipt", [None, {"cleanup_confirmed": False, "error": "cleanup failed"}]) @pytest.mark.parametrize("launch_started,cleanup", [(False, None), (True, None), (True, {"cleanup_confirmed": True})]) -async def test_owned_close_stops_without_receipt_or_filesystem_cleanup(agent, state, receipt, launch_started, cleanup): - state.owns_sandbox = True - state.launch_started = launch_started - state.cleanup = cleanup +async def test_owned_close_falls_back_to_stop_without_receipt(agent, state, receipt, launch_started, cleanup): + state.session.owns_sandbox = True + state.session.launch_started = launch_started + state.session.cleanup = cleanup state.observations = AgentObservationBundle(source="hermes") agent._session_records["session"] = _AgentSessionRecord(state=state, episode_id=state.request.episode_id) - download_json = mock_json_download(state.sandbox, return_value=receipt) + download_json = mock_json_download(state.session.sandbox, return_value=receipt) if receipt is None: download_json.side_effect = FileNotFoundError("missing cleanup receipt") - state.sandbox.exec.side_effect = AssertionError("Owned close must not depend on sandbox exec") + state.session.sandbox.exec.side_effect = AssertionError("Owned close must not depend on sandbox exec") close = AgentCloseSessionRequest(agent_session_id="session", episode_id=state.request.episode_id) response = await agent.close_agent_session(request(state), close) assert response.agent_observations == state.observations - assert state.sandbox_stopped + assert state.session.sandbox_stopped assert agent._session_records["session"].state is None assert await agent.close_agent_session(request(state), close) == response - state.sandbox.stop.assert_awaited_once() - state.sandbox.disconnect.assert_not_awaited() - state.sandbox.exec.assert_not_awaited() - download_json.assert_not_awaited() + state.session.sandbox.stop.assert_awaited_once() + state.session.sandbox.disconnect.assert_not_awaited() + if launch_started and cleanup is None: + state.session.sandbox.exec.assert_awaited_once() + assert download_json.await_count >= 1 + else: + state.session.sandbox.exec.assert_not_awaited() + download_json.assert_not_awaited() @pytest.mark.parametrize("failure", [RuntimeError, TimeoutError, asyncio.CancelledError]) async def test_owned_stop_failure_keeps_close_retryable(agent, state, failure): - state.owns_sandbox = True - state.launch_started = True + state.session.owns_sandbox = True + state.session.launch_started = True agent._session_records["session"] = _AgentSessionRecord(state=state, episode_id=state.request.episode_id) - state.sandbox.stop.side_effect = [failure("stop failed"), None] - download_json = mock_json_download(state.sandbox, side_effect=FileNotFoundError("missing cleanup receipt")) + state.session.sandbox.stop.side_effect = [failure("stop failed"), None] + download_json = mock_json_download(state.session.sandbox, side_effect=FileNotFoundError("missing cleanup receipt")) close = AgentCloseSessionRequest(agent_session_id="session", episode_id=state.request.episode_id) - with pytest.raises(failure, match="stop failed"): + with pytest.raises(failure): await agent.close_agent_session(request(state), close) assert agent._session_records["session"].state is state assert agent._session_records["session"].closing - assert state.launch_started and state.cleanup is None and not state.sandbox_stopped + assert state.session.launch_started and state.session.cleanup is None and not state.session.sandbox_stopped assert "session" not in agent._closed_session_records response = await agent.close_agent_session(request(state), close) assert await agent.close_agent_session(request(state), close) == response - assert state.sandbox.stop.await_count == 2 - assert state.sandbox_stopped - state.sandbox.disconnect.assert_not_awaited() - state.sandbox.exec.assert_not_awaited() - download_json.assert_not_awaited() + assert state.session.sandbox.stop.await_count == 2 + assert state.session.sandbox_stopped + state.session.sandbox.disconnect.assert_not_awaited() + assert state.session.sandbox.exec.await_count == 2 + assert download_json.await_count == 4 async def test_owned_stop_precedes_waiting_for_cancelled_activation(agent, state): - state.owns_sandbox = True - state.launch_started = True + state.session.owns_sandbox = True + state.session.launch_started = True agent.config.session_close_timeout_seconds = 0.1 started, stopped = asyncio.Event(), asyncio.Event() @@ -389,17 +396,17 @@ async def activation(): await stopped.wait() raise - state.sandbox.stop.side_effect = stopped.set - download_json = mock_json_download(state.sandbox, side_effect=FileNotFoundError("missing cleanup receipt")) + state.session.sandbox.stop.side_effect = stopped.set + download_json = mock_json_download(state.session.sandbox, side_effect=FileNotFoundError("missing cleanup receipt")) state.task = asyncio.create_task(activation()) await started.wait() try: await agent._close_agent_session_state(state) assert state.task.cancelled() - assert state.sandbox_stopped - state.sandbox.stop.assert_awaited_once() - state.sandbox.exec.assert_not_awaited() - download_json.assert_not_awaited() + assert state.session.sandbox_stopped + state.session.sandbox.stop.assert_awaited_once() + state.session.sandbox.exec.assert_awaited_once() + assert download_json.await_count == 2 finally: stopped.set() if not state.task.done(): @@ -410,8 +417,8 @@ async def activation(): @pytest.mark.parametrize("failure", [TimeoutError, asyncio.CancelledError]) async def test_unavailable_remote_fence_still_blocks_close(agent, state, failure): # Losing contact with the sandbox is not proof that its pending launch is fenced. - state.sandbox.exec.side_effect = [failure("launch status unavailable"), SimpleNamespace(return_code=0)] - mock_json_download(state.sandbox, side_effect=FileNotFoundError("missing cleanup receipt")) + state.session.sandbox.exec.side_effect = [failure("launch status unavailable"), SimpleNamespace(return_code=0)] + mock_json_download(state.session.sandbox, side_effect=FileNotFoundError("missing cleanup receipt")) state.upload_json = AsyncMock() with pytest.raises(failure): await agent._run_sandbox_episode( @@ -419,11 +426,11 @@ async def test_unavailable_remote_fence_still_blocks_close(agent, state, failure agent_session_id="session", state=state, ) - assert state.launch_started and state.cleanup is None and not state.sandbox_stopped - state.sandbox.exec.side_effect = None + assert state.session.launch_started and state.session.cleanup is None and not state.session.sandbox_stopped + state.session.sandbox.exec.side_effect = None with pytest.raises(RuntimeError, match="launch outcome is unknown"): await agent._close_agent_session_state(state) - state.sandbox.disconnect.assert_not_awaited() + state.session.sandbox.disconnect.assert_not_awaited() @pytest.fixture @@ -431,16 +438,16 @@ def local_runner(agent, state, monkeypatch, tmp_path): """Execute the actual launch/close shell commands and exchange files, without a remote provider.""" directory = tmp_path / "session" directory.mkdir() - state.directory = str(directory) - state.workdir = str(tmp_path) + state.session.directory = str(directory) + state.session.workdir = str(tmp_path) agent.config.session_close_timeout_seconds = 2 monkeypatch.setattr(hermes_sandbox, "_SANDBOX_PYTHON", sys.executable) monkeypatch.setattr( hermes_sandbox, "_SANDBOX_RUNNER", str(Path(hermes_sandbox.__file__).with_name("sandbox_runner.py")) ) monkeypatch.setattr(hermes_sandbox, "_SANDBOX_SUPERVISOR", hermes_sandbox.process_supervisor.__file__) - state.sandbox.upload.side_effect = shutil.copyfile - state.sandbox.download.side_effect = shutil.copyfile + state.session.sandbox.upload.side_effect = shutil.copyfile + state.session.sandbox.download.side_effect = shutil.copyfile async def execute(command, **kwargs): process = await asyncio.create_subprocess_exec( @@ -465,7 +472,7 @@ async def execute(command, **kwargs): os.killpg(process.pid, signal.SIGKILL) await process.wait() - state.sandbox.exec.side_effect = execute + state.session.sandbox.exec.side_effect = execute return execute @@ -484,7 +491,7 @@ async def queued_exec(command, **kwargs): raise OSError("provider failed before spawning the shell") return await local_runner(command, **kwargs) - state.sandbox.exec.side_effect = queued_exec + state.session.sandbox.exec.side_effect = queued_exec state.task = asyncio.create_task( agent._run_sandbox_episode( body=NeMoGymResponseCreateParamsNonStreaming(input="task"), agent_session_id="session", state=state @@ -495,8 +502,8 @@ async def queued_exec(command, **kwargs): state.task.cancel() with pytest.raises(asyncio.CancelledError if failure == "cancel" else OSError): await state.task - assert state.cleanup is not None and state.cleanup["cleanup_confirmed"] is True - directory = Path(state.directory) + assert state.session.cleanup is not None and state.session.cleanup["cleanup_confirmed"] is True + directory = Path(state.session.directory) assert (directory / "launch.claim").readlink() == Path("stop") assert json.loads((directory / "cleanup.json").read_text())["cleanup_confirmed"] is True @@ -507,25 +514,75 @@ async def queued_exec(command, **kwargs): close = AgentCloseSessionRequest(agent_session_id="session", episode_id=state.request.episode_id) first = await agent.close_agent_session(request(state), close) assert await agent.close_agent_session(request(state), close) == first - state.sandbox.disconnect.assert_awaited_once() + state.session.sandbox.disconnect.assert_awaited_once() assert not directory.exists() assert (await local_runner(commands[0])).return_code == 0 assert not directory.exists() +@pytest.mark.skipif(sys.platform != "linux", reason="The process supervisor requires Linux") +@pytest.mark.parametrize("owned", [False, True]) +async def test_close_preserves_interrupted_observations_before_release( + agent, state, local_runner, monkeypatch, tmp_path, owned +): + ready = tmp_path / "worker-ready" + worker = tmp_path / "worker.py" + worker.write_text( + "import json,os,pathlib,signal,sys,time\n" + "def interrupted(*args):\n" + " output = {'result': {'completed': False, 'interrupted': True, 'stop_reason': 'cancelled', " + "'messages': [{'role': 'assistant', 'content': 'partial answer'}]}, " + "'observations': {'invocations': [{'invocation_id': 'root', 'status': 'incomplete', " + "'model_response_ids': ['partial-completion']}]}, " + "'runtime': {'hostname': 'sandbox', 'pid': os.getpid()}}\n" + " pathlib.Path(sys.argv[2]).write_text(json.dumps(output))\n" + " sys.exit(0)\n" + "signal.signal(signal.SIGTERM, interrupted)\n" + f"pathlib.Path({str(ready)!r}).touch()\n" + "while True: time.sleep(0.01)\n" + ) + monkeypatch.setattr(hermes_sandbox, "_SANDBOX_RUNNER", str(worker)) + state.session.owns_sandbox = owned + agent._session_records["session"] = _AgentSessionRecord(state=state, episode_id=state.request.episode_id) + running = asyncio.create_task( + agent.responses(request(state), NeMoGymResponseCreateParamsNonStreaming(input="task")) + ) + try: + async with asyncio.timeout(5): + while not ready.exists(): + await asyncio.sleep(0.01) + close = AgentCloseSessionRequest(agent_session_id="session", episode_id=state.request.episode_id) + result = await agent.close_agent_session(request(state), close) + assert result.agent_observations.gaps == [] + invocation = result.agent_observations.records[0] + assert invocation.status == "incomplete" + assert invocation.model_calls[0].response_id == "partial-completion" + assert state.session.artifacts["result"]["messages"][0]["content"] == "partial answer" + assert state.session.cleanup["cleanup_confirmed"] is True + assert await agent.close_agent_session(request(state), close) == result + if owned: + state.session.sandbox.stop.assert_awaited_once() + else: + assert not Path(state.session.directory).exists() + state.session.sandbox.disconnect.assert_awaited_once() + finally: + await state.close(2) + await asyncio.gather(running, return_exceptions=True) + + async def test_close_can_recover_a_stop_claim_with_no_receipt(agent, state, local_runner): - directory = Path(state.directory) + directory = Path(state.session.directory) # The first close won the claim but was interrupted before publishing its receipt. (directory / "launch.claim").symlink_to("stop") - state.launch_started = True - await state.stop_runner(agent.config.session_close_timeout_seconds) - assert state.cleanup is not None and state.cleanup["cleanup_confirmed"] is True + state.session.launch_started = True + await state.session.stop_runner(timeout=agent.config.session_close_timeout_seconds) + assert state.session.cleanup is not None and state.session.cleanup["cleanup_confirmed"] is True assert json.loads((directory / "cleanup.json").read_text())["cleanup_confirmed"] is True @pytest.mark.parametrize("confirmed", [True, False]) async def test_receipt_download_failure_never_signals_a_reused_pid(agent, state, local_runner, tmp_path, confirmed): - directory = Path(state.directory) + directory = Path(state.session.directory) signaled = tmp_path / "unrelated-process-signaled" child = await asyncio.create_subprocess_exec( sys.executable, @@ -541,7 +598,7 @@ async def test_receipt_download_failure_never_signals_a_reused_pid(agent, state, (directory / "runner.pid").write_text(str(child.pid)) (directory / "launch.claim").symlink_to("launch") (directory / "cleanup.json").write_text(json.dumps({"cleanup_confirmed": confirmed, "error": None})) - original_read = state.sandbox.read_text + original_read = state.session.sandbox.read_text attempts = 0 async def transient_download(*args): @@ -551,13 +608,13 @@ async def transient_download(*args): raise OSError("transient download failure") return await original_read(*args) - state.sandbox.read_text = AsyncMock(side_effect=transient_download) - state.launch_started = True + state.session.sandbox.read_text = AsyncMock(side_effect=transient_download) + state.session.launch_started = True if confirmed: - await state.stop_runner(agent.config.session_close_timeout_seconds) + await state.session.stop_runner(timeout=agent.config.session_close_timeout_seconds) else: with pytest.raises(RuntimeError, match="cleanup was not confirmed"): - await state.stop_runner(agent.config.session_close_timeout_seconds) + await state.session.stop_runner(timeout=agent.config.session_close_timeout_seconds) await asyncio.sleep(0.05) assert not signaled.exists() assert child.returncode is None @@ -585,18 +642,18 @@ async def execute(): finally: assert confirmed.is_set(), "Provider exec was cancelled before remote cleanup" - async def terminate(_state): + async def terminate(*, timeout): assert not state.task.cancelling() - state.cleanup = {"cleanup_confirmed": True, "error": None, "return_code": None, "timed_out": False} + state.session.cleanup = {"cleanup_confirmed": True, "error": None, "return_code": None, "timed_out": False} confirmed.set() - state.stop_runner = AsyncMock(side_effect=terminate) + state.session.stop_runner = AsyncMock(side_effect=terminate) state.task = asyncio.create_task(execute()) await entered.wait() try: await agent._close_agent_session_state(state) assert state.task.cancelled() - state.sandbox.disconnect.assert_awaited_once() + state.session.sandbox.disconnect.assert_awaited_once() finally: confirmed.set() state.task.cancel() @@ -606,9 +663,9 @@ async def terminate(_state): async def test_close_retires_the_launch_path_before_removing_its_fence( agent, state, local_runner, monkeypatch, tmp_path ): - directory = Path(state.directory) - retired = Path(f"{state.directory}.closed") - state.launch_started = True + directory = Path(state.session.directory) + retired = Path(f"{state.session.directory}.closed") + state.session.launch_started = True commands = tmp_path / "bin" commands.mkdir() remove = commands / "rm" @@ -620,29 +677,29 @@ async def test_close_retires_the_launch_path_before_removing_its_fence( await agent._close_agent_session_state(state) assert not directory.exists() assert (retired / "launch.claim").readlink() == Path("stop") - state.sandbox.disconnect.assert_not_awaited() + state.session.sandbox.disconnect.assert_not_awaited() # Retry completes removal using the stable retired path, without reopening the launch path. await agent._close_agent_session_state(state) assert not retired.exists() - state.sandbox.disconnect.assert_awaited_once() + state.session.sandbox.disconnect.assert_awaited_once() async def test_launch_claim_without_pid_is_not_proof_of_cleanup(agent, state, local_runner): - directory = Path(state.directory) + directory = Path(state.session.directory) (directory / "launch.claim").symlink_to("launch") - state.launch_started = True + state.session.launch_started = True with pytest.raises(RuntimeError, match="launch outcome is unknown"): await agent._close_agent_session_state(state) - assert state.launch_started and state.cleanup is None and not state.sandbox_stopped + assert state.session.launch_started and state.session.cleanup is None and not state.session.sandbox_stopped assert not (directory / "cleanup.json").exists() - state.sandbox.disconnect.assert_not_awaited() + state.session.sandbox.disconnect.assert_not_awaited() @pytest.mark.parametrize("stop_timing", ["before-shell", "before-handler"]) async def test_stop_during_interpreter_startup_closes_without_starting_a_worker( agent, state, local_runner, monkeypatch, tmp_path, stop_timing ): - directory = Path(state.directory) + directory = Path(state.session.directory) ready = tmp_path / "before-handler" worker_started = tmp_path / "worker-started" wrapper = tmp_path / "delayed_supervisor.py" @@ -672,10 +729,10 @@ async def test_stop_during_interpreter_startup_closes_without_starting_a_worker( async with asyncio.timeout(3): while not ready.exists(): await asyncio.sleep(0.01) - await state.stop_runner(agent.config.session_close_timeout_seconds) + await state.session.stop_runner(timeout=agent.config.session_close_timeout_seconds) with pytest.raises(RuntimeError, match="exited without output"): await state.task - assert state.cleanup is not None and state.cleanup["cleanup_confirmed"] is True + assert state.session.cleanup is not None and state.session.cleanup["cleanup_confirmed"] is True assert not worker_started.exists() assert json.loads((directory / "cleanup.json").read_text()) == { "cleanup_confirmed": True, @@ -684,7 +741,7 @@ async def test_stop_during_interpreter_startup_closes_without_starting_a_worker( "timed_out": False, } await agent._close_agent_session_state(state) - state.sandbox.disconnect.assert_awaited_once() + state.session.sandbox.disconnect.assert_awaited_once() finally: if not state.task.done(): state.task.cancel() @@ -695,7 +752,7 @@ async def test_stop_during_interpreter_startup_closes_without_starting_a_worker( async def test_invalid_runtime_keeps_cleanup_confirmed(agent, state, runtime): state.upload_json = AsyncMock() mock_json_download( - state.sandbox, + state.session.sandbox, side_effect=[ {"cleanup_confirmed": True, "error": None}, {"result": {"completed": True, "messages": []}, "runtime": runtime}, @@ -705,47 +762,48 @@ async def test_invalid_runtime_keeps_cleanup_confirmed(agent, state, runtime): await agent._run_sandbox_episode( body=NeMoGymResponseCreateParamsNonStreaming(input="task"), agent_session_id="session", state=state ) - assert state.cleanup is not None and state.cleanup["cleanup_confirmed"] is True + assert state.session.cleanup is not None and state.session.cleanup["cleanup_confirmed"] is True await agent._close_agent_session_state(state) - state.sandbox.disconnect.assert_awaited_once() - state.sandbox.stop.assert_not_awaited() + state.session.sandbox.disconnect.assert_awaited_once() + state.session.sandbox.stop.assert_not_awaited() async def test_invalid_cleanup_schema_does_not_mark_runner_confirmed(agent, state): - state.launch_started = True + state.session.launch_started = True mock_json_download( - state.sandbox, return_value={"cleanup_confirmed": True, "error": None, "timed_out": False, "return_code": "0"} + state.session.sandbox, + return_value={"cleanup_confirmed": True, "error": None, "timed_out": False, "return_code": "0"}, ) with pytest.raises(ValidationError): - await state.stop_runner(agent.config.session_close_timeout_seconds) - assert state.launch_started and state.cleanup is None and not state.sandbox_stopped - state.sandbox.disconnect.assert_not_awaited() + await state.session.stop_runner(timeout=agent.config.session_close_timeout_seconds) + assert state.session.launch_started and state.session.cleanup is None and not state.session.sandbox_stopped + state.session.sandbox.disconnect.assert_not_awaited() @pytest.mark.parametrize("receipt", [{}, {"cleanup_confirmed": False}, {"cleanup_confirmed": "true"}]) async def test_runner_exit_without_cleanup_receipt_blocks_close(agent, state, receipt): - state.launch_started = True - download_json = mock_json_download(state.sandbox, return_value=receipt) + state.session.launch_started = True + download_json = mock_json_download(state.session.sandbox, return_value=receipt) with pytest.raises(RuntimeError, match="cleanup was not confirmed"): await agent._close_agent_session_state(state) - assert state.launch_started and state.cleanup is None and not state.sandbox_stopped - state.sandbox.disconnect.assert_not_awaited() + assert state.session.launch_started and state.session.cleanup is None and not state.session.sandbox_stopped + state.session.sandbox.disconnect.assert_not_awaited() download_json.return_value = {"cleanup_confirmed": True, "error": None} await agent._close_agent_session_state(state) - assert state.cleanup is not None and state.cleanup["cleanup_confirmed"] is True - state.sandbox.disconnect.assert_awaited_once() + assert state.session.cleanup is not None and state.session.cleanup["cleanup_confirmed"] is True + state.session.sandbox.disconnect.assert_awaited_once() @pytest.mark.parametrize("overrides", [{}, {"temperature": 0.0}]) async def test_session_prompt_and_limits_reach_runner(agent, state, overrides, tmp_path): - state.directory = str(tmp_path) + state.session.directory = str(tmp_path) agent.server_client.global_config_dict = { "model": {"responses_api_models": {"vllm_model": {"chat_template_kwargs": {"enable_thinking": False}}}} } agent.config.system_prompt = "Configured instruction" state.upload_json = AsyncMock() mock_json_download( - state.sandbox, + state.session.sandbox, side_effect=[ {"cleanup_confirmed": True, "error": None}, { @@ -758,13 +816,13 @@ async def test_session_prompt_and_limits_reach_runner(agent, state, overrides, t input="Fix the bug", instructions="Request instruction", **overrides ) await agent._run_sandbox_episode(body=body, agent_session_id="session", state=state) - launch_command = state.sandbox.exec.await_args_list[0].args[0] + launch_command = state.session.sandbox.exec.await_args_list[0].args[0] launch_args = shlex.split(launch_command) assert any(arg.endswith("/process_supervisor.py") for arg in launch_args) assert float(launch_args[launch_args.index("--timeout") + 1]) == agent.config.sandbox_runner_timeout_seconds cleanup_timeout = float(launch_args[launch_args.index("--cleanup-timeout") + 1]) assert cleanup_timeout == agent.config.session_close_timeout_seconds / 3 - assert state.sandbox.exec.await_args_list[0].kwargs["timeout_s"] > ( + assert state.session.sandbox.exec.await_args_list[0].kwargs["timeout_s"] > ( agent.config.sandbox_runner_timeout_seconds + 3 * cleanup_timeout ) # Execute the real launch prefix: cleanup must receive the shell's PID, not a literal "$". @@ -792,7 +850,7 @@ async def execute(command, **kwargs): events.append("exec") return SimpleNamespace(stdout="runner stderr", stderr="", return_code=0, error_type=None) - state.sandbox.exec.side_effect = execute + state.session.sandbox.exec.side_effect = execute async def download(sandbox, path): if path.endswith("/cleanup.json"): @@ -823,7 +881,7 @@ async def download(sandbox, path): "runtime": {"hostname": "sandbox", "pid": 123}, } - mock_json_download(state.sandbox, side_effect=download) + mock_json_download(state.session.sandbox, side_effect=download) activation = agent._run_sandbox_episode( body=NeMoGymResponseCreateParamsNonStreaming(input="task"), agent_session_id="session", state=state ) @@ -837,7 +895,7 @@ async def download(sandbox, path): with pytest.raises(RuntimeError, match="runner exited without output: runner stderr"): await activation assert events[:3] == ["exec", "cleanup", "output"] - assert state.cleanup is not None and state.cleanup["cleanup_confirmed"] is True + assert state.session.cleanup is not None and state.session.cleanup["cleanup_confirmed"] is True agent.server_client.post.assert_not_called() payload = state.upload_json.await_args.args[1] assert payload["model_base_url"] == "http://model:8000/ng-rollout/episode-a1/v1" @@ -923,7 +981,7 @@ def count_validation(self, body): agent._session_records["session"] = _AgentSessionRecord(state=state, episode_id=state.request.episode_id) state.upload_json = AsyncMock() mock_json_download( - state.sandbox, + state.session.sandbox, side_effect=[ {"cleanup_confirmed": True, "error": None}, {"result": result, "runtime": {"hostname": "sandbox", "pid": 123}}, @@ -999,7 +1057,7 @@ async def test_seed_binds_full_payload_and_is_serialized(agent, state): result = await agent.close_agent_session(SimpleNamespace(session={}), close) assert await agent.close_agent_session(SimpleNamespace(session={}), close) == result assert all(record.state is None for record in agent._session_records.values()) - state.sandbox.disconnect.assert_awaited_once() + state.session.sandbox.disconnect.assert_awaited_once() @pytest.mark.parametrize("marker", [None, "", 0, [], {}]) @@ -1030,7 +1088,7 @@ async def test_failed_setup_retains_handle_until_cleanup_confirmed( body.sandbox_access = None agent.config.sandbox_provider = "runtime" agent.config.sandbox_config = {"workdir": "/fallback"} - sandbox = state.sandbox + sandbox = state.session.sandbox factory = MagicMock(return_value=sandbox) factory.connect = AsyncMock(return_value=sandbox) monkeypatch.setattr(module, "AsyncSandbox", factory) diff --git a/tests/unit_tests/test_sandbox_session.py b/tests/unit_tests/test_sandbox_session.py new file mode 100644 index 00000000000..195c5c8c901 --- /dev/null +++ b/tests/unit_tests/test_sandbox_session.py @@ -0,0 +1,259 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +import json +from unittest.mock import AsyncMock + +import pytest + +from nemo_gym.sandbox import AsyncSandbox, SandboxExecResult +from nemo_gym.sandbox.session import SandboxCommand, SandboxSession + + +RECEIPT = {"cleanup_confirmed": True, "return_code": 0, "timed_out": False, "error": None} +OK = SandboxExecResult(stdout="", stderr="", return_code=0) + + +@pytest.fixture +def session(): + sandbox = AsyncMock(spec=AsyncSandbox) + sandbox.exec.return_value = OK + sandbox.read_text.return_value = json.dumps(RECEIPT) + return SandboxSession(sandbox=sandbox, directory="/session", workdir="/app", harness="test") + + +async def execute(session, *, collect=None, prepare=None, timeout=1): + return await session.execute( + prepare=prepare or AsyncMock(return_value=SandboxCommand(argv=["worker"])), + collect=collect or AsyncMock(return_value="transcript"), + timeout=10, + close_timeout=timeout, + ) + + +@pytest.mark.parametrize("owned", [False, True]) +async def test_completed_execution_captures_once_before_release(session, owned): + session.owns_sandbox = owned + collect = AsyncMock(return_value="transcript") + assert await execute(session, collect=collect) == "transcript" + assert session.cleanup == RECEIPT + assert session.artifacts == "transcript" + session.sandbox.stop.assert_not_awaited() + session.sandbox.disconnect.assert_not_awaited() + await asyncio.gather(session.close(timeout=1), session.close(timeout=1)) + await session.close(timeout=1) + collect.assert_awaited_once() + assert session.closed + if owned: + session.sandbox.stop.assert_awaited_once() + session.sandbox.disconnect.assert_not_awaited() + assert session.sandbox.exec.await_count == 1 + else: + session.sandbox.stop.assert_not_awaited() + session.sandbox.disconnect.assert_awaited_once() + assert "mv /session /session.closed" in session.sandbox.exec.await_args.args[0] + with pytest.raises(RuntimeError, match="closing or already activated"): + await execute(session) + + +@pytest.mark.parametrize("owned", [False, True]) +@pytest.mark.parametrize("cancel_execution", [False, True]) +async def test_interruption_captures_before_transport_cancellation_and_release(session, owned, cancel_execution): + session.owns_sandbox = owned + launched, collecting, captured = asyncio.Event(), asyncio.Event(), asyncio.Event() + events = [] + + async def provider_exec(command, **kwargs): + if "trap '' TERM" in command: + launched.set() + try: + await asyncio.Future() + finally: + assert session.cleanup == RECEIPT + events.append("cancel_exec") + events.append("remove_files") + return OK + + async def collect(): + assert session.cleanup == RECEIPT + events.append("collect") + collecting.set() + await captured.wait() + return "partial transcript" + + session.sandbox.exec.side_effect = provider_exec + session.sandbox.stop.side_effect = lambda: events.append("stop_sandbox") + session.sandbox.disconnect.side_effect = lambda: events.append("disconnect") + collector = AsyncMock(side_effect=collect) + running = asyncio.create_task(execute(session, collect=collector)) + await asyncio.wait_for(launched.wait(), 2) + if cancel_execution: + running.cancel() + closing = asyncio.create_task(session.close(timeout=1)) + await asyncio.wait_for(collecting.wait(), 2) + retry = asyncio.create_task(session.close(timeout=1)) + # Losing one close waiter must not cancel collection or release early. + closing.cancel() + with pytest.raises(asyncio.CancelledError): + await closing + assert events == ["collect"] + assert not session.closed + captured.set() + await asyncio.wait_for(retry, 2) + with pytest.raises(asyncio.CancelledError): + await running + assert session.artifacts == "partial transcript" + collector.assert_awaited_once() + assert events == ["collect", "cancel_exec", *(["stop_sandbox"] if owned else ["remove_files", "disconnect"])] + + +async def test_close_joins_collection_started_by_normal_completion(session): + collecting, finish = asyncio.Event(), asyncio.Event() + + async def collect(): + collecting.set() + await finish.wait() + return "snapshot" + + collector = AsyncMock(side_effect=collect) + running = asyncio.create_task(execute(session, collect=collector)) + await asyncio.wait_for(collecting.wait(), 2) + close = asyncio.create_task(session.close(timeout=1)) + await asyncio.sleep(0) + session.sandbox.disconnect.assert_not_awaited() + assert session.sandbox.exec.await_count == 1 + finish.set() + assert await running == "snapshot" + await close + collector.assert_awaited_once() + + +async def test_close_during_input_preparation_prevents_launch(session): + preparing = asyncio.Event() + + async def prepare(): + preparing.set() + await asyncio.Future() + + collector = AsyncMock() + running = asyncio.create_task(execute(session, prepare=prepare, collect=collector)) + await asyncio.wait_for(preparing.wait(), 2) + await session.close(timeout=1) + with pytest.raises(asyncio.CancelledError): + await running + assert not session.launch_started + assert session.cleanup is None + collector.assert_not_awaited() + session.sandbox.read_text.assert_not_awaited() + # Only session-directory removal ran. + assert session.sandbox.exec.await_count == 1 + assert "mv /session /session.closed" in session.sandbox.exec.await_args.args[0] + + +@pytest.mark.parametrize("failure", [OSError("transport failed"), "timeout", "unavailable"]) +async def test_transport_failure_preserves_partial_output_and_original_error(session, failure): + if isinstance(failure, Exception): + session.sandbox.exec.side_effect = [failure, OK] + else: + session.sandbox.exec.side_effect = [SandboxExecResult("", "transport", 1, failure), OK] + expected = OSError if isinstance(failure, Exception) else TimeoutError if failure == "timeout" else RuntimeError + with pytest.raises(expected): + await execute(session) + assert session.cleanup == RECEIPT + assert session.artifacts == "transcript" + await session.close(timeout=1) + assert session.closed + + +async def test_capture_failure_does_not_hide_execution_failure(session): + session.sandbox.exec.side_effect = [OSError("transport failed"), OK] + collector = AsyncMock(side_effect=ValueError("malformed snapshot")) + with pytest.raises(OSError, match="transport failed"): + await execute(session, collect=collector) + assert isinstance(session.capture_error, ValueError) + assert session.cleanup == RECEIPT + await session.close(timeout=1) + collector.assert_awaited_once() + assert session.closed + + +async def test_capture_timeout_does_not_block_resource_release(session): + async def collect(): + await asyncio.Future() + + with pytest.raises(TimeoutError): + await execute(session, collect=collect, timeout=0.02) + assert isinstance(session.capture_error, TimeoutError) + assert session.cleanup == RECEIPT + await session.close(timeout=1) + assert session.closed + + +async def test_borrowed_cleanup_failure_retains_state_and_retries_capture(session): + session.sandbox.read_text.return_value = json.dumps(RECEIPT | {"cleanup_confirmed": False}) + collector = AsyncMock(return_value="recovered transcript") + with pytest.raises(RuntimeError, match="cleanup was not confirmed"): + await execute(session, collect=collector) + with pytest.raises(RuntimeError, match="cleanup was not confirmed"): + await session.close(timeout=1) + assert session.closing and not session.closed + assert session.cleanup is None + collector.assert_not_awaited() + session.sandbox.disconnect.assert_not_awaited() + session.sandbox.stop.assert_not_awaited() + assert all("rm -rf" not in call.args[0] for call in session.sandbox.exec.await_args_list) + session.sandbox.read_text.return_value = json.dumps(RECEIPT) + await session.close(timeout=1) + collector.assert_awaited_once() + assert session.artifacts == "recovered transcript" + assert session.closed + + +async def test_owned_cleanup_deadline_falls_back_to_provider_stop(session): + session.owns_sandbox = True + launched = asyncio.Event() + + async def provider_exec(command, **kwargs): + launched.set() + await asyncio.Future() + + async def unavailable(path): + await asyncio.Future() + + session.sandbox.exec.side_effect = provider_exec + session.sandbox.read_text.side_effect = unavailable + collector = AsyncMock() + running = asyncio.create_task(execute(session, collect=collector, timeout=0.02)) + await asyncio.wait_for(launched.wait(), 2) + await asyncio.wait_for(session.close(timeout=0.02), 1) + with pytest.raises(asyncio.CancelledError): + await running + assert session.sandbox_stopped and session.closed + assert session.cleanup is None + assert session.capture_error is not None + collector.assert_not_awaited() + session.sandbox.stop.assert_awaited_once() + + +@pytest.mark.parametrize("owned", [False, True]) +async def test_release_failure_retries_without_recollecting(session, owned): + session.owns_sandbox = owned + release = session.sandbox.stop if owned else session.sandbox.disconnect + release.side_effect = [OSError("release failed"), None] + collector = AsyncMock(return_value="saved transcript") + await execute(session, collect=collector) + with pytest.raises(OSError, match="release failed"): + await session.close(timeout=1) + assert not session.closed + await session.close(timeout=1) + assert session.closed + assert release.await_count == 2 + collector.assert_awaited_once() + + +async def test_nonzero_worker_exit_still_passes_artifacts_to_adapter(session): + session.sandbox.exec.return_value = SandboxExecResult("", "worker error", 1) + session.sandbox.read_text.return_value = json.dumps(RECEIPT | {"return_code": 1}) + assert await execute(session) == "transcript" + assert session.cleanup["return_code"] == 1 From c4aabf01bb8dd3d839340e3abb785c51f9aa9b12 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 5 Oct 2026 17:13:01 -0700 Subject: [PATCH 26/44] fix(sandbox): validate bootstrap and tolerate diagnostic gaps Signed-off-by: yaoyu-33 --- nemo_gym/sandbox/api.py | 14 -- nemo_gym/sandbox/session.py | 6 +- nemo_gym/sandbox/supervisor_client.py | 54 ++++-- nemo_gym/sandbox/utils.py | 25 +++ responses_api_agents/hermes_agent/app.py | 26 ++- responses_api_agents/hermes_agent/sandbox.py | 17 +- .../hermes_agent/tests/test_app.py | 37 ++-- .../hermes_agent/tests/test_sandbox.py | 86 ++++++--- tests/unit_tests/test_global_config.py | 18 +- tests/unit_tests/test_sandbox_session.py | 53 +++-- .../test_sandbox_supervisor_bootstrap.py | 182 ++++++++++++++++++ .../test_sandbox_supervisor_client.py | 103 ++++++++-- 12 files changed, 499 insertions(+), 122 deletions(-) create mode 100644 tests/unit_tests/test_sandbox_supervisor_bootstrap.py diff --git a/nemo_gym/sandbox/api.py b/nemo_gym/sandbox/api.py index 81ae7e55b56..ebe556c69fb 100644 --- a/nemo_gym/sandbox/api.py +++ b/nemo_gym/sandbox/api.py @@ -576,20 +576,6 @@ async def upload(self, local_path: Path | str, remote_path: str) -> None: async def download(self, remote_path: str, local_path: Path | str) -> None: await self._provider.download_file(self._require_handle(), remote_path, Path(local_path)) - async def upload_text(self, path: str, *, text: str) -> None: - """Upload UTF-8 text through file transfer without shell interpolation.""" - with tempfile.TemporaryDirectory(prefix="sandbox-upload-") as directory: - source = Path(directory) / "payload" - source.write_text(text, encoding="utf-8") - await self.upload(source, path) - - async def read_text(self, path: str) -> str: - """Download UTF-8 text, replacing invalid bytes in partial process output.""" - with tempfile.TemporaryDirectory(prefix="sandbox-download-") as directory: - destination = Path(directory) / "payload" - await self.download(path, destination) - return destination.read_text(encoding="utf-8", errors="replace") - async def status(self) -> SandboxStatus: if self._handle is None: return SandboxStatus.UNKNOWN diff --git a/nemo_gym/sandbox/session.py b/nemo_gym/sandbox/session.py index a4e4d876856..1e0e897e351 100644 --- a/nemo_gym/sandbox/session.py +++ b/nemo_gym/sandbox/session.py @@ -26,7 +26,7 @@ class SandboxCommand: """Harness argv and the interpreter/path used to launch its supervisor.""" argv: list[str] - python: str = "python3" + python: str supervisor_path: str | None = None @@ -117,6 +117,10 @@ async def execute( LOG.exception("%s cleanup remains unconfirmed; close must retry", self.harness) raise await self._finish(timeout=close_timeout) + if result.return_code != 0 and self.capture_error is not None: + raise RuntimeError( + f"{self.harness} sandbox execution failed (exit {result.return_code}): {result.stderr or ''}" + ) from self.capture_error if self.capture_error is not None: raise self.capture_error # Collection either returned artifacts (including a valid None) or recorded an error. diff --git a/nemo_gym/sandbox/supervisor_client.py b/nemo_gym/sandbox/supervisor_client.py index 79d1a9481c4..44106bf0412 100644 --- a/nemo_gym/sandbox/supervisor_client.py +++ b/nemo_gym/sandbox/supervisor_client.py @@ -7,16 +7,19 @@ """ import json +import logging from shlex import join, quote from uuid import uuid4 -from pydantic import BaseModel, ConfigDict, TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from nemo_gym.sandbox import AsyncSandbox from nemo_gym.sandbox.process_supervisor import CleanupReceipt +from nemo_gym.sandbox.utils import read_text _CLEANUP_RECEIPT = TypeAdapter(CleanupReceipt) +LOG = logging.getLogger(__name__) class HarnessProcessInfo(BaseModel): @@ -33,14 +36,34 @@ class HarnessProcessInfo(BaseModel): def parse_cleanup_receipt(payload: object) -> CleanupReceipt: - """Validate the supervisor's existing schema without adding worker dependencies. + """Require positive cleanup evidence, tolerating malformed optional diagnostics. Older stop-before-launch receipts contain only acknowledgement and error. Normalize that form without inventing a successful worker exit code. """ - if isinstance(payload, dict) and payload.keys() == {"cleanup_confirmed", "error"}: + if not isinstance(payload, dict) or payload.get("cleanup_confirmed") is not True: + raise ValueError("Sandbox cleanup was not confirmed") + if payload.keys() == {"cleanup_confirmed", "error"}: payload = {**payload, "return_code": None, "timed_out": False} - return _CLEANUP_RECEIPT.validate_python(payload, strict=True, extra="forbid") + try: + return _CLEANUP_RECEIPT.validate_python(payload, strict=True, extra="forbid") + except ValidationError: + LOG.warning("Cleanup was confirmed, but its diagnostic fields did not match the receipt schema") + return { + "cleanup_confirmed": True, + "return_code": payload.get("return_code") if type(payload.get("return_code")) is int else None, + "timed_out": payload.get("timed_out") is True, + "error": payload.get("error") if isinstance(payload.get("error"), str) else None, + } + + +def parse_runtime_info(payload: object) -> HarnessProcessInfo | None: + """Read optional worker diagnostics without failing an otherwise valid episode.""" + try: + return HarnessProcessInfo.model_validate(payload) + except ValidationError: + LOG.warning("Harness runtime metadata is missing or malformed") + return None def supervised_launch_command( @@ -49,18 +72,22 @@ def supervised_launch_command( command: list[str], timeout: float, cleanup_timeout: float, - python: str = "python3", + python: str, supervisor_path: str | None = None, ) -> str: """Fence delayed launches and run a harness command under the shared supervisor. - Use a private interpreter and supervisor path when the harness installs its - own runtime. Supervisor and worker diagnostics are combined in runner.log. + The adapter must install or select the interpreter explicitly. Check that it + can load the supervisor before claiming a launch: a failed bootstrap cannot + write a cleanup receipt. Supervisor and worker diagnostics share runner.log. """ + supervisor = quote(supervisor_path or directory + "/process_supervisor.py") return ( + f"[ -d {quote(directory)} ] && [ ! -L {quote(directory + '/launch.claim')} ] || exit 0; " + f"{quote(python)} -I {supervisor} --help >/dev/null || exit $?; " f"trap '' TERM; ln -s launch {quote(directory + '/launch.claim')} 2>/dev/null || exit 0; " f"echo $$ > {quote(directory + '/runner.pid')} && " - f"exec {quote(python)} -I {quote(supervisor_path or directory + '/process_supervisor.py')} " + f"exec {quote(python)} -I {supervisor} " f"--timeout {timeout} --cleanup-timeout {cleanup_timeout} " f"--stop-file {quote(directory + '/runner.stop')} " f"--receipt {quote(directory + '/cleanup.json')} -- {join(command)} " @@ -79,10 +106,10 @@ async def stop_and_confirm_cleanup( """ receipt_path = f"{directory}/cleanup.json" try: - receipt = json.loads(await sandbox.read_text(receipt_path)) + receipt = json.loads(await read_text(sandbox, path=receipt_path)) except Exception: receipt = {} - if receipt.get("cleanup_confirmed") is not True: + if not isinstance(receipt, dict) or receipt.get("cleanup_confirmed") is not True: pid_path = quote(f"{directory}/runner.pid") stop_path = quote(f"{directory}/runner.stop") claim_path = quote(f"{directory}/launch.claim") @@ -100,11 +127,12 @@ async def stop_and_confirm_cleanup( ) await sandbox.exec(script, cwd=workdir, timeout_s=timeout + 5) try: - receipt = json.loads(await sandbox.read_text(receipt_path)) + receipt = json.loads(await read_text(sandbox, path=receipt_path)) except Exception as error: raise RuntimeError(f"{harness} launch outcome is unknown; cannot confirm termination") from error - if receipt.get("cleanup_confirmed") is not True: - raise RuntimeError(f"{harness} sandbox cleanup was not confirmed: {receipt.get('error')}") + if not isinstance(receipt, dict) or receipt.get("cleanup_confirmed") is not True: + detail = receipt.get("error") if isinstance(receipt, dict) else "cleanup receipt is not a JSON object" + raise RuntimeError(f"{harness} sandbox cleanup was not confirmed: {detail}") return parse_cleanup_receipt(receipt) diff --git a/nemo_gym/sandbox/utils.py b/nemo_gym/sandbox/utils.py index f9dd70a9c1e..7345921bba5 100644 --- a/nemo_gym/sandbox/utils.py +++ b/nemo_gym/sandbox/utils.py @@ -14,6 +14,31 @@ """Sandbox utility helpers.""" +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from nemo_gym.sandbox.api import AsyncSandbox + + +async def upload_text(sandbox: "AsyncSandbox", *, path: str, text: str) -> None: + """Upload UTF-8 text through file transfer without shell interpolation.""" + with tempfile.TemporaryDirectory(prefix="sandbox-upload-") as directory: + source = Path(directory) / "payload" + source.write_text(text, encoding="utf-8") + await sandbox.upload(source, path) + + +async def read_text(sandbox: "AsyncSandbox", *, path: str) -> str: + """Download UTF-8 text, replacing invalid bytes in partial process output.""" + with tempfile.TemporaryDirectory(prefix="sandbox-download-") as directory: + destination = Path(directory) / "payload" + await sandbox.download(path, destination) + return destination.read_text(encoding="utf-8", errors="replace") + + # Parallelism caps for CPU-limited sandboxes, each set by cpu_cap_env() to the # floored CPU limit (min 1). Tools size worker pools by host core count, not # the cgroup limit — Python's multiprocessing.cpu_count(), BLAS/OpenMP thread diff --git a/responses_api_agents/hermes_agent/app.py b/responses_api_agents/hermes_agent/app.py index 726dc2b96aa..213eb824477 100644 --- a/responses_api_agents/hermes_agent/app.py +++ b/responses_api_agents/hermes_agent/app.py @@ -71,6 +71,7 @@ from nemo_gym.sandbox.config import resolve_provider_config from nemo_gym.sandbox.providers import create_provider from nemo_gym.sandbox.session import SandboxSession +from nemo_gym.sandbox.supervisor_client import HarnessProcessInfo from nemo_gym.server_utils import get_response_json, raise_for_status from nemo_gym.tool_access import MCPToolAccess from responses_api_agents.hermes_agent.model_kwargs import _model_api_kwargs @@ -291,7 +292,9 @@ async def _close_agent_session_state(self, state: AgentSessionState) -> AgentClo if state.observations is None and output is not None: result = output.get("result") state.observations = self._sandbox_observations( - result if isinstance(result, dict) else {"failed": True}, output.get("observations") + result if isinstance(result, dict) else {"failed": True}, + output.get("observations"), + runtime_info=state.runtime_info, ) observations = state.observations or AgentObservationBundle( source="hermes", gaps=[ObservationGap(code="observation_capture_failed")] @@ -439,7 +442,10 @@ def _sandbox_observations( self, result: dict[str, Any], raw_observations: Any, + *, + runtime_info: HarnessProcessInfo | None, ) -> AgentObservationBundle: + gaps = [] if runtime_info is not None else [ObservationGap(code="runtime_info_unavailable")] if isinstance(raw_observations, dict): try: records: list[AgentInvocation | ToolCallObservation | ContextCompactionObservation] = [] @@ -473,17 +479,18 @@ def _sandbox_observations( return AgentObservationBundle( source="hermes", records=records, - gaps=[ObservationGap.model_validate(gap) for gap in raw_observations.get("gaps") or []], + gaps=[*gaps, *(ObservationGap.model_validate(gap) for gap in raw_observations.get("gaps") or [])], ) except Exception as error: LOG.exception("failed to validate sandbox Hermes observations") return AgentObservationBundle( source="hermes", gaps=[ + *gaps, ObservationGap( code="observation_capture_failed", detail=type(error).__name__, - ) + ), ], ) @@ -516,7 +523,7 @@ def _sandbox_observations( status="completed", ) ) - return AgentObservationBundle(source="hermes", records=records) + return AgentObservationBundle(source="hermes", records=records, gaps=gaps) async def _run_sandbox_episode( self, @@ -573,13 +580,16 @@ async def _run_sandbox_episode( response.metadata = { **(response.metadata or {}), "harness_execution": "sandbox", - "harness_hostname": runtime.hostname, - "harness_pid": str(runtime.pid), - "harness_python": runtime.python or "", } + if runtime is not None: + response.metadata.update( + harness_hostname=runtime.hostname, + harness_pid=str(runtime.pid), + harness_python=runtime.python or "", + ) return AgentEpisode( response=response, - observations=self._sandbox_observations(result, output.get("observations")), + observations=self._sandbox_observations(result, output.get("observations"), runtime_info=runtime), ) def _validate_request( diff --git a/responses_api_agents/hermes_agent/sandbox.py b/responses_api_agents/hermes_agent/sandbox.py index 9c8819b5ca9..dea7c1fb6f7 100644 --- a/responses_api_agents/hermes_agent/sandbox.py +++ b/responses_api_agents/hermes_agent/sandbox.py @@ -17,7 +17,8 @@ from nemo_gym.rollout_observability import AgentObservationBundle from nemo_gym.sandbox import process_supervisor from nemo_gym.sandbox.session import SandboxCommand, SandboxSession -from nemo_gym.sandbox.supervisor_client import HarnessProcessInfo +from nemo_gym.sandbox.supervisor_client import HarnessProcessInfo, parse_runtime_info +from nemo_gym.sandbox.utils import read_text, upload_text def _sandbox_hermes_install() -> tuple[str, str]: @@ -106,12 +107,12 @@ async def _install_runtime(self, timeout: float) -> None: async def upload_json(self, name: str, payload: dict[str, JsonValue]) -> None: """Write a Hermes input under the session directory using file transfer.""" - await self.session.sandbox.upload_text(f"{self.session.directory}/{name}", text=json.dumps(payload)) + await upload_text(self.session.sandbox, path=f"{self.session.directory}/{name}", text=json.dumps(payload)) async def read_json(self, name: str) -> dict[str, JsonValue]: """Read a Hermes output object without mixing it with the cleanup contract.""" path = f"{self.session.directory}/{name}" - payload = json.loads(await self.session.sandbox.read_text(path)) + payload = json.loads(await read_text(self.session.sandbox, path=path)) if not isinstance(payload, dict): raise TypeError(f"Hermes sandbox payload at {path} is not an object") return payload @@ -135,7 +136,7 @@ async def close(self, timeout: float) -> None: async def execute( self, payload: dict[str, JsonValue], *, timeout: float, close_timeout: float ) -> dict[str, JsonValue]: - """Use the common lifecycle, then validate Hermes-specific result metadata.""" + """Use the common lifecycle, then check the Hermes-specific result.""" output = await self.session.execute( prepare=lambda: self.prepare_execution(payload), collect=self.collect_artifacts, @@ -144,10 +145,6 @@ async def execute( ) if output.get("error") is not None: raise RuntimeError(f"Hermes sandbox runner failed: {output['error']}\n{output.get('traceback', '')}") - try: - self.runtime_info = HarnessProcessInfo.model_validate(output.get("runtime")) - except ValueError as error: - raise RuntimeError("Hermes sandbox runner returned invalid runtime metadata") from error return output async def prepare_execution(self, payload: dict[str, JsonValue]) -> SandboxCommand: @@ -167,7 +164,7 @@ async def prepare_execution(self, payload: dict[str, JsonValue]) -> SandboxComma async def collect_artifacts(self) -> dict[str, JsonValue]: """Copy Hermes output before close removes files, including interrupted output.""" try: - return await self.read_json("output.json") + output = await self.read_json("output.json") except Exception as error: logs = await self.session.sandbox.exec( f"cat {quote(self.session.directory + '/runner.log')} 2>/dev/null || true", @@ -175,3 +172,5 @@ async def collect_artifacts(self) -> dict[str, JsonValue]: timeout_s=30, ) raise RuntimeError(f"Hermes sandbox runner exited without output: {logs.stdout or ''}") from error + self.runtime_info = parse_runtime_info(output.get("runtime")) + return output diff --git a/responses_api_agents/hermes_agent/tests/test_app.py b/responses_api_agents/hermes_agent/tests/test_app.py index 5640d4392b1..69a6214df6d 100644 --- a/responses_api_agents/hermes_agent/tests/test_app.py +++ b/responses_api_agents/hermes_agent/tests/test_app.py @@ -15,7 +15,7 @@ import asyncio import json from pathlib import Path -from types import MethodType, SimpleNamespace +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -39,7 +39,7 @@ NeMoGymResponseReasoningItem, ) from nemo_gym.rollout_observability import AgentEpisode, AgentObservationBundle -from nemo_gym.sandbox import AsyncSandbox, SandboxExecResult, SandboxSpec +from nemo_gym.sandbox import SandboxExecResult, SandboxSpec from nemo_gym.sandbox.access import DirectSandboxConnection, SandboxAccess from nemo_gym.sandbox.session import SandboxSession from nemo_gym.server_utils import ServerClient @@ -134,7 +134,7 @@ async def test_sandbox_access_selects_runtime_provider(self, monkeypatch, with_m monkeypatch.setattr("responses_api_agents.hermes_agent.sandbox.shutil.which", lambda name: "/test/uv") monkeypatch.setattr( "responses_api_agents.hermes_agent.app.get_global_config_dict", - lambda: {"runtime": {"hostname": "sandbox", "pid": 123}}, + lambda: {"runtime": {}}, ) monkeypatch.setattr("responses_api_agents.hermes_agent.app.resolve_provider_config", resolve) monkeypatch.setattr("responses_api_agents.hermes_agent.app.create_provider", lambda config: provider) @@ -158,7 +158,7 @@ async def test_sandbox_access_selects_runtime_provider(self, monkeypatch, with_m ), ) - resolve.assert_called_once_with("runtime", {"runtime": {"hostname": "sandbox", "pid": 123}}) + resolve.assert_called_once_with("runtime", {"runtime": {}}) connect.assert_awaited_once_with( {"sandbox_id": "sandbox"}, provider=provider, @@ -188,7 +188,7 @@ async def exec_(command, **_kwargs): sandbox.exec.side_effect = exec_ monkeypatch.setattr( "responses_api_agents.hermes_agent.app.get_global_config_dict", - lambda: {"runtime": {"hostname": "sandbox", "pid": 123}}, + lambda: {"runtime": {}}, ) monkeypatch.setattr("responses_api_agents.hermes_agent.app.resolve_provider_config", MagicMock()) monkeypatch.setattr("responses_api_agents.hermes_agent.app.create_provider", lambda config: AsyncMock()) @@ -233,7 +233,7 @@ async def test_missing_sandbox_access_uses_configured_fallback(self, monkeypatch sandbox_factory = MagicMock(return_value=sandbox) monkeypatch.setattr( "responses_api_agents.hermes_agent.app.get_global_config_dict", - lambda: {"runtime": {"hostname": "sandbox", "pid": 123}}, + lambda: {"runtime": {}}, ) monkeypatch.setattr( "responses_api_agents.hermes_agent.app.resolve_provider_config", @@ -398,20 +398,24 @@ def _sandbox_session( session={"agent_session_id": "session"}, path_params={"rollout_id": seed.episode_id.capture_key}, ) - sandbox.upload_text = MethodType(AsyncSandbox.upload_text, sandbox) + original_download = sandbox.download - async def read_text(remote_path): + async def download(remote_path, local_path): if remote_path.endswith("/cleanup.json"): commands = getattr(sandbox, "commands", None) - return json.dumps( - { - "cleanup_confirmed": commands is None or any("kill -TERM" in cmd for cmd in commands), - "error": None, - } + Path(local_path).write_text( + json.dumps( + { + "cleanup_confirmed": commands is None or any("kill -TERM" in cmd for cmd in commands), + "error": None, + } + ), + encoding="utf-8", ) - return await AsyncSandbox.read_text(sandbox, remote_path) + else: + await original_download(remote_path, local_path) - sandbox.read_text = AsyncMock(side_effect=read_text) + sandbox.download = AsyncMock(side_effect=download) return hermes, request, seed async def test_sandbox_activation_calls_the_model_server_directly(self, monkeypatch) -> None: @@ -538,6 +542,9 @@ async def upload(self, *_args) -> None: async def disconnect(self) -> None: pass + async def download(self, remote_path, local_path) -> None: + raise FileNotFoundError("worker was interrupted") + # Mirrors AsyncSandbox.exec so an unsupported argument fails here too. async def exec( self, command, *, cwd=None, env=None, timeout_s=180, user=None, preserve_background_services=False diff --git a/responses_api_agents/hermes_agent/tests/test_sandbox.py b/responses_api_agents/hermes_agent/tests/test_sandbox.py index dc8a1ab7367..aedfb310a51 100644 --- a/responses_api_agents/hermes_agent/tests/test_sandbox.py +++ b/responses_api_agents/hermes_agent/tests/test_sandbox.py @@ -9,7 +9,7 @@ import signal import sys from pathlib import Path -from types import MethodType, SimpleNamespace +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock import pytest @@ -21,7 +21,6 @@ from nemo_gym.episode_types import EpisodeId, TaskId from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming from nemo_gym.rollout_observability import AgentEpisode, AgentObservationBundle -from nemo_gym.sandbox import AsyncSandbox from nemo_gym.sandbox.session import SandboxSession from nemo_gym.server_utils import ServerClient from responses_api_agents.hermes_agent import sandbox as hermes_sandbox @@ -59,8 +58,6 @@ def agent(monkeypatch): @pytest.fixture def state(): sandbox = AsyncMock() - sandbox.read_text = MethodType(AsyncSandbox.read_text, sandbox) - sandbox.upload_text = MethodType(AsyncSandbox.upload_text, sandbox) sandbox.exec.return_value = SimpleNamespace(return_code=0, stdout="", stderr="", error_type=None) return HermesSandboxSession( request=AgentSeedSessionRequest( @@ -83,13 +80,13 @@ def state(): def mock_json_download(sandbox, **kwargs): - """Supply JSON artifacts through the same text API as a real sandbox.""" + """Supply files through the real text utility's sandbox download boundary.""" download = AsyncMock(**kwargs) - async def read_text(path): - return json.dumps(await download(sandbox, path)) + async def write_download(path, destination): + Path(destination).write_text(json.dumps(await download(sandbox, path)), encoding="utf-8") - sandbox.read_text = AsyncMock(side_effect=read_text) + sandbox.download = AsyncMock(side_effect=write_download) return download @@ -598,7 +595,7 @@ async def test_receipt_download_failure_never_signals_a_reused_pid(agent, state, (directory / "runner.pid").write_text(str(child.pid)) (directory / "launch.claim").symlink_to("launch") (directory / "cleanup.json").write_text(json.dumps({"cleanup_confirmed": confirmed, "error": None})) - original_read = state.session.sandbox.read_text + original_download = state.session.sandbox.download attempts = 0 async def transient_download(*args): @@ -606,9 +603,9 @@ async def transient_download(*args): attempts += 1 if attempts == 1: raise OSError("transient download failure") - return await original_read(*args) + return await original_download(*args) - state.session.sandbox.read_text = AsyncMock(side_effect=transient_download) + state.session.sandbox.download = AsyncMock(side_effect=transient_download) state.session.launch_started = True if confirmed: await state.session.stop_runner(timeout=agent.config.session_close_timeout_seconds) @@ -705,6 +702,7 @@ async def test_stop_during_interpreter_startup_closes_without_starting_a_worker( wrapper = tmp_path / "delayed_supervisor.py" wrapper.write_text( "import pathlib,runpy,sys,time\n" + f"if '--help' in sys.argv: runpy.run_path({hermes_sandbox._SANDBOX_SUPERVISOR!r}, run_name='__main__')\n" f"pathlib.Path({str(ready)!r}).touch()\n" f"while not pathlib.Path({str(directory / 'runner.stop')!r}).exists(): time.sleep(0.01)\n" # Keep the interpreter in the pre-handler window while close sends TERM. @@ -749,35 +747,75 @@ async def test_stop_during_interpreter_startup_closes_without_starting_a_worker( @pytest.mark.parametrize("runtime", [None, {}, {"hostname": "sandbox", "pid": "123"}]) -async def test_invalid_runtime_keeps_cleanup_confirmed(agent, state, runtime): +async def test_invalid_runtime_preserves_response_and_cleanup(agent, state, runtime): state.upload_json = AsyncMock() mock_json_download( state.session.sandbox, side_effect=[ {"cleanup_confirmed": True, "error": None}, - {"result": {"completed": True, "messages": []}, "runtime": runtime}, + { + "result": { + "completed": True, + "messages": [{"role": "user", "content": "task"}, {"role": "assistant", "content": "done"}], + }, + "observations": {"invocations": [{"invocation_id": "root", "model_response_ids": ["completion"]}]}, + "runtime": runtime, + }, ], ) - with pytest.raises(RuntimeError, match="invalid runtime metadata"): + result = await agent._run_sandbox_episode( + body=NeMoGymResponseCreateParamsNonStreaming(input="task"), agent_session_id="session", state=state + ) + assert result.response.status == "completed" + assert result.response.output[-1].content[0].text == "done" + assert result.response.metadata["harness_execution"] == "sandbox" + assert not {"harness_hostname", "harness_pid", "harness_python"}.intersection(result.response.metadata) + assert result.observations.records[0].model_calls[0].response_id == "completion" + assert [gap.code for gap in result.observations.gaps] == ["runtime_info_unavailable"] + assert state.session.cleanup is not None and state.session.cleanup["cleanup_confirmed"] is True + close = await agent._close_agent_session_state(state) + assert close.agent_observations == result.observations + state.session.sandbox.disconnect.assert_awaited_once() + state.session.sandbox.stop.assert_not_awaited() + + +async def test_close_preserves_runtime_gap_after_runner_error(agent, state): + state.upload_json = AsyncMock() + mock_json_download( + state.session.sandbox, + side_effect=[ + {"cleanup_confirmed": True, "error": None}, + { + "error": "runner failed", + "observations": {"invocations": [{"invocation_id": "root", "model_response_ids": ["completion"]}]}, + "runtime": {"hostname": "sandbox", "pid": "invalid"}, + }, + ], + ) + with pytest.raises(RuntimeError, match="Hermes sandbox runner failed: runner failed"): await agent._run_sandbox_episode( body=NeMoGymResponseCreateParamsNonStreaming(input="task"), agent_session_id="session", state=state ) - assert state.session.cleanup is not None and state.session.cleanup["cleanup_confirmed"] is True - await agent._close_agent_session_state(state) + close = await agent._close_agent_session_state(state) + assert close.agent_observations.records[0].model_calls[0].response_id == "completion" + assert [gap.code for gap in close.agent_observations.gaps] == ["runtime_info_unavailable"] state.session.sandbox.disconnect.assert_awaited_once() - state.session.sandbox.stop.assert_not_awaited() -async def test_invalid_cleanup_schema_does_not_mark_runner_confirmed(agent, state): +async def test_invalid_cleanup_diagnostics_do_not_block_close(agent, state): state.session.launch_started = True mock_json_download( state.session.sandbox, return_value={"cleanup_confirmed": True, "error": None, "timed_out": False, "return_code": "0"}, ) - with pytest.raises(ValidationError): - await state.session.stop_runner(timeout=agent.config.session_close_timeout_seconds) - assert state.session.launch_started and state.session.cleanup is None and not state.session.sandbox_stopped - state.session.sandbox.disconnect.assert_not_awaited() + await agent._close_agent_session_state(state) + assert state.session.cleanup == { + "cleanup_confirmed": True, + "error": None, + "timed_out": False, + "return_code": None, + } + state.session.sandbox.disconnect.assert_awaited_once() @pytest.mark.parametrize("receipt", [{}, {"cleanup_confirmed": False}, {"cleanup_confirmed": "true"}]) @@ -795,8 +833,10 @@ async def test_runner_exit_without_cleanup_receipt_blocks_close(agent, state, re @pytest.mark.parametrize("overrides", [{}, {"temperature": 0.0}]) -async def test_session_prompt_and_limits_reach_runner(agent, state, overrides, tmp_path): +async def test_session_prompt_and_limits_reach_runner(agent, state, overrides, tmp_path, monkeypatch): state.session.directory = str(tmp_path) + monkeypatch.setattr(hermes_sandbox, "_SANDBOX_PYTHON", sys.executable) + monkeypatch.setattr(hermes_sandbox, "_SANDBOX_SUPERVISOR", hermes_sandbox.process_supervisor.__file__) agent.server_client.global_config_dict = { "model": {"responses_api_models": {"vllm_model": {"chat_template_kwargs": {"enable_thinking": False}}}} } diff --git a/tests/unit_tests/test_global_config.py b/tests/unit_tests/test_global_config.py index 5d09816d54a..0becc1afe9d 100644 --- a/tests/unit_tests/test_global_config.py +++ b/tests/unit_tests/test_global_config.py @@ -3028,9 +3028,21 @@ def test_raises_when_every_other_agent_is_self_contained(self) -> None: assert "no other agent instance to rehost it on" in str(exc_info.value) def test_real_benchmark_composes_onto_real_harness(self) -> None: - resolved = self._parse_config_paths( - "benchmarks/gpqa/config.yaml", - "responses_api_agents/hermes_agent/configs/hermes_agent.yaml", + # Native Hermes has no Resources binding; legacy harness swaps explicitly leave it unbound. + resolved = self._parse( + DictConfig( + { + "config_paths": [ + "benchmarks/gpqa/config.yaml", + "responses_api_agents/hermes_agent/configs/hermes_agent.yaml", + ], + "hermes_agent": { + "responses_api_agents": { + "hermes_agent": {"resources_server": {"type": "resources_servers", "name": "???"}} + } + }, + } + ) ) block = resolved[self._composed_name("gpqa_mcqa_simple_agent")]["responses_api_agents"]["hermes_agent"] diff --git a/tests/unit_tests/test_sandbox_session.py b/tests/unit_tests/test_sandbox_session.py index 195c5c8c901..1eb559e299e 100644 --- a/tests/unit_tests/test_sandbox_session.py +++ b/tests/unit_tests/test_sandbox_session.py @@ -7,7 +7,7 @@ import pytest -from nemo_gym.sandbox import AsyncSandbox, SandboxExecResult +from nemo_gym.sandbox import AsyncSandbox, SandboxExecResult, supervisor_client from nemo_gym.sandbox.session import SandboxCommand, SandboxSession @@ -16,16 +16,22 @@ @pytest.fixture -def session(): +def receipt_reader(monkeypatch): + reader = AsyncMock(return_value=json.dumps(RECEIPT)) + monkeypatch.setattr(supervisor_client, "read_text", reader) + return reader + + +@pytest.fixture +def session(receipt_reader): sandbox = AsyncMock(spec=AsyncSandbox) sandbox.exec.return_value = OK - sandbox.read_text.return_value = json.dumps(RECEIPT) return SandboxSession(sandbox=sandbox, directory="/session", workdir="/app", harness="test") async def execute(session, *, collect=None, prepare=None, timeout=1): return await session.execute( - prepare=prepare or AsyncMock(return_value=SandboxCommand(argv=["worker"])), + prepare=prepare or AsyncMock(return_value=SandboxCommand(argv=["worker"], python="/runtime/python")), collect=collect or AsyncMock(return_value="transcript"), timeout=10, close_timeout=timeout, @@ -129,7 +135,7 @@ async def collect(): collector.assert_awaited_once() -async def test_close_during_input_preparation_prevents_launch(session): +async def test_close_during_input_preparation_prevents_launch(session, receipt_reader): preparing = asyncio.Event() async def prepare(): @@ -145,7 +151,7 @@ async def prepare(): assert not session.launch_started assert session.cleanup is None collector.assert_not_awaited() - session.sandbox.read_text.assert_not_awaited() + receipt_reader.assert_not_awaited() # Only session-directory removal ran. assert session.sandbox.exec.await_count == 1 assert "mv /session /session.closed" in session.sandbox.exec.await_args.args[0] @@ -190,8 +196,8 @@ async def collect(): assert session.closed -async def test_borrowed_cleanup_failure_retains_state_and_retries_capture(session): - session.sandbox.read_text.return_value = json.dumps(RECEIPT | {"cleanup_confirmed": False}) +async def test_borrowed_cleanup_failure_retains_state_and_retries_capture(session, receipt_reader): + receipt_reader.return_value = json.dumps(RECEIPT | {"cleanup_confirmed": False}) collector = AsyncMock(return_value="recovered transcript") with pytest.raises(RuntimeError, match="cleanup was not confirmed"): await execute(session, collect=collector) @@ -203,14 +209,14 @@ async def test_borrowed_cleanup_failure_retains_state_and_retries_capture(sessio session.sandbox.disconnect.assert_not_awaited() session.sandbox.stop.assert_not_awaited() assert all("rm -rf" not in call.args[0] for call in session.sandbox.exec.await_args_list) - session.sandbox.read_text.return_value = json.dumps(RECEIPT) + receipt_reader.return_value = json.dumps(RECEIPT) await session.close(timeout=1) collector.assert_awaited_once() assert session.artifacts == "recovered transcript" assert session.closed -async def test_owned_cleanup_deadline_falls_back_to_provider_stop(session): +async def test_owned_cleanup_deadline_falls_back_to_provider_stop(session, receipt_reader): session.owns_sandbox = True launched = asyncio.Event() @@ -218,11 +224,11 @@ async def provider_exec(command, **kwargs): launched.set() await asyncio.Future() - async def unavailable(path): + async def unavailable(*args, **kwargs): await asyncio.Future() session.sandbox.exec.side_effect = provider_exec - session.sandbox.read_text.side_effect = unavailable + receipt_reader.side_effect = unavailable collector = AsyncMock() running = asyncio.create_task(execute(session, collect=collector, timeout=0.02)) await asyncio.wait_for(launched.wait(), 2) @@ -252,8 +258,25 @@ async def test_release_failure_retries_without_recollecting(session, owned): collector.assert_awaited_once() -async def test_nonzero_worker_exit_still_passes_artifacts_to_adapter(session): +@pytest.mark.parametrize("return_code", [1, None, "malformed"]) +async def test_nonzero_worker_exit_still_passes_artifacts_to_adapter(session, receipt_reader, return_code): session.sandbox.exec.return_value = SandboxExecResult("", "worker error", 1) - session.sandbox.read_text.return_value = json.dumps(RECEIPT | {"return_code": 1}) + receipt_reader.return_value = json.dumps(RECEIPT | {"return_code": return_code}) assert await execute(session) == "transcript" - assert session.cleanup["return_code"] == 1 + assert session.cleanup["return_code"] == (return_code if type(return_code) is int else None) + + +async def test_bootstrap_failure_preserves_stderr_and_still_releases(session, receipt_reader): + receipt_reader.return_value = json.dumps({"cleanup_confirmed": True, "error": None}) + session.sandbox.exec.return_value = SandboxExecResult("", "/runtime/python: not found", 127) + collector = AsyncMock(side_effect=FileNotFoundError("no worker output")) + with pytest.raises(RuntimeError, match="sandbox execution failed.*127.*python: not found") as error: + await execute(session, collect=collector) + assert isinstance(error.value.__cause__, FileNotFoundError) + assert session.cleanup["cleanup_confirmed"] is True + assert session.cleanup["return_code"] is None + collector.assert_awaited_once() + session.sandbox.exec.return_value = OK + await session.close(timeout=1) + assert session.closed + session.sandbox.disconnect.assert_awaited_once() diff --git a/tests/unit_tests/test_sandbox_supervisor_bootstrap.py b/tests/unit_tests/test_sandbox_supervisor_bootstrap.py new file mode 100644 index 00000000000..81008d6a6e5 --- /dev/null +++ b/tests/unit_tests/test_sandbox_supervisor_bootstrap.py @@ -0,0 +1,182 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Failed interpreter/supervisor bootstrap must remain safe to close.""" + +import asyncio +import json +import shutil +import sys +from pathlib import Path + +import pytest + +from nemo_gym.sandbox import process_supervisor +from nemo_gym.sandbox.providers.base import SandboxExecResult +from nemo_gym.sandbox.supervisor_client import ( + remove_session_directory, + stop_and_confirm_cleanup, + supervised_launch_command, +) + + +class LocalSandbox: + async def download(self, source: str, destination: Path) -> None: + shutil.copyfile(source, destination) + + async def exec(self, command: str, *, cwd: str | None = None, timeout_s: float = 30) -> SandboxExecResult: + process = await asyncio.create_subprocess_exec( + "sh", "-c", command, cwd=cwd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout_s) + return SandboxExecResult(stdout.decode(errors="replace"), stderr.decode(errors="replace"), process.returncode) + + +@pytest.mark.parametrize("failure", ["missing_python", "broken_python", "missing_supervisor", "broken_supervisor"]) +async def test_bootstrap_failure_preserves_diagnostics_and_allows_close(tmp_path: Path, failure: str) -> None: + directory = tmp_path / "session" + directory.mkdir() + python = sys.executable + supervisor = directory / "process_supervisor.py" + shutil.copyfile(process_supervisor.__file__, supervisor) + expected_error = "" + if failure == "missing_python": + python = str(tmp_path / "missing-python") + expected_error = "missing-python" + elif failure == "broken_python": + interpreter = tmp_path / "incompatible-python" + interpreter.write_text("#!/bin/sh\nprintf '%s\\n' 'incompatible interpreter' >&2\nexit 17\n") + interpreter.chmod(0o755) + python = str(interpreter) + expected_error = "incompatible interpreter" + elif failure == "missing_supervisor": + supervisor.unlink() + expected_error = "process_supervisor.py" + else: + supervisor.write_text("def invalid syntax\n") + expected_error = "SyntaxError" + sandbox = LocalSandbox() + command = supervised_launch_command( + directory=str(directory), + command=["touch", str(directory / "worker-started")], + timeout=1, + cleanup_timeout=1, + python=python, + ) + result = await sandbox.exec(command) + assert result.return_code != 0 + if failure == "broken_python": + assert result.return_code == 17 + assert expected_error in result.stderr + assert not (directory / "launch.claim").is_symlink() + assert not (directory / "runner.pid").exists() + assert not (directory / "worker-started").exists() + assert not (directory / "cleanup.json").exists() + + receipt = await stop_and_confirm_cleanup( + sandbox, directory=str(directory), workdir=None, timeout=1, harness="bootstrap-test" + ) + assert receipt["cleanup_confirmed"] is True + assert receipt["return_code"] is None + assert (directory / "launch.claim").readlink() == Path("stop") + + +@pytest.mark.parametrize("removed", [False, True]) +async def test_fenced_delayed_launch_does_not_require_the_runtime(tmp_path: Path, removed: bool) -> None: + directory = tmp_path / "session" + directory.mkdir() + (directory / "launch.claim").symlink_to("stop") + if removed: + shutil.rmtree(directory) + result = await LocalSandbox().exec( + supervised_launch_command( + directory=str(directory), + command=["touch", str(tmp_path / "worker-started")], + timeout=1, + cleanup_timeout=1, + python=str(tmp_path / "missing-python"), + ) + ) + assert result.return_code == 0 + assert result.stderr == "" + assert not (tmp_path / "worker-started").exists() + + +@pytest.mark.parametrize("remove_directory", [False, True]) +async def test_close_during_preflight_fences_later_launch(tmp_path: Path, remove_directory: bool) -> None: + directory = tmp_path / "session" + directory.mkdir() + preflight_started = tmp_path / "preflight-started" + continue_preflight = tmp_path / "continue-preflight" + worker_started = tmp_path / "worker-started" + supervisor = tmp_path / "supervisor.py" + supervisor.write_text( + "import pathlib, sys, time\n" + "if '--help' in sys.argv:\n" + f" pathlib.Path({str(preflight_started)!r}).touch()\n" + f" while not pathlib.Path({str(continue_preflight)!r}).exists():\n" + " time.sleep(0.01)\n" + "else:\n" + f" pathlib.Path({str(worker_started)!r}).touch()\n" + ) + sandbox = LocalSandbox() + launch = asyncio.create_task( + sandbox.exec( + supervised_launch_command( + directory=str(directory), + command=["worker"], + timeout=1, + cleanup_timeout=1, + python=sys.executable, + supervisor_path=str(supervisor), + ) + ) + ) + try: + async with asyncio.timeout(5): + while not preflight_started.exists(): + await asyncio.sleep(0.01) + receipt = await stop_and_confirm_cleanup( + sandbox, directory=str(directory), workdir=None, timeout=1, harness="bootstrap-test" + ) + assert receipt["cleanup_confirmed"] is True + assert receipt["return_code"] is None + if remove_directory: + await remove_session_directory( + sandbox, directory=str(directory), workdir=None, timeout=1, harness="bootstrap-test" + ) + finally: + continue_preflight.touch() + result = await asyncio.wait_for(launch, timeout=5) + assert result.return_code == 0 + assert not worker_started.exists() + + +async def test_bootstrap_quotes_private_paths_and_uses_the_same_interpreter(tmp_path: Path) -> None: + directory = tmp_path / "session's files" + directory.mkdir() + runtime = tmp_path / "runtime's $(touch unintended) files" + runtime.mkdir() + python = runtime / "python interpreter" + python.symlink_to(sys.executable) + supervisor = runtime / "supervisor's code.py" + arguments = directory / "arguments.json" + supervisor.write_text( + "import json, pathlib, sys\n" + "if '--help' not in sys.argv:\n" + f" pathlib.Path({str(arguments)!r}).write_text(json.dumps([sys.executable, *sys.argv[1:]]))\n" + ) + command = supervised_launch_command( + directory=str(directory), + command=["worker", "$(touch unintended)", "argument's value"], + timeout=1, + cleanup_timeout=1, + python=str(python), + supervisor_path=str(supervisor), + ) + result = await LocalSandbox().exec(command, cwd=str(tmp_path)) + assert result.return_code == 0, result.stderr + captured = json.loads(arguments.read_text()) + assert captured[0] == str(python) + assert captured[-3:] == ["worker", "$(touch unintended)", "argument's value"] + assert not (tmp_path / "unintended").exists() + assert (directory / "launch.claim").readlink() == Path("launch") diff --git a/tests/unit_tests/test_sandbox_supervisor_client.py b/tests/unit_tests/test_sandbox_supervisor_client.py index 7208022e40f..11c8eea8edc 100644 --- a/tests/unit_tests/test_sandbox_supervisor_client.py +++ b/tests/unit_tests/test_sandbox_supervisor_client.py @@ -12,20 +12,19 @@ import pytest from pydantic import ValidationError -from nemo_gym.sandbox import AsyncSandbox, process_supervisor +from nemo_gym.sandbox import process_supervisor from nemo_gym.sandbox.providers.base import SandboxExecResult from nemo_gym.sandbox.supervisor_client import ( HarnessProcessInfo, parse_cleanup_receipt, + parse_runtime_info, stop_and_confirm_cleanup, supervised_launch_command, ) +from nemo_gym.sandbox.utils import read_text, upload_text class LocalSandbox: - upload_text = AsyncSandbox.upload_text - read_text = AsyncSandbox.read_text - async def upload(self, source, destination): shutil.copyfile(source, destination) @@ -52,11 +51,11 @@ async def test_file_transport_keeps_contents_out_of_shell(session): sandbox.exec = AsyncMock(side_effect=AssertionError("file transfer must not invoke a shell")) path = str(directory / "input.json") payload = '{"prompt": "$(touch unwanted); `echo surprise`"}\n' - await sandbox.upload_text(path, text=payload) + await upload_text(sandbox, path=path, text=payload) assert Path(path).read_text() == payload - assert await sandbox.read_text(path) == payload + assert await read_text(sandbox, path=path) == payload Path(path).write_bytes(b"partial output\xff\n") - assert await sandbox.read_text(path) == "partial output\ufffd\n" + assert await read_text(sandbox, path=path) == "partial output\ufffd\n" sandbox.exec.assert_not_awaited() @@ -72,6 +71,35 @@ async def test_confirmed_receipt_does_not_signal_stored_pid(session): sandbox.exec.assert_not_awaited() +async def test_confirmed_receipt_with_malformed_diagnostics_still_allows_cleanup(session, caplog): + sandbox, directory = session + receipt = {"cleanup_confirmed": True, "return_code": "0", "error": {"secret": "do not log"}, "version": 2} + (directory / "cleanup.json").write_text(json.dumps(receipt)) + sandbox.exec = AsyncMock(side_effect=AssertionError("must not signal a possibly reused PID")) + assert await stop_and_confirm_cleanup( + sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + ) == {"cleanup_confirmed": True, "return_code": None, "timed_out": False, "error": None} + assert "diagnostic fields" in caplog.text + assert "do not log" not in caplog.text + sandbox.exec.assert_not_awaited() + + +@pytest.mark.parametrize("receipt", [[], "stopped", None, 1]) +async def test_nonobject_cleanup_receipt_is_unconfirmed_without_signalling_a_stale_pid(session, receipt): + sandbox, directory = session + path = directory / "cleanup.json" + path.write_text(json.dumps(receipt)) + (directory / "runner.pid").write_text("2147483647") + for _ in range(2): + with pytest.raises(RuntimeError, match="cleanup receipt is not a JSON object"): + await stop_and_confirm_cleanup( + sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + ) + assert json.loads(path.read_text()) == receipt + assert not (directory / "runner.stop").exists() + assert not (directory / "launch.claim").exists() + + async def test_stop_wins_claim_and_fences_delayed_launch(session): sandbox, directory = session receipt = await stop_and_confirm_cleanup( @@ -81,7 +109,11 @@ async def test_stop_wins_claim_and_fences_delayed_launch(session): assert (directory / "launch.claim").readlink() == Path("stop") assert (directory / "runner.stop").exists() command = supervised_launch_command( - directory=str(directory), command=["touch", str(directory / "started")], timeout=1, cleanup_timeout=1 + directory=str(directory), + command=["touch", str(directory / "started")], + timeout=1, + cleanup_timeout=1, + python=sys.executable, ) assert (await sandbox.exec(command)).return_code == 0 assert not (directory / "runner.pid").exists() @@ -106,13 +138,13 @@ async def test_missing_receipt_after_launch_is_not_cleanup_confirmation(session) @pytest.mark.skipif(sys.platform != "linux", reason="Linux supervisor contract") @pytest.mark.parametrize("private_runtime", [False, True]) -async def test_launch_supervises_worker_with_default_or_private_runtime(session, tmp_path, private_runtime): +async def test_launch_supervises_worker_with_selected_or_private_runtime(session, tmp_path, private_runtime): sandbox, directory = session runtime = tmp_path / "runtime's files" runtime.mkdir() supervisor = (runtime if private_runtime else directory) / "process_supervisor.py" shutil.copyfile(process_supervisor.__file__, supervisor) - options = {} + options = {"python": sys.executable} if private_runtime: python = runtime / "python interpreter" python.symlink_to(sys.executable) @@ -162,30 +194,42 @@ def test_cleanup_and_runtime_are_independent(): runtime = HarnessProcessInfo.model_validate({"hostname": "sandbox", "pid": 123}) assert runtime.hostname == "sandbox" and runtime.pid == 123 assert runtime.python is None - with pytest.raises(ValidationError): + with pytest.raises(ValueError, match="cleanup was not confirmed"): parse_cleanup_receipt(runtime.model_dump()) with pytest.raises(ValidationError): HarnessProcessInfo.model_validate(cleanup) - for field in cleanup: - with pytest.raises(ValidationError): - parse_cleanup_receipt({key: value for key, value in cleanup.items() if key != field}) @pytest.mark.parametrize( - "invalid", [{"return_code": "0"}, {"timed_out": "false"}, {"cleanup_confirmed": 1}, {"hostname": "worker"}] + "invalid,normalized", + [ + ({"return_code": "0"}, {"return_code": None}), + ({"return_code": True}, {"return_code": None}), + ({"timed_out": "false"}, {"timed_out": False}), + ({"error": {"message": "failed"}}, {"error": None}), + ({"hostname": "worker"}, {}), + ], ) -def test_cleanup_receipt_keeps_strict_validation(invalid): - with pytest.raises(ValidationError): - parse_cleanup_receipt( - {"return_code": 0, "timed_out": False, "cleanup_confirmed": True, "error": None, **invalid} - ) +def test_confirmed_cleanup_keeps_valid_diagnostics_and_ignores_malformed_fields(invalid, normalized): + cleanup = {"return_code": 0, "timed_out": False, "cleanup_confirmed": True, "error": None} + assert parse_cleanup_receipt(cleanup | invalid) == cleanup | normalized + + +@pytest.mark.parametrize( + "payload", + [None, [], "true", {}, {"cleanup_confirmed": False}, {"cleanup_confirmed": "true"}, {"cleanup_confirmed": 1}], +) +def test_cleanup_receipt_requires_positive_boolean_evidence(payload): + with pytest.raises(ValueError, match="cleanup was not confirmed"): + parse_cleanup_receipt(payload) def test_absent_exit_code_is_not_success(): full = {"return_code": None, "timed_out": False, "cleanup_confirmed": True, "error": None} assert parse_cleanup_receipt(full) == full assert parse_cleanup_receipt({"cleanup_confirmed": True, "error": None}) == full - with pytest.raises(ValidationError): + assert parse_cleanup_receipt({"cleanup_confirmed": True}) == full + with pytest.raises(ValueError, match="cleanup was not confirmed"): parse_cleanup_receipt({"return_code": 0, "error": None}) @@ -195,9 +239,26 @@ def test_runtime_info_keeps_strict_validation(invalid): HarnessProcessInfo.model_validate({"hostname": "sandbox", "pid": 123, **invalid}) +@pytest.mark.parametrize( + "payload", + [ + None, + [], + {}, + {"hostname": "sandbox", "pid": "123"}, + {"hostname": "sandbox", "pid": 123, "python": {"secret": "do not log"}}, + ], +) +def test_malformed_runtime_info_is_optional_and_logs_without_payload(payload, caplog): + assert parse_runtime_info(payload) is None + assert "runtime metadata" in caplog.text + assert "do not log" not in caplog.text + + def test_hermes_runtime_uses_the_same_schema(): payload = {"hostname": "sandbox", "pid": 123, "python": "/opt/hermes/bin/python"} assert HarnessProcessInfo.model_validate(payload).model_dump() == payload + assert parse_runtime_info(payload).model_dump() == payload for field in ("hostname", "pid"): with pytest.raises(ValidationError): HarnessProcessInfo.model_validate({key: value for key, value in payload.items() if key != field}) From 1a7dfb133f1563d64a4b5745e45dfe6d34ea27dd Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 6 Oct 2026 09:33:18 -0700 Subject: [PATCH 27/44] refactor(agent): move shared execution helpers into agent_utils Signed-off-by: yaoyu-33 --- nemo_gym/agent_utils/__init__.py | 8 ++++++++ nemo_gym/{sandbox => agent_utils}/process_supervisor.py | 0 .../session.py => agent_utils/sandbox_session.py} | 8 ++++---- nemo_gym/{sandbox => agent_utils}/supervisor_client.py | 4 ++-- responses_api_agents/hermes_agent/app.py | 4 ++-- responses_api_agents/hermes_agent/sandbox.py | 6 +++--- responses_api_agents/hermes_agent/tests/test_app.py | 2 +- responses_api_agents/hermes_agent/tests/test_sandbox.py | 2 +- .../hermes_agent/tests/test_sandbox_runner.py | 2 +- responses_api_agents/hermes_sandboxed_agent/app.py | 3 ++- .../hermes_sandboxed_agent/tests/test_app.py | 2 +- tests/unit_tests/test_local_provider.py | 3 ++- tests/unit_tests/test_process_supervisor.py | 2 +- tests/unit_tests/test_sandbox_session.py | 5 +++-- tests/unit_tests/test_sandbox_supervisor_bootstrap.py | 6 +++--- tests/unit_tests/test_sandbox_supervisor_client.py | 6 +++--- 16 files changed, 37 insertions(+), 26 deletions(-) create mode 100644 nemo_gym/agent_utils/__init__.py rename nemo_gym/{sandbox => agent_utils}/process_supervisor.py (100%) rename nemo_gym/{sandbox/session.py => agent_utils/sandbox_session.py} (98%) rename nemo_gym/{sandbox => agent_utils}/supervisor_client.py (97%) diff --git a/nemo_gym/agent_utils/__init__.py b/nemo_gym/agent_utils/__init__.py new file mode 100644 index 00000000000..20863e61152 --- /dev/null +++ b/nemo_gym/agent_utils/__init__.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Shared agent execution and supervision helpers. + +Sandbox sessions and supervisor clients run on the agent server. The standalone +process supervisor is uploaded into task sandboxes and uses only the standard +library. Core sandbox APIs and providers live in :mod:`nemo_gym.sandbox`. +""" diff --git a/nemo_gym/sandbox/process_supervisor.py b/nemo_gym/agent_utils/process_supervisor.py similarity index 100% rename from nemo_gym/sandbox/process_supervisor.py rename to nemo_gym/agent_utils/process_supervisor.py diff --git a/nemo_gym/sandbox/session.py b/nemo_gym/agent_utils/sandbox_session.py similarity index 98% rename from nemo_gym/sandbox/session.py rename to nemo_gym/agent_utils/sandbox_session.py index 1e0e897e351..63be6ee4afe 100644 --- a/nemo_gym/sandbox/session.py +++ b/nemo_gym/agent_utils/sandbox_session.py @@ -8,14 +8,14 @@ from dataclasses import dataclass, field from typing import cast -from nemo_gym.sandbox.api import AsyncSandbox -from nemo_gym.sandbox.process_supervisor import CleanupReceipt, exec_timeout -from nemo_gym.sandbox.providers.base import SandboxExecResult -from nemo_gym.sandbox.supervisor_client import ( +from nemo_gym.agent_utils.process_supervisor import CleanupReceipt, exec_timeout +from nemo_gym.agent_utils.supervisor_client import ( remove_session_directory, stop_and_confirm_cleanup, supervised_launch_command, ) +from nemo_gym.sandbox.api import AsyncSandbox +from nemo_gym.sandbox.providers.base import SandboxExecResult LOG = logging.getLogger(__name__) diff --git a/nemo_gym/sandbox/supervisor_client.py b/nemo_gym/agent_utils/supervisor_client.py similarity index 97% rename from nemo_gym/sandbox/supervisor_client.py rename to nemo_gym/agent_utils/supervisor_client.py index 44106bf0412..ab21c8760d7 100644 --- a/nemo_gym/sandbox/supervisor_client.py +++ b/nemo_gym/agent_utils/supervisor_client.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Controller-side transport for the shared sandbox process supervisor. -Session ownership and cancellation live in session.py; output parsing stays with the adapter. +Session ownership and cancellation live in sandbox_session.py; output parsing stays with the adapter. Unlike process_supervisor.py, this module is not uploaded to the task sandbox. """ @@ -13,8 +13,8 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from nemo_gym.agent_utils.process_supervisor import CleanupReceipt from nemo_gym.sandbox import AsyncSandbox -from nemo_gym.sandbox.process_supervisor import CleanupReceipt from nemo_gym.sandbox.utils import read_text diff --git a/responses_api_agents/hermes_agent/app.py b/responses_api_agents/hermes_agent/app.py index 213eb824477..b7455c8f115 100644 --- a/responses_api_agents/hermes_agent/app.py +++ b/responses_api_agents/hermes_agent/app.py @@ -30,6 +30,8 @@ from pydantic import ConfigDict, Field from toolsets import TOOLSETS # pyright: ignore[reportMissingImports] +from nemo_gym.agent_utils.sandbox_session import SandboxSession +from nemo_gym.agent_utils.supervisor_client import HarnessProcessInfo from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import ( AgentCloseSessionResponse, @@ -70,8 +72,6 @@ from nemo_gym.sandbox.access import DirectSandboxConnection from nemo_gym.sandbox.config import resolve_provider_config from nemo_gym.sandbox.providers import create_provider -from nemo_gym.sandbox.session import SandboxSession -from nemo_gym.sandbox.supervisor_client import HarnessProcessInfo from nemo_gym.server_utils import get_response_json, raise_for_status from nemo_gym.tool_access import MCPToolAccess from responses_api_agents.hermes_agent.model_kwargs import _model_api_kwargs diff --git a/responses_api_agents/hermes_agent/sandbox.py b/responses_api_agents/hermes_agent/sandbox.py index dea7c1fb6f7..6d1e19273c0 100644 --- a/responses_api_agents/hermes_agent/sandbox.py +++ b/responses_api_agents/hermes_agent/sandbox.py @@ -12,12 +12,12 @@ from pydantic import JsonValue +from nemo_gym.agent_utils import process_supervisor +from nemo_gym.agent_utils.sandbox_session import SandboxCommand, SandboxSession +from nemo_gym.agent_utils.supervisor_client import HarnessProcessInfo, parse_runtime_info from nemo_gym.base_responses_api_agent import AgentSessionState from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming from nemo_gym.rollout_observability import AgentObservationBundle -from nemo_gym.sandbox import process_supervisor -from nemo_gym.sandbox.session import SandboxCommand, SandboxSession -from nemo_gym.sandbox.supervisor_client import HarnessProcessInfo, parse_runtime_info from nemo_gym.sandbox.utils import read_text, upload_text diff --git a/responses_api_agents/hermes_agent/tests/test_app.py b/responses_api_agents/hermes_agent/tests/test_app.py index 69a6214df6d..b88b3f7f4d7 100644 --- a/responses_api_agents/hermes_agent/tests/test_app.py +++ b/responses_api_agents/hermes_agent/tests/test_app.py @@ -22,6 +22,7 @@ import yaml from fastapi import HTTPException +from nemo_gym.agent_utils.sandbox_session import SandboxSession from nemo_gym.base_responses_api_agent import ( AgentCloseSessionRequest, AgentCloseSessionResponse, @@ -41,7 +42,6 @@ from nemo_gym.rollout_observability import AgentEpisode, AgentObservationBundle from nemo_gym.sandbox import SandboxExecResult, SandboxSpec from nemo_gym.sandbox.access import DirectSandboxConnection, SandboxAccess -from nemo_gym.sandbox.session import SandboxSession from nemo_gym.server_utils import ServerClient from nemo_gym.tool_access import DirectHTTPToolAccess, MCPStreamableHTTPConnection, MCPToolAccess from responses_api_agents.hermes_agent.app import ( diff --git a/responses_api_agents/hermes_agent/tests/test_sandbox.py b/responses_api_agents/hermes_agent/tests/test_sandbox.py index aedfb310a51..2b9fdedcb4f 100644 --- a/responses_api_agents/hermes_agent/tests/test_sandbox.py +++ b/responses_api_agents/hermes_agent/tests/test_sandbox.py @@ -17,11 +17,11 @@ from fastapi.testclient import TestClient from pydantic import ValidationError +from nemo_gym.agent_utils.sandbox_session import SandboxSession from nemo_gym.base_responses_api_agent import AgentCloseSessionRequest, AgentSeedSessionRequest, _AgentSessionRecord from nemo_gym.episode_types import EpisodeId, TaskId from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming from nemo_gym.rollout_observability import AgentEpisode, AgentObservationBundle -from nemo_gym.sandbox.session import SandboxSession from nemo_gym.server_utils import ServerClient from responses_api_agents.hermes_agent import sandbox as hermes_sandbox from responses_api_agents.hermes_agent.app import ( diff --git a/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py b/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py index e12350413fd..a0fbdc177c2 100644 --- a/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py @@ -20,9 +20,9 @@ from run_agent import AIAgent from tools.mcp_tool import shutdown_mcp_servers +from nemo_gym.agent_utils import process_supervisor from nemo_gym.mcp_auto_exposure import TOKEN_HEADER, maybe_auto_expose from nemo_gym.openai_utils import NeMoGymChatCompletionCreateParamsNonStreaming -from nemo_gym.sandbox import process_supervisor from nemo_gym.server_utils import ServerClient from resources_servers.example_mcp_weather.app import ( ExampleMCPWeatherResourcesServer, diff --git a/responses_api_agents/hermes_sandboxed_agent/app.py b/responses_api_agents/hermes_sandboxed_agent/app.py index 50bfa578cac..b5847bf7ea7 100644 --- a/responses_api_agents/hermes_sandboxed_agent/app.py +++ b/responses_api_agents/hermes_sandboxed_agent/app.py @@ -14,12 +14,13 @@ from fastapi import HTTPException, Request from pydantic import ConfigDict, Field +from nemo_gym.agent_utils import process_supervisor from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import BaseResponsesAPIAgentConfig, SimpleResponsesAPIAgent from nemo_gym.config_types import ModelServerRef, ResourcesServerRef from nemo_gym.global_config import get_global_config_dict from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming -from nemo_gym.sandbox import AsyncSandbox, create_provider, process_supervisor +from nemo_gym.sandbox import AsyncSandbox, create_provider from nemo_gym.sandbox.config import resolve_provider_config from nemo_gym.server_utils import get_response_json, is_nemo_gym_fastapi_entrypoint, raise_for_status from responses_api_agents.hermes_sandboxed_agent.runner import classify_stop, split_input diff --git a/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py b/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py index 346480a0b1f..a53a6fa2b1a 100644 --- a/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py +++ b/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py @@ -9,9 +9,9 @@ import pytest +from nemo_gym.agent_utils.process_supervisor import DEFAULT_CLEANUP_TIMEOUT from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming from nemo_gym.sandbox import SandboxExecResult -from nemo_gym.sandbox.process_supervisor import DEFAULT_CLEANUP_TIMEOUT from nemo_gym.server_utils import ServerClient from responses_api_agents.hermes_sandboxed_agent.app import ( HermesSandboxedAgent, diff --git a/tests/unit_tests/test_local_provider.py b/tests/unit_tests/test_local_provider.py index 3818949006d..67ce10c1849 100644 --- a/tests/unit_tests/test_local_provider.py +++ b/tests/unit_tests/test_local_provider.py @@ -25,7 +25,8 @@ import pytest -from nemo_gym.sandbox import AsyncSandbox, SandboxSpec, SandboxStatus, create_provider, process_supervisor +from nemo_gym.agent_utils import process_supervisor +from nemo_gym.sandbox import AsyncSandbox, SandboxSpec, SandboxStatus, create_provider from nemo_gym.sandbox.providers.local import LocalProvider diff --git a/tests/unit_tests/test_process_supervisor.py b/tests/unit_tests/test_process_supervisor.py index 636b40b735c..a69aeb6ca87 100644 --- a/tests/unit_tests/test_process_supervisor.py +++ b/tests/unit_tests/test_process_supervisor.py @@ -16,7 +16,7 @@ import pytest -SUPERVISOR = Path(__file__).parents[2] / "nemo_gym/sandbox/process_supervisor.py" +SUPERVISOR = Path(__file__).parents[2] / "nemo_gym/agent_utils/process_supervisor.py" @pytest.mark.parametrize("timeout, cleanup", [(1, 0.1), (2700, 10), (21600, 100)]) diff --git a/tests/unit_tests/test_sandbox_session.py b/tests/unit_tests/test_sandbox_session.py index 1eb559e299e..cf4c5880ad0 100644 --- a/tests/unit_tests/test_sandbox_session.py +++ b/tests/unit_tests/test_sandbox_session.py @@ -7,8 +7,9 @@ import pytest -from nemo_gym.sandbox import AsyncSandbox, SandboxExecResult, supervisor_client -from nemo_gym.sandbox.session import SandboxCommand, SandboxSession +from nemo_gym.agent_utils import supervisor_client +from nemo_gym.agent_utils.sandbox_session import SandboxCommand, SandboxSession +from nemo_gym.sandbox import AsyncSandbox, SandboxExecResult RECEIPT = {"cleanup_confirmed": True, "return_code": 0, "timed_out": False, "error": None} diff --git a/tests/unit_tests/test_sandbox_supervisor_bootstrap.py b/tests/unit_tests/test_sandbox_supervisor_bootstrap.py index 81008d6a6e5..d454bfa76d4 100644 --- a/tests/unit_tests/test_sandbox_supervisor_bootstrap.py +++ b/tests/unit_tests/test_sandbox_supervisor_bootstrap.py @@ -10,13 +10,13 @@ import pytest -from nemo_gym.sandbox import process_supervisor -from nemo_gym.sandbox.providers.base import SandboxExecResult -from nemo_gym.sandbox.supervisor_client import ( +from nemo_gym.agent_utils import process_supervisor +from nemo_gym.agent_utils.supervisor_client import ( remove_session_directory, stop_and_confirm_cleanup, supervised_launch_command, ) +from nemo_gym.sandbox.providers.base import SandboxExecResult class LocalSandbox: diff --git a/tests/unit_tests/test_sandbox_supervisor_client.py b/tests/unit_tests/test_sandbox_supervisor_client.py index 11c8eea8edc..d298c91aa52 100644 --- a/tests/unit_tests/test_sandbox_supervisor_client.py +++ b/tests/unit_tests/test_sandbox_supervisor_client.py @@ -12,15 +12,15 @@ import pytest from pydantic import ValidationError -from nemo_gym.sandbox import process_supervisor -from nemo_gym.sandbox.providers.base import SandboxExecResult -from nemo_gym.sandbox.supervisor_client import ( +from nemo_gym.agent_utils import process_supervisor +from nemo_gym.agent_utils.supervisor_client import ( HarnessProcessInfo, parse_cleanup_receipt, parse_runtime_info, stop_and_confirm_cleanup, supervised_launch_command, ) +from nemo_gym.sandbox.providers.base import SandboxExecResult from nemo_gym.sandbox.utils import read_text, upload_text From 29e979f469e688c81108be9c8374dc833e040734 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 6 Oct 2026 10:37:18 -0700 Subject: [PATCH 28/44] feat(sandbox): add binary file transfer helpers Signed-off-by: yaoyu-33 --- nemo_gym/sandbox/utils.py | 16 +++++++ tests/unit_tests/test_sandbox_utils.py | 60 ++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 tests/unit_tests/test_sandbox_utils.py diff --git a/nemo_gym/sandbox/utils.py b/nemo_gym/sandbox/utils.py index 7345921bba5..da2977078dc 100644 --- a/nemo_gym/sandbox/utils.py +++ b/nemo_gym/sandbox/utils.py @@ -23,6 +23,22 @@ from nemo_gym.sandbox.api import AsyncSandbox +async def upload_bytes(sandbox: "AsyncSandbox", *, path: str, data: bytes) -> None: + """Upload bytes unchanged through file transfer without shell interpolation.""" + with tempfile.TemporaryDirectory(prefix="sandbox-upload-") as directory: + source = Path(directory) / "payload" + source.write_bytes(data) + await sandbox.upload(source, path) + + +async def read_bytes(sandbox: "AsyncSandbox", *, path: str) -> bytes: + """Download file contents unchanged, without text decoding.""" + with tempfile.TemporaryDirectory(prefix="sandbox-download-") as directory: + destination = Path(directory) / "payload" + await sandbox.download(path, destination) + return destination.read_bytes() + + async def upload_text(sandbox: "AsyncSandbox", *, path: str, text: str) -> None: """Upload UTF-8 text through file transfer without shell interpolation.""" with tempfile.TemporaryDirectory(prefix="sandbox-upload-") as directory: diff --git a/tests/unit_tests/test_sandbox_utils.py b/tests/unit_tests/test_sandbox_utils.py new file mode 100644 index 00000000000..0cc43445975 --- /dev/null +++ b/tests/unit_tests/test_sandbox_utils.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest + +from nemo_gym.sandbox import AsyncSandbox, SandboxSpec +from nemo_gym.sandbox.providers.local import LocalProvider +from nemo_gym.sandbox.utils import read_bytes, upload_bytes + + +pytestmark = pytest.mark.sandbox + + +@pytest.mark.parametrize("data", [b"", bytes(range(256)) + b"\r\n\r"], ids=["empty", "binary"]) +async def test_bytes_round_trip_preserves_content_and_cleans_temporary_files(tmp_path: Path, data: bytes) -> None: + path = tmp_path / "payload's $(literal).bin" + async with AsyncSandbox(LocalProvider(), SandboxSpec(workdir=str(tmp_path))) as sandbox: + await sandbox.start() + sandbox.upload = AsyncMock(wraps=sandbox.upload) + sandbox.download = AsyncMock(wraps=sandbox.download) + sandbox.exec = AsyncMock(side_effect=AssertionError("file transfer must not invoke a shell")) + + await upload_bytes(sandbox, path=str(path), data=data) + assert path.read_bytes() == data + assert await read_bytes(sandbox, path=str(path)) == data + + assert not Path(sandbox.upload.await_args.args[0]).parent.exists() + assert not Path(sandbox.download.await_args.args[1]).parent.exists() + sandbox.exec.assert_not_awaited() + + +@pytest.mark.parametrize("operation", ["upload", "download"]) +@pytest.mark.parametrize("error_type", [OSError, asyncio.CancelledError]) +async def test_bytes_transfer_cleans_temporary_files_on_error(operation: str, error_type: type[BaseException]) -> None: + sandbox = AsyncMock(spec=AsyncSandbox) + error = error_type("transfer interrupted") + temporary_paths: list[Path] = [] + + async def fail_transfer(source: str | Path, destination: str | Path) -> None: + local = Path(source if operation == "upload" else destination) + assert local.parent.is_dir() + if operation == "download": + local.write_bytes(b"partial") + temporary_paths.append(local) + raise error + + getattr(sandbox, operation).side_effect = fail_transfer + with pytest.raises(error_type) as raised: + if operation == "upload": + await upload_bytes(sandbox, path="/remote.bin", data=b"\x00\xff") + else: + await read_bytes(sandbox, path="/remote.bin") + + assert raised.value is error + assert len(temporary_paths) == 1 + assert not temporary_paths[0].parent.exists() From e1f36cdc8364c1d3f9e95f294f6a64b441c1d090 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 6 Oct 2026 11:01:20 -0700 Subject: [PATCH 29/44] refactor(sandbox): simplify utility imports and ordering Signed-off-by: yaoyu-33 --- nemo_gym/sandbox/utils.py | 69 +++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/nemo_gym/sandbox/utils.py b/nemo_gym/sandbox/utils.py index da2977078dc..3e695132da0 100644 --- a/nemo_gym/sandbox/utils.py +++ b/nemo_gym/sandbox/utils.py @@ -16,43 +16,8 @@ import tempfile from pathlib import Path -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from nemo_gym.sandbox.api import AsyncSandbox - - -async def upload_bytes(sandbox: "AsyncSandbox", *, path: str, data: bytes) -> None: - """Upload bytes unchanged through file transfer without shell interpolation.""" - with tempfile.TemporaryDirectory(prefix="sandbox-upload-") as directory: - source = Path(directory) / "payload" - source.write_bytes(data) - await sandbox.upload(source, path) - - -async def read_bytes(sandbox: "AsyncSandbox", *, path: str) -> bytes: - """Download file contents unchanged, without text decoding.""" - with tempfile.TemporaryDirectory(prefix="sandbox-download-") as directory: - destination = Path(directory) / "payload" - await sandbox.download(path, destination) - return destination.read_bytes() - - -async def upload_text(sandbox: "AsyncSandbox", *, path: str, text: str) -> None: - """Upload UTF-8 text through file transfer without shell interpolation.""" - with tempfile.TemporaryDirectory(prefix="sandbox-upload-") as directory: - source = Path(directory) / "payload" - source.write_text(text, encoding="utf-8") - await sandbox.upload(source, path) - - -async def read_text(sandbox: "AsyncSandbox", *, path: str) -> str: - """Download UTF-8 text, replacing invalid bytes in partial process output.""" - with tempfile.TemporaryDirectory(prefix="sandbox-download-") as directory: - destination = Path(directory) / "payload" - await sandbox.download(path, destination) - return destination.read_text(encoding="utf-8", errors="replace") +from nemo_gym.sandbox.api import AsyncSandbox # Parallelism caps for CPU-limited sandboxes, each set by cpu_cap_env() to the @@ -88,6 +53,38 @@ async def read_text(sandbox: "AsyncSandbox", *, path: str) -> str: ) +async def upload_bytes(sandbox: AsyncSandbox, *, path: str, data: bytes) -> None: + """Upload bytes unchanged through file transfer without shell interpolation.""" + with tempfile.TemporaryDirectory(prefix="sandbox-upload-") as directory: + source = Path(directory) / "payload" + source.write_bytes(data) + await sandbox.upload(source, path) + + +async def read_bytes(sandbox: AsyncSandbox, *, path: str) -> bytes: + """Download file contents unchanged, without text decoding.""" + with tempfile.TemporaryDirectory(prefix="sandbox-download-") as directory: + destination = Path(directory) / "payload" + await sandbox.download(path, destination) + return destination.read_bytes() + + +async def upload_text(sandbox: AsyncSandbox, *, path: str, text: str) -> None: + """Upload UTF-8 text through file transfer without shell interpolation.""" + with tempfile.TemporaryDirectory(prefix="sandbox-upload-") as directory: + source = Path(directory) / "payload" + source.write_text(text, encoding="utf-8") + await sandbox.upload(source, path) + + +async def read_text(sandbox: AsyncSandbox, *, path: str) -> str: + """Download UTF-8 text, replacing invalid bytes in partial process output.""" + with tempfile.TemporaryDirectory(prefix="sandbox-download-") as directory: + destination = Path(directory) / "payload" + await sandbox.download(path, destination) + return destination.read_text(encoding="utf-8", errors="replace") + + def cpu_cap_env(cpu: float | int | None) -> dict[str, str]: """CPU_CAP_ENV_VARS derived from a sandbox CPU limit; empty when unset.""" if cpu is None: From 5671efac93de71c56c37d4232ea4848eaf345207 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 6 Oct 2026 11:25:09 -0700 Subject: [PATCH 30/44] refactor(harness): centralize supervised session ownership Signed-off-by: yaoyu-33 --- nemo_gym/agent_utils/__init__.py | 8 - nemo_gym/harness/__init__.py | 15 ++ .../process_supervisor.py | 19 +- .../sandbox_session.py | 112 ++++++++---- .../supervisor_client.py | 108 +++++------ nemo_gym/sandbox/utils.py | 16 -- responses_api_agents/hermes_agent/app.py | 9 +- responses_api_agents/hermes_agent/sandbox.py | 61 ++++--- .../hermes_agent/sandbox_runner.py | 3 +- .../hermes_agent/tests/test_app.py | 26 +-- .../hermes_agent/tests/test_sandbox.py | 104 ++++++++--- .../hermes_agent/tests/test_sandbox_runner.py | 47 ++++- .../hermes_sandboxed_agent/app.py | 2 +- .../hermes_sandboxed_agent/tests/test_app.py | 2 +- tests/unit_tests/test_local_provider.py | 2 +- tests/unit_tests/test_process_supervisor.py | 2 +- tests/unit_tests/test_sandbox_session.py | 146 ++++++++++++++- .../test_sandbox_supervisor_bootstrap.py | 66 +++---- .../test_sandbox_supervisor_client.py | 170 ++++++++---------- tests/unit_tests/test_sandbox_utils.py | 60 ------- 20 files changed, 577 insertions(+), 401 deletions(-) delete mode 100644 nemo_gym/agent_utils/__init__.py create mode 100644 nemo_gym/harness/__init__.py rename nemo_gym/{agent_utils => harness}/process_supervisor.py (89%) rename nemo_gym/{agent_utils => harness}/sandbox_session.py (68%) rename nemo_gym/{agent_utils => harness}/supervisor_client.py (56%) delete mode 100644 tests/unit_tests/test_sandbox_utils.py diff --git a/nemo_gym/agent_utils/__init__.py b/nemo_gym/agent_utils/__init__.py deleted file mode 100644 index 20863e61152..00000000000 --- a/nemo_gym/agent_utils/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Shared agent execution and supervision helpers. - -Sandbox sessions and supervisor clients run on the agent server. The standalone -process supervisor is uploaded into task sandboxes and uses only the standard -library. Core sandbox APIs and providers live in :mod:`nemo_gym.sandbox`. -""" diff --git a/nemo_gym/harness/__init__.py b/nemo_gym/harness/__init__.py new file mode 100644 index 00000000000..3cb30910363 --- /dev/null +++ b/nemo_gym/harness/__init__.py @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Run harness processes inside task sandboxes under a shared supervisor. + +The harness adapter installs its runtime, stages activation input, and parses +harness output. SandboxSession uploads the supervisor, runs and stops the harness +process, collects artifacts, and releases the sandbox. The supervisor client +implements the shell protocol over session control files. All three run on the +agent server. + +Only process_supervisor.py is copied from this package into the task sandbox. It +uses the standard library to enforce deadlines, kill and reap descendants, and +write the cleanup receipt. Core sandbox APIs and providers live in +:mod:`nemo_gym.sandbox`. +""" diff --git a/nemo_gym/agent_utils/process_supervisor.py b/nemo_gym/harness/process_supervisor.py similarity index 89% rename from nemo_gym/agent_utils/process_supervisor.py rename to nemo_gym/harness/process_supervisor.py index a00a58765a3..7cea10e447b 100644 --- a/nemo_gym/agent_utils/process_supervisor.py +++ b/nemo_gym/harness/process_supervisor.py @@ -4,7 +4,7 @@ """Standalone Linux process supervisor to upload beside a sandboxed harness. Uses only the standard library; Gym need not be installed in the task sandbox. -The receipt confirms descendant cleanup independently of the worker's result. +The receipt confirms descendant cleanup independently of the harness result. """ from __future__ import annotations @@ -23,6 +23,7 @@ DEFAULT_CLEANUP_TIMEOUT = 10.0 +CLEANUP_PHASE_COUNT = 3 class CleanupReceipt(TypedDict): @@ -35,8 +36,12 @@ class CleanupReceipt(TypedDict): def exec_timeout(*, timeout: float, cleanup_timeout: float = DEFAULT_CLEANUP_TIMEOUT) -> float: - """Leave room for TERM grace, worker reaping, descendant draining, and receipt I/O.""" - return timeout + 3 * cleanup_timeout + 30 + """Reserve TERM grace, harness process reaping, and descendant draining. + + The extra 30 seconds cover startup, polling, and receipt I/O beyond those + bounded cleanup phases. Keep their count shared with the controller budget. + """ + return timeout + CLEANUP_PHASE_COUNT * cleanup_timeout + 30 def _drain_children(timeout: float) -> None: @@ -66,7 +71,7 @@ def _supervise( cleanup_timeout: float = DEFAULT_CLEANUP_TIMEOUT, stop_path: Path | None = None, ) -> CleanupReceipt: - """Enforce a worker deadline, then acknowledge cleanup after all descendants exit.""" + """Enforce a harness deadline, then acknowledge cleanup after all descendants exit.""" process = None subreaping = False stopping = False @@ -82,7 +87,7 @@ def interrupt(*_: object) -> None: # TERM can be ignored during interpreter startup. A durable stop marker fences # that window; later signals are handled without interrupting Popen. if stopping or (stop_path is not None and stop_path.exists()): - return receipt # The finally block confirms that no worker was launched. + return receipt # The finally block confirms that no harness process was launched. if sys.platform != "linux": raise RuntimeError("Sandbox process supervision requires Linux") # Leave the provider group when possible. An exec launcher may already make us @@ -99,7 +104,7 @@ def interrupt(*_: object) -> None: time.sleep(0.05) if process.poll() is None: receipt["timed_out"] = not stopping - # Let the worker checkpoint before the bounded hard cleanup. + # Let the harness checkpoint before the bounded hard cleanup. process.send_signal(signal.SIGTERM) try: process.wait(timeout=cleanup_timeout) @@ -143,7 +148,7 @@ def main() -> int: args = parser.parse_args() command = args.command[1:] if args.command[:1] == ["--"] else args.command if not command: - parser.error("a worker command is required after --") + parser.error("a harness command is required after --") receipt = _supervise(command, timeout=args.timeout, cleanup_timeout=args.cleanup_timeout, stop_path=args.stop_file) temporary = args.receipt.with_suffix(".tmp") temporary.write_text(json.dumps(receipt)) diff --git a/nemo_gym/agent_utils/sandbox_session.py b/nemo_gym/harness/sandbox_session.py similarity index 68% rename from nemo_gym/agent_utils/sandbox_session.py rename to nemo_gym/harness/sandbox_session.py index 63be6ee4afe..f80a3300e1c 100644 --- a/nemo_gym/agent_utils/sandbox_session.py +++ b/nemo_gym/harness/sandbox_session.py @@ -4,41 +4,60 @@ import asyncio import logging -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Coroutine from dataclasses import dataclass, field +from pathlib import Path from typing import cast -from nemo_gym.agent_utils.process_supervisor import CleanupReceipt, exec_timeout -from nemo_gym.agent_utils.supervisor_client import ( +from nemo_gym.harness import process_supervisor +from nemo_gym.harness.process_supervisor import CleanupReceipt +from nemo_gym.harness.supervisor_client import ( + OUTPUT_LOG_FILE, + STOP_REQUEST_FILE, + SUPERVISOR_FILE, remove_session_directory, stop_and_confirm_cleanup, supervised_launch_command, + supervision_timeouts, ) from nemo_gym.sandbox.api import AsyncSandbox from nemo_gym.sandbox.providers.base import SandboxExecResult +from nemo_gym.sandbox.utils import read_text LOG = logging.getLogger(__name__) +def _reuse_or_start[T]( + task: asyncio.Task[T] | None, factory: Callable[[], Coroutine[object, object, T]] +) -> asyncio.Task[T]: + if task is None or (task.done() and (task.cancelled() or task.exception() is not None)): + return asyncio.create_task(factory()) + return task + + @dataclass(frozen=True, kw_only=True) class SandboxCommand: - """Harness argv and the interpreter/path used to launch its supervisor.""" + """Harness argv and the installed interpreter used to launch its supervisor.""" argv: list[str] python: str - supervisor_path: str | None = None @dataclass(kw_only=True) class SandboxSession[Artifacts]: """Execute, stop, capture, then release an owned or borrowed sandbox. - Runtime installation belongs in the adapter's seed hook. ``prepare`` stages - activation input and returns a command; ``collect`` copies harness artifacts + Runtime installation belongs in the adapter's seed hook. ``stage_activation`` + stages input and returns a command; ``collect`` copies harness artifacts to the agent server after cleanup is confirmed. It may run a bounded snapshot command, but must not restart the harness. Parsing responses stays in the adapter. + A new session stages input and its supervisor, runs the harness process, then + stops and collects artifacts. Execution retains the sandbox until close + releases it. A failed borrowed cleanup remains closing and can be retried; + an owned sandbox can fall back to provider stop. Closed means release succeeded. + Each stop, collection and release phase is bounded by ``close_timeout`` (or the timeout passed to ``close``). Failed cleanup/release retains the handle for retry. Failed capture is recorded separately and does not prevent release. @@ -46,7 +65,7 @@ class SandboxSession[Artifacts]: """ sandbox: AsyncSandbox - directory: str + session_dir: str workdir: str | None harness: str owns_sandbox: bool = False @@ -55,19 +74,46 @@ class SandboxSession[Artifacts]: artifacts: Artifacts | None = field(default=None, init=False) capture_error: Exception | None = field(default=None, init=False) sandbox_stopped: bool = field(default=False, init=False) - closing: bool = field(default=False, init=False) - closed: bool = field(default=False, init=False) - _prepare_task: asyncio.Future[SandboxCommand] | None = field(default=None, init=False) + _stage_task: asyncio.Future[SandboxCommand] | None = field(default=None, init=False) _exec_task: asyncio.Task[SandboxExecResult] | None = field(default=None, init=False) _finalize_task: asyncio.Task[None] | None = field(default=None, init=False) _close_task: asyncio.Task[None] | None = field(default=None, init=False) _collect: Callable[[], Awaitable[Artifacts]] | None = field(default=None, init=False) _capture_attempted: bool = field(default=False, init=False) + @property + def closing(self) -> bool: + """Whether close has been requested, including a failed attempt.""" + return self._close_task is not None + + @property + def closed(self) -> bool: + """Whether sandbox release completed successfully.""" + task = self._close_task + return task is not None and task.done() and not task.cancelled() and task.exception() is None + + @property + def stop_request_path(self) -> str: + """Marker path an adapter may pass to its harness for cancellation diagnostics.""" + return f"{self.session_dir}/{STOP_REQUEST_FILE}" + + async def read_output_log(self) -> str: + """Read combined supervisor/harness diagnostics without masking the original failure.""" + try: + return await read_text(self.sandbox, path=f"{self.session_dir}/{OUTPUT_LOG_FILE}") + except Exception: + LOG.warning("Could not read %s harness output log", self.harness, exc_info=True) + return "" + + async def _stage(self, stage_activation: Callable[[], Awaitable[SandboxCommand]]) -> SandboxCommand: + command = await stage_activation() + await self.sandbox.upload(Path(process_supervisor.__file__), f"{self.session_dir}/{SUPERVISOR_FILE}") + return command + async def execute( self, *, - prepare: Callable[[], Awaitable[SandboxCommand]], + stage_activation: Callable[[], Awaitable[SandboxCommand]], collect: Callable[[], Awaitable[Artifacts]], timeout: float, close_timeout: float, @@ -78,30 +124,29 @@ async def execute( Callers that merely stop waiting (such as disconnected HTTP requests) must shield their shared activation task. """ - if self.closing or self._prepare_task is not None: + if self.closing or self._stage_task is not None: raise RuntimeError(f"{self.harness} sandbox session is closing or already activated") # Register both hooks before yielding, so close can fence preparation too. self._collect = collect - self._prepare_task = asyncio.ensure_future(prepare()) + self._stage_task = asyncio.create_task(self._stage(stage_activation)) try: - command = await asyncio.shield(self._prepare_task) + command = await asyncio.shield(self._stage_task) if self.closing or self._finalize_task is not None: raise RuntimeError(f"{self.harness} sandbox session closed before launch") - cleanup_timeout = close_timeout / 3 + cleanup_timeout, provider_timeout = supervision_timeouts(timeout=timeout, close_timeout=close_timeout) launch = supervised_launch_command( - directory=self.directory, + session_dir=self.session_dir, command=command.argv, timeout=timeout, cleanup_timeout=cleanup_timeout, python=command.python, - supervisor_path=command.supervisor_path, ) self.launch_started = True self._exec_task = asyncio.create_task( self.sandbox.exec( launch, cwd=self.workdir, - timeout_s=exec_timeout(timeout=timeout, cleanup_timeout=cleanup_timeout), + timeout_s=provider_timeout, ) ) result = await asyncio.shield(self._exec_task) @@ -109,7 +154,7 @@ async def execute( raise TimeoutError(f"{self.harness} sandbox supervisor exceeded its execution deadline") if result.error_type: raise RuntimeError(f"{self.harness} sandbox execution failed: {result.error_type}: {result.stderr}") - # A worker's nonzero exit can still carry useful, adapter-specific output. + # A harness process's nonzero exit can still carry useful, adapter-specific output. except BaseException: try: await self._finish(timeout=close_timeout) @@ -126,7 +171,7 @@ async def execute( # Collection either returned artifacts (including a valid None) or recorded an error. return cast(Artifacts, self.artifacts) - async def stop_runner(self, *, timeout: float) -> None: + async def stop_harness(self, *, timeout: float) -> None: """Fence a delayed launch or confirm descendant cleanup before transport cancellation.""" if self.sandbox_stopped or not self.launch_started or self.cleanup is not None: return @@ -134,7 +179,7 @@ async def stop_runner(self, *, timeout: float) -> None: async with asyncio.timeout(timeout): self.cleanup = await stop_and_confirm_cleanup( self.sandbox, - directory=self.directory, + session_dir=self.session_dir, workdir=self.workdir, timeout=timeout, harness=self.harness, @@ -143,11 +188,7 @@ async def stop_runner(self, *, timeout: float) -> None: raise RuntimeError(f"{self.harness} launch outcome is unknown; cleanup deadline exceeded") from error async def _finish(self, *, timeout: float) -> None: - if self._finalize_task is None or ( - self._finalize_task.done() - and (self._finalize_task.cancelled() or self._finalize_task.exception() is not None) - ): - self._finalize_task = asyncio.create_task(self._finalize(timeout=timeout)) + self._finalize_task = _reuse_or_start(self._finalize_task, lambda: self._finalize(timeout=timeout)) await asyncio.shield(self._finalize_task) async def _cancel_and_wait[T](self, task: asyncio.Future[T], *, timeout: float) -> None: @@ -164,9 +205,9 @@ async def _cancel_and_wait[T](self, task: asyncio.Future[T], *, timeout: float) # Execution/preparation errors are propagated by execute, independently of cleanup. async def _finalize(self, *, timeout: float) -> None: - if self._prepare_task is not None: - await self._cancel_and_wait(self._prepare_task, timeout=timeout) - await self.stop_runner(timeout=timeout) + if self._stage_task is not None: + await self._cancel_and_wait(self._stage_task, timeout=timeout) + await self.stop_harness(timeout=timeout) if self.launch_started and not self._capture_attempted and self._collect is not None: if self.sandbox_stopped: self.capture_error = RuntimeError(f"{self.harness} sandbox stopped before artifact capture") @@ -187,13 +228,7 @@ async def _release_exec(self, *, timeout: float) -> None: async def close(self, *, timeout: float) -> None: """Join finalization before release; concurrent/retried closes never duplicate capture.""" - self.closing = True - if self.closed: - return - if self._close_task is None or ( - self._close_task.done() and (self._close_task.cancelled() or self._close_task.exception() is not None) - ): - self._close_task = asyncio.create_task(self._close(timeout=timeout)) + self._close_task = _reuse_or_start(self._close_task, lambda: self._close(timeout=timeout)) await asyncio.shield(self._close_task) async def _close(self, *, timeout: float) -> None: @@ -216,10 +251,9 @@ async def _close(self, *, timeout: float) -> None: async with asyncio.timeout(timeout): await remove_session_directory( self.sandbox, - directory=self.directory, + session_dir=self.session_dir, workdir=self.workdir, timeout=timeout, harness=self.harness, ) await self.sandbox.disconnect() - self.closed = True diff --git a/nemo_gym/agent_utils/supervisor_client.py b/nemo_gym/harness/supervisor_client.py similarity index 56% rename from nemo_gym/agent_utils/supervisor_client.py rename to nemo_gym/harness/supervisor_client.py index ab21c8760d7..9bbce0e221a 100644 --- a/nemo_gym/agent_utils/supervisor_client.py +++ b/nemo_gym/harness/supervisor_client.py @@ -4,6 +4,19 @@ Session ownership and cancellation live in sandbox_session.py; output parsing stays with the adapter. Unlike process_supervisor.py, this module is not uploaded to the task sandbox. + +Session control files (all paths are relative to session_dir): + +===================== ==================== =========================================== +Filename Writer When +===================== ==================== =========================================== +process_supervisor.py Session Uploaded before launch +launch.claim Launch or stop shell Atomically claims launch or fences it +supervisor.pid Launch shell After claiming launch, before exec +stop.request Stop shell Before signalling the supervisor +cleanup.json Supervisor or stop After cleanup, or when stop fences launch +output.log Launch shell Captures supervisor and harness diagnostics +===================== ==================== =========================================== """ import json @@ -11,40 +24,28 @@ from shlex import join, quote from uuid import uuid4 -from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from pydantic import TypeAdapter, ValidationError -from nemo_gym.agent_utils.process_supervisor import CleanupReceipt +from nemo_gym.harness.process_supervisor import CLEANUP_PHASE_COUNT, CleanupReceipt, exec_timeout from nemo_gym.sandbox import AsyncSandbox from nemo_gym.sandbox.utils import read_text +SUPERVISOR_FILE = "process_supervisor.py" +LAUNCH_CLAIM_FILE = "launch.claim" +SUPERVISOR_PID_FILE = "supervisor.pid" +STOP_REQUEST_FILE = "stop.request" +CLEANUP_RECEIPT_FILE = "cleanup.json" +OUTPUT_LOG_FILE = "output.log" + _CLEANUP_RECEIPT = TypeAdapter(CleanupReceipt) LOG = logging.getLogger(__name__) -class HarnessProcessInfo(BaseModel): - """Identity of the supervised worker, which may be a shim that spawns the harness CLI. - - This is independent of supervisor cleanup and harness output. The PID is - diagnostic metadata, not the supervisor PID used to request cleanup. - """ - - model_config = ConfigDict(extra="forbid", strict=True) - hostname: str - pid: int - python: str | None = None - - def parse_cleanup_receipt(payload: object) -> CleanupReceipt: - """Require positive cleanup evidence, tolerating malformed optional diagnostics. - - Older stop-before-launch receipts contain only acknowledgement and error. - Normalize that form without inventing a successful worker exit code. - """ + """Require positive cleanup evidence, tolerating malformed optional diagnostics.""" if not isinstance(payload, dict) or payload.get("cleanup_confirmed") is not True: raise ValueError("Sandbox cleanup was not confirmed") - if payload.keys() == {"cleanup_confirmed", "error"}: - payload = {**payload, "return_code": None, "timed_out": False} try: return _CLEANUP_RECEIPT.validate_python(payload, strict=True, extra="forbid") except ValidationError: @@ -57,64 +58,67 @@ def parse_cleanup_receipt(payload: object) -> CleanupReceipt: } -def parse_runtime_info(payload: object) -> HarnessProcessInfo | None: - """Read optional worker diagnostics without failing an otherwise valid episode.""" - try: - return HarnessProcessInfo.model_validate(payload) - except ValidationError: - LOG.warning("Harness runtime metadata is missing or malformed") - return None +def supervision_timeouts(*, timeout: float, close_timeout: float) -> tuple[float, float]: + """Return the per-cleanup-phase and provider execution deadlines. + + Divide the close budget across TERM grace, harness process reaping, and + descendant draining. The provider deadline reserves all three phases after + the harness timeout, plus 30 seconds for startup, polling, and receipt I/O. + """ + cleanup_timeout = close_timeout / CLEANUP_PHASE_COUNT + return cleanup_timeout, exec_timeout(timeout=timeout, cleanup_timeout=cleanup_timeout) def supervised_launch_command( *, - directory: str, + session_dir: str, command: list[str], timeout: float, cleanup_timeout: float, python: str, - supervisor_path: str | None = None, ) -> str: """Fence delayed launches and run a harness command under the shared supervisor. The adapter must install or select the interpreter explicitly. Check that it can load the supervisor before claiming a launch: a failed bootstrap cannot - write a cleanup receipt. Supervisor and worker diagnostics share runner.log. + write a cleanup receipt. Supervisor and harness diagnostics share output.log. """ - supervisor = quote(supervisor_path or directory + "/process_supervisor.py") + supervisor = quote(f"{session_dir}/{SUPERVISOR_FILE}") + claim_path = quote(f"{session_dir}/{LAUNCH_CLAIM_FILE}") return ( - f"[ -d {quote(directory)} ] && [ ! -L {quote(directory + '/launch.claim')} ] || exit 0; " + f"[ -d {quote(session_dir)} ] && [ ! -L {claim_path} ] || exit 0; " f"{quote(python)} -I {supervisor} --help >/dev/null || exit $?; " - f"trap '' TERM; ln -s launch {quote(directory + '/launch.claim')} 2>/dev/null || exit 0; " - f"echo $$ > {quote(directory + '/runner.pid')} && " + f"trap '' TERM; ln -s launch {claim_path} 2>/dev/null || exit 0; " + f"echo $$ > {quote(f'{session_dir}/{SUPERVISOR_PID_FILE}')} && " f"exec {quote(python)} -I {supervisor} " f"--timeout {timeout} --cleanup-timeout {cleanup_timeout} " - f"--stop-file {quote(directory + '/runner.stop')} " - f"--receipt {quote(directory + '/cleanup.json')} -- {join(command)} " - f">{quote(directory + '/runner.log')} 2>&1" + f"--stop-file {quote(f'{session_dir}/{STOP_REQUEST_FILE}')} " + f"--receipt {quote(f'{session_dir}/{CLEANUP_RECEIPT_FILE}')} -- {join(command)} " + f">{quote(f'{session_dir}/{OUTPUT_LOG_FILE}')} 2>&1" ) async def stop_and_confirm_cleanup( - sandbox: AsyncSandbox, *, directory: str, workdir: str | None, timeout: float, harness: str + sandbox: AsyncSandbox, *, session_dir: str, workdir: str | None, timeout: float, harness: str ) -> CleanupReceipt: """Fence a pending launch or require explicit supervisor acknowledgement before teardown. - A stop that wins the launch claim writes a minimal receipt: no worker ran, so - runtime metadata and a return code are intentionally absent. Worker metadata - and harness output are validated separately by the adapter. + A stop that wins the launch claim writes the same receipt shape as the + supervisor, with no return code because no harness process ran. """ - receipt_path = f"{directory}/cleanup.json" + receipt_path = f"{session_dir}/{CLEANUP_RECEIPT_FILE}" try: receipt = json.loads(await read_text(sandbox, path=receipt_path)) except Exception: receipt = {} if not isinstance(receipt, dict) or receipt.get("cleanup_confirmed") is not True: - pid_path = quote(f"{directory}/runner.pid") - stop_path = quote(f"{directory}/runner.stop") - claim_path = quote(f"{directory}/launch.claim") + pid_path = quote(f"{session_dir}/{SUPERVISOR_PID_FILE}") + stop_path = quote(f"{session_dir}/{STOP_REQUEST_FILE}") + claim_path = quote(f"{session_dir}/{LAUNCH_CLAIM_FILE}") temporary = quote(f"{receipt_path}.{uuid4().hex}.tmp") - stopped = quote(json.dumps({"cleanup_confirmed": True, "error": None})) + stopped = quote( + json.dumps({"cleanup_confirmed": True, "return_code": None, "error": None, "timed_out": False}) + ) script = ( f"[ -f {quote(receipt_path)} ] && exit 0; " f"touch {stop_path} || exit 1; " @@ -137,7 +141,7 @@ async def stop_and_confirm_cleanup( async def remove_session_directory( - sandbox: AsyncSandbox, *, directory: str, workdir: str | None, timeout: float, harness: str + sandbox: AsyncSandbox, *, session_dir: str, workdir: str | None, timeout: float, harness: str ) -> None: """Remove adapter-owned files after cleanup, keeping delayed launches fenced. @@ -145,10 +149,10 @@ async def remove_session_directory( delayed launch could win the claim while recursive deletion is in progress. A failed removal leaves the retired path available for a close retry. """ - retired = f"{directory}.closed" + retired = f"{session_dir}.closed" result = await sandbox.exec( - f"if [ -d {quote(directory)} ]; then " - f"mv {quote(directory)} {quote(retired)} || exit 1; fi; rm -rf -- {quote(retired)}", + f"if [ -d {quote(session_dir)} ]; then " + f"mv {quote(session_dir)} {quote(retired)} || exit 1; fi; rm -rf -- {quote(retired)}", cwd=workdir, timeout_s=timeout, ) diff --git a/nemo_gym/sandbox/utils.py b/nemo_gym/sandbox/utils.py index 3e695132da0..4d98354ffca 100644 --- a/nemo_gym/sandbox/utils.py +++ b/nemo_gym/sandbox/utils.py @@ -53,22 +53,6 @@ ) -async def upload_bytes(sandbox: AsyncSandbox, *, path: str, data: bytes) -> None: - """Upload bytes unchanged through file transfer without shell interpolation.""" - with tempfile.TemporaryDirectory(prefix="sandbox-upload-") as directory: - source = Path(directory) / "payload" - source.write_bytes(data) - await sandbox.upload(source, path) - - -async def read_bytes(sandbox: AsyncSandbox, *, path: str) -> bytes: - """Download file contents unchanged, without text decoding.""" - with tempfile.TemporaryDirectory(prefix="sandbox-download-") as directory: - destination = Path(directory) / "payload" - await sandbox.download(path, destination) - return destination.read_bytes() - - async def upload_text(sandbox: AsyncSandbox, *, path: str, text: str) -> None: """Upload UTF-8 text through file transfer without shell interpolation.""" with tempfile.TemporaryDirectory(prefix="sandbox-upload-") as directory: diff --git a/responses_api_agents/hermes_agent/app.py b/responses_api_agents/hermes_agent/app.py index b7455c8f115..162e5672e33 100644 --- a/responses_api_agents/hermes_agent/app.py +++ b/responses_api_agents/hermes_agent/app.py @@ -30,8 +30,6 @@ from pydantic import ConfigDict, Field from toolsets import TOOLSETS # pyright: ignore[reportMissingImports] -from nemo_gym.agent_utils.sandbox_session import SandboxSession -from nemo_gym.agent_utils.supervisor_client import HarnessProcessInfo from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import ( AgentCloseSessionResponse, @@ -44,6 +42,7 @@ ) from nemo_gym.config_types import ModelServerRef, ResourcesServerRef from nemo_gym.global_config import get_global_config_dict +from nemo_gym.harness.sandbox_session import SandboxSession from nemo_gym.openai_utils import ( NeMoGymEasyInputMessage, NeMoGymFunctionCallOutput, @@ -76,7 +75,7 @@ from nemo_gym.tool_access import MCPToolAccess from responses_api_agents.hermes_agent.model_kwargs import _model_api_kwargs from responses_api_agents.hermes_agent.observability import HermesAgentObserver, normalize_hermes_messages -from responses_api_agents.hermes_agent.sandbox import HermesSandboxSession +from responses_api_agents.hermes_agent.sandbox import HarnessProcessInfo, HermesSandboxSession def _trajectory_to_output_items(messages, n_input): @@ -419,13 +418,13 @@ async def _initialize_agent_session_state( session=SandboxSession( sandbox=sandbox, workdir=workdir, - directory=session_dir, + session_dir=session_dir, owns_sandbox=owns_sandbox, harness="Hermes", ), ) try: - await state.prepare(install_timeout=self.config.sandbox_install_timeout_seconds) + await state.install_runtime(install_timeout=self.config.sandbox_install_timeout_seconds) except BaseException as error: try: await state.close(self.config.session_close_timeout_seconds) diff --git a/responses_api_agents/hermes_agent/sandbox.py b/responses_api_agents/hermes_agent/sandbox.py index 6d1e19273c0..dfd786189e4 100644 --- a/responses_api_agents/hermes_agent/sandbox.py +++ b/responses_api_agents/hermes_agent/sandbox.py @@ -5,22 +5,24 @@ import asyncio import importlib.metadata import json +import logging import shutil from dataclasses import dataclass from pathlib import Path from shlex import quote -from pydantic import JsonValue +from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError -from nemo_gym.agent_utils import process_supervisor -from nemo_gym.agent_utils.sandbox_session import SandboxCommand, SandboxSession -from nemo_gym.agent_utils.supervisor_client import HarnessProcessInfo, parse_runtime_info from nemo_gym.base_responses_api_agent import AgentSessionState +from nemo_gym.harness.sandbox_session import SandboxCommand, SandboxSession from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming from nemo_gym.rollout_observability import AgentObservationBundle from nemo_gym.sandbox.utils import read_text, upload_text +LOG = logging.getLogger(__name__) + + def _sandbox_hermes_install() -> tuple[str, str]: """Return the requirement the sandbox installs and the key that names its runtime directory. @@ -44,11 +46,28 @@ def _sandbox_hermes_install() -> tuple[str, str]: _SANDBOX_UV = f"{_SANDBOX_RUNTIME_DIR}/uv" _SANDBOX_PYTHON = f"{_SANDBOX_RUNTIME_DIR}/venv/bin/python" _SANDBOX_RUNNER = f"{_SANDBOX_RUNTIME_DIR}/sandbox_runner.py" -_SANDBOX_SUPERVISOR = f"{_SANDBOX_RUNTIME_DIR}/process_supervisor.py" _SANDBOX_OBSERVER = f"{_SANDBOX_RUNTIME_DIR}/sandbox_observer.py" _SANDBOX_MODEL_KWARGS = f"{_SANDBOX_RUNTIME_DIR}/model_kwargs.py" +class HarnessProcessInfo(BaseModel): + """Optional identity of the Hermes harness process, separate from cleanup evidence.""" + + model_config = ConfigDict(extra="forbid", strict=True) + hostname: str + pid: int + python: str | None = None + + +def parse_runtime_info(payload: object) -> HarnessProcessInfo | None: + """Read Hermes diagnostics without failing an otherwise valid episode.""" + try: + return HarnessProcessInfo.model_validate(payload) + except ValidationError: + LOG.warning("Hermes runtime metadata is missing or malformed") + return None + + @dataclass class HermesSandboxSession(AgentSessionState): """Hermes transport and cleanup; HTTP retry bookkeeping stays in the agent base.""" @@ -59,10 +78,10 @@ class HermesSandboxSession(AgentSessionState): task: asyncio.Task[NeMoGymResponse] | None = None runtime_info: HarnessProcessInfo | None = None - async def prepare(self, *, install_timeout: float) -> None: - """Reuse or install the pinned runtime and stage the Hermes worker files.""" + async def install_runtime(self, *, install_timeout: float) -> None: + """Reuse or install the pinned runtime and stage the Hermes harness files.""" prepared = await self.session.sandbox.exec( - f"mkdir -p {quote(_SANDBOX_RUNTIME_DIR)} {quote(self.session.directory)}", + f"mkdir -p {quote(_SANDBOX_RUNTIME_DIR)} {quote(self.session.session_dir)}", cwd=self.session.workdir, timeout_s=30, ) @@ -73,7 +92,6 @@ async def prepare(self, *, install_timeout: float) -> None: await self.session.sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), _SANDBOX_RUNNER) await self.session.sandbox.upload(Path(__file__).with_name("sandbox_observer.py"), _SANDBOX_OBSERVER) await self.session.sandbox.upload(Path(__file__).with_name("model_kwargs.py"), _SANDBOX_MODEL_KWARGS) - await self.session.sandbox.upload(Path(process_supervisor.__file__), _SANDBOX_SUPERVISOR) async def _runtime_installed(self) -> bool: """Whether the pinned Hermes and its MCP client import from its runtime path. @@ -107,11 +125,11 @@ async def _install_runtime(self, timeout: float) -> None: async def upload_json(self, name: str, payload: dict[str, JsonValue]) -> None: """Write a Hermes input under the session directory using file transfer.""" - await upload_text(self.session.sandbox, path=f"{self.session.directory}/{name}", text=json.dumps(payload)) + await upload_text(self.session.sandbox, path=f"{self.session.session_dir}/{name}", text=json.dumps(payload)) async def read_json(self, name: str) -> dict[str, JsonValue]: """Read a Hermes output object without mixing it with the cleanup contract.""" - path = f"{self.session.directory}/{name}" + path = f"{self.session.session_dir}/{name}" payload = json.loads(await read_text(self.session.sandbox, path=path)) if not isinstance(payload, dict): raise TypeError(f"Hermes sandbox payload at {path} is not an object") @@ -138,7 +156,7 @@ async def execute( ) -> dict[str, JsonValue]: """Use the common lifecycle, then check the Hermes-specific result.""" output = await self.session.execute( - prepare=lambda: self.prepare_execution(payload), + stage_activation=lambda: self.stage_activation(payload), collect=self.collect_artifacts, timeout=timeout, close_timeout=close_timeout, @@ -147,18 +165,17 @@ async def execute( raise RuntimeError(f"Hermes sandbox runner failed: {output['error']}\n{output.get('traceback', '')}") return output - async def prepare_execution(self, payload: dict[str, JsonValue]) -> SandboxCommand: - """Stage this activation's input and describe its worker command.""" - await self.upload_json("input.json", payload) + async def stage_activation(self, payload: dict[str, JsonValue]) -> SandboxCommand: + """Stage this activation's input and describe its harness process.""" + await self.upload_json("input.json", {**payload, "stop_request_path": self.session.stop_request_path}) return SandboxCommand( argv=[ _SANDBOX_PYTHON, _SANDBOX_RUNNER, - f"{self.session.directory}/input.json", - f"{self.session.directory}/output.json", + f"{self.session.session_dir}/input.json", + f"{self.session.session_dir}/output.json", ], python=_SANDBOX_PYTHON, - supervisor_path=_SANDBOX_SUPERVISOR, ) async def collect_artifacts(self) -> dict[str, JsonValue]: @@ -166,11 +183,7 @@ async def collect_artifacts(self) -> dict[str, JsonValue]: try: output = await self.read_json("output.json") except Exception as error: - logs = await self.session.sandbox.exec( - f"cat {quote(self.session.directory + '/runner.log')} 2>/dev/null || true", - cwd=self.session.workdir, - timeout_s=30, - ) - raise RuntimeError(f"Hermes sandbox runner exited without output: {logs.stdout or ''}") from error + logs = await self.session.read_output_log() + raise RuntimeError(f"Hermes sandbox runner exited without output: {logs}") from error self.runtime_info = parse_runtime_info(output.get("runtime")) return output diff --git a/responses_api_agents/hermes_agent/sandbox_runner.py b/responses_api_agents/hermes_agent/sandbox_runner.py index d240b045630..b7447361f40 100644 --- a/responses_api_agents/hermes_agent/sandbox_runner.py +++ b/responses_api_agents/hermes_agent/sandbox_runner.py @@ -125,7 +125,8 @@ def progress() -> dict[str, Any]: def interrupt(*_: object) -> None: nonlocal stop_reason # Close publishes this marker before signalling the supervisor. A deadline does not. - stop_reason = "cancelled" if (session_dir / "runner.stop").exists() else "wall_time" + stop_request_path = payload.get("stop_request_path") + stop_reason = "cancelled" if stop_request_path and Path(stop_request_path).exists() else "wall_time" # Save first: a blocked tool or API call may not unwind before the hard cleanup. if output_path is not None: partial = progress() diff --git a/responses_api_agents/hermes_agent/tests/test_app.py b/responses_api_agents/hermes_agent/tests/test_app.py index b88b3f7f4d7..40a13f6864f 100644 --- a/responses_api_agents/hermes_agent/tests/test_app.py +++ b/responses_api_agents/hermes_agent/tests/test_app.py @@ -22,7 +22,6 @@ import yaml from fastapi import HTTPException -from nemo_gym.agent_utils.sandbox_session import SandboxSession from nemo_gym.base_responses_api_agent import ( AgentCloseSessionRequest, AgentCloseSessionResponse, @@ -30,6 +29,7 @@ _AgentSessionRecord, ) from nemo_gym.episode_types import EpisodeId, TaskId +from nemo_gym.harness.sandbox_session import SandboxSession from nemo_gym.openai_utils import ( NeMoGymEasyInputMessage, NeMoGymFunctionCallOutput, @@ -165,14 +165,13 @@ async def test_sandbox_access_selects_runtime_provider(self, monkeypatch, with_m ) assert state.session.sandbox is sandbox assert state.session.workdir == "/app" - assert state.session.directory.startswith("/tmp/nemo-gym-hermes-sessions/") - assert len(state.session.directory.rsplit("/", 1)[-1]) == 32 + assert state.session.session_dir.startswith("/tmp/nemo-gym-hermes-sessions/") + assert len(state.session.session_dir.rsplit("/", 1)[-1]) == 32 assert sandbox.exec.await_count == 2 assert {call.args[0].name for call in sandbox.upload.await_args_list} == { "sandbox_runner.py", "sandbox_observer.py", "model_kwargs.py", - "process_supervisor.py", } async def test_seed_installs_hermes_when_it_does_not_import(self, monkeypatch) -> None: @@ -390,7 +389,7 @@ def _sandbox_session( hermes._session_records["session"] = _AgentSessionRecord( state=HermesSandboxSession( request=seed, - session=SandboxSession(sandbox=sandbox, workdir=None, directory="/session", harness="Hermes"), + session=SandboxSession(sandbox=sandbox, workdir=None, session_dir="/session", harness="Hermes"), ), episode_id=seed.episode_id, ) @@ -424,7 +423,7 @@ def __init__(self) -> None: self.uploaded: dict = {} async def upload(self, local_path, remote_path) -> None: - self.uploaded[remote_path] = json.loads(Path(local_path).read_text()) + self.uploaded[remote_path] = Path(local_path).read_text() # Mirrors AsyncSandbox.exec so an unsupported argument fails here too. async def exec( @@ -442,7 +441,8 @@ async def download(self, remote_path, local_path) -> None: response = await hermes.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="fix bug")) - runner_input = sandbox.uploaded["/session/input.json"] + runner_input = json.loads(sandbox.uploaded["/session/input.json"]) + assert "/session/process_supervisor.py" in sandbox.uploaded assert runner_input["model_base_url"] == f"http://model-server:1/ng-rollout/{seed.episode_id.capture_key}/v1" assert runner_input["user_message"] == "fix bug" assert response.metadata["harness_execution"] == "sandbox" @@ -453,7 +453,7 @@ def __init__(self) -> None: self.uploaded: dict = {} async def upload(self, local_path, remote_path) -> None: - self.uploaded[remote_path] = json.loads(Path(local_path).read_text()) + self.uploaded[remote_path] = Path(local_path).read_text() async def exec( self, command, *, cwd=None, env=None, timeout_s=180, user=None, preserve_background_services=False @@ -488,7 +488,7 @@ async def download(self, remote_path, local_path) -> None: request, NeMoGymResponseCreateParamsNonStreaming(input="what is the weather") ) - runner_input = sandbox.uploaded["/session/input.json"] + runner_input = json.loads(sandbox.uploaded["/session/input.json"]) assert runner_input["mcp_servers"] == ["weather", "search"] assert runner_input["required_mcp_servers"] == ["weather"] assert runner_input["enabled_toolsets"] == ["terminal", "weather", "search"] @@ -517,7 +517,7 @@ async def exec( self, command, *, cwd=None, env=None, timeout_s=180, user=None, preserve_background_services=False ) -> SandboxExecResult: self.commands.append(command) - if "runner.pid" in command and "exec " in command: + if "supervisor.pid" in command and "exec " in command: return SandboxExecResult(stdout=None, stderr="timed out", return_code=124, error_type="timeout") return SandboxExecResult(stdout="", stderr="", return_code=0) @@ -527,7 +527,7 @@ async def exec( with pytest.raises(TimeoutError, match="exceeded its execution deadline"): await hermes.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="fix bug")) - stop = [command for command in sandbox.commands if "runner.stop" in command and "kill -TERM" in command] + stop = [command for command in sandbox.commands if "stop.request" in command and "kill -TERM" in command] assert len(stop) == 1 async def test_close_during_activation_stops_the_runner(self, monkeypatch) -> None: @@ -550,7 +550,7 @@ async def exec( self, command, *, cwd=None, env=None, timeout_s=180, user=None, preserve_background_services=False ) -> SandboxExecResult: self.commands.append(command) - if "runner.pid" in command and "exec " in command: + if "supervisor.pid" in command and "exec " in command: self.runner_started.set() await asyncio.Event().wait() return SandboxExecResult(stdout="", stderr="", return_code=0) @@ -569,7 +569,7 @@ async def exec( timeout=5, ) - stop = [command for command in sandbox.commands if "runner.stop" in command and "kill -TERM" in command] + stop = [command for command in sandbox.commands if "stop.request" in command and "kill -TERM" in command] assert len(stop) == 1 assert hermes._session_records["session"].state is None with pytest.raises(asyncio.CancelledError): diff --git a/responses_api_agents/hermes_agent/tests/test_sandbox.py b/responses_api_agents/hermes_agent/tests/test_sandbox.py index 2b9fdedcb4f..6ad6d520885 100644 --- a/responses_api_agents/hermes_agent/tests/test_sandbox.py +++ b/responses_api_agents/hermes_agent/tests/test_sandbox.py @@ -17,9 +17,11 @@ from fastapi.testclient import TestClient from pydantic import ValidationError -from nemo_gym.agent_utils.sandbox_session import SandboxSession from nemo_gym.base_responses_api_agent import AgentCloseSessionRequest, AgentSeedSessionRequest, _AgentSessionRecord from nemo_gym.episode_types import EpisodeId, TaskId +from nemo_gym.harness import process_supervisor +from nemo_gym.harness.sandbox_session import SandboxSession +from nemo_gym.harness.supervisor_client import STOP_REQUEST_FILE, SUPERVISOR_FILE, parse_cleanup_receipt from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming from nemo_gym.rollout_observability import AgentEpisode, AgentObservationBundle from nemo_gym.server_utils import ServerClient @@ -29,7 +31,7 @@ HermesAgentConfig, HermesAgentRunRequest, ) -from responses_api_agents.hermes_agent.sandbox import HermesSandboxSession +from responses_api_agents.hermes_agent.sandbox import HarnessProcessInfo, HermesSandboxSession, parse_runtime_info @pytest.fixture @@ -74,7 +76,7 @@ def state(): }, ), session=SandboxSession( - sandbox=sandbox, workdir="/app", directory="/tmp/nemo-gym-hermes-sessions/session", harness="Hermes" + sandbox=sandbox, workdir="/app", session_dir="/tmp/nemo-gym-hermes-sessions/session", harness="Hermes" ), ) @@ -435,14 +437,13 @@ def local_runner(agent, state, monkeypatch, tmp_path): """Execute the actual launch/close shell commands and exchange files, without a remote provider.""" directory = tmp_path / "session" directory.mkdir() - state.session.directory = str(directory) + state.session.session_dir = str(directory) state.session.workdir = str(tmp_path) agent.config.session_close_timeout_seconds = 2 monkeypatch.setattr(hermes_sandbox, "_SANDBOX_PYTHON", sys.executable) monkeypatch.setattr( hermes_sandbox, "_SANDBOX_RUNNER", str(Path(hermes_sandbox.__file__).with_name("sandbox_runner.py")) ) - monkeypatch.setattr(hermes_sandbox, "_SANDBOX_SUPERVISOR", hermes_sandbox.process_supervisor.__file__) state.session.sandbox.upload.side_effect = shutil.copyfile state.session.sandbox.download.side_effect = shutil.copyfile @@ -500,13 +501,13 @@ async def queued_exec(command, **kwargs): with pytest.raises(asyncio.CancelledError if failure == "cancel" else OSError): await state.task assert state.session.cleanup is not None and state.session.cleanup["cleanup_confirmed"] is True - directory = Path(state.session.directory) + directory = Path(state.session.session_dir) assert (directory / "launch.claim").readlink() == Path("stop") assert json.loads((directory / "cleanup.json").read_text())["cleanup_confirmed"] is True # A delayed delivery cannot launch, before or after close removes the session directory. assert (await local_runner(commands[0])).return_code == 0 - assert not (directory / "runner.pid").exists() + assert not (directory / "supervisor.pid").exists() agent._session_records["session"] = _AgentSessionRecord(state=state, episode_id=state.request.episode_id) close = AgentCloseSessionRequest(agent_session_id="session", episode_id=state.request.episode_id) first = await agent.close_agent_session(request(state), close) @@ -560,7 +561,7 @@ async def test_close_preserves_interrupted_observations_before_release( if owned: state.session.sandbox.stop.assert_awaited_once() else: - assert not Path(state.session.directory).exists() + assert not Path(state.session.session_dir).exists() state.session.sandbox.disconnect.assert_awaited_once() finally: await state.close(2) @@ -568,18 +569,18 @@ async def test_close_preserves_interrupted_observations_before_release( async def test_close_can_recover_a_stop_claim_with_no_receipt(agent, state, local_runner): - directory = Path(state.session.directory) + directory = Path(state.session.session_dir) # The first close won the claim but was interrupted before publishing its receipt. (directory / "launch.claim").symlink_to("stop") state.session.launch_started = True - await state.session.stop_runner(timeout=agent.config.session_close_timeout_seconds) + await state.session.stop_harness(timeout=agent.config.session_close_timeout_seconds) assert state.session.cleanup is not None and state.session.cleanup["cleanup_confirmed"] is True assert json.loads((directory / "cleanup.json").read_text())["cleanup_confirmed"] is True @pytest.mark.parametrize("confirmed", [True, False]) async def test_receipt_download_failure_never_signals_a_reused_pid(agent, state, local_runner, tmp_path, confirmed): - directory = Path(state.session.directory) + directory = Path(state.session.session_dir) signaled = tmp_path / "unrelated-process-signaled" child = await asyncio.create_subprocess_exec( sys.executable, @@ -592,7 +593,7 @@ async def test_receipt_download_failure_never_signals_a_reused_pid(agent, state, ) try: assert await child.stdout.readline() == b"ready\n" - (directory / "runner.pid").write_text(str(child.pid)) + (directory / "supervisor.pid").write_text(str(child.pid)) (directory / "launch.claim").symlink_to("launch") (directory / "cleanup.json").write_text(json.dumps({"cleanup_confirmed": confirmed, "error": None})) original_download = state.session.sandbox.download @@ -608,10 +609,10 @@ async def transient_download(*args): state.session.sandbox.download = AsyncMock(side_effect=transient_download) state.session.launch_started = True if confirmed: - await state.session.stop_runner(timeout=agent.config.session_close_timeout_seconds) + await state.session.stop_harness(timeout=agent.config.session_close_timeout_seconds) else: with pytest.raises(RuntimeError, match="cleanup was not confirmed"): - await state.session.stop_runner(timeout=agent.config.session_close_timeout_seconds) + await state.session.stop_harness(timeout=agent.config.session_close_timeout_seconds) await asyncio.sleep(0.05) assert not signaled.exists() assert child.returncode is None @@ -644,7 +645,7 @@ async def terminate(*, timeout): state.session.cleanup = {"cleanup_confirmed": True, "error": None, "return_code": None, "timed_out": False} confirmed.set() - state.session.stop_runner = AsyncMock(side_effect=terminate) + state.session.stop_harness = AsyncMock(side_effect=terminate) state.task = asyncio.create_task(execute()) await entered.wait() try: @@ -660,8 +661,8 @@ async def terminate(*, timeout): async def test_close_retires_the_launch_path_before_removing_its_fence( agent, state, local_runner, monkeypatch, tmp_path ): - directory = Path(state.session.directory) - retired = Path(f"{state.session.directory}.closed") + directory = Path(state.session.session_dir) + retired = Path(f"{state.session.session_dir}.closed") state.session.launch_started = True commands = tmp_path / "bin" commands.mkdir() @@ -682,7 +683,7 @@ async def test_close_retires_the_launch_path_before_removing_its_fence( async def test_launch_claim_without_pid_is_not_proof_of_cleanup(agent, state, local_runner): - directory = Path(state.session.directory) + directory = Path(state.session.session_dir) (directory / "launch.claim").symlink_to("launch") state.session.launch_started = True with pytest.raises(RuntimeError, match="launch outcome is unknown"): @@ -696,27 +697,29 @@ async def test_launch_claim_without_pid_is_not_proof_of_cleanup(agent, state, lo async def test_stop_during_interpreter_startup_closes_without_starting_a_worker( agent, state, local_runner, monkeypatch, tmp_path, stop_timing ): - directory = Path(state.session.directory) + directory = Path(state.session.session_dir) ready = tmp_path / "before-handler" worker_started = tmp_path / "worker-started" wrapper = tmp_path / "delayed_supervisor.py" wrapper.write_text( "import pathlib,runpy,sys,time\n" - f"if '--help' in sys.argv: runpy.run_path({hermes_sandbox._SANDBOX_SUPERVISOR!r}, run_name='__main__')\n" + f"if '--help' in sys.argv: runpy.run_path({process_supervisor.__file__!r}, run_name='__main__')\n" f"pathlib.Path({str(ready)!r}).touch()\n" - f"while not pathlib.Path({str(directory / 'runner.stop')!r}).exists(): time.sleep(0.01)\n" + f"while not pathlib.Path({str(directory / STOP_REQUEST_FILE)!r}).exists(): time.sleep(0.01)\n" # Keep the interpreter in the pre-handler window while close sends TERM. "time.sleep(0.15)\n" - f"runner=runpy.run_path({hermes_sandbox._SANDBOX_SUPERVISOR!r})\n" + f"runner=runpy.run_path({process_supervisor.__file__!r})\n" "def unexpected_worker(*args, **kwargs):\n" f" pathlib.Path({str(worker_started)!r}).touch()\n" " raise AssertionError('Worker must not start after the stop marker')\n" "runner['subprocess'].Popen=unexpected_worker\n" "raise SystemExit(runner['main']())\n" ) - monkeypatch.setattr(hermes_sandbox, "_SANDBOX_SUPERVISOR", str(wrapper)) + state.session.sandbox.upload.side_effect = lambda source, destination: shutil.copyfile( + wrapper if Path(source).name == SUPERVISOR_FILE else source, destination + ) if stop_timing == "before-shell": - (directory / "runner.stop").touch() + (directory / STOP_REQUEST_FILE).touch() state.task = asyncio.create_task( agent._run_sandbox_episode( body=NeMoGymResponseCreateParamsNonStreaming(input="task"), agent_session_id="session", state=state @@ -727,7 +730,7 @@ async def test_stop_during_interpreter_startup_closes_without_starting_a_worker( async with asyncio.timeout(3): while not ready.exists(): await asyncio.sleep(0.01) - await state.session.stop_runner(timeout=agent.config.session_close_timeout_seconds) + await state.session.stop_harness(timeout=agent.config.session_close_timeout_seconds) with pytest.raises(RuntimeError, match="exited without output"): await state.task assert state.session.cleanup is not None and state.session.cleanup["cleanup_confirmed"] is True @@ -746,6 +749,49 @@ async def test_stop_during_interpreter_startup_closes_without_starting_a_worker( await asyncio.gather(state.task, return_exceptions=True) +def test_cleanup_and_runtime_are_independent(): + cleanup = {"return_code": 0, "timed_out": False, "cleanup_confirmed": True, "error": None} + assert parse_cleanup_receipt(cleanup) == cleanup + runtime = HarnessProcessInfo.model_validate({"hostname": "sandbox", "pid": 123}) + assert runtime.hostname == "sandbox" and runtime.pid == 123 + assert runtime.python is None + with pytest.raises(ValueError, match="cleanup was not confirmed"): + parse_cleanup_receipt(runtime.model_dump()) + with pytest.raises(ValidationError): + HarnessProcessInfo.model_validate(cleanup) + + +@pytest.mark.parametrize("invalid", [{"pid": "123"}, {"hostname": 123}, {"python": 123}, {"return_code": 0}]) +def test_runtime_info_keeps_strict_validation(invalid): + with pytest.raises(ValidationError): + HarnessProcessInfo.model_validate({"hostname": "sandbox", "pid": 123, **invalid}) + + +@pytest.mark.parametrize( + "payload", + [ + None, + [], + {}, + {"hostname": "sandbox", "pid": "123"}, + {"hostname": "sandbox", "pid": 123, "python": {"secret": "do not log"}}, + ], +) +def test_malformed_runtime_info_is_optional_and_logs_without_payload(payload, caplog): + assert parse_runtime_info(payload) is None + assert "runtime metadata" in caplog.text + assert "do not log" not in caplog.text + + +def test_hermes_runtime_uses_the_same_schema(): + payload = {"hostname": "sandbox", "pid": 123, "python": "/opt/hermes/bin/python"} + assert HarnessProcessInfo.model_validate(payload).model_dump() == payload + assert parse_runtime_info(payload).model_dump() == payload + for field in ("hostname", "pid"): + with pytest.raises(ValidationError): + HarnessProcessInfo.model_validate({key: value for key, value in payload.items() if key != field}) + + @pytest.mark.parametrize("runtime", [None, {}, {"hostname": "sandbox", "pid": "123"}]) async def test_invalid_runtime_preserves_response_and_cleanup(agent, state, runtime): state.upload_json = AsyncMock() @@ -834,9 +880,9 @@ async def test_runner_exit_without_cleanup_receipt_blocks_close(agent, state, re @pytest.mark.parametrize("overrides", [{}, {"temperature": 0.0}]) async def test_session_prompt_and_limits_reach_runner(agent, state, overrides, tmp_path, monkeypatch): - state.session.directory = str(tmp_path) + state.session.session_dir = str(tmp_path) monkeypatch.setattr(hermes_sandbox, "_SANDBOX_PYTHON", sys.executable) - monkeypatch.setattr(hermes_sandbox, "_SANDBOX_SUPERVISOR", hermes_sandbox.process_supervisor.__file__) + state.session.sandbox.upload.side_effect = shutil.copyfile agent.server_client.global_config_dict = { "model": {"responses_api_models": {"vllm_model": {"chat_template_kwargs": {"enable_thinking": False}}}} } @@ -870,8 +916,9 @@ async def test_session_prompt_and_limits_reach_runner(agent, state, overrides, t assert separator process = await asyncio.create_subprocess_exec("sh", "-c", launch_prefix) assert await process.wait() == 0 - assert int((tmp_path / "runner.pid").read_text()) == process.pid + assert int((tmp_path / "supervisor.pid").read_text()) == process.pid payload = state.upload_json.await_args.args[1] + assert payload["stop_request_path"] == state.session.stop_request_path assert payload["user_message"] == "Fix the bug" assert payload["history"] == [] assert payload["system_message"] == "Configured instruction\n\nRequest instruction" @@ -884,6 +931,7 @@ async def test_session_prompt_and_limits_reach_runner(agent, state, overrides, t @pytest.mark.parametrize("hermes_error", [None, "Model generated invalid tool call"]) async def test_exec_reads_final_output_after_confirmed_cleanup(agent, state, output_available, hermes_error): state.upload_json = AsyncMock() + state.session.read_output_log = AsyncMock(return_value="runner stderr") events = [] async def execute(command, **kwargs): diff --git a/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py b/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py index a0fbdc177c2..9c781b9540a 100644 --- a/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py @@ -4,6 +4,7 @@ import asyncio import json import os +import signal import socket import subprocess import sys @@ -20,7 +21,8 @@ from run_agent import AIAgent from tools.mcp_tool import shutdown_mcp_servers -from nemo_gym.agent_utils import process_supervisor +from nemo_gym.harness import process_supervisor +from nemo_gym.harness.supervisor_client import STOP_REQUEST_FILE from nemo_gym.mcp_auto_exposure import TOKEN_HEADER, maybe_auto_expose from nemo_gym.openai_utils import NeMoGymChatCompletionCreateParamsNonStreaming from nemo_gym.server_utils import ServerClient @@ -136,6 +138,42 @@ def _payload(model_base_url: str, **overrides) -> dict: } +@pytest.mark.parametrize("marker", ["present", "absent", "omitted"]) +def test_interruption_uses_supplied_stop_request_path(tmp_path, monkeypatch, restore_process_globals, marker): + from responses_api_agents.hermes_agent import sandbox_runner + + class Agent: + def __init__(self, **kwargs): + self._session_messages = [{"role": "assistant", "content": "partial answer"}] + + def _build_api_kwargs(self, messages): + return {} + + def run_conversation(self, *args, **kwargs): + signal.getsignal(signal.SIGTERM)(signal.SIGTERM, None) + return {"completed": True, "messages": self._session_messages} + + def interrupt(self, message): + assert message == ("sandbox cancellation" if marker == "present" else "sandbox timeout") + + monkeypatch.setattr("run_agent.AIAgent", Agent) + monkeypatch.setattr(sandbox_runner, "_use_model_server", lambda _url: None) + payload = _payload("http://unused/v1") + if marker != "omitted": + marker_path = tmp_path / "custom-stop-marker" + payload["stop_request_path"] = str(marker_path) + if marker == "present": + marker_path.touch() + output_path = tmp_path / "output.json" + result = _run(payload, tmp_path, output_path=output_path)["result"] + expected = "cancelled" if marker == "present" else "wall_time" + assert result["stop_reason"] == expected + assert result["interrupted"] and not result["completed"] + checkpoint = json.loads(output_path.read_text())["result"] + assert checkpoint["stop_reason"] == expected + assert checkpoint["messages"][-1]["content"] == "partial answer" + + def test_granted_mcp_tools_reach_the_seeded_resources_session(tmp_path, restore_process_globals) -> None: resources = ExampleMCPWeatherResourcesServer( config=ExampleMCPWeatherResourcesServerConfig( @@ -346,7 +384,8 @@ def test_worker_stop_checkpoints_partial_work_before_hard_cleanup(tmp_path, coop from responses_api_agents.hermes_agent import sandbox_runner input_path, output_path = tmp_path / "input.json", tmp_path / "output.json" - input_path.write_text(json.dumps(_payload("http://unused/v1"))) + stop_request_path = tmp_path / STOP_REQUEST_FILE + input_path.write_text(json.dumps({**_payload("http://unused/v1"), "stop_request_path": str(stop_request_path)})) # Run the real worker/observer under the real supervisor, with a deterministic slow harness. driver = """ import os, pathlib, sys, time, types @@ -385,7 +424,7 @@ def run_conversation(self, *args, **kwargs): "--cleanup-timeout", "0.5", "--stop-file", - str(tmp_path / "runner.stop"), + str(stop_request_path), "--receipt", str(tmp_path / "cleanup.json"), "--", @@ -410,7 +449,7 @@ def run_conversation(self, *args, **kwargs): while not (tmp_path / "model.patch").exists() and process.poll() is None and time.monotonic() < deadline: time.sleep(0.01) assert (tmp_path / "model.patch").exists() - (tmp_path / "runner.stop").touch() + stop_request_path.touch() process.terminate() _, stderr = process.communicate(timeout=10) assert process.returncode == 0, stderr diff --git a/responses_api_agents/hermes_sandboxed_agent/app.py b/responses_api_agents/hermes_sandboxed_agent/app.py index b5847bf7ea7..ca82b9769fa 100644 --- a/responses_api_agents/hermes_sandboxed_agent/app.py +++ b/responses_api_agents/hermes_sandboxed_agent/app.py @@ -14,11 +14,11 @@ from fastapi import HTTPException, Request from pydantic import ConfigDict, Field -from nemo_gym.agent_utils import process_supervisor from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import BaseResponsesAPIAgentConfig, SimpleResponsesAPIAgent from nemo_gym.config_types import ModelServerRef, ResourcesServerRef from nemo_gym.global_config import get_global_config_dict +from nemo_gym.harness import process_supervisor from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming from nemo_gym.sandbox import AsyncSandbox, create_provider from nemo_gym.sandbox.config import resolve_provider_config diff --git a/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py b/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py index a53a6fa2b1a..e9c1d188026 100644 --- a/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py +++ b/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py @@ -9,7 +9,7 @@ import pytest -from nemo_gym.agent_utils.process_supervisor import DEFAULT_CLEANUP_TIMEOUT +from nemo_gym.harness.process_supervisor import DEFAULT_CLEANUP_TIMEOUT from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming from nemo_gym.sandbox import SandboxExecResult from nemo_gym.server_utils import ServerClient diff --git a/tests/unit_tests/test_local_provider.py b/tests/unit_tests/test_local_provider.py index 67ce10c1849..e027a99570b 100644 --- a/tests/unit_tests/test_local_provider.py +++ b/tests/unit_tests/test_local_provider.py @@ -25,7 +25,7 @@ import pytest -from nemo_gym.agent_utils import process_supervisor +from nemo_gym.harness import process_supervisor from nemo_gym.sandbox import AsyncSandbox, SandboxSpec, SandboxStatus, create_provider from nemo_gym.sandbox.providers.local import LocalProvider diff --git a/tests/unit_tests/test_process_supervisor.py b/tests/unit_tests/test_process_supervisor.py index a69aeb6ca87..f73f3bef6a7 100644 --- a/tests/unit_tests/test_process_supervisor.py +++ b/tests/unit_tests/test_process_supervisor.py @@ -16,7 +16,7 @@ import pytest -SUPERVISOR = Path(__file__).parents[2] / "nemo_gym/agent_utils/process_supervisor.py" +SUPERVISOR = Path(__file__).parents[2] / "nemo_gym/harness/process_supervisor.py" @pytest.mark.parametrize("timeout, cleanup", [(1, 0.1), (2700, 10), (21600, 100)]) diff --git a/tests/unit_tests/test_sandbox_session.py b/tests/unit_tests/test_sandbox_session.py index cf4c5880ad0..4677da28ac5 100644 --- a/tests/unit_tests/test_sandbox_session.py +++ b/tests/unit_tests/test_sandbox_session.py @@ -7,8 +7,8 @@ import pytest -from nemo_gym.agent_utils import supervisor_client -from nemo_gym.agent_utils.sandbox_session import SandboxCommand, SandboxSession +from nemo_gym.harness import supervisor_client +from nemo_gym.harness.sandbox_session import SandboxCommand, SandboxSession from nemo_gym.sandbox import AsyncSandbox, SandboxExecResult @@ -27,12 +27,13 @@ def receipt_reader(monkeypatch): def session(receipt_reader): sandbox = AsyncMock(spec=AsyncSandbox) sandbox.exec.return_value = OK - return SandboxSession(sandbox=sandbox, directory="/session", workdir="/app", harness="test") + return SandboxSession(sandbox=sandbox, session_dir="/session", workdir="/app", harness="test") -async def execute(session, *, collect=None, prepare=None, timeout=1): +async def execute(session, *, collect=None, stage_activation=None, timeout=1): return await session.execute( - prepare=prepare or AsyncMock(return_value=SandboxCommand(argv=["worker"], python="/runtime/python")), + stage_activation=stage_activation + or AsyncMock(return_value=SandboxCommand(argv=["worker"], python="/runtime/python")), collect=collect or AsyncMock(return_value="transcript"), timeout=10, close_timeout=timeout, @@ -139,12 +140,12 @@ async def collect(): async def test_close_during_input_preparation_prevents_launch(session, receipt_reader): preparing = asyncio.Event() - async def prepare(): + async def stage_activation(): preparing.set() await asyncio.Future() collector = AsyncMock() - running = asyncio.create_task(execute(session, prepare=prepare, collect=collector)) + running = asyncio.create_task(execute(session, stage_activation=stage_activation, collect=collector)) await asyncio.wait_for(preparing.wait(), 2) await session.close(timeout=1) with pytest.raises(asyncio.CancelledError): @@ -268,7 +269,7 @@ async def test_nonzero_worker_exit_still_passes_artifacts_to_adapter(session, re async def test_bootstrap_failure_preserves_stderr_and_still_releases(session, receipt_reader): - receipt_reader.return_value = json.dumps({"cleanup_confirmed": True, "error": None}) + receipt_reader.return_value = json.dumps(RECEIPT | {"return_code": None}) session.sandbox.exec.return_value = SandboxExecResult("", "/runtime/python: not found", 127) collector = AsyncMock(side_effect=FileNotFoundError("no worker output")) with pytest.raises(RuntimeError, match="sandbox execution failed.*127.*python: not found") as error: @@ -281,3 +282,132 @@ async def test_bootstrap_failure_preserves_stderr_and_still_releases(session, re await session.close(timeout=1) assert session.closed session.sandbox.disconnect.assert_awaited_once() + + +async def test_session_uploads_supervisor_before_launch(session): + from pathlib import Path + + from nemo_gym.harness import process_supervisor + from nemo_gym.harness.supervisor_client import SUPERVISOR_FILE + + events = [] + + async def upload(source, destination): + assert Path(source).read_bytes() == Path(process_supervisor.__file__).read_bytes() + assert destination == f"/session/{SUPERVISOR_FILE}" + events.append("upload") + + async def launch(*args, **kwargs): + assert events == ["upload"] + events.append("launch") + return OK + + session.sandbox.upload.side_effect = upload + session.sandbox.exec.side_effect = launch + assert await execute(session) == "transcript" + assert events == ["upload", "launch"] + assert not session.closing and not session.closed + session.sandbox.stop.assert_not_awaited() + session.sandbox.disconnect.assert_not_awaited() + + +@pytest.mark.parametrize("owned", [False, True]) +async def test_close_during_supervisor_upload_joins_staging_before_release(session, receipt_reader, owned): + session.owns_sandbox = owned + uploading = asyncio.Event() + events = [] + + async def upload(*args): + uploading.set() + try: + await asyncio.Future() + finally: + events.append("upload_settled") + + session.sandbox.upload.side_effect = upload + release = session.sandbox.stop if owned else session.sandbox.disconnect + release.side_effect = lambda: events.append("release") + collector = AsyncMock() + running = asyncio.create_task(execute(session, collect=collector)) + await asyncio.wait_for(uploading.wait(), 2) + await session.close(timeout=1) + with pytest.raises(asyncio.CancelledError): + await running + assert events == ["upload_settled", "release"] + assert not session.launch_started + assert session.closed and session.closing + collector.assert_not_awaited() + receipt_reader.assert_not_awaited() + assert all("exec " not in call.args[0] for call in session.sandbox.exec.await_args_list) + + +async def test_supervisor_upload_failure_preserves_error_and_allows_close(session, receipt_reader): + session.sandbox.upload.side_effect = OSError("supervisor upload failed") + collector = AsyncMock() + with pytest.raises(OSError, match="supervisor upload failed"): + await execute(session, collect=collector) + assert not session.launch_started + assert not session.closing + collector.assert_not_awaited() + receipt_reader.assert_not_awaited() + session.sandbox.exec.assert_not_awaited() + await session.close(timeout=1) + assert session.closed + session.sandbox.disconnect.assert_awaited_once() + + +async def test_cancelled_close_stays_closing_and_can_retry(session): + release_started = asyncio.Event() + + async def disconnect(): + release_started.set() + await asyncio.Future() + + session.sandbox.disconnect.side_effect = disconnect + closing = asyncio.create_task(session.close(timeout=1)) + await asyncio.wait_for(release_started.wait(), 2) + session._close_task.cancel() + with pytest.raises(asyncio.CancelledError): + await closing + assert session.closing and not session.closed + with pytest.raises(RuntimeError, match="closing or already activated"): + await execute(session) + session.sandbox.disconnect.side_effect = None + await session.close(timeout=1) + assert session.closed + + +async def test_failure_log_is_read_before_session_release(session, monkeypatch): + from nemo_gym.harness import sandbox_session + + async def read_log(sandbox, *, path): + sandbox.disconnect.assert_not_awaited() + assert path == "/session/output.log" + return "diagnostic output" + + monkeypatch.setattr(sandbox_session, "read_text", read_log) + + async def collect(): + log = await session.read_output_log() + raise FileNotFoundError(f"Missing artifacts: {log}") + + with pytest.raises(FileNotFoundError, match="Missing artifacts: diagnostic output"): + await execute(session, collect=collect) + await session.close(timeout=1) + session.sandbox.disconnect.assert_awaited_once() + + +async def test_missing_failure_log_does_not_mask_capture_failure(session, monkeypatch): + from nemo_gym.harness import sandbox_session + + monkeypatch.setattr(sandbox_session, "read_text", AsyncMock(side_effect=OSError("download failed"))) + + async def collect(): + log = await session.read_output_log() + raise FileNotFoundError(f"Missing artifacts: {log}") + + with pytest.raises(FileNotFoundError, match="Missing artifacts"): + await execute(session, collect=collect) + assert session.cleanup == RECEIPT + await session.close(timeout=1) + assert session.closed diff --git a/tests/unit_tests/test_sandbox_supervisor_bootstrap.py b/tests/unit_tests/test_sandbox_supervisor_bootstrap.py index d454bfa76d4..4706eee60e4 100644 --- a/tests/unit_tests/test_sandbox_supervisor_bootstrap.py +++ b/tests/unit_tests/test_sandbox_supervisor_bootstrap.py @@ -10,8 +10,12 @@ import pytest -from nemo_gym.agent_utils import process_supervisor -from nemo_gym.agent_utils.supervisor_client import ( +from nemo_gym.harness import process_supervisor +from nemo_gym.harness.supervisor_client import ( + CLEANUP_RECEIPT_FILE, + LAUNCH_CLAIM_FILE, + SUPERVISOR_FILE, + SUPERVISOR_PID_FILE, remove_session_directory, stop_and_confirm_cleanup, supervised_launch_command, @@ -33,10 +37,10 @@ async def exec(self, command: str, *, cwd: str | None = None, timeout_s: float = @pytest.mark.parametrize("failure", ["missing_python", "broken_python", "missing_supervisor", "broken_supervisor"]) async def test_bootstrap_failure_preserves_diagnostics_and_allows_close(tmp_path: Path, failure: str) -> None: - directory = tmp_path / "session" - directory.mkdir() + session_dir = tmp_path / "session" + session_dir.mkdir() python = sys.executable - supervisor = directory / "process_supervisor.py" + supervisor = session_dir / SUPERVISOR_FILE shutil.copyfile(process_supervisor.__file__, supervisor) expected_error = "" if failure == "missing_python": @@ -56,8 +60,8 @@ async def test_bootstrap_failure_preserves_diagnostics_and_allows_close(tmp_path expected_error = "SyntaxError" sandbox = LocalSandbox() command = supervised_launch_command( - directory=str(directory), - command=["touch", str(directory / "worker-started")], + session_dir=str(session_dir), + command=["touch", str(session_dir / "worker-started")], timeout=1, cleanup_timeout=1, python=python, @@ -67,29 +71,29 @@ async def test_bootstrap_failure_preserves_diagnostics_and_allows_close(tmp_path if failure == "broken_python": assert result.return_code == 17 assert expected_error in result.stderr - assert not (directory / "launch.claim").is_symlink() - assert not (directory / "runner.pid").exists() - assert not (directory / "worker-started").exists() - assert not (directory / "cleanup.json").exists() + assert not (session_dir / LAUNCH_CLAIM_FILE).is_symlink() + assert not (session_dir / SUPERVISOR_PID_FILE).exists() + assert not (session_dir / "worker-started").exists() + assert not (session_dir / CLEANUP_RECEIPT_FILE).exists() receipt = await stop_and_confirm_cleanup( - sandbox, directory=str(directory), workdir=None, timeout=1, harness="bootstrap-test" + sandbox, session_dir=str(session_dir), workdir=None, timeout=1, harness="bootstrap-test" ) assert receipt["cleanup_confirmed"] is True assert receipt["return_code"] is None - assert (directory / "launch.claim").readlink() == Path("stop") + assert (session_dir / LAUNCH_CLAIM_FILE).readlink() == Path("stop") @pytest.mark.parametrize("removed", [False, True]) async def test_fenced_delayed_launch_does_not_require_the_runtime(tmp_path: Path, removed: bool) -> None: - directory = tmp_path / "session" - directory.mkdir() - (directory / "launch.claim").symlink_to("stop") + session_dir = tmp_path / "session" + session_dir.mkdir() + (session_dir / LAUNCH_CLAIM_FILE).symlink_to("stop") if removed: - shutil.rmtree(directory) + shutil.rmtree(session_dir) result = await LocalSandbox().exec( supervised_launch_command( - directory=str(directory), + session_dir=str(session_dir), command=["touch", str(tmp_path / "worker-started")], timeout=1, cleanup_timeout=1, @@ -103,12 +107,12 @@ async def test_fenced_delayed_launch_does_not_require_the_runtime(tmp_path: Path @pytest.mark.parametrize("remove_directory", [False, True]) async def test_close_during_preflight_fences_later_launch(tmp_path: Path, remove_directory: bool) -> None: - directory = tmp_path / "session" - directory.mkdir() + session_dir = tmp_path / "session" + session_dir.mkdir() preflight_started = tmp_path / "preflight-started" continue_preflight = tmp_path / "continue-preflight" worker_started = tmp_path / "worker-started" - supervisor = tmp_path / "supervisor.py" + supervisor = session_dir / SUPERVISOR_FILE supervisor.write_text( "import pathlib, sys, time\n" "if '--help' in sys.argv:\n" @@ -122,12 +126,11 @@ async def test_close_during_preflight_fences_later_launch(tmp_path: Path, remove launch = asyncio.create_task( sandbox.exec( supervised_launch_command( - directory=str(directory), + session_dir=str(session_dir), command=["worker"], timeout=1, cleanup_timeout=1, python=sys.executable, - supervisor_path=str(supervisor), ) ) ) @@ -136,13 +139,13 @@ async def test_close_during_preflight_fences_later_launch(tmp_path: Path, remove while not preflight_started.exists(): await asyncio.sleep(0.01) receipt = await stop_and_confirm_cleanup( - sandbox, directory=str(directory), workdir=None, timeout=1, harness="bootstrap-test" + sandbox, session_dir=str(session_dir), workdir=None, timeout=1, harness="bootstrap-test" ) assert receipt["cleanup_confirmed"] is True assert receipt["return_code"] is None if remove_directory: await remove_session_directory( - sandbox, directory=str(directory), workdir=None, timeout=1, harness="bootstrap-test" + sandbox, session_dir=str(session_dir), workdir=None, timeout=1, harness="bootstrap-test" ) finally: continue_preflight.touch() @@ -152,26 +155,25 @@ async def test_close_during_preflight_fences_later_launch(tmp_path: Path, remove async def test_bootstrap_quotes_private_paths_and_uses_the_same_interpreter(tmp_path: Path) -> None: - directory = tmp_path / "session's files" - directory.mkdir() + session_dir = tmp_path / "session's $(touch unintended) files" + session_dir.mkdir() runtime = tmp_path / "runtime's $(touch unintended) files" runtime.mkdir() python = runtime / "python interpreter" python.symlink_to(sys.executable) - supervisor = runtime / "supervisor's code.py" - arguments = directory / "arguments.json" + supervisor = session_dir / SUPERVISOR_FILE + arguments = session_dir / "arguments.json" supervisor.write_text( "import json, pathlib, sys\n" "if '--help' not in sys.argv:\n" f" pathlib.Path({str(arguments)!r}).write_text(json.dumps([sys.executable, *sys.argv[1:]]))\n" ) command = supervised_launch_command( - directory=str(directory), + session_dir=str(session_dir), command=["worker", "$(touch unintended)", "argument's value"], timeout=1, cleanup_timeout=1, python=str(python), - supervisor_path=str(supervisor), ) result = await LocalSandbox().exec(command, cwd=str(tmp_path)) assert result.return_code == 0, result.stderr @@ -179,4 +181,4 @@ async def test_bootstrap_quotes_private_paths_and_uses_the_same_interpreter(tmp_ assert captured[0] == str(python) assert captured[-3:] == ["worker", "$(touch unintended)", "argument's value"] assert not (tmp_path / "unintended").exists() - assert (directory / "launch.claim").readlink() == Path("launch") + assert (session_dir / LAUNCH_CLAIM_FILE).readlink() == Path("launch") diff --git a/tests/unit_tests/test_sandbox_supervisor_client.py b/tests/unit_tests/test_sandbox_supervisor_client.py index d298c91aa52..654f32d3815 100644 --- a/tests/unit_tests/test_sandbox_supervisor_client.py +++ b/tests/unit_tests/test_sandbox_supervisor_client.py @@ -10,15 +10,19 @@ from unittest.mock import AsyncMock import pytest -from pydantic import ValidationError -from nemo_gym.agent_utils import process_supervisor -from nemo_gym.agent_utils.supervisor_client import ( - HarnessProcessInfo, +from nemo_gym.harness import process_supervisor +from nemo_gym.harness.supervisor_client import ( + CLEANUP_RECEIPT_FILE, + LAUNCH_CLAIM_FILE, + OUTPUT_LOG_FILE, + STOP_REQUEST_FILE, + SUPERVISOR_FILE, + SUPERVISOR_PID_FILE, parse_cleanup_receipt, - parse_runtime_info, stop_and_confirm_cleanup, supervised_launch_command, + supervision_timeouts, ) from nemo_gym.sandbox.providers.base import SandboxExecResult from nemo_gym.sandbox.utils import read_text, upload_text @@ -41,15 +45,15 @@ async def exec(self, command, *, cwd=None, timeout_s=30): @pytest.fixture def session(tmp_path): - directory = tmp_path / "session's files" - directory.mkdir() - return LocalSandbox(), directory + session_dir = tmp_path / "session's files" + session_dir.mkdir() + return LocalSandbox(), session_dir async def test_file_transport_keeps_contents_out_of_shell(session): - sandbox, directory = session + sandbox, session_dir = session sandbox.exec = AsyncMock(side_effect=AssertionError("file transfer must not invoke a shell")) - path = str(directory / "input.json") + path = str(session_dir / "input.json") payload = '{"prompt": "$(touch unwanted); `echo surprise`"}\n' await upload_text(sandbox, path=path, text=payload) assert Path(path).read_text() == payload @@ -60,24 +64,24 @@ async def test_file_transport_keeps_contents_out_of_shell(session): async def test_confirmed_receipt_does_not_signal_stored_pid(session): - sandbox, directory = session - receipt = {"cleanup_confirmed": True, "error": None} - (directory / "cleanup.json").write_text(json.dumps(receipt)) - (directory / "runner.pid").write_text("12345") + sandbox, session_dir = session + receipt = {"cleanup_confirmed": True, "error": None, "return_code": None, "timed_out": False} + (session_dir / CLEANUP_RECEIPT_FILE).write_text(json.dumps(receipt)) + (session_dir / SUPERVISOR_PID_FILE).write_text("12345") sandbox.exec = AsyncMock(side_effect=AssertionError("must not signal a possibly reused PID")) assert await stop_and_confirm_cleanup( - sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + sandbox, session_dir=str(session_dir), workdir=str(session_dir.parent), timeout=1, harness="test" ) == parse_cleanup_receipt(receipt) sandbox.exec.assert_not_awaited() async def test_confirmed_receipt_with_malformed_diagnostics_still_allows_cleanup(session, caplog): - sandbox, directory = session + sandbox, session_dir = session receipt = {"cleanup_confirmed": True, "return_code": "0", "error": {"secret": "do not log"}, "version": 2} - (directory / "cleanup.json").write_text(json.dumps(receipt)) + (session_dir / CLEANUP_RECEIPT_FILE).write_text(json.dumps(receipt)) sandbox.exec = AsyncMock(side_effect=AssertionError("must not signal a possibly reused PID")) assert await stop_and_confirm_cleanup( - sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + sandbox, session_dir=str(session_dir), workdir=str(session_dir.parent), timeout=1, harness="test" ) == {"cleanup_confirmed": True, "return_code": None, "timed_out": False, "error": None} assert "diagnostic fields" in caplog.text assert "do not log" not in caplog.text @@ -86,71 +90,70 @@ async def test_confirmed_receipt_with_malformed_diagnostics_still_allows_cleanup @pytest.mark.parametrize("receipt", [[], "stopped", None, 1]) async def test_nonobject_cleanup_receipt_is_unconfirmed_without_signalling_a_stale_pid(session, receipt): - sandbox, directory = session - path = directory / "cleanup.json" + sandbox, session_dir = session + path = session_dir / CLEANUP_RECEIPT_FILE path.write_text(json.dumps(receipt)) - (directory / "runner.pid").write_text("2147483647") + (session_dir / SUPERVISOR_PID_FILE).write_text("2147483647") for _ in range(2): with pytest.raises(RuntimeError, match="cleanup receipt is not a JSON object"): await stop_and_confirm_cleanup( - sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + sandbox, session_dir=str(session_dir), workdir=str(session_dir.parent), timeout=1, harness="test" ) assert json.loads(path.read_text()) == receipt - assert not (directory / "runner.stop").exists() - assert not (directory / "launch.claim").exists() + assert not (session_dir / STOP_REQUEST_FILE).exists() + assert not (session_dir / LAUNCH_CLAIM_FILE).exists() -async def test_stop_wins_claim_and_fences_delayed_launch(session): - sandbox, directory = session +async def test_stop_wins_claim_and_fences_delayed_launch(session, caplog): + sandbox, session_dir = session receipt = await stop_and_confirm_cleanup( - sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + sandbox, session_dir=str(session_dir), workdir=str(session_dir.parent), timeout=1, harness="test" ) assert receipt == {"cleanup_confirmed": True, "error": None, "return_code": None, "timed_out": False} - assert (directory / "launch.claim").readlink() == Path("stop") - assert (directory / "runner.stop").exists() + assert json.loads((session_dir / CLEANUP_RECEIPT_FILE).read_text()) == receipt + assert "diagnostic fields" not in caplog.text + assert (session_dir / LAUNCH_CLAIM_FILE).readlink() == Path("stop") + assert (session_dir / STOP_REQUEST_FILE).exists() command = supervised_launch_command( - directory=str(directory), - command=["touch", str(directory / "started")], + session_dir=str(session_dir), + command=["touch", str(session_dir / "started")], timeout=1, cleanup_timeout=1, python=sys.executable, ) assert (await sandbox.exec(command)).return_code == 0 - assert not (directory / "runner.pid").exists() - assert not (directory / "runner.log").exists() - assert not (directory / "started").exists() - # A fenced launch is safe to close, without pretending a worker exited successfully. + assert not (session_dir / SUPERVISOR_PID_FILE).exists() + assert not (session_dir / OUTPUT_LOG_FILE).exists() + assert not (session_dir / "started").exists() + # A fenced launch is safe to close, without claiming a successful harness exit. assert receipt["return_code"] is None - with pytest.raises(ValidationError): - HarnessProcessInfo.model_validate(receipt) async def test_missing_receipt_after_launch_is_not_cleanup_confirmation(session): - sandbox, directory = session - (directory / "launch.claim").symlink_to("launch") + sandbox, session_dir = session + (session_dir / LAUNCH_CLAIM_FILE).symlink_to("launch") with pytest.raises(RuntimeError, match="test launch outcome is unknown"): await stop_and_confirm_cleanup( - sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + sandbox, session_dir=str(session_dir), workdir=str(session_dir.parent), timeout=1, harness="test" ) - assert (directory / "launch.claim").readlink() == Path("launch") - assert not (directory / "cleanup.json").exists() + assert (session_dir / LAUNCH_CLAIM_FILE).readlink() == Path("launch") + assert not (session_dir / CLEANUP_RECEIPT_FILE).exists() @pytest.mark.skipif(sys.platform != "linux", reason="Linux supervisor contract") -@pytest.mark.parametrize("private_runtime", [False, True]) -async def test_launch_supervises_worker_with_selected_or_private_runtime(session, tmp_path, private_runtime): - sandbox, directory = session +@pytest.mark.parametrize("private_python", [False, True]) +async def test_launch_supervises_harness_with_selected_or_private_python(session, tmp_path, private_python): + sandbox, session_dir = session runtime = tmp_path / "runtime's files" runtime.mkdir() - supervisor = (runtime if private_runtime else directory) / "process_supervisor.py" + supervisor = session_dir / SUPERVISOR_FILE shutil.copyfile(process_supervisor.__file__, supervisor) - options = {"python": sys.executable} - if private_runtime: + python = Path(sys.executable) + if private_python: python = runtime / "python interpreter" python.symlink_to(sys.executable) - options = {"python": str(python), "supervisor_path": str(supervisor)} command = supervised_launch_command( - directory=str(directory), + session_dir=str(session_dir), command=[ sys.executable, "-c", @@ -159,45 +162,41 @@ async def test_launch_supervises_worker_with_selected_or_private_runtime(session ], timeout=5, cleanup_timeout=1, - **options, + python=str(python), ) launched = await sandbox.exec(command) assert launched.return_code == 0, launched.stderr receipt = await stop_and_confirm_cleanup( - sandbox, directory=str(directory), workdir=str(tmp_path), timeout=2, harness="test" + sandbox, session_dir=str(session_dir), workdir=str(tmp_path), timeout=2, harness="test" ) assert receipt == {"cleanup_confirmed": True, "error": None, "return_code": 0, "timed_out": False} - assert set((directory / "runner.log").read_text().splitlines()) == {"$(literal)", "stderr"} + assert set((session_dir / OUTPUT_LOG_FILE).read_text().splitlines()) == {"$(literal)", "stderr"} @pytest.mark.parametrize("confirmed", [False, "true", 1]) async def test_unconfirmed_cleanup_preserves_receipt_for_retry(session, confirmed): - sandbox, directory = session - receipt_path = directory / "cleanup.json" + sandbox, session_dir = session + receipt_path = session_dir / CLEANUP_RECEIPT_FILE receipt = {"cleanup_confirmed": confirmed, "error": "descendants remain"} receipt_path.write_text(json.dumps(receipt)) with pytest.raises(RuntimeError, match="test sandbox cleanup was not confirmed: descendants remain"): await stop_and_confirm_cleanup( - sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + sandbox, session_dir=str(session_dir), workdir=str(session_dir.parent), timeout=1, harness="test" ) assert json.loads(receipt_path.read_text()) == receipt - receipt = {"cleanup_confirmed": True, "error": None} + receipt = {"cleanup_confirmed": True, "error": None, "return_code": None, "timed_out": False} receipt_path.write_text(json.dumps(receipt)) assert await stop_and_confirm_cleanup( - sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" + sandbox, session_dir=str(session_dir), workdir=str(session_dir.parent), timeout=1, harness="test" ) == parse_cleanup_receipt(receipt) -def test_cleanup_and_runtime_are_independent(): - cleanup = {"return_code": 0, "timed_out": False, "cleanup_confirmed": True, "error": None} - assert parse_cleanup_receipt(cleanup) == cleanup - runtime = HarnessProcessInfo.model_validate({"hostname": "sandbox", "pid": 123}) - assert runtime.hostname == "sandbox" and runtime.pid == 123 - assert runtime.python is None - with pytest.raises(ValueError, match="cleanup was not confirmed"): - parse_cleanup_receipt(runtime.model_dump()) - with pytest.raises(ValidationError): - HarnessProcessInfo.model_validate(cleanup) +@pytest.mark.parametrize("timeout,close_timeout", [(1, 0.3), (2700, 30), (21600, 300)]) +def test_supervision_timeouts_reserve_cleanup_and_provider_margin(timeout, close_timeout): + cleanup_timeout, provider_timeout = supervision_timeouts(timeout=timeout, close_timeout=close_timeout) + assert cleanup_timeout * process_supervisor.CLEANUP_PHASE_COUNT == pytest.approx(close_timeout) + assert provider_timeout == pytest.approx(timeout + close_timeout + 30) + assert provider_timeout == process_supervisor.exec_timeout(timeout=timeout, cleanup_timeout=cleanup_timeout) @pytest.mark.parametrize( @@ -224,41 +223,12 @@ def test_cleanup_receipt_requires_positive_boolean_evidence(payload): parse_cleanup_receipt(payload) -def test_absent_exit_code_is_not_success(): +def test_absent_exit_code_is_not_success(caplog): full = {"return_code": None, "timed_out": False, "cleanup_confirmed": True, "error": None} assert parse_cleanup_receipt(full) == full + assert "diagnostic fields" not in caplog.text assert parse_cleanup_receipt({"cleanup_confirmed": True, "error": None}) == full + assert "diagnostic fields" in caplog.text assert parse_cleanup_receipt({"cleanup_confirmed": True}) == full with pytest.raises(ValueError, match="cleanup was not confirmed"): parse_cleanup_receipt({"return_code": 0, "error": None}) - - -@pytest.mark.parametrize("invalid", [{"pid": "123"}, {"hostname": 123}, {"python": 123}, {"return_code": 0}]) -def test_runtime_info_keeps_strict_validation(invalid): - with pytest.raises(ValidationError): - HarnessProcessInfo.model_validate({"hostname": "sandbox", "pid": 123, **invalid}) - - -@pytest.mark.parametrize( - "payload", - [ - None, - [], - {}, - {"hostname": "sandbox", "pid": "123"}, - {"hostname": "sandbox", "pid": 123, "python": {"secret": "do not log"}}, - ], -) -def test_malformed_runtime_info_is_optional_and_logs_without_payload(payload, caplog): - assert parse_runtime_info(payload) is None - assert "runtime metadata" in caplog.text - assert "do not log" not in caplog.text - - -def test_hermes_runtime_uses_the_same_schema(): - payload = {"hostname": "sandbox", "pid": 123, "python": "/opt/hermes/bin/python"} - assert HarnessProcessInfo.model_validate(payload).model_dump() == payload - assert parse_runtime_info(payload).model_dump() == payload - for field in ("hostname", "pid"): - with pytest.raises(ValidationError): - HarnessProcessInfo.model_validate({key: value for key, value in payload.items() if key != field}) diff --git a/tests/unit_tests/test_sandbox_utils.py b/tests/unit_tests/test_sandbox_utils.py deleted file mode 100644 index 0cc43445975..00000000000 --- a/tests/unit_tests/test_sandbox_utils.py +++ /dev/null @@ -1,60 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import asyncio -from pathlib import Path -from unittest.mock import AsyncMock - -import pytest - -from nemo_gym.sandbox import AsyncSandbox, SandboxSpec -from nemo_gym.sandbox.providers.local import LocalProvider -from nemo_gym.sandbox.utils import read_bytes, upload_bytes - - -pytestmark = pytest.mark.sandbox - - -@pytest.mark.parametrize("data", [b"", bytes(range(256)) + b"\r\n\r"], ids=["empty", "binary"]) -async def test_bytes_round_trip_preserves_content_and_cleans_temporary_files(tmp_path: Path, data: bytes) -> None: - path = tmp_path / "payload's $(literal).bin" - async with AsyncSandbox(LocalProvider(), SandboxSpec(workdir=str(tmp_path))) as sandbox: - await sandbox.start() - sandbox.upload = AsyncMock(wraps=sandbox.upload) - sandbox.download = AsyncMock(wraps=sandbox.download) - sandbox.exec = AsyncMock(side_effect=AssertionError("file transfer must not invoke a shell")) - - await upload_bytes(sandbox, path=str(path), data=data) - assert path.read_bytes() == data - assert await read_bytes(sandbox, path=str(path)) == data - - assert not Path(sandbox.upload.await_args.args[0]).parent.exists() - assert not Path(sandbox.download.await_args.args[1]).parent.exists() - sandbox.exec.assert_not_awaited() - - -@pytest.mark.parametrize("operation", ["upload", "download"]) -@pytest.mark.parametrize("error_type", [OSError, asyncio.CancelledError]) -async def test_bytes_transfer_cleans_temporary_files_on_error(operation: str, error_type: type[BaseException]) -> None: - sandbox = AsyncMock(spec=AsyncSandbox) - error = error_type("transfer interrupted") - temporary_paths: list[Path] = [] - - async def fail_transfer(source: str | Path, destination: str | Path) -> None: - local = Path(source if operation == "upload" else destination) - assert local.parent.is_dir() - if operation == "download": - local.write_bytes(b"partial") - temporary_paths.append(local) - raise error - - getattr(sandbox, operation).side_effect = fail_transfer - with pytest.raises(error_type) as raised: - if operation == "upload": - await upload_bytes(sandbox, path="/remote.bin", data=b"\x00\xff") - else: - await read_bytes(sandbox, path="/remote.bin") - - assert raised.value is error - assert len(temporary_paths) == 1 - assert not temporary_paths[0].parent.exists() From 2093960be523e5b2ce615e30b4db492f9ca67a14 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 6 Oct 2026 12:11:02 -0700 Subject: [PATCH 31/44] refactor(agent): restore agent_utils package name Signed-off-by: yaoyu-33 --- nemo_gym/{harness => agent_utils}/__init__.py | 0 .../{harness => agent_utils}/process_supervisor.py | 0 nemo_gym/{harness => agent_utils}/sandbox_session.py | 6 +++--- .../{harness => agent_utils}/supervisor_client.py | 2 +- responses_api_agents/hermes_agent/app.py | 2 +- responses_api_agents/hermes_agent/sandbox.py | 2 +- responses_api_agents/hermes_agent/tests/test_app.py | 2 +- .../hermes_agent/tests/test_sandbox.py | 6 +++--- .../hermes_agent/tests/test_sandbox_runner.py | 4 ++-- responses_api_agents/hermes_sandboxed_agent/app.py | 2 +- .../hermes_sandboxed_agent/tests/test_app.py | 2 +- tests/unit_tests/test_local_provider.py | 2 +- tests/unit_tests/test_process_supervisor.py | 2 +- tests/unit_tests/test_sandbox_session.py | 12 ++++++------ .../unit_tests/test_sandbox_supervisor_bootstrap.py | 4 ++-- tests/unit_tests/test_sandbox_supervisor_client.py | 4 ++-- 16 files changed, 26 insertions(+), 26 deletions(-) rename nemo_gym/{harness => agent_utils}/__init__.py (100%) rename nemo_gym/{harness => agent_utils}/process_supervisor.py (100%) rename nemo_gym/{harness => agent_utils}/sandbox_session.py (98%) rename nemo_gym/{harness => agent_utils}/supervisor_client.py (98%) diff --git a/nemo_gym/harness/__init__.py b/nemo_gym/agent_utils/__init__.py similarity index 100% rename from nemo_gym/harness/__init__.py rename to nemo_gym/agent_utils/__init__.py diff --git a/nemo_gym/harness/process_supervisor.py b/nemo_gym/agent_utils/process_supervisor.py similarity index 100% rename from nemo_gym/harness/process_supervisor.py rename to nemo_gym/agent_utils/process_supervisor.py diff --git a/nemo_gym/harness/sandbox_session.py b/nemo_gym/agent_utils/sandbox_session.py similarity index 98% rename from nemo_gym/harness/sandbox_session.py rename to nemo_gym/agent_utils/sandbox_session.py index f80a3300e1c..1b2923c5050 100644 --- a/nemo_gym/harness/sandbox_session.py +++ b/nemo_gym/agent_utils/sandbox_session.py @@ -9,9 +9,9 @@ from pathlib import Path from typing import cast -from nemo_gym.harness import process_supervisor -from nemo_gym.harness.process_supervisor import CleanupReceipt -from nemo_gym.harness.supervisor_client import ( +from nemo_gym.agent_utils import process_supervisor +from nemo_gym.agent_utils.process_supervisor import CleanupReceipt +from nemo_gym.agent_utils.supervisor_client import ( OUTPUT_LOG_FILE, STOP_REQUEST_FILE, SUPERVISOR_FILE, diff --git a/nemo_gym/harness/supervisor_client.py b/nemo_gym/agent_utils/supervisor_client.py similarity index 98% rename from nemo_gym/harness/supervisor_client.py rename to nemo_gym/agent_utils/supervisor_client.py index 9bbce0e221a..689f52c91e0 100644 --- a/nemo_gym/harness/supervisor_client.py +++ b/nemo_gym/agent_utils/supervisor_client.py @@ -26,7 +26,7 @@ from pydantic import TypeAdapter, ValidationError -from nemo_gym.harness.process_supervisor import CLEANUP_PHASE_COUNT, CleanupReceipt, exec_timeout +from nemo_gym.agent_utils.process_supervisor import CLEANUP_PHASE_COUNT, CleanupReceipt, exec_timeout from nemo_gym.sandbox import AsyncSandbox from nemo_gym.sandbox.utils import read_text diff --git a/responses_api_agents/hermes_agent/app.py b/responses_api_agents/hermes_agent/app.py index 162e5672e33..51be6400277 100644 --- a/responses_api_agents/hermes_agent/app.py +++ b/responses_api_agents/hermes_agent/app.py @@ -30,6 +30,7 @@ from pydantic import ConfigDict, Field from toolsets import TOOLSETS # pyright: ignore[reportMissingImports] +from nemo_gym.agent_utils.sandbox_session import SandboxSession from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import ( AgentCloseSessionResponse, @@ -42,7 +43,6 @@ ) from nemo_gym.config_types import ModelServerRef, ResourcesServerRef from nemo_gym.global_config import get_global_config_dict -from nemo_gym.harness.sandbox_session import SandboxSession from nemo_gym.openai_utils import ( NeMoGymEasyInputMessage, NeMoGymFunctionCallOutput, diff --git a/responses_api_agents/hermes_agent/sandbox.py b/responses_api_agents/hermes_agent/sandbox.py index dfd786189e4..50a040dbbcd 100644 --- a/responses_api_agents/hermes_agent/sandbox.py +++ b/responses_api_agents/hermes_agent/sandbox.py @@ -13,8 +13,8 @@ from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError +from nemo_gym.agent_utils.sandbox_session import SandboxCommand, SandboxSession from nemo_gym.base_responses_api_agent import AgentSessionState -from nemo_gym.harness.sandbox_session import SandboxCommand, SandboxSession from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming from nemo_gym.rollout_observability import AgentObservationBundle from nemo_gym.sandbox.utils import read_text, upload_text diff --git a/responses_api_agents/hermes_agent/tests/test_app.py b/responses_api_agents/hermes_agent/tests/test_app.py index 40a13f6864f..d518d09bb94 100644 --- a/responses_api_agents/hermes_agent/tests/test_app.py +++ b/responses_api_agents/hermes_agent/tests/test_app.py @@ -22,6 +22,7 @@ import yaml from fastapi import HTTPException +from nemo_gym.agent_utils.sandbox_session import SandboxSession from nemo_gym.base_responses_api_agent import ( AgentCloseSessionRequest, AgentCloseSessionResponse, @@ -29,7 +30,6 @@ _AgentSessionRecord, ) from nemo_gym.episode_types import EpisodeId, TaskId -from nemo_gym.harness.sandbox_session import SandboxSession from nemo_gym.openai_utils import ( NeMoGymEasyInputMessage, NeMoGymFunctionCallOutput, diff --git a/responses_api_agents/hermes_agent/tests/test_sandbox.py b/responses_api_agents/hermes_agent/tests/test_sandbox.py index 6ad6d520885..9ce7e05d091 100644 --- a/responses_api_agents/hermes_agent/tests/test_sandbox.py +++ b/responses_api_agents/hermes_agent/tests/test_sandbox.py @@ -17,11 +17,11 @@ from fastapi.testclient import TestClient from pydantic import ValidationError +from nemo_gym.agent_utils import process_supervisor +from nemo_gym.agent_utils.sandbox_session import SandboxSession +from nemo_gym.agent_utils.supervisor_client import STOP_REQUEST_FILE, SUPERVISOR_FILE, parse_cleanup_receipt from nemo_gym.base_responses_api_agent import AgentCloseSessionRequest, AgentSeedSessionRequest, _AgentSessionRecord from nemo_gym.episode_types import EpisodeId, TaskId -from nemo_gym.harness import process_supervisor -from nemo_gym.harness.sandbox_session import SandboxSession -from nemo_gym.harness.supervisor_client import STOP_REQUEST_FILE, SUPERVISOR_FILE, parse_cleanup_receipt from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming from nemo_gym.rollout_observability import AgentEpisode, AgentObservationBundle from nemo_gym.server_utils import ServerClient diff --git a/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py b/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py index 9c781b9540a..ccee5738fec 100644 --- a/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py @@ -21,8 +21,8 @@ from run_agent import AIAgent from tools.mcp_tool import shutdown_mcp_servers -from nemo_gym.harness import process_supervisor -from nemo_gym.harness.supervisor_client import STOP_REQUEST_FILE +from nemo_gym.agent_utils import process_supervisor +from nemo_gym.agent_utils.supervisor_client import STOP_REQUEST_FILE from nemo_gym.mcp_auto_exposure import TOKEN_HEADER, maybe_auto_expose from nemo_gym.openai_utils import NeMoGymChatCompletionCreateParamsNonStreaming from nemo_gym.server_utils import ServerClient diff --git a/responses_api_agents/hermes_sandboxed_agent/app.py b/responses_api_agents/hermes_sandboxed_agent/app.py index ca82b9769fa..b5847bf7ea7 100644 --- a/responses_api_agents/hermes_sandboxed_agent/app.py +++ b/responses_api_agents/hermes_sandboxed_agent/app.py @@ -14,11 +14,11 @@ from fastapi import HTTPException, Request from pydantic import ConfigDict, Field +from nemo_gym.agent_utils import process_supervisor from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import BaseResponsesAPIAgentConfig, SimpleResponsesAPIAgent from nemo_gym.config_types import ModelServerRef, ResourcesServerRef from nemo_gym.global_config import get_global_config_dict -from nemo_gym.harness import process_supervisor from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming from nemo_gym.sandbox import AsyncSandbox, create_provider from nemo_gym.sandbox.config import resolve_provider_config diff --git a/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py b/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py index e9c1d188026..a53a6fa2b1a 100644 --- a/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py +++ b/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py @@ -9,7 +9,7 @@ import pytest -from nemo_gym.harness.process_supervisor import DEFAULT_CLEANUP_TIMEOUT +from nemo_gym.agent_utils.process_supervisor import DEFAULT_CLEANUP_TIMEOUT from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming from nemo_gym.sandbox import SandboxExecResult from nemo_gym.server_utils import ServerClient diff --git a/tests/unit_tests/test_local_provider.py b/tests/unit_tests/test_local_provider.py index e027a99570b..67ce10c1849 100644 --- a/tests/unit_tests/test_local_provider.py +++ b/tests/unit_tests/test_local_provider.py @@ -25,7 +25,7 @@ import pytest -from nemo_gym.harness import process_supervisor +from nemo_gym.agent_utils import process_supervisor from nemo_gym.sandbox import AsyncSandbox, SandboxSpec, SandboxStatus, create_provider from nemo_gym.sandbox.providers.local import LocalProvider diff --git a/tests/unit_tests/test_process_supervisor.py b/tests/unit_tests/test_process_supervisor.py index f73f3bef6a7..a69aeb6ca87 100644 --- a/tests/unit_tests/test_process_supervisor.py +++ b/tests/unit_tests/test_process_supervisor.py @@ -16,7 +16,7 @@ import pytest -SUPERVISOR = Path(__file__).parents[2] / "nemo_gym/harness/process_supervisor.py" +SUPERVISOR = Path(__file__).parents[2] / "nemo_gym/agent_utils/process_supervisor.py" @pytest.mark.parametrize("timeout, cleanup", [(1, 0.1), (2700, 10), (21600, 100)]) diff --git a/tests/unit_tests/test_sandbox_session.py b/tests/unit_tests/test_sandbox_session.py index 4677da28ac5..7a210edfa72 100644 --- a/tests/unit_tests/test_sandbox_session.py +++ b/tests/unit_tests/test_sandbox_session.py @@ -7,8 +7,8 @@ import pytest -from nemo_gym.harness import supervisor_client -from nemo_gym.harness.sandbox_session import SandboxCommand, SandboxSession +from nemo_gym.agent_utils import supervisor_client +from nemo_gym.agent_utils.sandbox_session import SandboxCommand, SandboxSession from nemo_gym.sandbox import AsyncSandbox, SandboxExecResult @@ -287,8 +287,8 @@ async def test_bootstrap_failure_preserves_stderr_and_still_releases(session, re async def test_session_uploads_supervisor_before_launch(session): from pathlib import Path - from nemo_gym.harness import process_supervisor - from nemo_gym.harness.supervisor_client import SUPERVISOR_FILE + from nemo_gym.agent_utils import process_supervisor + from nemo_gym.agent_utils.supervisor_client import SUPERVISOR_FILE events = [] @@ -378,7 +378,7 @@ async def disconnect(): async def test_failure_log_is_read_before_session_release(session, monkeypatch): - from nemo_gym.harness import sandbox_session + from nemo_gym.agent_utils import sandbox_session async def read_log(sandbox, *, path): sandbox.disconnect.assert_not_awaited() @@ -398,7 +398,7 @@ async def collect(): async def test_missing_failure_log_does_not_mask_capture_failure(session, monkeypatch): - from nemo_gym.harness import sandbox_session + from nemo_gym.agent_utils import sandbox_session monkeypatch.setattr(sandbox_session, "read_text", AsyncMock(side_effect=OSError("download failed"))) diff --git a/tests/unit_tests/test_sandbox_supervisor_bootstrap.py b/tests/unit_tests/test_sandbox_supervisor_bootstrap.py index 4706eee60e4..015d48b79b8 100644 --- a/tests/unit_tests/test_sandbox_supervisor_bootstrap.py +++ b/tests/unit_tests/test_sandbox_supervisor_bootstrap.py @@ -10,8 +10,8 @@ import pytest -from nemo_gym.harness import process_supervisor -from nemo_gym.harness.supervisor_client import ( +from nemo_gym.agent_utils import process_supervisor +from nemo_gym.agent_utils.supervisor_client import ( CLEANUP_RECEIPT_FILE, LAUNCH_CLAIM_FILE, SUPERVISOR_FILE, diff --git a/tests/unit_tests/test_sandbox_supervisor_client.py b/tests/unit_tests/test_sandbox_supervisor_client.py index 654f32d3815..dfb9cddb550 100644 --- a/tests/unit_tests/test_sandbox_supervisor_client.py +++ b/tests/unit_tests/test_sandbox_supervisor_client.py @@ -11,8 +11,8 @@ import pytest -from nemo_gym.harness import process_supervisor -from nemo_gym.harness.supervisor_client import ( +from nemo_gym.agent_utils import process_supervisor +from nemo_gym.agent_utils.supervisor_client import ( CLEANUP_RECEIPT_FILE, LAUNCH_CLAIM_FILE, OUTPUT_LOG_FILE, From a309dfb9a5cb8c65b6bd7366260c189c3dcd3284 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 6 Oct 2026 13:07:16 -0700 Subject: [PATCH 32/44] fix(hermes): default owned sandbox lifetime Signed-off-by: yaoyu-33 --- responses_api_agents/hermes_agent/app.py | 8 ++++- .../hermes_agent/tests/test_app.py | 36 ++++++++++++++++--- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/responses_api_agents/hermes_agent/app.py b/responses_api_agents/hermes_agent/app.py index 51be6400277..dd84e038ead 100644 --- a/responses_api_agents/hermes_agent/app.py +++ b/responses_api_agents/hermes_agent/app.py @@ -405,7 +405,13 @@ async def _initialize_agent_session_state( try: if owns_sandbox: sandbox = AsyncSandbox(provider) - await sandbox.start(SandboxSpec(**self.config.sandbox_config)) + sandbox_config = self.config.sandbox_config.copy() + # Provider TTLs outlive this server; reserve ten minutes for setup/cleanup overhead. + sandbox_config.setdefault( + "ttl_s", + self.config.sandbox_install_timeout_seconds + self.config.sandbox_runner_timeout_seconds + 600, + ) + await sandbox.start(SandboxSpec(**sandbox_config)) else: sandbox = await AsyncSandbox.connect(connection.descriptor, provider=provider) except BaseException: diff --git a/responses_api_agents/hermes_agent/tests/test_app.py b/responses_api_agents/hermes_agent/tests/test_app.py index d518d09bb94..3656428b132 100644 --- a/responses_api_agents/hermes_agent/tests/test_app.py +++ b/responses_api_agents/hermes_agent/tests/test_app.py @@ -122,7 +122,10 @@ def test_configured_model_overrides_server_name(self) -> None: @pytest.mark.parametrize("with_mcp", [False, True]) async def test_sandbox_access_selects_runtime_provider(self, monkeypatch, with_mcp) -> None: hermes = HermesAgent( - config=_config(enabled_toolsets=["terminal", "web"]), + config=_config( + enabled_toolsets=["terminal", "web"], + sandbox_config={"ttl_s": 123, "workdir": "/unused"}, + ), server_client=MagicMock(spec=ServerClient), ) provider_config = {"opensandbox": {"connection": {}}} @@ -163,7 +166,10 @@ async def test_sandbox_access_selects_runtime_provider(self, monkeypatch, with_m {"sandbox_id": "sandbox"}, provider=provider, ) + sandbox.start.assert_not_awaited() + assert hermes.config.sandbox_config == {"ttl_s": 123, "workdir": "/unused"} assert state.session.sandbox is sandbox + assert state.session.owns_sandbox is False assert state.session.workdir == "/app" assert state.session.session_dir.startswith("/tmp/nemo-gym-hermes-sessions/") assert len(state.session.session_dir.rsplit("/", 1)[-1]) == 32 @@ -217,12 +223,30 @@ async def exec_(command, **_kwargs): assert sum("import run_agent" in command for command in commands) == 2 assert sandbox.upload.await_args_list[0].args[0] == "/usr/bin/uv" - async def test_missing_sandbox_access_uses_configured_fallback(self, monkeypatch) -> None: + @pytest.mark.parametrize( + "sandbox_config, install_timeout, runner_timeout, expected_ttl", + [ + ({}, 900, 21600, 23100), + ({"workdir": "/fallback"}, 2.5, 3.25, 605.75), + ({"workdir": "/fallback", "ttl_s": 42}, 900, 21600, 42), + ({"workdir": "/fallback", "ttl_s": None}, 900, 21600, None), + ], + ) + async def test_missing_sandbox_access_uses_configured_fallback( + self, + monkeypatch: pytest.MonkeyPatch, + sandbox_config: dict[str, object], + install_timeout: float, + runner_timeout: float, + expected_ttl: float | None, + ) -> None: hermes = HermesAgent( config=_config( enabled_toolsets=["terminal", "web"], sandbox_provider="runtime", - sandbox_config={"workdir": "/fallback"}, + sandbox_config=sandbox_config, + sandbox_install_timeout_seconds=install_timeout, + sandbox_runner_timeout_seconds=runner_timeout, ), server_client=MagicMock(spec=ServerClient), ) @@ -230,6 +254,7 @@ async def test_missing_sandbox_access_uses_configured_fallback(self, monkeypatch sandbox = AsyncMock() sandbox.exec.return_value = MagicMock(return_code=0, stdout="", stderr="") sandbox_factory = MagicMock(return_value=sandbox) + original_config = hermes.config.sandbox_config.copy() monkeypatch.setattr( "responses_api_agents.hermes_agent.app.get_global_config_dict", lambda: {"runtime": {}}, @@ -252,9 +277,10 @@ async def test_missing_sandbox_access_uses_configured_fallback(self, monkeypatch ) sandbox_factory.assert_called_once_with(provider) - sandbox.start.assert_awaited_once_with(SandboxSpec(workdir="/fallback")) + sandbox.start.assert_awaited_once_with(SandboxSpec(**{**original_config, "ttl_s": expected_ttl})) + assert hermes.config.sandbox_config == original_config assert state.session.sandbox is sandbox - assert state.session.workdir == "/fallback" + assert state.session.workdir == original_config.get("workdir") assert state.session.owns_sandbox is True await hermes._close_agent_session_state(state) sandbox.stop.assert_awaited_once() From a67ebd73b79e6b6567315e79f3149070677b687a Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 6 Oct 2026 13:40:14 -0700 Subject: [PATCH 33/44] refactor(openclaw): adopt shared sandbox session lifecycle Signed-off-by: yaoyu-33 --- nemo_gym/agent_utils/__init__.py | 15 + .../process_supervisor.py | 19 +- nemo_gym/agent_utils/sandbox_session.py | 259 +++++++++++ nemo_gym/agent_utils/supervisor_client.py | 160 +++++++ nemo_gym/sandbox/runner.py | 110 ----- nemo_gym/sandbox/utils.py | 22 + responses_api_agents/hermes_agent/app.py | 3 +- .../hermes_agent/tests/test_sandbox_runner.py | 2 +- .../hermes_sandboxed_agent/app.py | 3 +- .../hermes_sandboxed_agent/tests/test_app.py | 2 +- responses_api_agents/openclaw_agent/app.py | 72 +-- .../openclaw_agent/sandbox.py | 218 ++++----- .../tests/test_exec_transport.py | 47 +- .../tests/test_lifecycle_contract.py | 61 ++- .../openclaw_agent/tests/test_sandbox.py | 127 ++++-- .../tests/test_sandbox_runner.py | 6 +- tests/unit_tests/test_local_provider.py | 3 +- tests/unit_tests/test_process_supervisor.py | 2 +- .../test_sandbox_runner_transport.py | 166 ------- tests/unit_tests/test_sandbox_session.py | 413 ++++++++++++++++++ .../test_sandbox_supervisor_bootstrap.py | 184 ++++++++ .../test_sandbox_supervisor_client.py | 234 ++++++++++ 22 files changed, 1601 insertions(+), 527 deletions(-) create mode 100644 nemo_gym/agent_utils/__init__.py rename nemo_gym/{sandbox => agent_utils}/process_supervisor.py (89%) create mode 100644 nemo_gym/agent_utils/sandbox_session.py create mode 100644 nemo_gym/agent_utils/supervisor_client.py delete mode 100644 nemo_gym/sandbox/runner.py delete mode 100644 tests/unit_tests/test_sandbox_runner_transport.py create mode 100644 tests/unit_tests/test_sandbox_session.py create mode 100644 tests/unit_tests/test_sandbox_supervisor_bootstrap.py create mode 100644 tests/unit_tests/test_sandbox_supervisor_client.py diff --git a/nemo_gym/agent_utils/__init__.py b/nemo_gym/agent_utils/__init__.py new file mode 100644 index 00000000000..3cb30910363 --- /dev/null +++ b/nemo_gym/agent_utils/__init__.py @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Run harness processes inside task sandboxes under a shared supervisor. + +The harness adapter installs its runtime, stages activation input, and parses +harness output. SandboxSession uploads the supervisor, runs and stops the harness +process, collects artifacts, and releases the sandbox. The supervisor client +implements the shell protocol over session control files. All three run on the +agent server. + +Only process_supervisor.py is copied from this package into the task sandbox. It +uses the standard library to enforce deadlines, kill and reap descendants, and +write the cleanup receipt. Core sandbox APIs and providers live in +:mod:`nemo_gym.sandbox`. +""" diff --git a/nemo_gym/sandbox/process_supervisor.py b/nemo_gym/agent_utils/process_supervisor.py similarity index 89% rename from nemo_gym/sandbox/process_supervisor.py rename to nemo_gym/agent_utils/process_supervisor.py index a00a58765a3..7cea10e447b 100644 --- a/nemo_gym/sandbox/process_supervisor.py +++ b/nemo_gym/agent_utils/process_supervisor.py @@ -4,7 +4,7 @@ """Standalone Linux process supervisor to upload beside a sandboxed harness. Uses only the standard library; Gym need not be installed in the task sandbox. -The receipt confirms descendant cleanup independently of the worker's result. +The receipt confirms descendant cleanup independently of the harness result. """ from __future__ import annotations @@ -23,6 +23,7 @@ DEFAULT_CLEANUP_TIMEOUT = 10.0 +CLEANUP_PHASE_COUNT = 3 class CleanupReceipt(TypedDict): @@ -35,8 +36,12 @@ class CleanupReceipt(TypedDict): def exec_timeout(*, timeout: float, cleanup_timeout: float = DEFAULT_CLEANUP_TIMEOUT) -> float: - """Leave room for TERM grace, worker reaping, descendant draining, and receipt I/O.""" - return timeout + 3 * cleanup_timeout + 30 + """Reserve TERM grace, harness process reaping, and descendant draining. + + The extra 30 seconds cover startup, polling, and receipt I/O beyond those + bounded cleanup phases. Keep their count shared with the controller budget. + """ + return timeout + CLEANUP_PHASE_COUNT * cleanup_timeout + 30 def _drain_children(timeout: float) -> None: @@ -66,7 +71,7 @@ def _supervise( cleanup_timeout: float = DEFAULT_CLEANUP_TIMEOUT, stop_path: Path | None = None, ) -> CleanupReceipt: - """Enforce a worker deadline, then acknowledge cleanup after all descendants exit.""" + """Enforce a harness deadline, then acknowledge cleanup after all descendants exit.""" process = None subreaping = False stopping = False @@ -82,7 +87,7 @@ def interrupt(*_: object) -> None: # TERM can be ignored during interpreter startup. A durable stop marker fences # that window; later signals are handled without interrupting Popen. if stopping or (stop_path is not None and stop_path.exists()): - return receipt # The finally block confirms that no worker was launched. + return receipt # The finally block confirms that no harness process was launched. if sys.platform != "linux": raise RuntimeError("Sandbox process supervision requires Linux") # Leave the provider group when possible. An exec launcher may already make us @@ -99,7 +104,7 @@ def interrupt(*_: object) -> None: time.sleep(0.05) if process.poll() is None: receipt["timed_out"] = not stopping - # Let the worker checkpoint before the bounded hard cleanup. + # Let the harness checkpoint before the bounded hard cleanup. process.send_signal(signal.SIGTERM) try: process.wait(timeout=cleanup_timeout) @@ -143,7 +148,7 @@ def main() -> int: args = parser.parse_args() command = args.command[1:] if args.command[:1] == ["--"] else args.command if not command: - parser.error("a worker command is required after --") + parser.error("a harness command is required after --") receipt = _supervise(command, timeout=args.timeout, cleanup_timeout=args.cleanup_timeout, stop_path=args.stop_file) temporary = args.receipt.with_suffix(".tmp") temporary.write_text(json.dumps(receipt)) diff --git a/nemo_gym/agent_utils/sandbox_session.py b/nemo_gym/agent_utils/sandbox_session.py new file mode 100644 index 00000000000..1b2923c5050 --- /dev/null +++ b/nemo_gym/agent_utils/sandbox_session.py @@ -0,0 +1,259 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Agent-server lifecycle for one supervised harness activation.""" + +import asyncio +import logging +from collections.abc import Awaitable, Callable, Coroutine +from dataclasses import dataclass, field +from pathlib import Path +from typing import cast + +from nemo_gym.agent_utils import process_supervisor +from nemo_gym.agent_utils.process_supervisor import CleanupReceipt +from nemo_gym.agent_utils.supervisor_client import ( + OUTPUT_LOG_FILE, + STOP_REQUEST_FILE, + SUPERVISOR_FILE, + remove_session_directory, + stop_and_confirm_cleanup, + supervised_launch_command, + supervision_timeouts, +) +from nemo_gym.sandbox.api import AsyncSandbox +from nemo_gym.sandbox.providers.base import SandboxExecResult +from nemo_gym.sandbox.utils import read_text + + +LOG = logging.getLogger(__name__) + + +def _reuse_or_start[T]( + task: asyncio.Task[T] | None, factory: Callable[[], Coroutine[object, object, T]] +) -> asyncio.Task[T]: + if task is None or (task.done() and (task.cancelled() or task.exception() is not None)): + return asyncio.create_task(factory()) + return task + + +@dataclass(frozen=True, kw_only=True) +class SandboxCommand: + """Harness argv and the installed interpreter used to launch its supervisor.""" + + argv: list[str] + python: str + + +@dataclass(kw_only=True) +class SandboxSession[Artifacts]: + """Execute, stop, capture, then release an owned or borrowed sandbox. + + Runtime installation belongs in the adapter's seed hook. ``stage_activation`` + stages input and returns a command; ``collect`` copies harness artifacts + to the agent server after cleanup is confirmed. It may run a bounded snapshot + command, but must not restart the harness. Parsing responses stays in the adapter. + + A new session stages input and its supervisor, runs the harness process, then + stops and collects artifacts. Execution retains the sandbox until close + releases it. A failed borrowed cleanup remains closing and can be retried; + an owned sandbox can fall back to provider stop. Closed means release succeeded. + + Each stop, collection and release phase is bounded by ``close_timeout`` (or + the timeout passed to ``close``). Failed cleanup/release retains the handle + for retry. Failed capture is recorded separately and does not prevent release. + HTTP request binding and activation retries belong to the agent session layer. + """ + + sandbox: AsyncSandbox + session_dir: str + workdir: str | None + harness: str + owns_sandbox: bool = False + launch_started: bool = field(default=False, init=False) + cleanup: CleanupReceipt | None = field(default=None, init=False) + artifacts: Artifacts | None = field(default=None, init=False) + capture_error: Exception | None = field(default=None, init=False) + sandbox_stopped: bool = field(default=False, init=False) + _stage_task: asyncio.Future[SandboxCommand] | None = field(default=None, init=False) + _exec_task: asyncio.Task[SandboxExecResult] | None = field(default=None, init=False) + _finalize_task: asyncio.Task[None] | None = field(default=None, init=False) + _close_task: asyncio.Task[None] | None = field(default=None, init=False) + _collect: Callable[[], Awaitable[Artifacts]] | None = field(default=None, init=False) + _capture_attempted: bool = field(default=False, init=False) + + @property + def closing(self) -> bool: + """Whether close has been requested, including a failed attempt.""" + return self._close_task is not None + + @property + def closed(self) -> bool: + """Whether sandbox release completed successfully.""" + task = self._close_task + return task is not None and task.done() and not task.cancelled() and task.exception() is None + + @property + def stop_request_path(self) -> str: + """Marker path an adapter may pass to its harness for cancellation diagnostics.""" + return f"{self.session_dir}/{STOP_REQUEST_FILE}" + + async def read_output_log(self) -> str: + """Read combined supervisor/harness diagnostics without masking the original failure.""" + try: + return await read_text(self.sandbox, path=f"{self.session_dir}/{OUTPUT_LOG_FILE}") + except Exception: + LOG.warning("Could not read %s harness output log", self.harness, exc_info=True) + return "" + + async def _stage(self, stage_activation: Callable[[], Awaitable[SandboxCommand]]) -> SandboxCommand: + command = await stage_activation() + await self.sandbox.upload(Path(process_supervisor.__file__), f"{self.session_dir}/{SUPERVISOR_FILE}") + return command + + async def execute( + self, + *, + stage_activation: Callable[[], Awaitable[SandboxCommand]], + collect: Callable[[], Awaitable[Artifacts]], + timeout: float, + close_timeout: float, + ) -> Artifacts: + """Run once, preserving captured artifacts even when execution raises. + + Cancellation stops the remote harness before releasing provider exec. + Callers that merely stop waiting (such as disconnected HTTP requests) + must shield their shared activation task. + """ + if self.closing or self._stage_task is not None: + raise RuntimeError(f"{self.harness} sandbox session is closing or already activated") + # Register both hooks before yielding, so close can fence preparation too. + self._collect = collect + self._stage_task = asyncio.create_task(self._stage(stage_activation)) + try: + command = await asyncio.shield(self._stage_task) + if self.closing or self._finalize_task is not None: + raise RuntimeError(f"{self.harness} sandbox session closed before launch") + cleanup_timeout, provider_timeout = supervision_timeouts(timeout=timeout, close_timeout=close_timeout) + launch = supervised_launch_command( + session_dir=self.session_dir, + command=command.argv, + timeout=timeout, + cleanup_timeout=cleanup_timeout, + python=command.python, + ) + self.launch_started = True + self._exec_task = asyncio.create_task( + self.sandbox.exec( + launch, + cwd=self.workdir, + timeout_s=provider_timeout, + ) + ) + result = await asyncio.shield(self._exec_task) + if result.error_type == "timeout": + raise TimeoutError(f"{self.harness} sandbox supervisor exceeded its execution deadline") + if result.error_type: + raise RuntimeError(f"{self.harness} sandbox execution failed: {result.error_type}: {result.stderr}") + # A harness process's nonzero exit can still carry useful, adapter-specific output. + except BaseException: + try: + await self._finish(timeout=close_timeout) + except Exception: + LOG.exception("%s cleanup remains unconfirmed; close must retry", self.harness) + raise + await self._finish(timeout=close_timeout) + if result.return_code != 0 and self.capture_error is not None: + raise RuntimeError( + f"{self.harness} sandbox execution failed (exit {result.return_code}): {result.stderr or ''}" + ) from self.capture_error + if self.capture_error is not None: + raise self.capture_error + # Collection either returned artifacts (including a valid None) or recorded an error. + return cast(Artifacts, self.artifacts) + + async def stop_harness(self, *, timeout: float) -> None: + """Fence a delayed launch or confirm descendant cleanup before transport cancellation.""" + if self.sandbox_stopped or not self.launch_started or self.cleanup is not None: + return + try: + async with asyncio.timeout(timeout): + self.cleanup = await stop_and_confirm_cleanup( + self.sandbox, + session_dir=self.session_dir, + workdir=self.workdir, + timeout=timeout, + harness=self.harness, + ) + except TimeoutError as error: + raise RuntimeError(f"{self.harness} launch outcome is unknown; cleanup deadline exceeded") from error + + async def _finish(self, *, timeout: float) -> None: + self._finalize_task = _reuse_or_start(self._finalize_task, lambda: self._finalize(timeout=timeout)) + await asyncio.shield(self._finalize_task) + + async def _cancel_and_wait[T](self, task: asyncio.Future[T], *, timeout: float) -> None: + if not task.done(): + task.cancel() + try: + await asyncio.wait_for(asyncio.shield(task), timeout=timeout) + except asyncio.CancelledError: + if not task.cancelled(): + raise + except Exception: + if not task.done(): + raise + # Execution/preparation errors are propagated by execute, independently of cleanup. + + async def _finalize(self, *, timeout: float) -> None: + if self._stage_task is not None: + await self._cancel_and_wait(self._stage_task, timeout=timeout) + await self.stop_harness(timeout=timeout) + if self.launch_started and not self._capture_attempted and self._collect is not None: + if self.sandbox_stopped: + self.capture_error = RuntimeError(f"{self.harness} sandbox stopped before artifact capture") + else: + try: + async with asyncio.timeout(timeout): + self.artifacts = await self._collect() + self.capture_error = None + except Exception as error: + self.capture_error = error + LOG.exception("%s artifact capture failed after confirmed cleanup", self.harness) + self._capture_attempted = True + await self._release_exec(timeout=timeout) + + async def _release_exec(self, *, timeout: float) -> None: + if (self.cleanup is not None or self.sandbox_stopped) and self._exec_task is not None: + await self._cancel_and_wait(self._exec_task, timeout=timeout) + + async def close(self, *, timeout: float) -> None: + """Join finalization before release; concurrent/retried closes never duplicate capture.""" + self._close_task = _reuse_or_start(self._close_task, lambda: self._close(timeout=timeout)) + await asyncio.shield(self._close_task) + + async def _close(self, *, timeout: float) -> None: + try: + await self._finish(timeout=timeout) + except Exception: + if not self.owns_sandbox: + raise + # Container teardown remains the authority if graceful cleanup is unavailable. + LOG.exception("%s graceful cleanup failed; stopping owned sandbox", self.harness) + if not self._capture_attempted: + self.capture_error = RuntimeError(f"{self.harness} artifact capture unavailable during forced stop") + if self.owns_sandbox: + if not self.sandbox_stopped: + async with asyncio.timeout(timeout): + await self.sandbox.stop() + self.sandbox_stopped = True + await self._release_exec(timeout=timeout) + else: + async with asyncio.timeout(timeout): + await remove_session_directory( + self.sandbox, + session_dir=self.session_dir, + workdir=self.workdir, + timeout=timeout, + harness=self.harness, + ) + await self.sandbox.disconnect() diff --git a/nemo_gym/agent_utils/supervisor_client.py b/nemo_gym/agent_utils/supervisor_client.py new file mode 100644 index 00000000000..689f52c91e0 --- /dev/null +++ b/nemo_gym/agent_utils/supervisor_client.py @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Controller-side transport for the shared sandbox process supervisor. + +Session ownership and cancellation live in sandbox_session.py; output parsing stays with the adapter. +Unlike process_supervisor.py, this module is not uploaded to the task sandbox. + +Session control files (all paths are relative to session_dir): + +===================== ==================== =========================================== +Filename Writer When +===================== ==================== =========================================== +process_supervisor.py Session Uploaded before launch +launch.claim Launch or stop shell Atomically claims launch or fences it +supervisor.pid Launch shell After claiming launch, before exec +stop.request Stop shell Before signalling the supervisor +cleanup.json Supervisor or stop After cleanup, or when stop fences launch +output.log Launch shell Captures supervisor and harness diagnostics +===================== ==================== =========================================== +""" + +import json +import logging +from shlex import join, quote +from uuid import uuid4 + +from pydantic import TypeAdapter, ValidationError + +from nemo_gym.agent_utils.process_supervisor import CLEANUP_PHASE_COUNT, CleanupReceipt, exec_timeout +from nemo_gym.sandbox import AsyncSandbox +from nemo_gym.sandbox.utils import read_text + + +SUPERVISOR_FILE = "process_supervisor.py" +LAUNCH_CLAIM_FILE = "launch.claim" +SUPERVISOR_PID_FILE = "supervisor.pid" +STOP_REQUEST_FILE = "stop.request" +CLEANUP_RECEIPT_FILE = "cleanup.json" +OUTPUT_LOG_FILE = "output.log" + +_CLEANUP_RECEIPT = TypeAdapter(CleanupReceipt) +LOG = logging.getLogger(__name__) + + +def parse_cleanup_receipt(payload: object) -> CleanupReceipt: + """Require positive cleanup evidence, tolerating malformed optional diagnostics.""" + if not isinstance(payload, dict) or payload.get("cleanup_confirmed") is not True: + raise ValueError("Sandbox cleanup was not confirmed") + try: + return _CLEANUP_RECEIPT.validate_python(payload, strict=True, extra="forbid") + except ValidationError: + LOG.warning("Cleanup was confirmed, but its diagnostic fields did not match the receipt schema") + return { + "cleanup_confirmed": True, + "return_code": payload.get("return_code") if type(payload.get("return_code")) is int else None, + "timed_out": payload.get("timed_out") is True, + "error": payload.get("error") if isinstance(payload.get("error"), str) else None, + } + + +def supervision_timeouts(*, timeout: float, close_timeout: float) -> tuple[float, float]: + """Return the per-cleanup-phase and provider execution deadlines. + + Divide the close budget across TERM grace, harness process reaping, and + descendant draining. The provider deadline reserves all three phases after + the harness timeout, plus 30 seconds for startup, polling, and receipt I/O. + """ + cleanup_timeout = close_timeout / CLEANUP_PHASE_COUNT + return cleanup_timeout, exec_timeout(timeout=timeout, cleanup_timeout=cleanup_timeout) + + +def supervised_launch_command( + *, + session_dir: str, + command: list[str], + timeout: float, + cleanup_timeout: float, + python: str, +) -> str: + """Fence delayed launches and run a harness command under the shared supervisor. + + The adapter must install or select the interpreter explicitly. Check that it + can load the supervisor before claiming a launch: a failed bootstrap cannot + write a cleanup receipt. Supervisor and harness diagnostics share output.log. + """ + supervisor = quote(f"{session_dir}/{SUPERVISOR_FILE}") + claim_path = quote(f"{session_dir}/{LAUNCH_CLAIM_FILE}") + return ( + f"[ -d {quote(session_dir)} ] && [ ! -L {claim_path} ] || exit 0; " + f"{quote(python)} -I {supervisor} --help >/dev/null || exit $?; " + f"trap '' TERM; ln -s launch {claim_path} 2>/dev/null || exit 0; " + f"echo $$ > {quote(f'{session_dir}/{SUPERVISOR_PID_FILE}')} && " + f"exec {quote(python)} -I {supervisor} " + f"--timeout {timeout} --cleanup-timeout {cleanup_timeout} " + f"--stop-file {quote(f'{session_dir}/{STOP_REQUEST_FILE}')} " + f"--receipt {quote(f'{session_dir}/{CLEANUP_RECEIPT_FILE}')} -- {join(command)} " + f">{quote(f'{session_dir}/{OUTPUT_LOG_FILE}')} 2>&1" + ) + + +async def stop_and_confirm_cleanup( + sandbox: AsyncSandbox, *, session_dir: str, workdir: str | None, timeout: float, harness: str +) -> CleanupReceipt: + """Fence a pending launch or require explicit supervisor acknowledgement before teardown. + + A stop that wins the launch claim writes the same receipt shape as the + supervisor, with no return code because no harness process ran. + """ + receipt_path = f"{session_dir}/{CLEANUP_RECEIPT_FILE}" + try: + receipt = json.loads(await read_text(sandbox, path=receipt_path)) + except Exception: + receipt = {} + if not isinstance(receipt, dict) or receipt.get("cleanup_confirmed") is not True: + pid_path = quote(f"{session_dir}/{SUPERVISOR_PID_FILE}") + stop_path = quote(f"{session_dir}/{STOP_REQUEST_FILE}") + claim_path = quote(f"{session_dir}/{LAUNCH_CLAIM_FILE}") + temporary = quote(f"{receipt_path}.{uuid4().hex}.tmp") + stopped = quote( + json.dumps({"cleanup_confirmed": True, "return_code": None, "error": None, "timed_out": False}) + ) + script = ( + f"[ -f {quote(receipt_path)} ] && exit 0; " + f"touch {stop_path} || exit 1; " + f"ln -s stop {claim_path} 2>/dev/null || true; " + f'if [ "$(readlink {claim_path})" = stop ]; then ' + f"printf '%s' {stopped} > {temporary} && mv {temporary} {quote(receipt_path)}; exit $?; fi; " + f'if [ -s {pid_path} ]; then kill -TERM "$(cat {pid_path})" 2>/dev/null || true; fi; ' + f"for _ in $(seq 1 {max(1, int(timeout))}); do " + f"[ -f {quote(receipt_path)} ] && exit 0; sleep 1; done; exit 1" + ) + await sandbox.exec(script, cwd=workdir, timeout_s=timeout + 5) + try: + receipt = json.loads(await read_text(sandbox, path=receipt_path)) + except Exception as error: + raise RuntimeError(f"{harness} launch outcome is unknown; cannot confirm termination") from error + if not isinstance(receipt, dict) or receipt.get("cleanup_confirmed") is not True: + detail = receipt.get("error") if isinstance(receipt, dict) else "cleanup receipt is not a JSON object" + raise RuntimeError(f"{harness} sandbox cleanup was not confirmed: {detail}") + return parse_cleanup_receipt(receipt) + + +async def remove_session_directory( + sandbox: AsyncSandbox, *, session_dir: str, workdir: str | None, timeout: float, harness: str +) -> None: + """Remove adapter-owned files after cleanup, keeping delayed launches fenced. + + Retire the directory atomically before unlinking its claim. Otherwise a + delayed launch could win the claim while recursive deletion is in progress. + A failed removal leaves the retired path available for a close retry. + """ + retired = f"{session_dir}.closed" + result = await sandbox.exec( + f"if [ -d {quote(session_dir)} ]; then " + f"mv {quote(session_dir)} {quote(retired)} || exit 1; fi; rm -rf -- {quote(retired)}", + cwd=workdir, + timeout_s=timeout, + ) + if result.return_code != 0 or result.error_type: + raise RuntimeError(f"Could not remove {harness} session files") diff --git a/nemo_gym/sandbox/runner.py b/nemo_gym/sandbox/runner.py deleted file mode 100644 index a609fe964da..00000000000 --- a/nemo_gym/sandbox/runner.py +++ /dev/null @@ -1,110 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Controller-side transport for the shared sandbox process supervisor. - -Session ownership, cancellation, and harness output parsing stay with the adapter. -Unlike process_supervisor.py, this module is not uploaded to the task sandbox. -""" - -import json -import tempfile -from pathlib import Path -from shlex import join, quote -from uuid import uuid4 - -from pydantic import BaseModel, ConfigDict, TypeAdapter - -from nemo_gym.sandbox import AsyncSandbox -from nemo_gym.sandbox.process_supervisor import CleanupReceipt - - -_CLEANUP_RECEIPT = TypeAdapter(CleanupReceipt) - - -class RunnerRuntimeInfo(BaseModel): - """Worker identity, independent of supervisor cleanup and harness output.""" - - model_config = ConfigDict(extra="forbid", strict=True) - hostname: str - pid: int - python: str | None = None - - -def parse_cleanup_receipt(payload: object) -> CleanupReceipt: - """Validate the supervisor's existing schema without adding worker dependencies. - - Older stop-before-launch receipts contain only acknowledgement and error. - Normalize that form without inventing a successful worker exit code. - """ - if isinstance(payload, dict) and payload.keys() == {"cleanup_confirmed", "error"}: - payload = {**payload, "return_code": None, "timed_out": False} - return _CLEANUP_RECEIPT.validate_python(payload, strict=True, extra="forbid") - - -async def upload_text(sandbox: AsyncSandbox, *, path: str, text: str) -> None: - """Upload adapter-owned text without interpolating its contents into a shell command.""" - with tempfile.TemporaryDirectory(prefix="sandbox-runner-upload-") as directory: - source = Path(directory) / "payload" - source.write_text(text) - await sandbox.upload(source, path) - - -async def read_text(sandbox: AsyncSandbox, *, path: str) -> str: - """Download an adapter-owned artifact, preserving non-UTF8 output with replacement.""" - with tempfile.TemporaryDirectory(prefix="sandbox-runner-download-") as directory: - destination = Path(directory) / "payload" - await sandbox.download(path, destination) - return destination.read_text(errors="replace") - - -def supervisor_command(*, directory: str, command: list[str], timeout: float, cleanup_timeout: float) -> str: - """Fence delayed launches and run a harness command under the shared supervisor.""" - return ( - f"trap '' TERM; ln -s launch {quote(directory + '/launch.claim')} 2>/dev/null || exit 0; " - f"echo $$ > {quote(directory + '/runner.pid')} && " - f"exec python3 -I {quote(directory + '/process_supervisor.py')} " - f"--timeout {timeout} --cleanup-timeout {cleanup_timeout} " - f"--stop-file {quote(directory + '/runner.stop')} " - f"--receipt {quote(directory + '/cleanup.json')} -- {join(command)} " - f">{quote(directory + '/runner.log')} 2>&1" - ) - - -async def confirm_runner_cleanup( - sandbox: AsyncSandbox, *, directory: str, workdir: str, timeout: float, harness: str -) -> CleanupReceipt: - """Fence a pending launch or require explicit supervisor acknowledgement before teardown. - - A stop that wins the launch claim writes a minimal receipt: no worker ran, so - runtime metadata and a return code are intentionally absent. Worker metadata - and harness output are validated separately by the adapter. - """ - receipt_path = f"{directory}/cleanup.json" - try: - receipt = json.loads(await read_text(sandbox, path=receipt_path)) - except Exception: - receipt = {} - if receipt.get("cleanup_confirmed") is not True: - pid_path = quote(f"{directory}/runner.pid") - stop_path = quote(f"{directory}/runner.stop") - claim_path = quote(f"{directory}/launch.claim") - temporary = quote(f"{receipt_path}.{uuid4().hex}.tmp") - stopped = quote(json.dumps({"cleanup_confirmed": True, "error": None})) - script = ( - f"[ -f {quote(receipt_path)} ] && exit 0; " - f"touch {stop_path} || exit 1; " - f"ln -s stop {claim_path} 2>/dev/null || true; " - f'if [ "$(readlink {claim_path})" = stop ]; then ' - f"printf '%s' {stopped} > {temporary} && mv {temporary} {quote(receipt_path)}; exit $?; fi; " - f'if [ -s {pid_path} ]; then kill -TERM "$(cat {pid_path})" 2>/dev/null || true; fi; ' - f"for _ in $(seq 1 {max(1, int(timeout))}); do " - f"[ -f {quote(receipt_path)} ] && exit 0; sleep 1; done; exit 1" - ) - await sandbox.exec(script, cwd=workdir, timeout_s=timeout + 5) - try: - receipt = json.loads(await read_text(sandbox, path=receipt_path)) - except Exception as error: - raise RuntimeError(f"{harness} launch outcome is unknown; cannot confirm termination") from error - if receipt.get("cleanup_confirmed") is not True: - raise RuntimeError(f"{harness} sandbox cleanup was not confirmed: {receipt.get('error')}") - return parse_cleanup_receipt(receipt) diff --git a/nemo_gym/sandbox/utils.py b/nemo_gym/sandbox/utils.py index f9dd70a9c1e..4d98354ffca 100644 --- a/nemo_gym/sandbox/utils.py +++ b/nemo_gym/sandbox/utils.py @@ -14,6 +14,12 @@ """Sandbox utility helpers.""" +import tempfile +from pathlib import Path + +from nemo_gym.sandbox.api import AsyncSandbox + + # Parallelism caps for CPU-limited sandboxes, each set by cpu_cap_env() to the # floored CPU limit (min 1). Tools size worker pools by host core count, not # the cgroup limit — Python's multiprocessing.cpu_count(), BLAS/OpenMP thread @@ -47,6 +53,22 @@ ) +async def upload_text(sandbox: AsyncSandbox, *, path: str, text: str) -> None: + """Upload UTF-8 text through file transfer without shell interpolation.""" + with tempfile.TemporaryDirectory(prefix="sandbox-upload-") as directory: + source = Path(directory) / "payload" + source.write_text(text, encoding="utf-8") + await sandbox.upload(source, path) + + +async def read_text(sandbox: AsyncSandbox, *, path: str) -> str: + """Download UTF-8 text, replacing invalid bytes in partial process output.""" + with tempfile.TemporaryDirectory(prefix="sandbox-download-") as directory: + destination = Path(directory) / "payload" + await sandbox.download(path, destination) + return destination.read_text(encoding="utf-8", errors="replace") + + def cpu_cap_env(cpu: float | int | None) -> dict[str, str]: """CPU_CAP_ENV_VARS derived from a sandbox CPU limit; empty when unset.""" if cpu is None: diff --git a/responses_api_agents/hermes_agent/app.py b/responses_api_agents/hermes_agent/app.py index 45355923db7..3913835a8f0 100644 --- a/responses_api_agents/hermes_agent/app.py +++ b/responses_api_agents/hermes_agent/app.py @@ -36,6 +36,7 @@ from pydantic import ConfigDict, Field from toolsets import TOOLSETS # pyright: ignore[reportMissingImports] +from nemo_gym.agent_utils import process_supervisor from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import ( AgentCloseSessionResponse, @@ -72,7 +73,7 @@ ObservationGap, ToolCallObservation, ) -from nemo_gym.sandbox import AsyncSandbox, SandboxSpec, process_supervisor +from nemo_gym.sandbox import AsyncSandbox, SandboxSpec from nemo_gym.sandbox.access import DirectSandboxConnection from nemo_gym.sandbox.config import resolve_provider_config from nemo_gym.sandbox.providers import create_provider diff --git a/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py b/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py index e12350413fd..a0fbdc177c2 100644 --- a/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py @@ -20,9 +20,9 @@ from run_agent import AIAgent from tools.mcp_tool import shutdown_mcp_servers +from nemo_gym.agent_utils import process_supervisor from nemo_gym.mcp_auto_exposure import TOKEN_HEADER, maybe_auto_expose from nemo_gym.openai_utils import NeMoGymChatCompletionCreateParamsNonStreaming -from nemo_gym.sandbox import process_supervisor from nemo_gym.server_utils import ServerClient from resources_servers.example_mcp_weather.app import ( ExampleMCPWeatherResourcesServer, diff --git a/responses_api_agents/hermes_sandboxed_agent/app.py b/responses_api_agents/hermes_sandboxed_agent/app.py index 50bfa578cac..b5847bf7ea7 100644 --- a/responses_api_agents/hermes_sandboxed_agent/app.py +++ b/responses_api_agents/hermes_sandboxed_agent/app.py @@ -14,12 +14,13 @@ from fastapi import HTTPException, Request from pydantic import ConfigDict, Field +from nemo_gym.agent_utils import process_supervisor from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import BaseResponsesAPIAgentConfig, SimpleResponsesAPIAgent from nemo_gym.config_types import ModelServerRef, ResourcesServerRef from nemo_gym.global_config import get_global_config_dict from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming -from nemo_gym.sandbox import AsyncSandbox, create_provider, process_supervisor +from nemo_gym.sandbox import AsyncSandbox, create_provider from nemo_gym.sandbox.config import resolve_provider_config from nemo_gym.server_utils import get_response_json, is_nemo_gym_fastapi_entrypoint, raise_for_status from responses_api_agents.hermes_sandboxed_agent.runner import classify_stop, split_input diff --git a/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py b/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py index 346480a0b1f..a53a6fa2b1a 100644 --- a/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py +++ b/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py @@ -9,9 +9,9 @@ import pytest +from nemo_gym.agent_utils.process_supervisor import DEFAULT_CLEANUP_TIMEOUT from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming from nemo_gym.sandbox import SandboxExecResult -from nemo_gym.sandbox.process_supervisor import DEFAULT_CLEANUP_TIMEOUT from nemo_gym.server_utils import ServerClient from responses_api_agents.hermes_sandboxed_agent.app import ( HermesSandboxedAgent, diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index 3ce8023d25a..2cc19449cf1 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -34,6 +34,7 @@ from fastapi import HTTPException, Request from pydantic import ConfigDict, Field, PrivateAttr +from nemo_gym.agent_utils.sandbox_session import SandboxSession from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import ( AgentCloseSessionResponse, @@ -64,7 +65,7 @@ AgentObservationBundle, ObservationGap, ) -from nemo_gym.sandbox import AsyncSandbox, SandboxSpec, create_provider, process_supervisor +from nemo_gym.sandbox import AsyncSandbox, SandboxSpec, create_provider from nemo_gym.sandbox.access import DirectSandboxConnection from nemo_gym.sandbox.config import resolve_provider_config from nemo_gym.server_utils import get_global_config_dict, get_response_json, raise_for_status @@ -494,7 +495,13 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Open raise directory = f"/tmp/nemo-gym-openclaw-sessions/{uuid4().hex}" runtime = f"/tmp/nemo-gym-openclaw-node-22.19.0-{self.config.openclaw_version}" - state = OpenClawSandboxSession(body, sandbox, directory, runtime, workdir=workdir, owns_sandbox=owns_sandbox) + state = OpenClawSandboxSession( + request=body, + session=SandboxSession( + sandbox=sandbox, session_dir=directory, workdir=workdir, harness="OpenClaw", owns_sandbox=owns_sandbox + ), + runtime=runtime, + ) prepared_directory = False try: if owns_sandbox: @@ -541,16 +548,13 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Open f"{installed.stderr or installed.stdout}" ) await sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), f"{directory}/sandbox_runner.py") - await sandbox.upload(Path(process_supervisor.__file__), f"{directory}/process_supervisor.py") except BaseException as error: try: if prepared_directory or owns_sandbox: await state.close(self.config.session_close_timeout_seconds) else: # The path check may have rejected overlap with task-owned storage. - state.closing = True await sandbox.disconnect() - state.closed = True except BaseException: LOG.exception("OpenClaw seed cleanup failed; retaining session %s", body.agent_session_id) raise AgentSessionSetupError(state, error=error) from error @@ -652,7 +656,7 @@ async def _sandbox_response( { "agents": { "defaults": { - "workspace": state.workdir, + "workspace": state.session.workdir, "skipBootstrap": True, "skills": [], "model": {"primary": self._effective_model()}, @@ -667,7 +671,6 @@ async def _sandbox_response( "plugins": {"enabled": False}, } ) - await state.upload_json("home/.openclaw/openclaw.json", native_config) command = [ f"{state.runtime}/node/bin/node", f"{state.runtime}/openclaw/node_modules/openclaw/openclaw.mjs", @@ -677,26 +680,27 @@ async def _sandbox_response( "--agent", "main", "--session-id", - state.directory.rsplit("/", 1)[-1], + state.session.session_dir.rsplit("/", 1)[-1], "--thinking", self.config.thinking, "--model", self._effective_model(), "--message-file", - f"{state.directory}/prompt.txt", + f"{state.session.session_dir}/prompt.txt", "--timeout", str(self.config.timeout), ] payload = { - "directory": state.directory, + "config": native_config, + "directory": state.session.session_dir, "command": command, "prompt": f"{system}\n\n{prompt}" if system else prompt, - "cwd": state.workdir, + "cwd": state.session.workdir, "env": { - "HOME": f"{state.directory}/home", - "XDG_CACHE_HOME": f"{state.directory}/home/.cache", - "OPENCLAW_STATE_DIR": f"{state.directory}/home/.openclaw", - "OPENCLAW_CONFIG_PATH": f"{state.directory}/home/.openclaw/openclaw.json", + "HOME": f"{state.session.session_dir}/home", + "XDG_CACHE_HOME": f"{state.session.session_dir}/home/.cache", + "OPENCLAW_STATE_DIR": f"{state.session.session_dir}/home/.openclaw", + "OPENCLAW_CONFIG_PATH": f"{state.session.session_dir}/home/.openclaw/openclaw.json", "OPENCLAW_TELEMETRY": "0", "CLAWHUB_DISABLE_TELEMETRY": "1", "OPENCLAW_EXEC_SHELL_SNAPSHOT": "0", @@ -704,7 +708,7 @@ async def _sandbox_response( } try: async with self.sem: - stdout = await state.execute( + await state.execute( payload, timeout=self.config.timeout, close_timeout=self.config.session_close_timeout_seconds ) except BaseException: @@ -715,7 +719,7 @@ async def _sandbox_response( except Exception: LOG.exception("Could not salvage interrupted OpenClaw observations") raise - response = await self._collect_sandbox_response(state, body, prompt=prompt, system=system, stdout=stdout) + response = await self._collect_sandbox_response(state, body, prompt=prompt, system=system) if response.error is not None: # Preserve the transcript in session observations, but let the environment # classify runtime/provider failures before it invokes the verifier. @@ -729,20 +733,16 @@ async def _collect_sandbox_response( *, prompt: str, system: str, - stdout: str = "", ) -> NeMoGymResponse: gaps = [] - if not stdout: - try: - stdout = await state.read_text("stdout.log") - except Exception: - gaps.append(ObservationGap(code="agent_stdout_unavailable")) - session_id = state.directory.rsplit("/", 1)[-1] - events = [] - try: - transcript = await state.read_text(f"home/.openclaw/agents/main/sessions/{session_id}.jsonl") - events = parse_openclaw_session_events(transcript) - except Exception: + artifacts = state.session.artifacts or {} + stdout = artifacts.get("stdout.log", "") + if "stdout.log" not in artifacts: + gaps.append(ObservationGap(code="agent_stdout_unavailable")) + session_id = state.session.session_dir.rsplit("/", 1)[-1] + transcript_name = f"home/.openclaw/agents/main/sessions/{session_id}.jsonl" + events = parse_openclaw_session_events(artifacts.get(transcript_name, "")) + if transcript_name not in artifacts: gaps.append(ObservationGap(code="agent_transcript_unavailable")) # include_input=True preserves reasoning fields; only remove the input # messages when projecting the transcript into Responses output. @@ -837,8 +837,12 @@ def count(name: str) -> int: ObservationGap(code="model_call_usage_unavailable", detail="Only CLI envelope totals available") ) gaps.append(ObservationGap(code="reasoning_token_usage_unavailable")) - result = state.cleanup + result = state.session.cleanup runtime = state.runtime_info + if runtime is None: + gaps.append(ObservationGap(code="runtime_info_unavailable")) + if result is not None and result["return_code"] is None: + gaps.append(ObservationGap(code="worker_exit_code_unavailable")) error = result["error"] if result else "OpenClaw activation ended without a cleanup receipt" last = assistants[-1] if assistants else {} if last.get("stopReason") == "error" or ( @@ -846,10 +850,7 @@ def count(name: str) -> int: ): error = error or last.get("errorMessage") or f"OpenClaw model call {last['stopReason']}" if result and result["return_code"] and not result["timed_out"]: - try: - stderr = (await state.read_text("stderr.log"))[-4000:] - except Exception: - stderr = "stderr unavailable" + stderr = artifacts.get("stderr.log", "stderr unavailable")[-4000:] error = error or f"OpenClaw exited {result['return_code']}: {stderr}" incomplete = bool(result and result["timed_out"]) or last.get("stopReason") == "length" if not incomplete and not error and last.get("stopReason") != "stop": @@ -875,8 +876,7 @@ def count(name: str) -> int: ), metadata={ "harness_execution": "sandbox", - "harness_hostname": runtime.hostname if runtime else "unknown", - "harness_pid": str(runtime.pid) if runtime else "unknown", + **({"harness_hostname": runtime.hostname, "harness_pid": str(runtime.pid)} if runtime else {}), "openclaw_version": self.config.openclaw_version, }, ) diff --git a/responses_api_agents/openclaw_agent/sandbox.py b/responses_api_agents/openclaw_agent/sandbox.py index d30a6832d59..378cdc1d4fb 100644 --- a/responses_api_agents/openclaw_agent/sandbox.py +++ b/responses_api_agents/openclaw_agent/sandbox.py @@ -1,28 +1,22 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""OpenClaw execution in borrowed or agent-owned sandboxes.""" +"""OpenClaw artifacts and request state over the shared sandbox lifecycle.""" import asyncio import json import logging -from dataclasses import dataclass, field -from shlex import quote +from dataclasses import dataclass -from pydantic import JsonValue +from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError +from nemo_gym.agent_utils.sandbox_session import SandboxCommand, SandboxSession from nemo_gym.base_responses_api_agent import AgentSessionState from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming from nemo_gym.rollout_observability import AgentObservationBundle -from nemo_gym.sandbox import AsyncSandbox, process_supervisor -from nemo_gym.sandbox.process_supervisor import CleanupReceipt -from nemo_gym.sandbox.providers.base import SandboxExecResult -from nemo_gym.sandbox.runner import ( - RunnerRuntimeInfo, - confirm_runner_cleanup, - read_text, - supervisor_command, - upload_text, -) +from nemo_gym.sandbox.utils import read_text, upload_text + + +LOG = logging.getLogger(__name__) # Executed by sandbox Python before uploading any adapter-owned files. Resolve @@ -39,136 +33,100 @@ """ +class HarnessProcessInfo(BaseModel): + """Optional OpenClaw shim identity, independent of supervisor cleanup.""" + + model_config = ConfigDict(extra="forbid", strict=True) + hostname: str + pid: int + python: str | None = None + + +def parse_runtime_info(payload: object) -> HarnessProcessInfo | None: + """Read OpenClaw diagnostics without failing an otherwise valid episode.""" + try: + return HarnessProcessInfo.model_validate(payload) + except ValidationError: + LOG.warning("OpenClaw runtime metadata is missing or malformed") + return None + + @dataclass class OpenClawSandboxSession(AgentSessionState): - """Worker-local session with retryable, fail-closed runner teardown.""" + """OpenClaw request/output state; SandboxSession owns execution and teardown.""" - sandbox: AsyncSandbox - directory: str + session: SandboxSession[dict[str, str]] runtime: str - workdir: str = field(kw_only=True) - owns_sandbox: bool = field(default=False, kw_only=True) - sandbox_stopped: bool = False task: asyncio.Task[NeMoGymResponse] | None = None - exec_task: asyncio.Task[SandboxExecResult] | None = None - cleanup: CleanupReceipt | None = None - runtime_info: RunnerRuntimeInfo | None = None + runtime_info: HarnessProcessInfo | None = None observations: AgentObservationBundle | None = None activation_request: NeMoGymResponseCreateParamsNonStreaming | None = None - closing: bool = False - launch_started: bool = False - closed: bool = False - close_lock: asyncio.Lock = field(default_factory=asyncio.Lock) async def upload_json(self, name: str, payload: JsonValue) -> None: - """Upload adapter-owned data beneath this session's directory.""" - await upload_text(self.sandbox, path=f"{self.directory}/{name}", text=json.dumps(payload)) + """Stage adapter input using the shared text-transfer utility.""" + await upload_text(self.session.sandbox, path=f"{self.session.session_dir}/{name}", text=json.dumps(payload)) async def read_text(self, name: str) -> str: - """Read adapter-owned output independently of cleanup confirmation.""" - return await read_text(self.sandbox, path=f"{self.directory}/{name}") - - async def stop_runner(self, timeout: float) -> None: - """Fence a delayed launch or require the shared supervisor's cleanup receipt.""" - if self.sandbox_stopped or not self.launch_started or self.cleanup is not None: - return - self.cleanup = await confirm_runner_cleanup( - self.sandbox, directory=self.directory, workdir=self.workdir, timeout=timeout, harness="OpenClaw" - ) + """Read adapter output using file transfer.""" + return await read_text(self.session.sandbox, path=f"{self.session.session_dir}/{name}") async def close(self, timeout: float) -> None: - """Stop owned sandboxes; only stop harness work and disconnect borrowed ones.""" - async with self.close_lock: - if self.closed: - return - self.closing = True - # Cancelling provider exec can kill the supervisor. Obtain its - # descendant-cleanup receipt before cancelling the response task. - if self.owns_sandbox: - # The provider is the cleanup authority for an agent-owned sandbox. - # Keep the handle retryable if stop fails or times out. - if not self.sandbox_stopped: - await asyncio.wait_for(self.sandbox.stop(), timeout=timeout) - self.sandbox_stopped = True - else: - await self.stop_runner(timeout) - if self.task is not None: - if not self.task.done() and not self.task.cancelling(): - self.task.cancel() - try: - await asyncio.wait_for(asyncio.shield(self.task), timeout=timeout) - except asyncio.CancelledError: - if not self.task.cancelled(): - raise - except Exception: - if not self.task.done(): - raise - if self.exec_task is not None: - # A confirmed receipt makes it safe to cancel a stuck provider - # response; transport completion is not another cleanup gate. - if not self.exec_task.done(): - self.exec_task.cancel() - try: - await asyncio.wait_for(asyncio.shield(self.exec_task), timeout=timeout) - except asyncio.CancelledError: - if not self.exec_task.cancelled(): - raise - except Exception: - if not self.exec_task.done(): - raise - # Transport failure is not cleanup failure once the receipt is confirmed. - if self.owns_sandbox: - self.closed = True - return - retired = f"{self.directory}.closed" - result = await self.sandbox.exec( - f"if [ -d {quote(self.directory)} ]; then " - f"mv {quote(self.directory)} {quote(retired)} || exit 1; fi; rm -rf -- {quote(retired)}", - timeout_s=timeout, - ) - if result.return_code != 0 or getattr(result, "error_type", None): - raise RuntimeError(f"Could not remove OpenClaw session files: {result.stderr}") - await self.sandbox.disconnect() - self.closed = True - - async def execute(self, payload: dict[str, JsonValue], *, timeout: float, close_timeout: float) -> str: - """Run the supervisor through provider-neutral exec, without a PTY.""" - await self.upload_json("input.json", payload) - cleanup_timeout = close_timeout / 3 - command = supervisor_command( - directory=self.directory, - command=["python3", "-I", f"{self.directory}/sandbox_runner.py", f"{self.directory}/input.json"], + """Capture and release before cancelling the HTTP activation.""" + await self.session.close(timeout=timeout) + if self.task is not None and not self.task.done() and not self.task.cancelling(): + self.task.cancel() + if self.task is not None: + try: + await asyncio.wait_for(asyncio.shield(self.task), timeout=timeout) + except asyncio.CancelledError: + if not self.task.cancelled(): + raise + except Exception: + if not self.task.done(): + raise + # The agent base replays the activation error independently of close. + + async def execute(self, payload: dict[str, JsonValue], *, timeout: float, close_timeout: float) -> dict[str, str]: + """Run through the common lifecycle; the adapter validates terminal events.""" + artifacts = await self.session.execute( + stage_activation=lambda: self.stage_activation(payload), + collect=self.collect_artifacts, timeout=timeout, - cleanup_timeout=cleanup_timeout, + close_timeout=close_timeout, ) - self.launch_started = True - try: - # The runner enforces its own deadline and reaps descendants. - # Leave extra time for cleanup and transport before provider timeout. - self.exec_task = asyncio.create_task( - self.sandbox.exec( - command, - cwd=self.workdir, - timeout_s=process_supervisor.exec_timeout(timeout=timeout, cleanup_timeout=cleanup_timeout), - ) - ) - # HTTP cancellation must not propagate into provider exec before - # the supervisor has stopped and reaped the harness descendants. - launched = await asyncio.shield(self.exec_task) - if getattr(launched, "error_type", None) == "timeout": - raise TimeoutError("OpenClaw sandbox supervisor exceeded its execution deadline") - if launched.return_code != 0 or getattr(launched, "error_type", None): - raise RuntimeError(f"OpenClaw sandbox supervisor failed: {launched.stderr}") - except BaseException: + if self.session.closing: + raise asyncio.CancelledError + return artifacts + + async def stage_activation(self, payload: dict[str, JsonValue]) -> SandboxCommand: + """Stage the invocation and return the OpenClaw worker command.""" + await self.upload_json("home/.openclaw/openclaw.json", payload["config"]) + await self.upload_json("input.json", payload) + return SandboxCommand( + python="python3", + argv=[ + "python3", + "-I", + f"{self.session.session_dir}/sandbox_runner.py", + f"{self.session.session_dir}/input.json", + ], + ) + + async def collect_artifacts(self) -> dict[str, str]: + """Retain transcript and diagnostics before releasing the task sandbox.""" + artifacts = {} + session_id = self.session.session_dir.rsplit("/", 1)[-1] + for name in ("stdout.log", "stderr.log", f"home/.openclaw/agents/main/sessions/{session_id}.jsonl"): try: - await self.stop_runner(close_timeout) + artifacts[name] = await self.read_text(name) except Exception: - logging.getLogger(__name__).exception("OpenClaw cleanup remains unconfirmed; close must retry") - raise - else: - await self.stop_runner(close_timeout) - # A fenced launch can be safely closed without ever running a worker. - if self.cleanup is None or self.cleanup["return_code"] is None: - raise RuntimeError("OpenClaw sandbox runner has no worker exit code") - self.runtime_info = RunnerRuntimeInfo.model_validate_json(await self.read_text("runtime.json")) - return await self.read_text("stdout.log") + LOG.warning("OpenClaw artifact unavailable: %s", name, exc_info=True) + try: + runtime = json.loads(await self.read_text("runtime.json")) + except Exception: + runtime = None + self.runtime_info = parse_runtime_info(runtime) + if "stdout.log" not in artifacts and not any(name.endswith(".jsonl") for name in artifacts): + logs = await self.session.read_output_log() + raise RuntimeError(f"OpenClaw sandbox runner returned no valid result: {logs}") + return artifacts diff --git a/responses_api_agents/openclaw_agent/tests/test_exec_transport.py b/responses_api_agents/openclaw_agent/tests/test_exec_transport.py index 3d028645fed..bf39c3e9bba 100644 --- a/responses_api_agents/openclaw_agent/tests/test_exec_transport.py +++ b/responses_api_agents/openclaw_agent/tests/test_exec_transport.py @@ -12,7 +12,7 @@ import pytest -from nemo_gym.sandbox import process_supervisor +from nemo_gym.agent_utils.sandbox_session import SandboxSession from nemo_gym.sandbox.providers.base import SandboxExecResult from responses_api_agents.openclaw_agent import sandbox_runner from responses_api_agents.openclaw_agent.sandbox import OpenClawSandboxSession @@ -40,7 +40,7 @@ async def disconnect(self): self.disconnected = True async def exec(self, command, *, cwd=None, timeout_s=30): - launch = command.startswith("trap '' TERM;") + launch = "--receipt" in command and "process_supervisor.py" in command if launch: self.deadline = timeout_s self.delayed_command = command @@ -71,17 +71,22 @@ def make_session(tmp_path): request.sandbox_access.workdir = str(workdir) provider = ExecOnlySandbox() state = OpenClawSandboxSession( - request, provider, str(directory), str(tmp_path / "runtime"), workdir=request.sandbox_access.workdir + request=request, + session=SandboxSession( + sandbox=provider, session_dir=str(directory), workdir=request.sandbox_access.workdir, harness="OpenClaw" + ), + runtime=str(tmp_path / "runtime"), ) shutil.copyfile(sandbox_runner.__file__, directory / "sandbox_runner.py") - shutil.copyfile(process_supervisor.__file__, directory / "process_supervisor.py") + assert not (directory / "process_supervisor.py").exists() return state, provider, workdir def payload(state, code, timeout=0.5): return { - "directory": state.directory, + "directory": state.session.session_dir, "prompt": "task", + "config": {}, "cwd": state.request.sandbox_access.workdir, "command": [sys.executable, "-c", code], "env": {}, @@ -116,9 +121,9 @@ async def test_exec_only_supervision_reaps_detached_child(tmp_path, ending): await asyncio.gather(task, return_exceptions=True) else: await task - assert state.cleanup["cleanup_confirmed"] is True + assert state.session.cleanup["cleanup_confirmed"] is True if ending == "timeout": - assert state.cleanup["timed_out"] is True + assert state.session.cleanup["timed_out"] is True with pytest.raises(ProcessLookupError): os.kill(int((workdir / "child.pid").read_text()), 0) assert provider.cancelled_launch is False @@ -126,9 +131,9 @@ async def test_exec_only_supervision_reaps_detached_child(tmp_path, ending): await state.close(3) await state.close(3) assert provider.disconnected - assert not Path(state.directory).exists() + assert not Path(state.session.session_dir).exists() finally: - if not state.closed: + if not state.session.closed: await state.close(3) @@ -137,27 +142,27 @@ async def test_lost_launch_is_fenced_even_after_directory_retirement(tmp_path): provider.lost_launch = True with pytest.raises(TimeoutError, match="lost launch response"): await state.execute(payload(state, "open('started','w').close()"), timeout=0.5, close_timeout=3) - assert state.cleanup["cleanup_confirmed"] is True - assert state.cleanup["return_code"] is None - assert state.cleanup["error"] is None + assert state.session.cleanup["cleanup_confirmed"] is True + assert state.session.cleanup["return_code"] is None + assert state.session.cleanup["error"] is None assert state.runtime_info is None await state.close(3) provider.lost_launch = False await provider.exec(provider.delayed_command, cwd=str(workdir)) assert not (workdir / "started").exists() - assert not Path(state.directory).exists() + assert not Path(state.session.session_dir).exists() async def test_failed_receipt_keeps_files_and_can_retry(tmp_path): state, provider, _ = make_session(tmp_path) - state.launch_started = True + state.session.launch_started = True receipt = { "return_code": 1, "timed_out": False, "cleanup_confirmed": False, "error": "descendants remain", } - path = Path(state.directory) / "cleanup.json" + path = Path(state.session.session_dir) / "cleanup.json" path.write_text(json.dumps(receipt)) with pytest.raises(RuntimeError, match="cleanup was not confirmed"): await state.close(3) @@ -171,17 +176,17 @@ async def test_failed_receipt_keeps_files_and_can_retry(tmp_path): async def test_confirmed_cleanup_cancels_stuck_transport(tmp_path): state, provider, _ = make_session(tmp_path) - state.launch_started = True + state.session.launch_started = True receipt = { "return_code": 0, "timed_out": False, "cleanup_confirmed": True, "error": None, } - (Path(state.directory) / "cleanup.json").write_text(json.dumps(receipt)) - state.exec_task = asyncio.create_task(asyncio.Event().wait()) + (Path(state.session.session_dir) / "cleanup.json").write_text(json.dumps(receipt)) + state.session._exec_task = asyncio.create_task(asyncio.Event().wait()) await asyncio.sleep(0) await state.close(0.5) - assert state.exec_task.cancelled() - assert state.closed and provider.disconnected - assert not Path(state.directory).exists() + assert state.session._exec_task.cancelled() + assert state.session.closed and provider.disconnected + assert not Path(state.session.session_dir).exists() diff --git a/responses_api_agents/openclaw_agent/tests/test_lifecycle_contract.py b/responses_api_agents/openclaw_agent/tests/test_lifecycle_contract.py index f4678802c2b..164054dc25a 100644 --- a/responses_api_agents/openclaw_agent/tests/test_lifecycle_contract.py +++ b/responses_api_agents/openclaw_agent/tests/test_lifecycle_contract.py @@ -6,7 +6,6 @@ import pytest from fastapi import Request -from pydantic import ValidationError from responses_api_agents.openclaw_agent.tests.test_sandbox import seed, setup # noqa: F401 @@ -16,7 +15,14 @@ async def session(setup): body = seed() await agent.seed_agent_session(Request({"type": "http", "session": {}}), body) state = agent._session_records[body.agent_session_id].state - payload = {"directory": state.directory, "cwd": state.workdir, "prompt": "task", "command": [], "env": {}} + payload = { + "directory": state.session.session_dir, + "cwd": state.session.workdir, + "prompt": "task", + "config": {}, + "command": [], + "env": {}, + } return state, sandbox, payload @@ -29,35 +35,38 @@ async def session(setup): {"hostname": "worker", "pid": 123, "return_code": 0}, ], ) -async def test_invalid_runtime_fails_execution_but_not_confirmed_close(setup, runtime): +async def test_invalid_runtime_preserves_output_and_confirmed_close(setup, runtime): state, sandbox, payload = await session(setup) sandbox.runtime_info = runtime - with pytest.raises(ValidationError): - await state.execute(payload, timeout=5, close_timeout=1) - assert state.cleanup == {"cleanup_confirmed": True, "return_code": 0, "error": None, "timed_out": False} + output = await state.execute(payload, timeout=5, close_timeout=1) + assert output + assert state.runtime_info is None + assert state.session.cleanup == {"cleanup_confirmed": True, "return_code": 0, "error": None, "timed_out": False} await state.close(1) - assert state.closed + assert state.session.closed sandbox.disconnect.assert_awaited_once() sandbox.stop.assert_not_awaited() -@pytest.mark.parametrize("invalid", [{"return_code": "0"}, {"cleanup_confirmed": 1}, {"hostname": "mixed"}]) +@pytest.mark.parametrize( + "invalid", [{"cleanup_confirmed": 1}, {"cleanup_confirmed": "true"}, {"cleanup_confirmed": False}] +) async def test_bad_cleanup_keeps_state_retryable_without_reading_runtime(setup, invalid): state, sandbox, payload = await session(setup) sandbox.result.update(invalid) - with pytest.raises((RuntimeError, ValidationError)): + with pytest.raises(RuntimeError): await state.execute(payload, timeout=5, close_timeout=1) - assert state.cleanup is None + assert state.session.cleanup is None assert state.runtime_info is None - with pytest.raises((RuntimeError, ValidationError)): + with pytest.raises(RuntimeError): await state.close(1) - assert not state.closed + assert not state.session.closed sandbox.disconnect.assert_not_awaited() - sandbox.files[f"{state.directory}/cleanup.json"] = json.dumps({"cleanup_confirmed": True, "error": None}) + sandbox.files[f"{state.session.session_dir}/cleanup.json"] = json.dumps({"cleanup_confirmed": True, "error": None}) # A stop-before-launch receipt suffices for cleanup, not execution success. await state.close(1) - assert state.closed - assert state.runtime_info is None + assert state.session.closed + assert state.runtime_info is not None async def test_output_failure_does_not_erase_cleanup_confirmation(setup): @@ -65,14 +74,28 @@ async def test_output_failure_does_not_erase_cleanup_confirmation(setup): download = sandbox.download async def fail_output(source, destination): - if source.endswith("/stdout.log"): + if source.endswith("/stdout.log") or source.endswith(".jsonl"): raise RuntimeError("output unreadable") return await download(source, destination) sandbox.download = fail_output - with pytest.raises(RuntimeError, match="output unreadable"): + with pytest.raises(RuntimeError, match="returned no valid result"): await state.execute(payload, timeout=5, close_timeout=1) - assert state.cleanup["cleanup_confirmed"] is True - assert state.cleanup["error"] is None + assert state.session.cleanup["cleanup_confirmed"] is True + assert state.session.cleanup["error"] is None + await state.close(1) + sandbox.disconnect.assert_awaited_once() + + +@pytest.mark.parametrize("diagnostics", [{"return_code": "0"}, {"hostname": "extra"}, {"timed_out": 1}]) +async def test_diagnostic_schema_drift_does_not_block_capture_or_release(setup, diagnostics): + state, sandbox, payload = await session(setup) + sandbox.result.update(diagnostics) + output = await state.execute(payload, timeout=5, close_timeout=1) + assert output + assert state.session.cleanup["cleanup_confirmed"] is True + if "return_code" in diagnostics: + assert state.session.cleanup["return_code"] is None await state.close(1) + assert state.session.closed sandbox.disconnect.assert_awaited_once() diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index 9e386561a49..b305320aa0d 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -15,10 +15,10 @@ from omegaconf import OmegaConf from pydantic import ValidationError +from nemo_gym.agent_utils.supervisor_client import parse_cleanup_receipt from nemo_gym.base_responses_api_agent import AgentCloseSessionRequest, AgentSeedSessionRequest from nemo_gym.episode_types import EpisodeId, TaskId from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming -from nemo_gym.sandbox.runner import parse_cleanup_receipt from nemo_gym.server_utils import ServerClient from responses_api_agents.openclaw_agent.app import OpenClawAgent, OpenClawAgentConfig, _unique_usage_messages @@ -111,9 +111,9 @@ def pty(self): raise AssertionError("This provider supports exec only; no PTY API") async def run_exec(self, command, **kwargs): - if command.startswith("trap '' TERM;"): + if "--receipt" in command and "process_supervisor.py" in command: return await self.launch(command=command, **kwargs) - if "runner.stop" in command: + if "stop.request" in command: await self.request_stop() return self.exec.return_value @@ -199,7 +199,7 @@ def test_http_native_flow_runs_openclaw_in_borrowed_sandbox(setup, model_timeout created = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")) assert created.status_code == 200, created.text session_id = created.json()["agent_session_id"] - directory = agent._session_records[session_id].state.directory + directory = agent._session_records[session_id].state.session.session_dir installer = f"{directory}/install_openclaw_runtime.sh" assert installer in sandbox.files assert agent.config.resources_server is None @@ -701,9 +701,9 @@ async def test_unknown_launch_outcome_fails_closed(setup): async def test_install_failure_disconnects_without_stopping_owner(setup): agent, sandbox = setup sandbox.exec.side_effect = [ - SimpleNamespace(return_code=0), + SimpleNamespace(return_code=0, error_type=None), SimpleNamespace(return_code=1, stderr="npm failed"), - SimpleNamespace(return_code=0), + SimpleNamespace(return_code=0, error_type=None), ] request = Request({"type": "http", "session": {}}) with pytest.raises(RuntimeError, match="npm failed"): @@ -717,9 +717,9 @@ async def test_install_failure_disconnects_without_stopping_owner(setup): async def test_cancelled_install_never_publishes_session_or_launches_openclaw(setup): agent, sandbox = setup sandbox.exec.side_effect = [ - SimpleNamespace(return_code=0), + SimpleNamespace(return_code=0, error_type=None), asyncio.CancelledError(), - SimpleNamespace(return_code=0), + SimpleNamespace(return_code=0, error_type=None), ] request = Request({"type": "http", "session": {}}) with pytest.raises(asyncio.CancelledError): @@ -752,7 +752,7 @@ async def test_cleanup_command_provider_error_blocks_close_and_can_retry(setup): SimpleNamespace(return_code=0, error_type="timeout", stderr="cleanup timed out"), SimpleNamespace(return_code=0, error_type=None, stderr=""), ] - with pytest.raises(RuntimeError, match="cleanup timed out"): + with pytest.raises(RuntimeError, match="Could not remove OpenClaw session files"): await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) sandbox.disconnect.assert_not_awaited() await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) @@ -1100,12 +1100,12 @@ async def test_invalid_envelope_usage_does_not_discard_valid_transcript(setup): request, session_id, task = await activate(agent, sandbox) await task state = agent._session_records[session_id].state + state.session.artifacts["stdout.log"] = json.dumps({"meta": {"agentMeta": {"usage": {"input": "not-a-number"}}}}) response = await agent._collect_sandbox_response( state, NeMoGymResponseCreateParamsNonStreaming(input="task"), prompt="task", system="", - stdout=json.dumps({"meta": {"agentMeta": {"usage": {"input": "not-a-number"}}}}), ) assert response.status == "completed" assert response.output[-1].content[0].text == "Fixed" @@ -1132,18 +1132,19 @@ async def test_native_envelope_fallback_preserves_known_and_unknown_cache_detail request, session_id, task = await activate(agent, sandbox) await task state = agent._session_records[session_id].state - sandbox.files[f"{state.directory}/home/.openclaw/agents/main/sessions/{Path(state.directory).name}.jsonl"] = "" + state.session.artifacts = { + "stdout.log": json.dumps( + { + "payloads": [{"text": "Retained final output."}], + "meta": {"agentMeta": {"usage": {"input": 10, "output": 3, **cache}}}, + } + ) + } response = await agent._collect_sandbox_response( state, NeMoGymResponseCreateParamsNonStreaming(input="task"), prompt="task", system="", - stdout=json.dumps( - { - "payloads": [{"text": "Retained final output."}], - "meta": {"agentMeta": {"usage": {"input": 10, "output": 3, **cache}}}, - } - ), ) assert response.output[0].content[0].text == "Retained final output." assert response.usage.input_tokens == 10 + (expected_cache or 0) @@ -1308,7 +1309,7 @@ async def test_caller_id_never_controls_filesystem_path(setup): request = Request({"type": "http", "session": {}}) response = await agent.seed_agent_session(request, body) assert response.agent_session_id == body.agent_session_id - directory = agent._session_records[body.agent_session_id].state.directory + directory = agent._session_records[body.agent_session_id].state.session.session_dir assert Path(directory).parent == Path("/tmp/nemo-gym-openclaw-sessions") assert len(Path(directory).name) == 32 await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(body.agent_session_id))) @@ -1482,9 +1483,9 @@ async def test_failed_seed_with_confirmed_cleanup_accepts_cookie_less_close(setu agent, sandbox = setup body = seed() sandbox.exec.side_effect = [ - SimpleNamespace(return_code=0), + SimpleNamespace(return_code=0, error_type=None), SimpleNamespace(return_code=1, stderr="npm unavailable"), - SimpleNamespace(return_code=0), + SimpleNamespace(return_code=0, error_type=None), ] with pytest.raises(RuntimeError, match="npm unavailable"): await agent.seed_agent_session(Request({"type": "http", "session": {}}), body) @@ -1499,14 +1500,14 @@ async def test_failed_seed_cleanup_retains_state_until_explicit_retry(setup): agent, sandbox = setup body = seed() sandbox.exec.side_effect = [ - SimpleNamespace(return_code=0), + SimpleNamespace(return_code=0, error_type=None), SimpleNamespace(return_code=1, stderr="npm unavailable"), - SimpleNamespace(return_code=0), + SimpleNamespace(return_code=0, error_type=None), ] sandbox.disconnect.side_effect = RuntimeError("disconnect unavailable") with pytest.raises(RuntimeError, match="npm unavailable"): await agent.seed_agent_session(Request({"type": "http", "session": {}}), body) - assert agent._session_records[body.agent_session_id].state.closing + assert agent._session_records[body.agent_session_id].state.session.closing assert body.agent_session_id not in agent._closed_session_records with pytest.raises(HTTPException, match="closing"): await agent.seed_agent_session(Request({"type": "http", "session": {}}), body) @@ -1568,8 +1569,8 @@ def test_sandbox_source_controls_ownership_and_native_routing(setup, owned): created = client.post("/v1/agent_sessions", json=body.model_dump(mode="json")) assert created.status_code == 200, created.text state = agent._session_records[body.agent_session_id].state - assert state.owns_sandbox is owned - assert state.workdir == sandbox.expected_workdir + assert state.session.owns_sandbox is owned + assert state.session.workdir == sandbox.expected_workdir resolve.assert_called_once_with("agent-provider" if owned else "sandbox", {}) workspace_calls = [call for call in sandbox.exec.await_args_list if call.args[0].startswith("mkdir -p --")] assert len(workspace_calls) == int(owned) @@ -1618,13 +1619,13 @@ def test_owned_stop_failure_blocks_close_until_retry(setup): created = client.post("/v1/agent_sessions", json=body.model_dump(mode="json")) assert created.status_code == 200, created.text state = agent._session_records[body.agent_session_id].state - assert state.workdir == "/workspace" + assert state.session.workdir == "/workspace" # An owned sandbox can be destroyed even when no runner receipt was returned. - state.launch_started = True + state.session.launch_started = True sandbox.stop.side_effect = [RuntimeError("provider stop failed"), None] close_request = {"agent_session_id": body.agent_session_id, "episode_id": body.episode_id.model_dump()} assert client.post("/v1/agent_sessions/close", json=close_request).status_code == 500 - assert not state.closed + assert not state.session.closed assert ( client.post(f"/ng-rollout/{body.episode_id.capture_key}/v1/responses", json={"input": "task"}).status_code == 409 @@ -1666,7 +1667,7 @@ async def fail_at_stage(command, **kwargs): assert error.value is original assert client.post("/v1/agent_sessions", json=body.model_dump(mode="json")).status_code == 409 state = agent._session_records[body.agent_session_id].state - assert state.closing and not state.closed + assert state.session.closing and not state.session.closed response = client.post( "/v1/agent_sessions/close", json={"agent_session_id": body.agent_session_id, "episode_id": body.episode_id.model_dump()}, @@ -1706,3 +1707,71 @@ def test_owned_workdir_is_validated_before_creation(setup, workdir): assert client.post("/v1/agent_sessions", json=body.model_dump(mode="json")).status_code == 422 factory.assert_not_called() sandbox.exec.assert_not_awaited() + + +@pytest.mark.parametrize("owned", [False, True]) +@pytest.mark.parametrize("invalid_runtime", [False, True]) +async def test_close_retains_capture_after_provider_release(setup, tmp_path, owned, invalid_runtime): + agent, sandbox = setup + body = seed() + if owned: + body.sandbox_access = None + agent.config.sandbox_provider = "agent-provider" + agent.config.sandbox_config = {"image": "test-image"} + if invalid_runtime: + sandbox.runtime_info = {"hostname": "worker", "pid": "not-an-integer"} + sandbox.result["return_code"] = "not-an-integer" + sandbox.start = AsyncMock() + + request = Request({"type": "http", "session": {}, "path_params": {"rollout_id": "openclaw-smoke-a2"}}) + with patch("responses_api_agents.openclaw_agent.app.AsyncSandbox", return_value=sandbox) as factory: + factory.connect = AsyncMock(return_value=sandbox) + await agent.seed_agent_session(request, body) + state = agent._session_records[body.agent_session_id].state + directory = state.session.session_dir + assert f"{directory}/process_supervisor.py" not in sandbox.files + sandbox.blocked = True + task = asyncio.create_task(agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="task"))) + await asyncio.wait_for(sandbox.started.wait(), 2) + assert f"{directory}/process_supervisor.py" in sandbox.files + + async def release(): + assert state.session.cleanup["cleanup_confirmed"] is True + assert state.session.artifacts is not None + sandbox.files.clear() + sandbox.download = AsyncMock(side_effect=AssertionError("output read after release")) + + sandbox.stop.side_effect = release + sandbox.disconnect.side_effect = release + close = AgentCloseSessionRequest(**close_body(body.agent_session_id)) + response = await agent.close_agent_session(request, close) + with pytest.raises(asyncio.CancelledError): + await task + assert response.agent_observations is not None + invocations = [record for record in response.agent_observations.records if record.kind == "agent_invocation"] + assert any(record.conversation for record in invocations) + assert state.session.closed + assert await agent.close_agent_session(request, close) == response + (sandbox.stop if owned else sandbox.disconnect).assert_awaited_once() + (sandbox.disconnect if owned else sandbox.stop).assert_not_awaited() + + +@pytest.mark.parametrize("diagnostics", [{"return_code": "0"}, {"extra": 1}]) +def test_optional_diagnostics_preserve_valid_terminal_result(setup, diagnostics): + agent, sandbox = setup + sandbox.result.update(diagnostics) + sandbox.runtime_info = {"pid": "invalid"} + with TestClient(agent.setup_webserver()) as client: + created = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")) + created.raise_for_status() + response = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}) + response.raise_for_status() + assert response.json()["status"] == "completed" + assert response.json()["output"][-1]["content"][0]["text"] == "Fixed" + assert "harness_hostname" not in response.json()["metadata"] + assert "harness_pid" not in response.json()["metadata"] + closed = client.post("/v1/agent_sessions/close", json=close_body(created.json()["agent_session_id"])) + closed.raise_for_status() + gaps = {gap["code"] for gap in closed.json()["agent_observations"]["gaps"]} + assert "runtime_info_unavailable" in gaps + assert ("worker_exit_code_unavailable" in gaps) == ("return_code" in diagnostics) diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py b/responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py index b8a9243216a..b665088499f 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py @@ -12,7 +12,7 @@ import pytest -from nemo_gym.sandbox import process_supervisor +from nemo_gym.agent_utils import process_supervisor from responses_api_agents.openclaw_agent import sandbox_runner @@ -42,7 +42,7 @@ def launch(tmp_path, code, *, timeout=3, python=sys.executable, env=None, superv "--receipt", str(tmp_path / "cleanup.json"), "--stop-file", - str(tmp_path / "runner.stop"), + str(tmp_path / "stop.request"), "--", *command, ] @@ -124,7 +124,7 @@ def test_spawn_error_does_not_prevent_cleanup(tmp_path): @linux_only def test_stop_marker_fences_launch_without_inventing_worker_identity(tmp_path): - (tmp_path / "runner.stop").touch() + (tmp_path / "stop.request").touch() summary = result(tmp_path, launch(tmp_path, "open('unexpected', 'w').close()")) assert summary["cleanup_confirmed"] is True assert summary["return_code"] is None diff --git a/tests/unit_tests/test_local_provider.py b/tests/unit_tests/test_local_provider.py index 3818949006d..67ce10c1849 100644 --- a/tests/unit_tests/test_local_provider.py +++ b/tests/unit_tests/test_local_provider.py @@ -25,7 +25,8 @@ import pytest -from nemo_gym.sandbox import AsyncSandbox, SandboxSpec, SandboxStatus, create_provider, process_supervisor +from nemo_gym.agent_utils import process_supervisor +from nemo_gym.sandbox import AsyncSandbox, SandboxSpec, SandboxStatus, create_provider from nemo_gym.sandbox.providers.local import LocalProvider diff --git a/tests/unit_tests/test_process_supervisor.py b/tests/unit_tests/test_process_supervisor.py index 636b40b735c..a69aeb6ca87 100644 --- a/tests/unit_tests/test_process_supervisor.py +++ b/tests/unit_tests/test_process_supervisor.py @@ -16,7 +16,7 @@ import pytest -SUPERVISOR = Path(__file__).parents[2] / "nemo_gym/sandbox/process_supervisor.py" +SUPERVISOR = Path(__file__).parents[2] / "nemo_gym/agent_utils/process_supervisor.py" @pytest.mark.parametrize("timeout, cleanup", [(1, 0.1), (2700, 10), (21600, 100)]) diff --git a/tests/unit_tests/test_sandbox_runner_transport.py b/tests/unit_tests/test_sandbox_runner_transport.py deleted file mode 100644 index 840a5c4945e..00000000000 --- a/tests/unit_tests/test_sandbox_runner_transport.py +++ /dev/null @@ -1,166 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Transport contracts shared by sandboxed harness controllers.""" - -import asyncio -import json -import shutil -from pathlib import Path -from unittest.mock import AsyncMock - -import pytest -from pydantic import ValidationError - -from nemo_gym.sandbox.providers.base import SandboxExecResult -from nemo_gym.sandbox.runner import ( - RunnerRuntimeInfo, - confirm_runner_cleanup, - parse_cleanup_receipt, - read_text, - supervisor_command, - upload_text, -) - - -class LocalSandbox: - async def upload(self, source, destination): - shutil.copyfile(source, destination) - - async def download(self, source, destination): - shutil.copyfile(source, destination) - - async def exec(self, command, *, cwd=None, timeout_s=30): - process = await asyncio.create_subprocess_exec( - "sh", "-c", command, cwd=cwd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE - ) - stdout, stderr = await asyncio.wait_for(process.communicate(), timeout_s) - return SandboxExecResult(stdout.decode(errors="replace"), stderr.decode(errors="replace"), process.returncode) - - -@pytest.fixture -def session(tmp_path): - directory = tmp_path / "session's files" - directory.mkdir() - return LocalSandbox(), directory - - -async def test_file_transport_keeps_contents_out_of_shell(session): - sandbox, directory = session - sandbox.exec = AsyncMock(side_effect=AssertionError("file transfer must not invoke a shell")) - path = str(directory / "input.json") - payload = '{"prompt": "$(touch unwanted); `echo surprise`"}\n' - await upload_text(sandbox, path=path, text=payload) - assert Path(path).read_text() == payload - assert await read_text(sandbox, path=path) == payload - Path(path).write_bytes(b"partial output\xff\n") - assert await read_text(sandbox, path=path) == "partial output\ufffd\n" - sandbox.exec.assert_not_awaited() - - -async def test_confirmed_receipt_does_not_signal_stored_pid(session): - sandbox, directory = session - receipt = {"cleanup_confirmed": True, "error": None} - (directory / "cleanup.json").write_text(json.dumps(receipt)) - (directory / "runner.pid").write_text("12345") - sandbox.exec = AsyncMock(side_effect=AssertionError("must not signal a possibly reused PID")) - assert await confirm_runner_cleanup( - sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" - ) == parse_cleanup_receipt(receipt) - sandbox.exec.assert_not_awaited() - - -async def test_stop_wins_claim_and_fences_delayed_launch(session): - sandbox, directory = session - receipt = await confirm_runner_cleanup( - sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" - ) - assert receipt == {"cleanup_confirmed": True, "error": None, "return_code": None, "timed_out": False} - assert (directory / "launch.claim").readlink() == Path("stop") - assert (directory / "runner.stop").exists() - command = supervisor_command( - directory=str(directory), command=["touch", str(directory / "started")], timeout=1, cleanup_timeout=1 - ) - assert (await sandbox.exec(command)).return_code == 0 - assert not (directory / "runner.pid").exists() - assert not (directory / "runner.log").exists() - assert not (directory / "started").exists() - # A fenced launch is safe to close, without pretending a worker exited successfully. - assert receipt["return_code"] is None - with pytest.raises(ValidationError): - RunnerRuntimeInfo.model_validate(receipt) - - -async def test_missing_receipt_after_launch_is_not_cleanup_confirmation(session): - sandbox, directory = session - (directory / "launch.claim").symlink_to("launch") - with pytest.raises(RuntimeError, match="test launch outcome is unknown"): - await confirm_runner_cleanup( - sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" - ) - assert (directory / "launch.claim").readlink() == Path("launch") - assert not (directory / "cleanup.json").exists() - - -@pytest.mark.parametrize("confirmed", [False, "true", 1]) -async def test_unconfirmed_cleanup_preserves_receipt_for_retry(session, confirmed): - sandbox, directory = session - receipt_path = directory / "cleanup.json" - receipt = {"cleanup_confirmed": confirmed, "error": "descendants remain"} - receipt_path.write_text(json.dumps(receipt)) - with pytest.raises(RuntimeError, match="test sandbox cleanup was not confirmed: descendants remain"): - await confirm_runner_cleanup( - sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" - ) - assert json.loads(receipt_path.read_text()) == receipt - receipt = {"cleanup_confirmed": True, "error": None} - receipt_path.write_text(json.dumps(receipt)) - assert await confirm_runner_cleanup( - sandbox, directory=str(directory), workdir=str(directory.parent), timeout=1, harness="test" - ) == parse_cleanup_receipt(receipt) - - -def test_cleanup_and_runtime_are_independent(): - cleanup = {"return_code": 0, "timed_out": False, "cleanup_confirmed": True, "error": None} - assert parse_cleanup_receipt(cleanup) == cleanup - runtime = RunnerRuntimeInfo.model_validate({"hostname": "sandbox", "pid": 123}) - assert runtime.hostname == "sandbox" and runtime.pid == 123 - assert runtime.python is None - with pytest.raises(ValidationError): - parse_cleanup_receipt(runtime.model_dump()) - with pytest.raises(ValidationError): - RunnerRuntimeInfo.model_validate(cleanup) - for field in cleanup: - with pytest.raises(ValidationError): - parse_cleanup_receipt({key: value for key, value in cleanup.items() if key != field}) - - -@pytest.mark.parametrize( - "invalid", [{"return_code": "0"}, {"timed_out": "false"}, {"cleanup_confirmed": 1}, {"hostname": "worker"}] -) -def test_cleanup_receipt_keeps_strict_validation(invalid): - with pytest.raises(ValidationError): - parse_cleanup_receipt( - {"return_code": 0, "timed_out": False, "cleanup_confirmed": True, "error": None, **invalid} - ) - - -def test_absent_exit_code_is_not_success(): - full = {"return_code": None, "timed_out": False, "cleanup_confirmed": True, "error": None} - assert parse_cleanup_receipt(full) == full - assert parse_cleanup_receipt({"cleanup_confirmed": True, "error": None}) == full - with pytest.raises(ValidationError): - parse_cleanup_receipt({"return_code": 0, "error": None}) - - -@pytest.mark.parametrize("invalid", [{"pid": "123"}, {"hostname": 123}, {"python": 123}, {"return_code": 0}]) -def test_runtime_info_keeps_strict_validation(invalid): - with pytest.raises(ValidationError): - RunnerRuntimeInfo.model_validate({"hostname": "sandbox", "pid": 123, **invalid}) - - -def test_hermes_runtime_uses_the_same_schema(): - payload = {"hostname": "sandbox", "pid": 123, "python": "/opt/hermes/bin/python"} - assert RunnerRuntimeInfo.model_validate(payload).model_dump() == payload - for field in ("hostname", "pid"): - with pytest.raises(ValidationError): - RunnerRuntimeInfo.model_validate({key: value for key, value in payload.items() if key != field}) diff --git a/tests/unit_tests/test_sandbox_session.py b/tests/unit_tests/test_sandbox_session.py new file mode 100644 index 00000000000..7a210edfa72 --- /dev/null +++ b/tests/unit_tests/test_sandbox_session.py @@ -0,0 +1,413 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +import json +from unittest.mock import AsyncMock + +import pytest + +from nemo_gym.agent_utils import supervisor_client +from nemo_gym.agent_utils.sandbox_session import SandboxCommand, SandboxSession +from nemo_gym.sandbox import AsyncSandbox, SandboxExecResult + + +RECEIPT = {"cleanup_confirmed": True, "return_code": 0, "timed_out": False, "error": None} +OK = SandboxExecResult(stdout="", stderr="", return_code=0) + + +@pytest.fixture +def receipt_reader(monkeypatch): + reader = AsyncMock(return_value=json.dumps(RECEIPT)) + monkeypatch.setattr(supervisor_client, "read_text", reader) + return reader + + +@pytest.fixture +def session(receipt_reader): + sandbox = AsyncMock(spec=AsyncSandbox) + sandbox.exec.return_value = OK + return SandboxSession(sandbox=sandbox, session_dir="/session", workdir="/app", harness="test") + + +async def execute(session, *, collect=None, stage_activation=None, timeout=1): + return await session.execute( + stage_activation=stage_activation + or AsyncMock(return_value=SandboxCommand(argv=["worker"], python="/runtime/python")), + collect=collect or AsyncMock(return_value="transcript"), + timeout=10, + close_timeout=timeout, + ) + + +@pytest.mark.parametrize("owned", [False, True]) +async def test_completed_execution_captures_once_before_release(session, owned): + session.owns_sandbox = owned + collect = AsyncMock(return_value="transcript") + assert await execute(session, collect=collect) == "transcript" + assert session.cleanup == RECEIPT + assert session.artifacts == "transcript" + session.sandbox.stop.assert_not_awaited() + session.sandbox.disconnect.assert_not_awaited() + await asyncio.gather(session.close(timeout=1), session.close(timeout=1)) + await session.close(timeout=1) + collect.assert_awaited_once() + assert session.closed + if owned: + session.sandbox.stop.assert_awaited_once() + session.sandbox.disconnect.assert_not_awaited() + assert session.sandbox.exec.await_count == 1 + else: + session.sandbox.stop.assert_not_awaited() + session.sandbox.disconnect.assert_awaited_once() + assert "mv /session /session.closed" in session.sandbox.exec.await_args.args[0] + with pytest.raises(RuntimeError, match="closing or already activated"): + await execute(session) + + +@pytest.mark.parametrize("owned", [False, True]) +@pytest.mark.parametrize("cancel_execution", [False, True]) +async def test_interruption_captures_before_transport_cancellation_and_release(session, owned, cancel_execution): + session.owns_sandbox = owned + launched, collecting, captured = asyncio.Event(), asyncio.Event(), asyncio.Event() + events = [] + + async def provider_exec(command, **kwargs): + if "trap '' TERM" in command: + launched.set() + try: + await asyncio.Future() + finally: + assert session.cleanup == RECEIPT + events.append("cancel_exec") + events.append("remove_files") + return OK + + async def collect(): + assert session.cleanup == RECEIPT + events.append("collect") + collecting.set() + await captured.wait() + return "partial transcript" + + session.sandbox.exec.side_effect = provider_exec + session.sandbox.stop.side_effect = lambda: events.append("stop_sandbox") + session.sandbox.disconnect.side_effect = lambda: events.append("disconnect") + collector = AsyncMock(side_effect=collect) + running = asyncio.create_task(execute(session, collect=collector)) + await asyncio.wait_for(launched.wait(), 2) + if cancel_execution: + running.cancel() + closing = asyncio.create_task(session.close(timeout=1)) + await asyncio.wait_for(collecting.wait(), 2) + retry = asyncio.create_task(session.close(timeout=1)) + # Losing one close waiter must not cancel collection or release early. + closing.cancel() + with pytest.raises(asyncio.CancelledError): + await closing + assert events == ["collect"] + assert not session.closed + captured.set() + await asyncio.wait_for(retry, 2) + with pytest.raises(asyncio.CancelledError): + await running + assert session.artifacts == "partial transcript" + collector.assert_awaited_once() + assert events == ["collect", "cancel_exec", *(["stop_sandbox"] if owned else ["remove_files", "disconnect"])] + + +async def test_close_joins_collection_started_by_normal_completion(session): + collecting, finish = asyncio.Event(), asyncio.Event() + + async def collect(): + collecting.set() + await finish.wait() + return "snapshot" + + collector = AsyncMock(side_effect=collect) + running = asyncio.create_task(execute(session, collect=collector)) + await asyncio.wait_for(collecting.wait(), 2) + close = asyncio.create_task(session.close(timeout=1)) + await asyncio.sleep(0) + session.sandbox.disconnect.assert_not_awaited() + assert session.sandbox.exec.await_count == 1 + finish.set() + assert await running == "snapshot" + await close + collector.assert_awaited_once() + + +async def test_close_during_input_preparation_prevents_launch(session, receipt_reader): + preparing = asyncio.Event() + + async def stage_activation(): + preparing.set() + await asyncio.Future() + + collector = AsyncMock() + running = asyncio.create_task(execute(session, stage_activation=stage_activation, collect=collector)) + await asyncio.wait_for(preparing.wait(), 2) + await session.close(timeout=1) + with pytest.raises(asyncio.CancelledError): + await running + assert not session.launch_started + assert session.cleanup is None + collector.assert_not_awaited() + receipt_reader.assert_not_awaited() + # Only session-directory removal ran. + assert session.sandbox.exec.await_count == 1 + assert "mv /session /session.closed" in session.sandbox.exec.await_args.args[0] + + +@pytest.mark.parametrize("failure", [OSError("transport failed"), "timeout", "unavailable"]) +async def test_transport_failure_preserves_partial_output_and_original_error(session, failure): + if isinstance(failure, Exception): + session.sandbox.exec.side_effect = [failure, OK] + else: + session.sandbox.exec.side_effect = [SandboxExecResult("", "transport", 1, failure), OK] + expected = OSError if isinstance(failure, Exception) else TimeoutError if failure == "timeout" else RuntimeError + with pytest.raises(expected): + await execute(session) + assert session.cleanup == RECEIPT + assert session.artifacts == "transcript" + await session.close(timeout=1) + assert session.closed + + +async def test_capture_failure_does_not_hide_execution_failure(session): + session.sandbox.exec.side_effect = [OSError("transport failed"), OK] + collector = AsyncMock(side_effect=ValueError("malformed snapshot")) + with pytest.raises(OSError, match="transport failed"): + await execute(session, collect=collector) + assert isinstance(session.capture_error, ValueError) + assert session.cleanup == RECEIPT + await session.close(timeout=1) + collector.assert_awaited_once() + assert session.closed + + +async def test_capture_timeout_does_not_block_resource_release(session): + async def collect(): + await asyncio.Future() + + with pytest.raises(TimeoutError): + await execute(session, collect=collect, timeout=0.02) + assert isinstance(session.capture_error, TimeoutError) + assert session.cleanup == RECEIPT + await session.close(timeout=1) + assert session.closed + + +async def test_borrowed_cleanup_failure_retains_state_and_retries_capture(session, receipt_reader): + receipt_reader.return_value = json.dumps(RECEIPT | {"cleanup_confirmed": False}) + collector = AsyncMock(return_value="recovered transcript") + with pytest.raises(RuntimeError, match="cleanup was not confirmed"): + await execute(session, collect=collector) + with pytest.raises(RuntimeError, match="cleanup was not confirmed"): + await session.close(timeout=1) + assert session.closing and not session.closed + assert session.cleanup is None + collector.assert_not_awaited() + session.sandbox.disconnect.assert_not_awaited() + session.sandbox.stop.assert_not_awaited() + assert all("rm -rf" not in call.args[0] for call in session.sandbox.exec.await_args_list) + receipt_reader.return_value = json.dumps(RECEIPT) + await session.close(timeout=1) + collector.assert_awaited_once() + assert session.artifacts == "recovered transcript" + assert session.closed + + +async def test_owned_cleanup_deadline_falls_back_to_provider_stop(session, receipt_reader): + session.owns_sandbox = True + launched = asyncio.Event() + + async def provider_exec(command, **kwargs): + launched.set() + await asyncio.Future() + + async def unavailable(*args, **kwargs): + await asyncio.Future() + + session.sandbox.exec.side_effect = provider_exec + receipt_reader.side_effect = unavailable + collector = AsyncMock() + running = asyncio.create_task(execute(session, collect=collector, timeout=0.02)) + await asyncio.wait_for(launched.wait(), 2) + await asyncio.wait_for(session.close(timeout=0.02), 1) + with pytest.raises(asyncio.CancelledError): + await running + assert session.sandbox_stopped and session.closed + assert session.cleanup is None + assert session.capture_error is not None + collector.assert_not_awaited() + session.sandbox.stop.assert_awaited_once() + + +@pytest.mark.parametrize("owned", [False, True]) +async def test_release_failure_retries_without_recollecting(session, owned): + session.owns_sandbox = owned + release = session.sandbox.stop if owned else session.sandbox.disconnect + release.side_effect = [OSError("release failed"), None] + collector = AsyncMock(return_value="saved transcript") + await execute(session, collect=collector) + with pytest.raises(OSError, match="release failed"): + await session.close(timeout=1) + assert not session.closed + await session.close(timeout=1) + assert session.closed + assert release.await_count == 2 + collector.assert_awaited_once() + + +@pytest.mark.parametrize("return_code", [1, None, "malformed"]) +async def test_nonzero_worker_exit_still_passes_artifacts_to_adapter(session, receipt_reader, return_code): + session.sandbox.exec.return_value = SandboxExecResult("", "worker error", 1) + receipt_reader.return_value = json.dumps(RECEIPT | {"return_code": return_code}) + assert await execute(session) == "transcript" + assert session.cleanup["return_code"] == (return_code if type(return_code) is int else None) + + +async def test_bootstrap_failure_preserves_stderr_and_still_releases(session, receipt_reader): + receipt_reader.return_value = json.dumps(RECEIPT | {"return_code": None}) + session.sandbox.exec.return_value = SandboxExecResult("", "/runtime/python: not found", 127) + collector = AsyncMock(side_effect=FileNotFoundError("no worker output")) + with pytest.raises(RuntimeError, match="sandbox execution failed.*127.*python: not found") as error: + await execute(session, collect=collector) + assert isinstance(error.value.__cause__, FileNotFoundError) + assert session.cleanup["cleanup_confirmed"] is True + assert session.cleanup["return_code"] is None + collector.assert_awaited_once() + session.sandbox.exec.return_value = OK + await session.close(timeout=1) + assert session.closed + session.sandbox.disconnect.assert_awaited_once() + + +async def test_session_uploads_supervisor_before_launch(session): + from pathlib import Path + + from nemo_gym.agent_utils import process_supervisor + from nemo_gym.agent_utils.supervisor_client import SUPERVISOR_FILE + + events = [] + + async def upload(source, destination): + assert Path(source).read_bytes() == Path(process_supervisor.__file__).read_bytes() + assert destination == f"/session/{SUPERVISOR_FILE}" + events.append("upload") + + async def launch(*args, **kwargs): + assert events == ["upload"] + events.append("launch") + return OK + + session.sandbox.upload.side_effect = upload + session.sandbox.exec.side_effect = launch + assert await execute(session) == "transcript" + assert events == ["upload", "launch"] + assert not session.closing and not session.closed + session.sandbox.stop.assert_not_awaited() + session.sandbox.disconnect.assert_not_awaited() + + +@pytest.mark.parametrize("owned", [False, True]) +async def test_close_during_supervisor_upload_joins_staging_before_release(session, receipt_reader, owned): + session.owns_sandbox = owned + uploading = asyncio.Event() + events = [] + + async def upload(*args): + uploading.set() + try: + await asyncio.Future() + finally: + events.append("upload_settled") + + session.sandbox.upload.side_effect = upload + release = session.sandbox.stop if owned else session.sandbox.disconnect + release.side_effect = lambda: events.append("release") + collector = AsyncMock() + running = asyncio.create_task(execute(session, collect=collector)) + await asyncio.wait_for(uploading.wait(), 2) + await session.close(timeout=1) + with pytest.raises(asyncio.CancelledError): + await running + assert events == ["upload_settled", "release"] + assert not session.launch_started + assert session.closed and session.closing + collector.assert_not_awaited() + receipt_reader.assert_not_awaited() + assert all("exec " not in call.args[0] for call in session.sandbox.exec.await_args_list) + + +async def test_supervisor_upload_failure_preserves_error_and_allows_close(session, receipt_reader): + session.sandbox.upload.side_effect = OSError("supervisor upload failed") + collector = AsyncMock() + with pytest.raises(OSError, match="supervisor upload failed"): + await execute(session, collect=collector) + assert not session.launch_started + assert not session.closing + collector.assert_not_awaited() + receipt_reader.assert_not_awaited() + session.sandbox.exec.assert_not_awaited() + await session.close(timeout=1) + assert session.closed + session.sandbox.disconnect.assert_awaited_once() + + +async def test_cancelled_close_stays_closing_and_can_retry(session): + release_started = asyncio.Event() + + async def disconnect(): + release_started.set() + await asyncio.Future() + + session.sandbox.disconnect.side_effect = disconnect + closing = asyncio.create_task(session.close(timeout=1)) + await asyncio.wait_for(release_started.wait(), 2) + session._close_task.cancel() + with pytest.raises(asyncio.CancelledError): + await closing + assert session.closing and not session.closed + with pytest.raises(RuntimeError, match="closing or already activated"): + await execute(session) + session.sandbox.disconnect.side_effect = None + await session.close(timeout=1) + assert session.closed + + +async def test_failure_log_is_read_before_session_release(session, monkeypatch): + from nemo_gym.agent_utils import sandbox_session + + async def read_log(sandbox, *, path): + sandbox.disconnect.assert_not_awaited() + assert path == "/session/output.log" + return "diagnostic output" + + monkeypatch.setattr(sandbox_session, "read_text", read_log) + + async def collect(): + log = await session.read_output_log() + raise FileNotFoundError(f"Missing artifacts: {log}") + + with pytest.raises(FileNotFoundError, match="Missing artifacts: diagnostic output"): + await execute(session, collect=collect) + await session.close(timeout=1) + session.sandbox.disconnect.assert_awaited_once() + + +async def test_missing_failure_log_does_not_mask_capture_failure(session, monkeypatch): + from nemo_gym.agent_utils import sandbox_session + + monkeypatch.setattr(sandbox_session, "read_text", AsyncMock(side_effect=OSError("download failed"))) + + async def collect(): + log = await session.read_output_log() + raise FileNotFoundError(f"Missing artifacts: {log}") + + with pytest.raises(FileNotFoundError, match="Missing artifacts"): + await execute(session, collect=collect) + assert session.cleanup == RECEIPT + await session.close(timeout=1) + assert session.closed diff --git a/tests/unit_tests/test_sandbox_supervisor_bootstrap.py b/tests/unit_tests/test_sandbox_supervisor_bootstrap.py new file mode 100644 index 00000000000..015d48b79b8 --- /dev/null +++ b/tests/unit_tests/test_sandbox_supervisor_bootstrap.py @@ -0,0 +1,184 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Failed interpreter/supervisor bootstrap must remain safe to close.""" + +import asyncio +import json +import shutil +import sys +from pathlib import Path + +import pytest + +from nemo_gym.agent_utils import process_supervisor +from nemo_gym.agent_utils.supervisor_client import ( + CLEANUP_RECEIPT_FILE, + LAUNCH_CLAIM_FILE, + SUPERVISOR_FILE, + SUPERVISOR_PID_FILE, + remove_session_directory, + stop_and_confirm_cleanup, + supervised_launch_command, +) +from nemo_gym.sandbox.providers.base import SandboxExecResult + + +class LocalSandbox: + async def download(self, source: str, destination: Path) -> None: + shutil.copyfile(source, destination) + + async def exec(self, command: str, *, cwd: str | None = None, timeout_s: float = 30) -> SandboxExecResult: + process = await asyncio.create_subprocess_exec( + "sh", "-c", command, cwd=cwd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout_s) + return SandboxExecResult(stdout.decode(errors="replace"), stderr.decode(errors="replace"), process.returncode) + + +@pytest.mark.parametrize("failure", ["missing_python", "broken_python", "missing_supervisor", "broken_supervisor"]) +async def test_bootstrap_failure_preserves_diagnostics_and_allows_close(tmp_path: Path, failure: str) -> None: + session_dir = tmp_path / "session" + session_dir.mkdir() + python = sys.executable + supervisor = session_dir / SUPERVISOR_FILE + shutil.copyfile(process_supervisor.__file__, supervisor) + expected_error = "" + if failure == "missing_python": + python = str(tmp_path / "missing-python") + expected_error = "missing-python" + elif failure == "broken_python": + interpreter = tmp_path / "incompatible-python" + interpreter.write_text("#!/bin/sh\nprintf '%s\\n' 'incompatible interpreter' >&2\nexit 17\n") + interpreter.chmod(0o755) + python = str(interpreter) + expected_error = "incompatible interpreter" + elif failure == "missing_supervisor": + supervisor.unlink() + expected_error = "process_supervisor.py" + else: + supervisor.write_text("def invalid syntax\n") + expected_error = "SyntaxError" + sandbox = LocalSandbox() + command = supervised_launch_command( + session_dir=str(session_dir), + command=["touch", str(session_dir / "worker-started")], + timeout=1, + cleanup_timeout=1, + python=python, + ) + result = await sandbox.exec(command) + assert result.return_code != 0 + if failure == "broken_python": + assert result.return_code == 17 + assert expected_error in result.stderr + assert not (session_dir / LAUNCH_CLAIM_FILE).is_symlink() + assert not (session_dir / SUPERVISOR_PID_FILE).exists() + assert not (session_dir / "worker-started").exists() + assert not (session_dir / CLEANUP_RECEIPT_FILE).exists() + + receipt = await stop_and_confirm_cleanup( + sandbox, session_dir=str(session_dir), workdir=None, timeout=1, harness="bootstrap-test" + ) + assert receipt["cleanup_confirmed"] is True + assert receipt["return_code"] is None + assert (session_dir / LAUNCH_CLAIM_FILE).readlink() == Path("stop") + + +@pytest.mark.parametrize("removed", [False, True]) +async def test_fenced_delayed_launch_does_not_require_the_runtime(tmp_path: Path, removed: bool) -> None: + session_dir = tmp_path / "session" + session_dir.mkdir() + (session_dir / LAUNCH_CLAIM_FILE).symlink_to("stop") + if removed: + shutil.rmtree(session_dir) + result = await LocalSandbox().exec( + supervised_launch_command( + session_dir=str(session_dir), + command=["touch", str(tmp_path / "worker-started")], + timeout=1, + cleanup_timeout=1, + python=str(tmp_path / "missing-python"), + ) + ) + assert result.return_code == 0 + assert result.stderr == "" + assert not (tmp_path / "worker-started").exists() + + +@pytest.mark.parametrize("remove_directory", [False, True]) +async def test_close_during_preflight_fences_later_launch(tmp_path: Path, remove_directory: bool) -> None: + session_dir = tmp_path / "session" + session_dir.mkdir() + preflight_started = tmp_path / "preflight-started" + continue_preflight = tmp_path / "continue-preflight" + worker_started = tmp_path / "worker-started" + supervisor = session_dir / SUPERVISOR_FILE + supervisor.write_text( + "import pathlib, sys, time\n" + "if '--help' in sys.argv:\n" + f" pathlib.Path({str(preflight_started)!r}).touch()\n" + f" while not pathlib.Path({str(continue_preflight)!r}).exists():\n" + " time.sleep(0.01)\n" + "else:\n" + f" pathlib.Path({str(worker_started)!r}).touch()\n" + ) + sandbox = LocalSandbox() + launch = asyncio.create_task( + sandbox.exec( + supervised_launch_command( + session_dir=str(session_dir), + command=["worker"], + timeout=1, + cleanup_timeout=1, + python=sys.executable, + ) + ) + ) + try: + async with asyncio.timeout(5): + while not preflight_started.exists(): + await asyncio.sleep(0.01) + receipt = await stop_and_confirm_cleanup( + sandbox, session_dir=str(session_dir), workdir=None, timeout=1, harness="bootstrap-test" + ) + assert receipt["cleanup_confirmed"] is True + assert receipt["return_code"] is None + if remove_directory: + await remove_session_directory( + sandbox, session_dir=str(session_dir), workdir=None, timeout=1, harness="bootstrap-test" + ) + finally: + continue_preflight.touch() + result = await asyncio.wait_for(launch, timeout=5) + assert result.return_code == 0 + assert not worker_started.exists() + + +async def test_bootstrap_quotes_private_paths_and_uses_the_same_interpreter(tmp_path: Path) -> None: + session_dir = tmp_path / "session's $(touch unintended) files" + session_dir.mkdir() + runtime = tmp_path / "runtime's $(touch unintended) files" + runtime.mkdir() + python = runtime / "python interpreter" + python.symlink_to(sys.executable) + supervisor = session_dir / SUPERVISOR_FILE + arguments = session_dir / "arguments.json" + supervisor.write_text( + "import json, pathlib, sys\n" + "if '--help' not in sys.argv:\n" + f" pathlib.Path({str(arguments)!r}).write_text(json.dumps([sys.executable, *sys.argv[1:]]))\n" + ) + command = supervised_launch_command( + session_dir=str(session_dir), + command=["worker", "$(touch unintended)", "argument's value"], + timeout=1, + cleanup_timeout=1, + python=str(python), + ) + result = await LocalSandbox().exec(command, cwd=str(tmp_path)) + assert result.return_code == 0, result.stderr + captured = json.loads(arguments.read_text()) + assert captured[0] == str(python) + assert captured[-3:] == ["worker", "$(touch unintended)", "argument's value"] + assert not (tmp_path / "unintended").exists() + assert (session_dir / LAUNCH_CLAIM_FILE).readlink() == Path("launch") diff --git a/tests/unit_tests/test_sandbox_supervisor_client.py b/tests/unit_tests/test_sandbox_supervisor_client.py new file mode 100644 index 00000000000..dfb9cddb550 --- /dev/null +++ b/tests/unit_tests/test_sandbox_supervisor_client.py @@ -0,0 +1,234 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Transport contracts shared by sandboxed harness controllers.""" + +import asyncio +import json +import shutil +import sys +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest + +from nemo_gym.agent_utils import process_supervisor +from nemo_gym.agent_utils.supervisor_client import ( + CLEANUP_RECEIPT_FILE, + LAUNCH_CLAIM_FILE, + OUTPUT_LOG_FILE, + STOP_REQUEST_FILE, + SUPERVISOR_FILE, + SUPERVISOR_PID_FILE, + parse_cleanup_receipt, + stop_and_confirm_cleanup, + supervised_launch_command, + supervision_timeouts, +) +from nemo_gym.sandbox.providers.base import SandboxExecResult +from nemo_gym.sandbox.utils import read_text, upload_text + + +class LocalSandbox: + async def upload(self, source, destination): + shutil.copyfile(source, destination) + + async def download(self, source, destination): + shutil.copyfile(source, destination) + + async def exec(self, command, *, cwd=None, timeout_s=30): + process = await asyncio.create_subprocess_exec( + "sh", "-c", command, cwd=cwd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout_s) + return SandboxExecResult(stdout.decode(errors="replace"), stderr.decode(errors="replace"), process.returncode) + + +@pytest.fixture +def session(tmp_path): + session_dir = tmp_path / "session's files" + session_dir.mkdir() + return LocalSandbox(), session_dir + + +async def test_file_transport_keeps_contents_out_of_shell(session): + sandbox, session_dir = session + sandbox.exec = AsyncMock(side_effect=AssertionError("file transfer must not invoke a shell")) + path = str(session_dir / "input.json") + payload = '{"prompt": "$(touch unwanted); `echo surprise`"}\n' + await upload_text(sandbox, path=path, text=payload) + assert Path(path).read_text() == payload + assert await read_text(sandbox, path=path) == payload + Path(path).write_bytes(b"partial output\xff\n") + assert await read_text(sandbox, path=path) == "partial output\ufffd\n" + sandbox.exec.assert_not_awaited() + + +async def test_confirmed_receipt_does_not_signal_stored_pid(session): + sandbox, session_dir = session + receipt = {"cleanup_confirmed": True, "error": None, "return_code": None, "timed_out": False} + (session_dir / CLEANUP_RECEIPT_FILE).write_text(json.dumps(receipt)) + (session_dir / SUPERVISOR_PID_FILE).write_text("12345") + sandbox.exec = AsyncMock(side_effect=AssertionError("must not signal a possibly reused PID")) + assert await stop_and_confirm_cleanup( + sandbox, session_dir=str(session_dir), workdir=str(session_dir.parent), timeout=1, harness="test" + ) == parse_cleanup_receipt(receipt) + sandbox.exec.assert_not_awaited() + + +async def test_confirmed_receipt_with_malformed_diagnostics_still_allows_cleanup(session, caplog): + sandbox, session_dir = session + receipt = {"cleanup_confirmed": True, "return_code": "0", "error": {"secret": "do not log"}, "version": 2} + (session_dir / CLEANUP_RECEIPT_FILE).write_text(json.dumps(receipt)) + sandbox.exec = AsyncMock(side_effect=AssertionError("must not signal a possibly reused PID")) + assert await stop_and_confirm_cleanup( + sandbox, session_dir=str(session_dir), workdir=str(session_dir.parent), timeout=1, harness="test" + ) == {"cleanup_confirmed": True, "return_code": None, "timed_out": False, "error": None} + assert "diagnostic fields" in caplog.text + assert "do not log" not in caplog.text + sandbox.exec.assert_not_awaited() + + +@pytest.mark.parametrize("receipt", [[], "stopped", None, 1]) +async def test_nonobject_cleanup_receipt_is_unconfirmed_without_signalling_a_stale_pid(session, receipt): + sandbox, session_dir = session + path = session_dir / CLEANUP_RECEIPT_FILE + path.write_text(json.dumps(receipt)) + (session_dir / SUPERVISOR_PID_FILE).write_text("2147483647") + for _ in range(2): + with pytest.raises(RuntimeError, match="cleanup receipt is not a JSON object"): + await stop_and_confirm_cleanup( + sandbox, session_dir=str(session_dir), workdir=str(session_dir.parent), timeout=1, harness="test" + ) + assert json.loads(path.read_text()) == receipt + assert not (session_dir / STOP_REQUEST_FILE).exists() + assert not (session_dir / LAUNCH_CLAIM_FILE).exists() + + +async def test_stop_wins_claim_and_fences_delayed_launch(session, caplog): + sandbox, session_dir = session + receipt = await stop_and_confirm_cleanup( + sandbox, session_dir=str(session_dir), workdir=str(session_dir.parent), timeout=1, harness="test" + ) + assert receipt == {"cleanup_confirmed": True, "error": None, "return_code": None, "timed_out": False} + assert json.loads((session_dir / CLEANUP_RECEIPT_FILE).read_text()) == receipt + assert "diagnostic fields" not in caplog.text + assert (session_dir / LAUNCH_CLAIM_FILE).readlink() == Path("stop") + assert (session_dir / STOP_REQUEST_FILE).exists() + command = supervised_launch_command( + session_dir=str(session_dir), + command=["touch", str(session_dir / "started")], + timeout=1, + cleanup_timeout=1, + python=sys.executable, + ) + assert (await sandbox.exec(command)).return_code == 0 + assert not (session_dir / SUPERVISOR_PID_FILE).exists() + assert not (session_dir / OUTPUT_LOG_FILE).exists() + assert not (session_dir / "started").exists() + # A fenced launch is safe to close, without claiming a successful harness exit. + assert receipt["return_code"] is None + + +async def test_missing_receipt_after_launch_is_not_cleanup_confirmation(session): + sandbox, session_dir = session + (session_dir / LAUNCH_CLAIM_FILE).symlink_to("launch") + with pytest.raises(RuntimeError, match="test launch outcome is unknown"): + await stop_and_confirm_cleanup( + sandbox, session_dir=str(session_dir), workdir=str(session_dir.parent), timeout=1, harness="test" + ) + assert (session_dir / LAUNCH_CLAIM_FILE).readlink() == Path("launch") + assert not (session_dir / CLEANUP_RECEIPT_FILE).exists() + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux supervisor contract") +@pytest.mark.parametrize("private_python", [False, True]) +async def test_launch_supervises_harness_with_selected_or_private_python(session, tmp_path, private_python): + sandbox, session_dir = session + runtime = tmp_path / "runtime's files" + runtime.mkdir() + supervisor = session_dir / SUPERVISOR_FILE + shutil.copyfile(process_supervisor.__file__, supervisor) + python = Path(sys.executable) + if private_python: + python = runtime / "python interpreter" + python.symlink_to(sys.executable) + command = supervised_launch_command( + session_dir=str(session_dir), + command=[ + sys.executable, + "-c", + "import sys; print(sys.argv[1]); print('stderr', file=sys.stderr)", + "$(literal)", + ], + timeout=5, + cleanup_timeout=1, + python=str(python), + ) + launched = await sandbox.exec(command) + assert launched.return_code == 0, launched.stderr + receipt = await stop_and_confirm_cleanup( + sandbox, session_dir=str(session_dir), workdir=str(tmp_path), timeout=2, harness="test" + ) + assert receipt == {"cleanup_confirmed": True, "error": None, "return_code": 0, "timed_out": False} + assert set((session_dir / OUTPUT_LOG_FILE).read_text().splitlines()) == {"$(literal)", "stderr"} + + +@pytest.mark.parametrize("confirmed", [False, "true", 1]) +async def test_unconfirmed_cleanup_preserves_receipt_for_retry(session, confirmed): + sandbox, session_dir = session + receipt_path = session_dir / CLEANUP_RECEIPT_FILE + receipt = {"cleanup_confirmed": confirmed, "error": "descendants remain"} + receipt_path.write_text(json.dumps(receipt)) + with pytest.raises(RuntimeError, match="test sandbox cleanup was not confirmed: descendants remain"): + await stop_and_confirm_cleanup( + sandbox, session_dir=str(session_dir), workdir=str(session_dir.parent), timeout=1, harness="test" + ) + assert json.loads(receipt_path.read_text()) == receipt + receipt = {"cleanup_confirmed": True, "error": None, "return_code": None, "timed_out": False} + receipt_path.write_text(json.dumps(receipt)) + assert await stop_and_confirm_cleanup( + sandbox, session_dir=str(session_dir), workdir=str(session_dir.parent), timeout=1, harness="test" + ) == parse_cleanup_receipt(receipt) + + +@pytest.mark.parametrize("timeout,close_timeout", [(1, 0.3), (2700, 30), (21600, 300)]) +def test_supervision_timeouts_reserve_cleanup_and_provider_margin(timeout, close_timeout): + cleanup_timeout, provider_timeout = supervision_timeouts(timeout=timeout, close_timeout=close_timeout) + assert cleanup_timeout * process_supervisor.CLEANUP_PHASE_COUNT == pytest.approx(close_timeout) + assert provider_timeout == pytest.approx(timeout + close_timeout + 30) + assert provider_timeout == process_supervisor.exec_timeout(timeout=timeout, cleanup_timeout=cleanup_timeout) + + +@pytest.mark.parametrize( + "invalid,normalized", + [ + ({"return_code": "0"}, {"return_code": None}), + ({"return_code": True}, {"return_code": None}), + ({"timed_out": "false"}, {"timed_out": False}), + ({"error": {"message": "failed"}}, {"error": None}), + ({"hostname": "worker"}, {}), + ], +) +def test_confirmed_cleanup_keeps_valid_diagnostics_and_ignores_malformed_fields(invalid, normalized): + cleanup = {"return_code": 0, "timed_out": False, "cleanup_confirmed": True, "error": None} + assert parse_cleanup_receipt(cleanup | invalid) == cleanup | normalized + + +@pytest.mark.parametrize( + "payload", + [None, [], "true", {}, {"cleanup_confirmed": False}, {"cleanup_confirmed": "true"}, {"cleanup_confirmed": 1}], +) +def test_cleanup_receipt_requires_positive_boolean_evidence(payload): + with pytest.raises(ValueError, match="cleanup was not confirmed"): + parse_cleanup_receipt(payload) + + +def test_absent_exit_code_is_not_success(caplog): + full = {"return_code": None, "timed_out": False, "cleanup_confirmed": True, "error": None} + assert parse_cleanup_receipt(full) == full + assert "diagnostic fields" not in caplog.text + assert parse_cleanup_receipt({"cleanup_confirmed": True, "error": None}) == full + assert "diagnostic fields" in caplog.text + assert parse_cleanup_receipt({"cleanup_confirmed": True}) == full + with pytest.raises(ValueError, match="cleanup was not confirmed"): + parse_cleanup_receipt({"return_code": 0, "error": None}) From a88a216eaba0497b4f8aff9cff30f204ada11c7e Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 6 Oct 2026 13:46:38 -0700 Subject: [PATCH 34/44] test(openclaw): prepare seeded config directory in exec fixture Signed-off-by: yaoyu-33 --- .../openclaw_agent/tests/test_exec_transport.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/responses_api_agents/openclaw_agent/tests/test_exec_transport.py b/responses_api_agents/openclaw_agent/tests/test_exec_transport.py index bf39c3e9bba..ca7044adab3 100644 --- a/responses_api_agents/openclaw_agent/tests/test_exec_transport.py +++ b/responses_api_agents/openclaw_agent/tests/test_exec_transport.py @@ -64,7 +64,8 @@ async def exec(self, command, *, cwd=None, timeout_s=30): def make_session(tmp_path): directory = tmp_path / "session" - directory.mkdir() + # Mirror the directory prepared by the real seed hook before activation. + (directory / "home/.openclaw").mkdir(parents=True) workdir = tmp_path / "task" workdir.mkdir() request = seed() From d5e6ad20b733faeeb3a35cabc4288c13609d4b11 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 6 Oct 2026 13:53:21 -0700 Subject: [PATCH 35/44] fix(config): preserve native Hermes templates during migration Signed-off-by: yaoyu-33 --- .../add_legacy_agent_environment_servers.py | 14 ++-- ...st_add_legacy_agent_environment_servers.py | 69 +++++++++++++++++++ 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/scripts/add_legacy_agent_environment_servers.py b/scripts/add_legacy_agent_environment_servers.py index be03db0419d..3c3fc9ca23c 100644 --- a/scripts/add_legacy_agent_environment_servers.py +++ b/scripts/add_legacy_agent_environment_servers.py @@ -98,9 +98,10 @@ def is_rename(instance: dict) -> bool: def needs_environment_server(instance: dict) -> bool: """True for an agent instance a run dispatches to, so it needs a server in front of it. - Two kinds are skipped. An unbound template leaves `resources_server.name` unset for - composition to fill. A shared overlay names several benchmarks' agents to override one field - on each; without an entrypoint or an `_inherit_from` supplying one, that name is not a server + Unbound templates leave `resources_server.name` unset for composition to fill, or + use Hermes' native session interface with no Resources binding. A shared overlay names + several benchmarks' agents to override one field on each; without an entrypoint or an + `_inherit_from` supplying one, that name is not a server a run can start, and declaring a server for it strands the reference in every run that merges the overlay without the agent. """ @@ -110,7 +111,12 @@ def needs_environment_server(instance: dict) -> bool: agent = instance.get("responses_api_agents", {}).get(agent_type) if agent_type else None if not isinstance(agent, dict): return False - if (agent.get("resources_server") or {}).get("name") == "???": + resources_server = agent.get("resources_server", {}) + if (resources_server or {}).get("name") == "???": + return False + # Explicitly unbound Hermes uses native sessions; its compatibility /run requires Resources. + # A legacy relay here would conflict with the Environment Server supplied by composition. + if agent_type == "hermes_agent" and resources_server is None: return False return bool(agent.get("entrypoint") or instance.get("_inherit_from")) diff --git a/tests/unit_tests/test_add_legacy_agent_environment_servers.py b/tests/unit_tests/test_add_legacy_agent_environment_servers.py index 7f437509b7e..7b6666a05b9 100644 --- a/tests/unit_tests/test_add_legacy_agent_environment_servers.py +++ b/tests/unit_tests/test_add_legacy_agent_environment_servers.py @@ -72,6 +72,75 @@ def test_migrates_a_config_outside_the_repository(tmp_path: Path) -> None: assert config.read_text().startswith(AGENT_CONFIG) # existing content, comments included, is untouched +@pytest.mark.parametrize("resources", [None, {"type": "resources_servers", "name": "my_resources"}]) +@pytest.mark.parametrize("agent_type", ["hermes_agent", "osworld_agent"]) +def test_migration_respects_hermes_native_session_template( + tmp_path: Path, resources: dict[str, str] | None, agent_type: str +) -> None: + config = tmp_path / "agent.yaml" + config.write_text( + yaml.safe_dump( + { + "my_agent": { + "responses_api_agents": { + agent_type: { + "entrypoint": "app.py", + "resources_server": resources, + } + } + } + } + ) + ) + before = config.read_text() + + assert migration.main([str(config)]) == 0 + + if agent_type == "hermes_agent" and resources is None: + assert config.read_text() == before + else: + assert _environment_servers(yaml.safe_load(config.read_text())) == {"my_agent": ["my_environment_server"]} + + +def test_hermes_overlay_keeps_its_inherited_resources_binding(tmp_path: Path) -> None: + base = tmp_path / "base.yaml" + base.write_text(AGENT_CONFIG.replace("simple_agent", "hermes_agent")) + overlay = tmp_path / "overlay.yaml" + overlay.write_text( + "renamed_agent:\n" + " _inherit_from: my_hermes_agent\n" + " responses_api_agents:\n" + " hermes_agent:\n" + " max_turns: 7\n" + ) + + assert migration.main([str(base), str(overlay)]) == 0 + + resolved = _parse(base, overlay, strict=True) + assert _environment_servers_by_agent(resolved) == {"renamed_agent": ["renamed_environment_server"]} + assert resolved.renamed_agent.responses_api_agents.hermes_agent.resources_server.name == "my_resources" + + +def test_default_hermes_composes_with_exactly_one_native_environment(tmp_path: Path) -> None: + config = tmp_path / "hermes.yaml" + default = SCRIPT.parents[1] / "responses_api_agents/hermes_agent/configs/hermes_agent.yaml" + config.write_text(default.read_text()) + composition = tmp_path / "run.yaml" + composition.write_text( + "policy_model_name: test-model\n" + + _server_fronting("hermes_agent", name="native_environment", server_type="single_agent_turn_legacy") + ) + before = config.read_text() + + # Migrating the standalone harness must not install a second, legacy route. + assert migration.main([str(config)]) == 0 + + assert config.read_text() == before + resolved = _parse(config, composition, strict=True) + assert _environment_servers_by_agent(resolved) == {"hermes_agent": ["native_environment"]} + assert resolved.hermes_agent.responses_api_agents.hermes_agent.resources_server is None + + def test_check_reports_without_writing(tmp_path: Path, capsys: pytest.CaptureFixture) -> None: config = tmp_path / "my_run.yaml" config.write_text(AGENT_CONFIG) From 0db6de06c9d37dd93e1c7ce3338ebed19e9d4df8 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 6 Oct 2026 14:01:04 -0700 Subject: [PATCH 36/44] fix(config): preserve native openclaw templates during migration Signed-off-by: yaoyu-33 --- .../configs/openclaw_agent_native.yaml | 1 + .../add_legacy_agent_environment_servers.py | 14 +++-- ...st_add_legacy_agent_environment_servers.py | 54 +++++++++++++++++++ 3 files changed, 65 insertions(+), 4 deletions(-) diff --git a/responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml b/responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml index 2807560aef4..db87e307bab 100644 --- a/responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml +++ b/responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml @@ -3,6 +3,7 @@ openclaw_agent_native: openclaw_agent: entrypoint: app.py num_workers: 1 + resources_server: null model_server: {type: responses_api_models, name: policy_model} model: ${policy_model_name} openclaw_version: 2026.6.11 diff --git a/scripts/add_legacy_agent_environment_servers.py b/scripts/add_legacy_agent_environment_servers.py index be03db0419d..465ec256d17 100644 --- a/scripts/add_legacy_agent_environment_servers.py +++ b/scripts/add_legacy_agent_environment_servers.py @@ -98,9 +98,10 @@ def is_rename(instance: dict) -> bool: def needs_environment_server(instance: dict) -> bool: """True for an agent instance a run dispatches to, so it needs a server in front of it. - Two kinds are skipped. An unbound template leaves `resources_server.name` unset for - composition to fill. A shared overlay names several benchmarks' agents to override one field - on each; without an entrypoint or an `_inherit_from` supplying one, that name is not a server + Unbound templates leave `resources_server.name` unset for composition to fill, or + explicitly select a native session interface without Resources. A shared overlay names + several benchmarks' agents to override one field on each; without an entrypoint or an + `_inherit_from` supplying one, that name is not a server a run can start, and declaring a server for it strands the reference in every run that merges the overlay without the agent. """ @@ -110,7 +111,12 @@ def needs_environment_server(instance: dict) -> bool: agent = instance.get("responses_api_agents", {}).get(agent_type) if agent_type else None if not isinstance(agent, dict): return False - if (agent.get("resources_server") or {}).get("name") == "???": + resources_server = agent.get("resources_server", {}) + if (resources_server or {}).get("name") == "???": + return False + # These adapters require Resources for compatibility /run; explicit null is a native template. + # An omitted binding can inherit Resources and must still migrate. + if agent_type in {"hermes_agent", "openclaw_agent", "opencode_agent"} and resources_server is None: return False return bool(agent.get("entrypoint") or instance.get("_inherit_from")) diff --git a/tests/unit_tests/test_add_legacy_agent_environment_servers.py b/tests/unit_tests/test_add_legacy_agent_environment_servers.py index 37f133799de..2dc3393eabe 100644 --- a/tests/unit_tests/test_add_legacy_agent_environment_servers.py +++ b/tests/unit_tests/test_add_legacy_agent_environment_servers.py @@ -72,6 +72,60 @@ def test_migrates_a_config_outside_the_repository(tmp_path: Path) -> None: assert config.read_text().startswith(AGENT_CONFIG) # existing content, comments included, is untouched +@pytest.mark.parametrize("resources", [None, {"type": "resources_servers", "name": "my_resources"}]) +@pytest.mark.parametrize("agent_type", ["hermes_agent", "openclaw_agent", "opencode_agent", "osworld_agent"]) +def test_migration_respects_native_session_templates( + tmp_path: Path, resources: dict[str, str] | None, agent_type: str +) -> None: + config = tmp_path / "agent.yaml" + config.write_text( + yaml.safe_dump( + { + "my_agent": { + "responses_api_agents": {agent_type: {"entrypoint": "app.py", "resources_server": resources}} + } + } + ) + ) + before = config.read_text() + assert migration.main([str(config)]) == 0 + if agent_type != "osworld_agent" and resources is None: + assert config.read_text() == before + else: + assert _environment_servers(yaml.safe_load(config.read_text())) == {"my_agent": ["my_environment_server"]} + + +@pytest.mark.parametrize("agent_type", ["hermes_agent", "openclaw_agent", "opencode_agent"]) +def test_native_capable_overlay_keeps_inherited_resources(tmp_path: Path, agent_type: str) -> None: + base = tmp_path / "base.yaml" + base.write_text(AGENT_CONFIG.replace("simple_agent", agent_type)) + overlay = tmp_path / "overlay.yaml" + overlay.write_text( + f"renamed_agent:\n _inherit_from: my_{agent_type}\n responses_api_agents:\n {agent_type}: {{}}\n" + ) + assert migration.main([str(base), str(overlay)]) == 0 + resolved = _parse(base, overlay, strict=True) + assert _environment_servers_by_agent(resolved) == {"renamed_agent": ["renamed_environment_server"]} + assert resolved.renamed_agent.responses_api_agents[agent_type].resources_server.name == "my_resources" + + +def test_native_openclaw_template_composes_with_one_environment(tmp_path: Path) -> None: + config = tmp_path / "agent.yaml" + default = SCRIPT.parents[1] / "responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml" + config.write_text(default.read_text()) + composition = tmp_path / "run.yaml" + composition.write_text( + "policy_model_name: test-model\n" + + _server_fronting("openclaw_agent_native", name="native_environment", server_type="single_agent_turn_legacy") + ) + before = config.read_text() + assert migration.main([str(config)]) == 0 + assert config.read_text() == before + resolved = _parse(config, composition, strict=True) + assert _environment_servers_by_agent(resolved) == {"openclaw_agent_native": ["native_environment"]} + assert resolved.openclaw_agent_native.responses_api_agents.openclaw_agent.resources_server is None + + def test_check_reports_without_writing(tmp_path: Path, capsys: pytest.CaptureFixture) -> None: config = tmp_path / "my_run.yaml" config.write_text(AGENT_CONFIG) From e9fff6ae576ce8888e9e0fe8eb71ddcfd44054b0 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 6 Oct 2026 16:35:13 -0700 Subject: [PATCH 37/44] fix(openclaw): align shared model errors and native routing Signed-off-by: yaoyu-33 --- nemo_gym/base_responses_api_model.py | 28 ++++++++---- .../openai_model/tests/test_app.py | 43 +++++++++++++++++++ responses_api_models/vllm_model/app.py | 6 +-- .../vllm_model/tests/test_app.py | 8 ++-- .../add_legacy_agent_environment_servers.py | 8 +++- ...st_add_legacy_agent_environment_servers.py | 6 ++- .../test_responses_api_model_streaming.py | 26 ++++++++++- 7 files changed, 104 insertions(+), 21 deletions(-) diff --git a/nemo_gym/base_responses_api_model.py b/nemo_gym/base_responses_api_model.py index 46e0204bdf9..932a4d8d511 100644 --- a/nemo_gym/base_responses_api_model.py +++ b/nemo_gym/base_responses_api_model.py @@ -42,6 +42,7 @@ from uuid import uuid4 import orjson +from aiohttp import ClientResponseError from fastapi import Body, FastAPI, HTTPException, Request, Response from fastapi.exceptions import RequestValidationError from fastapi.responses import StreamingResponse @@ -259,6 +260,16 @@ async def _stream_served_response(self, response: Any, events: Iterable[str | by def setup_webserver(self) -> FastAPI: app = FastAPI() + @app.exception_handler(ClientResponseError) + async def upstream_error(request: Request, error: ClientResponseError) -> Response: + # Preserve the provider's structured error while the buffered response + # is still uncommitted, including context limits and rate limits. + return Response( + content=getattr(error, "response_content", error.message), + status_code=error.status, + media_type=(error.headers or {}).get("Content-Type", "application/json"), + ) + self.setup_session_middleware(app) capture_config = ModelCallCaptureConfig.model_validate(self.server_client.global_config_dict) install_model_call_capture( @@ -315,10 +326,9 @@ async def responses_dispatch(self, request: Request, body: dict = Body()): always do), the request is first sanitized from the streaming wire dialect (extra bookkeeping fields, ``namespace`` tool specs — see ``nemo_gym.responses_streaming``), delegated to the same ``responses()``, and the complete response is re-emitted as a - synthesized Responses SSE event stream. A ``responses()`` failure on this path is turned - into a terminal ``response.failed`` event rather than an HTTP 500 (bad-request validation - and an ``HTTPException`` raised by ``responses()`` still fail eagerly with their status, before - the stream is committed). + synthesized Responses SSE event stream. Upstream HTTP errors and ``HTTPException`` + retain their status before the stream starts. Other ``responses()`` failures become terminal + ``response.failed`` events; bad-request validation still fails eagerly. """ if not body.get("stream"): params = _validate_responses_params(body) @@ -350,13 +360,13 @@ async def _stream_responses( ) response_json["output"] = restore_namespace_tool_calls(response_json.get("output") or [], ns_map) return await self._stream_served_response(response_json, synthesize_responses_sse(response_json)) - except HTTPException: - # A status the server chose to return, such as a propagated provider error, is raised - # before the stream is committed, so the client receives it like a validation error. + except (ClientResponseError, HTTPException): + # Generation is buffered: no headers or SSE bytes have been sent yet. + # Let the shared HTTP handler retain the upstream error code and body. raise except Exception as exc: - # The streaming contract is already the response's shape, so a backend failure must be a - # terminal response.failed event, not an HTTP 500 the client would see as a broken stream. + # Keep the existing terminal-SSE fallback for failures without an + # upstream HTTP response to preserve. logger.exception("responses() failed while serving a streaming /v1/responses request") return StreamingResponse( synthesize_responses_failure_sse(str(exc)), diff --git a/responses_api_models/openai_model/tests/test_app.py b/responses_api_models/openai_model/tests/test_app.py index d26b5e9226a..33f3a27705c 100644 --- a/responses_api_models/openai_model/tests/test_app.py +++ b/responses_api_models/openai_model/tests/test_app.py @@ -109,6 +109,49 @@ async def test_retry_configuration_is_scoped_to_model_server(self): assert judge._client.max_http_attempts == 5 assert policy._client.max_http_attempts == 3 + @pytest.mark.parametrize( + "status,message,code", + [ + (400, "This model's maximum context length is 262144 tokens.", "context_length_exceeded"), + (429, "Rate limit exceeded", "rate_limit_exceeded"), + (503, "Model service unavailable", "server_error"), + ], + ) + @pytest.mark.parametrize( + "api,stream", + [("chat/completions", False), ("chat/completions", True), ("responses", False), ("responses", True)], + ) + def test_upstream_error_preserves_status_and_body(self, tmp_path, status, message, code, api, stream): + server = self._setup_server() + server.server_client.global_config_dict = { + "observability_enabled": True, + "model_call_capture_dir": str(tmp_path), + } + payload = {"error": {"message": message, "type": "upstream_error", "code": code}} + error = ClientResponseError(MagicMock(), (), status=status, message=message) + error.response_content = json.dumps(payload).encode() + server._client = MagicMock(spec=NeMoGymAsyncOpenAI) + server._client.create_chat_completion = AsyncMock(side_effect=error) + server._client.create_response = AsyncMock(side_effect=error) + body = ( + {"messages": [{"role": "user", "content": "hello"}]} if api == "chat/completions" else {"input": "hello"} + ) + if stream: + body["stream"] = True + + app = server.setup_webserver() + server.setup_exception_middleware(app) + with TestClient(app) as client: + response = client.post(f"/ng-rollout/upstream-error/v1/{api}", json=body) + + assert response.status_code == status + assert response.headers["content-type"] == "application/json" + assert response.content == error.response_content + assert response.json() == payload + calls = read_model_call_records(CaptureStore(tmp_path), "upstream-error") + assert len(calls) == 1 + assert calls[0].status_code == status + async def test_chat_completions(self, monkeypatch: MonkeyPatch, tmp_path) -> None: server = self._setup_server() server.server_client.global_config_dict = { diff --git a/responses_api_models/vllm_model/app.py b/responses_api_models/vllm_model/app.py index a8708f9109d..bf362a08411 100644 --- a/responses_api_models/vllm_model/app.py +++ b/responses_api_models/vllm_model/app.py @@ -69,9 +69,9 @@ def _log_unhandled_engine_error(request: Request, status: int, body: str, route: str) -> None: """Log an engine answer the server does not handle, once, before it is re-raised. - The caller receives only the HTTP status. The engine's body names the rejected field or - the engine's reason, and the rollout id (set by the capture route's prefix) names the - caller, so both are kept here together with the request path. + The shared model error handler preserves the upstream status and body. The log + additionally ties that error to the request path and rollout id (set by the + capture route's prefix). """ LOG.warning( "engine answered %s to a %s for %s (rollout %s): %s", diff --git a/responses_api_models/vllm_model/tests/test_app.py b/responses_api_models/vllm_model/tests/test_app.py index 9ac55de76ce..b75c3e2dd92 100644 --- a/responses_api_models/vllm_model/tests/test_app.py +++ b/responses_api_models/vllm_model/tests/test_app.py @@ -879,9 +879,8 @@ def test_context_overflow_propagation_flag( def test_an_engine_error_the_server_does_not_handle_is_logged_with_its_body( self, monkeypatch: MonkeyPatch, caplog, use_completions_api: bool ) -> None: - # Any engine answer but the handled context-length 400 reaches the caller as a plain - # server error that carries only the status; the log keeps the status, the request path, - # the rollout id, and the engine's body. + # Unhandled engine errors preserve the upstream status/body for SDK classifiers. + # The warning also retains the request path and rollout id for diagnosis. server = self._setup_server(monkeypatch, use_completions_api=use_completions_api) request_info = MagicMock(real_url="http://vllm.test/v1/chat/completions") error = ClientResponseError(request_info, (), status=422, message="Unprocessable Entity") @@ -902,7 +901,8 @@ def test_an_engine_error_the_server_does_not_handle_is_logged_with_its_body( json={"model": "dummy_model", "messages": [{"role": "user", "content": "hi"}]}, ) - assert response.status_code == 500 + assert response.status_code == 422 + assert response.content == error.response_content logged = [ record.getMessage() for record in caplog.records diff --git a/scripts/add_legacy_agent_environment_servers.py b/scripts/add_legacy_agent_environment_servers.py index 465ec256d17..f8a097b26c9 100644 --- a/scripts/add_legacy_agent_environment_servers.py +++ b/scripts/add_legacy_agent_environment_servers.py @@ -114,9 +114,13 @@ def needs_environment_server(instance: dict) -> bool: resources_server = agent.get("resources_server", {}) if (resources_server or {}).get("name") == "???": return False - # These adapters require Resources for compatibility /run; explicit null is a native template. + # Explicitly unbound native templates do not use the compatibility /run route. + # A legacy relay would conflict with the Environment Server supplied by composition. # An omitted binding can inherit Resources and must still migrate. - if agent_type in {"hermes_agent", "openclaw_agent", "opencode_agent"} and resources_server is None: + if ( + agent_type in {"hermes_agent", "pi_agent", "codex_agent", "openclaw_agent", "opencode_agent"} + and resources_server is None + ): return False return bool(agent.get("entrypoint") or instance.get("_inherit_from")) diff --git a/tests/unit_tests/test_add_legacy_agent_environment_servers.py b/tests/unit_tests/test_add_legacy_agent_environment_servers.py index 70a231c3335..c1a80a59d4b 100644 --- a/tests/unit_tests/test_add_legacy_agent_environment_servers.py +++ b/tests/unit_tests/test_add_legacy_agent_environment_servers.py @@ -73,7 +73,9 @@ def test_migrates_a_config_outside_the_repository(tmp_path: Path) -> None: @pytest.mark.parametrize("resources", [None, {"type": "resources_servers", "name": "my_resources"}]) -@pytest.mark.parametrize("agent_type", ["hermes_agent", "openclaw_agent", "opencode_agent", "osworld_agent"]) +@pytest.mark.parametrize( + "agent_type", ["hermes_agent", "pi_agent", "codex_agent", "openclaw_agent", "opencode_agent", "osworld_agent"] +) def test_migration_respects_native_session_templates( tmp_path: Path, resources: dict[str, str] | None, agent_type: str ) -> None: @@ -95,7 +97,7 @@ def test_migration_respects_native_session_templates( assert _environment_servers(yaml.safe_load(config.read_text())) == {"my_agent": ["my_environment_server"]} -@pytest.mark.parametrize("agent_type", ["hermes_agent", "openclaw_agent", "opencode_agent"]) +@pytest.mark.parametrize("agent_type", ["hermes_agent", "pi_agent", "codex_agent", "openclaw_agent", "opencode_agent"]) def test_native_capable_overlay_keeps_inherited_resources(tmp_path: Path, agent_type: str) -> None: base = tmp_path / "base.yaml" base.write_text(AGENT_CONFIG.replace("simple_agent", agent_type)) diff --git a/tests/unit_tests/test_responses_api_model_streaming.py b/tests/unit_tests/test_responses_api_model_streaming.py index 30239fafcf0..3ff89db1424 100644 --- a/tests/unit_tests/test_responses_api_model_streaming.py +++ b/tests/unit_tests/test_responses_api_model_streaming.py @@ -23,10 +23,11 @@ import json from time import time -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock, patch from uuid import uuid4 import pytest +from aiohttp import ClientResponseError from fastapi import Body, HTTPException, Request from fastapi.testclient import TestClient from pydantic import TypeAdapter, ValidationError @@ -780,6 +781,29 @@ def test_streaming_backend_error_yields_response_failed(self) -> None: assert payload["response"]["status"] == "failed" assert "backend exploded" in payload["response"]["error"]["message"] + @pytest.mark.parametrize("stream", [False, True]) + @pytest.mark.parametrize( + "status,content_type,content", + [ + (400, "application/json", b'{"error":{"code":"context_length_exceeded"}}'), + (429, "application/json", b'{"error":{"code":"rate_limit_exceeded"}}'), + (502, "text/plain", b"Upstream unavailable"), + ], + ) + def test_upstream_http_error_is_preserved_before_stream_starts( + self, stream: bool, status: int, content_type: str, content: bytes + ) -> None: + error = ClientResponseError( + MagicMock(), (), status=status, message="upstream failed", headers={"Content-Type": content_type} + ) + error.response_content = content + with patch.object(_EchoModel, "responses", AsyncMock(side_effect=error)): + client, _ = _client(_EchoModel) + response = client.post("/v1/responses", json={"input": "task", "stream": stream}) + assert response.status_code == status + assert response.headers["content-type"].split(";")[0] == content_type + assert response.content == content + def test_streaming_http_exception_keeps_its_status(self) -> None: # An HTTPException is a status the server chose to return; it is raised before the # stream is committed, so it is not converted into a response.failed event. From c0136aee87c7ef37789607d0ee27561ef1b46934 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Thu, 8 Oct 2026 08:56:38 -0700 Subject: [PATCH 38/44] fix(openclaw): align native bootstrap and model error policy Signed-off-by: yaoyu-33 --- AGENTS.md | 5 ++ nemo_gym/base_responses_api_model.py | 6 ++ responses_api_agents/openclaw_agent/README.md | 6 +- responses_api_agents/openclaw_agent/app.py | 54 +++++++----- .../openclaw_agent/tests/test_bootstrap.py | 82 +++++++++++++++++++ .../openclaw_agent/tests/test_sandbox.py | 49 +++++++++-- .../litellm_model/tests/test_app.py | 21 +++++ responses_api_models/openai_model/app.py | 4 + .../openai_model/tests/test_app.py | 26 +++++- ...st_add_legacy_agent_environment_servers.py | 3 +- 10 files changed, 225 insertions(+), 31 deletions(-) create mode 100644 responses_api_agents/openclaw_agent/tests/test_bootstrap.py diff --git a/AGENTS.md b/AGENTS.md index ac244dc3a9e..e29f52cd4ca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,6 +105,11 @@ When an environment requires an external tool (compiler, runtime, etc.), auto-in 3. For tests: add a `pytest_configure` hook in `conftest.py` that calls `ensure_()` before collection, so `skipif(shutil.which("tool") is None)` markers see the installed tool 4. Build-from-source scripts should be idempotent (skip if artifacts exist) and install into a local prefix (e.g. `./` in the server dir, gitignored) +For sandboxed harnesses, install the harness runtime inside the sandbox during agent-session setup, before activation; +do not install an unused copy on the agent-server host. An optional local execution path may install its host runtime +on first use instead of startup. Document that first-call setup, share concurrent installation attempts, surface +installation failures, and allow a later call to retry. Required host tools still follow the startup rule above. + ## Common Commands for Building & Testing Environments ```bash diff --git a/nemo_gym/base_responses_api_model.py b/nemo_gym/base_responses_api_model.py index 932a4d8d511..220cda477af 100644 --- a/nemo_gym/base_responses_api_model.py +++ b/nemo_gym/base_responses_api_model.py @@ -257,11 +257,17 @@ async def _stream_served_response(self, response: Any, events: Iterable[str | by headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, ) + def _should_propagate_upstream_http_error(self, error: ClientResponseError) -> bool: + """Allow adapters to retain an explicit upstream-status compatibility policy.""" + return True + def setup_webserver(self) -> FastAPI: app = FastAPI() @app.exception_handler(ClientResponseError) async def upstream_error(request: Request, error: ClientResponseError) -> Response: + if not self._should_propagate_upstream_http_error(error): + raise error # Preserve the provider's structured error while the buffered response # is still uncommitted, including context limits and rate limits. return Response( diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index 3170a301fd2..532e79c9c1f 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -161,13 +161,15 @@ to `1200` for a 20-minute model-call limit. Increase the separate `timeout` and deadline when a task needs more total time; increasing those alone does not raise the model-idle limit. The installer supports Linux glibc on x86_64/aarch64 and musl on x86_64, with Python 3.8 or later for the -supervisor. Node 22.19.0 and OpenClaw are installed under +supervisor. Session setup installs missing Python and Bash with apt-get or apk +when running as root; otherwise preinstall Python 3.8+ and Bash. Existing tools +are preserved. Node 22.19.0 and OpenClaw are installed under `/tmp/nemo-gym-openclaw-node-22.19.0-`. Node archives are checked against upstream SHA-256 sums; the installed package version is verified before caching and when reusing a cached runtime. A version-scoped `flock` serializes runtime installation and cache validation across session setup attempts. Missing `flock`, `curl`, CA certificates, `tar`, `gzip`, `sha256sum`, and `awk` are installed with `apt-get` or `apk` only when running as root. Other images receive an actionable prerequisite error. Installation errors include -the failed command, exit status, and stderr. Supporting these architectures does not establish +the failed command, exit status, provider error, stdout, and stderr. Supporting these architectures does not establish validation on every task image; record the actual image digest and runtime versions with each run. The runner uses ordinary sandbox `exec`; no PTY/session API is required. Its internal deadline diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index 687e495116c..2de819cfc76 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -389,6 +389,31 @@ def _extract_instruction(body_input) -> tuple[str, Optional[str]]: return user_message, system_message +def _sandbox_prepare_command(workdir: str, directory: str, runtime: str) -> str: + # Bootstrap with POSIX sh; the runtime installer itself requires Bash. + bootstrap = """set -eu +set -- +command -v python3 >/dev/null 2>&1 || set -- "$@" python3 +command -v bash >/dev/null 2>&1 || set -- "$@" bash +if [ "$#" -gt 0 ]; then + [ "$(id -u)" = 0 ] || { echo "Native OpenClaw requires $*: preinstall these tools or use a root image." >&2; exit 1; } + if command -v apk >/dev/null 2>&1; then + apk add --no-cache "$@" + elif command -v apt-get >/dev/null 2>&1; then + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "$@" + else + echo "Native OpenClaw requires $*: preinstall these tools (automatic installation requires apt-get or apk)." >&2 + exit 1 + fi +fi +""" + check_paths = shlex.join( + ["python3", "-I", "-c", _SANDBOX_PATH_CHECK, workdir, str(Path(directory).parent), runtime] + ) + return bootstrap + f"{check_paths} && mkdir -p {shlex.quote(directory + '/home/.openclaw')}" + + class OpenClawAgentConfig(BaseResponsesAPIAgentConfig): resources_server: Optional[ResourcesServerRef] = None model_server: Optional[ModelServerRef] = None @@ -512,24 +537,15 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Open raise RuntimeError( f"Cannot create OpenClaw sandbox workdir {workdir}: {workspace.stderr or workspace.stdout}" ) - check_paths = shlex.join( - [ - "python3", - "-c", - _SANDBOX_PATH_CHECK, - workdir, - "/tmp/nemo-gym-openclaw-sessions", - runtime, - ] - ) prepared = await sandbox.exec( - "command -v python3 >/dev/null 2>&1 || " - "{ echo 'Native OpenClaw requires Python >=3.8 in the task image' >&2; exit 1; }; " - f"{check_paths} && mkdir -p {shlex.quote(directory + '/home/.openclaw')}", - timeout_s=30, + _sandbox_prepare_command(workdir, directory, runtime), + timeout_s=self.config.sandbox_install_timeout_seconds, ) - if prepared.return_code != 0 or getattr(prepared, "error_type", None): - raise RuntimeError(prepared.stderr or "Cannot create OpenClaw sandbox session directory") + if prepared.return_code != 0 or prepared.error_type: + raise RuntimeError( + f"OpenClaw sandbox preparation failed (exit {prepared.return_code}, error={prepared.error_type}): " + f"stderr={prepared.stderr[-16000:]}; stdout={prepared.stdout[-16000:]}" + ) prepared_directory = True # Install only the agent runtime in the existing task sandbox. # Resources has already prepared the task repository and its dependencies. @@ -541,11 +557,11 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Open cwd=workdir, timeout_s=self.config.sandbox_install_timeout_seconds, ) - if installed.return_code != 0 or getattr(installed, "error_type", None): + if installed.return_code != 0 or installed.error_type: raise RuntimeError( - f"OpenClaw runtime setup failed (exit {installed.return_code}): " + f"OpenClaw runtime setup failed (exit {installed.return_code}, error={installed.error_type}): " f"bash {directory}/{installer} {runtime} {self.config.openclaw_version}\n" - f"{installed.stderr or installed.stdout}" + f"stderr={installed.stderr[-16000:]}; stdout={installed.stdout[-16000:]}" ) await sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), f"{directory}/sandbox_runner.py") except BaseException as error: diff --git a/responses_api_agents/openclaw_agent/tests/test_bootstrap.py b/responses_api_agents/openclaw_agent/tests/test_bootstrap.py new file mode 100644 index 00000000000..df450abd01b --- /dev/null +++ b/responses_api_agents/openclaw_agent/tests/test_bootstrap.py @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import shlex +import shutil +import subprocess +import sys + +import pytest + +from responses_api_agents.openclaw_agent.app import _sandbox_prepare_command + + +@pytest.mark.parametrize("package_manager", ["apk", "apt-get"]) +@pytest.mark.parametrize("missing", [(), ("bash",), ("python3",), ("python3", "bash")]) +@pytest.mark.parametrize("failure", [None, "nonroot", "install", "unavailable"]) +def test_posix_bootstrap_installs_only_missing_prerequisites(tmp_path, package_manager, missing, failure): + """Exercise the actual preparation through sh without invoking a host package manager.""" + bindir = tmp_path / "bin" + bindir.mkdir() + binaries = {"python3": sys.executable, "bash": shutil.which("bash"), "mkdir": shutil.which("mkdir")} + assert all(binaries.values()) + for name, executable in binaries.items(): + if name not in missing: + (bindir / name).symlink_to(executable) + (bindir / "ln").symlink_to(shutil.which("ln")) + uid = bindir / "id" + uid.write_text(f"#!/bin/sh\necho {1000 if failure == 'nonroot' else 0}\n") + uid.chmod(0o755) + manager = bindir / package_manager + manager.write_text( + '#!/bin/sh\nprintf "%s\\n" "$*" >> "$TEST_ROOT/packages.log"\n' + + ("exit 100\n" if failure == "install" else "") + + 'for name in "$@"; do\n' + + f'case "$name" in\n python3) ln -s {shlex.quote(sys.executable)} "$TEST_ROOT/bin/python3" ;;\n' + + f' bash) ln -s {shlex.quote(binaries["bash"])} "$TEST_ROOT/bin/bash" ;;\n' + + "esac\ndone\n" + ) + manager.chmod(0o755) + if failure == "unavailable": + manager.unlink() + directory = tmp_path / "sessions" / "session with spaces" + task = tmp_path / "task" + task.mkdir() + command = _sandbox_prepare_command(str(task), str(directory), str(tmp_path / "runtime")) + result = subprocess.run( + ["/bin/sh", "-c", command], + env=os.environ | {"PATH": str(bindir), "TEST_ROOT": str(tmp_path)}, + capture_output=True, + text=True, + errors="replace", + timeout=10, + ) + if failure and missing: + assert result.returncode == (100 if failure == "install" else 1), result.stderr + assert not directory.exists() + if failure != "install": + assert "preinstall" in result.stderr + assert not (tmp_path / "packages.log").exists() + return + assert result.returncode == 0, result.stderr + assert (directory / "home/.openclaw").is_dir() + packages = " ".join(missing) + if missing: + assert (tmp_path / "packages.log").read_text().splitlines() == ( + [f"add --no-cache {packages}"] + if package_manager == "apk" + else ["update", f"install -y --no-install-recommends {packages}"] + ) + else: + assert not (tmp_path / "packages.log").exists() + bash = subprocess.run( + ["/bin/sh", "-c", "bash -c 'printf ready'"], + env=os.environ | {"PATH": str(bindir)}, + capture_output=True, + text=True, + errors="replace", + timeout=10, + ) + assert bash.returncode == 0, bash.stderr + assert bash.stdout == "ready" diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index b305320aa0d..218bc113028 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -186,6 +186,38 @@ def setup(): yield agent, sandbox +@pytest.mark.parametrize("stage", ["prepare", "install"]) +@pytest.mark.parametrize("error_type", ["timeout", "sandbox"]) +@pytest.mark.parametrize("owned", [False, True]) +async def test_provider_error_type_blocks_setup_even_with_zero_exit(setup, stage, error_type, owned): + agent, sandbox = setup + request = Request({"type": "http", "session": {}}) + body = seed() + if owned: + agent.config.sandbox_provider = "sandbox" + agent.config.sandbox_config = {"image": "test-image", "workdir": "/app"} + body = body.model_copy(update={"sandbox_access": None}) + sandbox.start = AsyncMock() + ok = SimpleNamespace(return_code=0, error_type=None, stdout="", stderr="") + error = SimpleNamespace(return_code=0, error_type=error_type, stdout="bootstrap output", stderr="provider failed") + sandbox.exec.side_effect = ([ok] if owned else []) + ([error, ok] if stage == "prepare" else [ok, error, ok]) + with patch("responses_api_agents.openclaw_agent.app.AsyncSandbox", return_value=sandbox) as sandbox_class: + sandbox_class.connect = AsyncMock(return_value=sandbox) + with pytest.raises(RuntimeError, match=f"error={error_type}") as failed: + await agent.seed_agent_session(request, body) + assert "bootstrap output" in str(failed.value) + assert "provider failed" in str(failed.value) + assert not request.session + sandbox.launch.assert_not_awaited() + if owned: + sandbox.start.assert_awaited_once() + sandbox.stop.assert_awaited_once() + else: + sandbox.disconnect.assert_awaited_once() + sandbox.stop.assert_not_awaited() + assert not any(record.state is not None for record in agent._session_records.values()) + + def close_body(session_id): return {"agent_session_id": session_id, "episode_id": seed().episode_id.model_dump()} @@ -442,15 +474,18 @@ def test_wall_limit_returns_gradable_partial_output(setup, stop_reason): assert client.post("/v1/agent_sessions/close", json=close_body(session_id)).status_code == 200 +@pytest.mark.parametrize("stop_reason", ["stop", "length", "aborted"]) @pytest.mark.parametrize("failure", ["runner", "exit"]) -def test_runtime_failures_cannot_become_successful_responses(setup, failure): +def test_runtime_failures_cannot_become_successful_responses(setup, failure, stop_reason): agent, sandbox = setup sandbox.result.update(error="runner failed" if failure == "runner" else None, return_code=1) + sandbox.events = events(stop_reason=stop_reason) with TestClient(agent.setup_webserver()) as client: session_id = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).json()["agent_session_id"] response = client.post("/ng-rollout/openclaw-smoke-a2/v1/responses", json={"input": "task"}) assert response.status_code == 502 - assert ("runner failed" if failure == "runner" else "exited 1") in response.json()["detail"] + expected = "runner failed" if failure == "runner" else "aborted" if stop_reason == "aborted" else "exited 1" + assert expected in response.json()["detail"] closed = client.post("/v1/agent_sessions/close", json=close_body(session_id)) invocation = closed.json()["agent_observations"]["records"][0] assert invocation["status"] == "failed" @@ -702,7 +737,7 @@ async def test_install_failure_disconnects_without_stopping_owner(setup): agent, sandbox = setup sandbox.exec.side_effect = [ SimpleNamespace(return_code=0, error_type=None), - SimpleNamespace(return_code=1, stderr="npm failed"), + SimpleNamespace(return_code=1, error_type=None, stdout="", stderr="npm failed"), SimpleNamespace(return_code=0, error_type=None), ] request = Request({"type": "http", "session": {}}) @@ -1219,7 +1254,9 @@ def test_workdir_validation_accepts_separate_repository_and_rejects_missing(tmp_ async def test_path_check_failure_never_uploads_or_installs(setup): agent, sandbox = setup - sandbox.exec.return_value = SimpleNamespace(return_code=1, stderr="task workdir overlaps adapter-owned storage") + sandbox.exec.return_value = SimpleNamespace( + return_code=1, error_type=None, stdout="", stderr="task workdir overlaps adapter-owned storage" + ) request = Request({"type": "http", "session": {}}) with pytest.raises(RuntimeError, match="overlaps adapter-owned storage"): await agent.seed_agent_session(request, seed()) @@ -1484,7 +1521,7 @@ async def test_failed_seed_with_confirmed_cleanup_accepts_cookie_less_close(setu body = seed() sandbox.exec.side_effect = [ SimpleNamespace(return_code=0, error_type=None), - SimpleNamespace(return_code=1, stderr="npm unavailable"), + SimpleNamespace(return_code=1, error_type=None, stdout="", stderr="npm unavailable"), SimpleNamespace(return_code=0, error_type=None), ] with pytest.raises(RuntimeError, match="npm unavailable"): @@ -1501,7 +1538,7 @@ async def test_failed_seed_cleanup_retains_state_until_explicit_retry(setup): body = seed() sandbox.exec.side_effect = [ SimpleNamespace(return_code=0, error_type=None), - SimpleNamespace(return_code=1, stderr="npm unavailable"), + SimpleNamespace(return_code=1, error_type=None, stdout="", stderr="npm unavailable"), SimpleNamespace(return_code=0, error_type=None), ] sandbox.disconnect.side_effect = RuntimeError("disconnect unavailable") diff --git a/responses_api_models/litellm_model/tests/test_app.py b/responses_api_models/litellm_model/tests/test_app.py index dafb0f1e247..ea6b3aed21c 100644 --- a/responses_api_models/litellm_model/tests/test_app.py +++ b/responses_api_models/litellm_model/tests/test_app.py @@ -510,3 +510,24 @@ async def test_responses_propagates_configured_http_status(self) -> None: assert exc_info.value.status_code == 400 assert exc_info.value.detail == {"error": {"code": "context_length_exceeded"}} + + @pytest.mark.parametrize("allowed,expected", [([], 500), ([429], 500), ([400], 400)]) + @pytest.mark.parametrize("api", ["responses", "chat/completions"]) + @pytest.mark.parametrize("stream", [False, True]) + def test_http_routes_respect_propagation_policy(self, allowed, expected, api, stream) -> None: + server = _make_server(propagate_upstream_http_status_codes=allowed) + error = ClientResponseError(SimpleNamespace(real_url="https://litellm.example/v1"), (), status=400) + error.response_content = b'{"error":{"code":"context_length_exceeded"}}' + server._client = MagicMock(spec=NeMoGymAsyncOpenAI) + server._client.create_response = AsyncMock(side_effect=error) + server._client.create_chat_completion = AsyncMock(side_effect=error) + app = server.setup_webserver() + server.setup_exception_middleware(app) + body = {"input": "hello"} if api == "responses" else {"messages": [{"role": "user", "content": "hello"}]} + body["stream"] = stream + response = TestClient(app).post(f"/v1/{api}", json=body) + assert response.status_code == expected + if expected == 400: + assert response.json() == {"detail": {"error": {"code": "context_length_exceeded"}}} + operation = server._client.create_response if api == "responses" else server._client.create_chat_completion + operation.assert_awaited_once() diff --git a/responses_api_models/openai_model/app.py b/responses_api_models/openai_model/app.py index b0297256472..a8f31e936bd 100644 --- a/responses_api_models/openai_model/app.py +++ b/responses_api_models/openai_model/app.py @@ -232,6 +232,10 @@ class SimpleModelServer(SimpleResponsesAPIModel): ray_enabled = False config: SimpleModelServerConfig + def _should_propagate_upstream_http_error(self, error: ClientResponseError) -> bool: + # OpenAI and subclasses such as LiteLLM retain their opt-in HTTP policy. + return error.status in self.config.propagate_upstream_http_status_codes + def model_post_init(self, context): self._client = NeMoGymAsyncOpenAI( base_url=self.config.openai_base_url, diff --git a/responses_api_models/openai_model/tests/test_app.py b/responses_api_models/openai_model/tests/test_app.py index 33f3a27705c..9a6b254c95c 100644 --- a/responses_api_models/openai_model/tests/test_app.py +++ b/responses_api_models/openai_model/tests/test_app.py @@ -122,7 +122,7 @@ async def test_retry_configuration_is_scoped_to_model_server(self): [("chat/completions", False), ("chat/completions", True), ("responses", False), ("responses", True)], ) def test_upstream_error_preserves_status_and_body(self, tmp_path, status, message, code, api, stream): - server = self._setup_server() + server = self._setup_server(propagate_upstream_http_status_codes=[status]) server.server_client.global_config_dict = { "observability_enabled": True, "model_call_capture_dir": str(tmp_path), @@ -146,12 +146,32 @@ def test_upstream_error_preserves_status_and_body(self, tmp_path, status, messag assert response.status_code == status assert response.headers["content-type"] == "application/json" - assert response.content == error.response_content - assert response.json() == payload + assert response.json() == {"detail": payload} calls = read_model_call_records(CaptureStore(tmp_path), "upstream-error") assert len(calls) == 1 assert calls[0].status_code == status + @pytest.mark.parametrize("allowed", [[], [401]]) + @pytest.mark.parametrize("status", [400, 429, 503]) + @pytest.mark.parametrize("api", ["responses", "chat/completions"]) + @pytest.mark.parametrize("stream", [False, True]) + def test_unselected_upstream_http_status_keeps_compatibility(self, allowed, status, api, stream): + server = self._setup_server(propagate_upstream_http_status_codes=allowed) + error = ClientResponseError(SimpleNamespace(real_url="https://provider.example/v1"), (), status=status) + error.response_content = b'{"error":{"code":"provider_error"}}' + server._client = MagicMock(spec=NeMoGymAsyncOpenAI) + server._client.create_response = AsyncMock(side_effect=error) + server._client.create_chat_completion = AsyncMock(side_effect=error) + body = {"input": "hello"} if api == "responses" else {"messages": [{"role": "user", "content": "hello"}]} + body["stream"] = stream + app = server.setup_webserver() + server.setup_exception_middleware(app) + with TestClient(app) as client: + response = client.post(f"/v1/{api}", json=body) + assert response.status_code == 500 + operation = server._client.create_response if api == "responses" else server._client.create_chat_completion + operation.assert_awaited_once() + async def test_chat_completions(self, monkeypatch: MonkeyPatch, tmp_path) -> None: server = self._setup_server() server.server_client.global_config_dict = { diff --git a/tests/unit_tests/test_add_legacy_agent_environment_servers.py b/tests/unit_tests/test_add_legacy_agent_environment_servers.py index af6c80de5cc..d4606623bc2 100644 --- a/tests/unit_tests/test_add_legacy_agent_environment_servers.py +++ b/tests/unit_tests/test_add_legacy_agent_environment_servers.py @@ -107,7 +107,8 @@ def test_native_capable_overlay_keeps_inherited_resources(tmp_path: Path, agent_ ) assert migration.main([str(base), str(overlay)]) == 0 resolved = _parse(base, overlay, strict=True) - assert _environment_servers_by_agent(resolved) == {"renamed_agent": ["renamed_environment_server"]} + # Main retargets the agent reference while preserving the environment's public name. + assert _environment_servers_by_agent(resolved) == {"renamed_agent": ["my_environment_server"]} assert resolved.renamed_agent.responses_api_agents[agent_type].resources_server.name == "my_resources" From f757424acf5c9155c8f2083c6233ed0994af04f3 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Thu, 8 Oct 2026 09:55:06 -0700 Subject: [PATCH 39/44] refactor(openclaw): align session and local config names Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/README.md | 19 +++++----- .../configs/openclaw_agent.yaml | 38 +++++++------------ .../configs/openclaw_agent_native.yaml | 14 ------- .../configs/openclaw_local_agent.yaml | 34 +++++++++++++++++ .../openclaw_agent/tests/test_app.py | 15 +++++++- .../openclaw_agent/tests/test_sandbox.py | 4 +- ...st_add_legacy_agent_environment_servers.py | 8 ++-- 7 files changed, 76 insertions(+), 56 deletions(-) delete mode 100644 responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml create mode 100644 responses_api_agents/openclaw_agent/configs/openclaw_local_agent.yaml diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index 532e79c9c1f..7b7ac815a91 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -94,7 +94,8 @@ The host installer settings above apply only to local CLI calls, with installati until the first call. Native sessions use their separate in-sandbox installer and the exact configured `openclaw_version`; host environment overrides do not change that runtime. -See `configs/openclaw_agent.yaml`. +Use `configs/openclaw_agent.yaml` for EnvironmentServer sessions and +`configs/openclaw_local_agent.yaml` for the local CLI settings above. ## Native EnvironmentServer sessions @@ -110,7 +111,7 @@ with an already prepared SWE-Pro JSONL and a reachable policy model, create a ru ```yaml config_paths: - resources_servers/swebench_pro/configs/swebench_pro.yaml - - responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml + - responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml - environment_servers/single_agent_turn_legacy/configs/single_agent_turn_legacy.yaml # Route existing prepared flat rows through native session orchestration. @@ -121,18 +122,18 @@ single_agent_turn_legacy: environment_servers: single_agent_turn_legacy: resources_server: {type: resources_servers, name: swebench_pro_resources_server} - agent_server: {type: responses_api_agents, name: openclaw_agent_native} + agent_server: {type: responses_api_agents, name: openclaw_agent} ``` -Save it as `native-run.yaml`, then run: +Save it as `run.yaml`, then run: ```bash -gym env start --config native-run.yaml --model-type openai_model +gym env start --config run.yaml --model-type openai_model NEMO_GYM_ALLOW_UNSUPPORTED_PAIRING=1 \ - gym eval run --no-serve --config native-run.yaml --model-type openai_model \ - --agent openclaw_agent_native \ - --input /path/to/prepared-swe-pro.jsonl --output outputs/openclaw-native.jsonl --limit 1 + gym eval run --no-serve --config run.yaml --model-type openai_model \ + --agent openclaw_agent \ + --input /path/to/prepared-swe-pro.jsonl --output outputs/openclaw-rollout.jsonl --limit 1 ``` Supply the sandbox provider config through `env.yaml` or an additional `--config` file, and ensure @@ -156,7 +157,7 @@ route is embedded in OpenClaw's Chat Completions provider configuration to retai The native adapter sets OpenClaw's provider `timeoutSeconds` to `model_timeout_seconds` (600 seconds by default). This raises the upstream 120-second idle watchdog so buffered reasoning responses can -finish. For example, set `openclaw_agent_native.responses_api_agents.openclaw_agent.model_timeout_seconds` +finish. For example, set `openclaw_agent.responses_api_agents.openclaw_agent.model_timeout_seconds` to `1200` for a 20-minute model-call limit. Increase the separate `timeout` and EnvironmentServer episode deadline when a task needs more total time; increasing those alone does not raise the model-idle limit. diff --git a/responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml b/responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml index 99e6874e004..5a777e39cf9 100644 --- a/responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml +++ b/responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml @@ -1,30 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# EnvironmentServer sessions. Use openclaw_local_agent.yaml for host CLI execution. openclaw_agent: responses_api_agents: openclaw_agent: entrypoint: app.py - resources_server: - type: resources_servers - name: ??? - concurrency: 32 - command: openclaw - model: nvinf/nvidia/meta/llama-3.3-70b-instruct - workspace_root: outputs/openclaw_agent/workspaces - openclaw_agent_id: main - thinking: "off" - system_prompt: null - setup_timeout: 900 - timeout: 900 - extra_args: [] - env: {} - openclaw_config: - models: - providers: - nvinf: - api: openai-completions - baseUrl: https://integrate.api.nvidia.com/v1 - apiKey: ${policy_api_key} - models: - - id: nvidia/meta/llama-3.3-70b-instruct - name: nvidia/meta/llama-3.3-70b-instruct - api: openai-completions + num_workers: 1 + resources_server: null + model_server: {type: responses_api_models, name: policy_model} + model: ${policy_model_name} openclaw_version: 2026.6.11 + timeout: 900 + model_timeout_seconds: 600 + sandbox_install_timeout_seconds: 900 + session_close_timeout_seconds: 60 + session_close_retry_window_seconds: 300 diff --git a/responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml b/responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml deleted file mode 100644 index db87e307bab..00000000000 --- a/responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml +++ /dev/null @@ -1,14 +0,0 @@ -openclaw_agent_native: - responses_api_agents: - openclaw_agent: - entrypoint: app.py - num_workers: 1 - resources_server: null - model_server: {type: responses_api_models, name: policy_model} - model: ${policy_model_name} - openclaw_version: 2026.6.11 - timeout: 900 - model_timeout_seconds: 600 - sandbox_install_timeout_seconds: 900 - session_close_timeout_seconds: 60 - session_close_retry_window_seconds: 300 diff --git a/responses_api_agents/openclaw_agent/configs/openclaw_local_agent.yaml b/responses_api_agents/openclaw_agent/configs/openclaw_local_agent.yaml new file mode 100644 index 00000000000..652fac50f91 --- /dev/null +++ b/responses_api_agents/openclaw_agent/configs/openclaw_local_agent.yaml @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Host CLI example. Use openclaw_agent.yaml for EnvironmentServer sessions. +openclaw_agent: + responses_api_agents: + openclaw_agent: + entrypoint: app.py + resources_server: + type: resources_servers + name: ??? + concurrency: 32 + command: openclaw + model: nvinf/nvidia/meta/llama-3.3-70b-instruct + workspace_root: outputs/openclaw_agent/workspaces + openclaw_agent_id: main + thinking: "off" + system_prompt: null + setup_timeout: 900 + timeout: 900 + extra_args: [] + env: {} + openclaw_config: + models: + providers: + nvinf: + api: openai-completions + baseUrl: https://integrate.api.nvidia.com/v1 + apiKey: ${policy_api_key} + models: + - id: nvidia/meta/llama-3.3-70b-instruct + name: nvidia/meta/llama-3.3-70b-instruct + api: openai-completions + openclaw_version: 2026.6.11 diff --git a/responses_api_agents/openclaw_agent/tests/test_app.py b/responses_api_agents/openclaw_agent/tests/test_app.py index 047ec88942b..951d6def673 100644 --- a/responses_api_agents/openclaw_agent/tests/test_app.py +++ b/responses_api_agents/openclaw_agent/tests/test_app.py @@ -957,11 +957,22 @@ def test_module_parses(self) -> None: app_path = Path(__file__).resolve().parent.parent / "app.py" compile(app_path.read_text(), str(app_path), "exec") - def test_config_yaml_parses(self) -> None: + def test_default_config_selects_environment_sessions(self) -> None: cfg_path = Path(__file__).resolve().parent.parent / "configs" / "openclaw_agent.yaml" data = yaml.safe_load(cfg_path.read_text()) - assert "openclaw_agent" in data + assert set(data) == {"openclaw_agent"} + inner = data["openclaw_agent"]["responses_api_agents"]["openclaw_agent"] + assert inner["entrypoint"] == "app.py" + assert inner["resources_server"] is None + assert inner["model_server"] == {"type": "responses_api_models", "name": "policy_model"} + assert inner["model"] == "${policy_model_name}" + + def test_local_config_yaml_parses(self) -> None: + cfg_path = Path(__file__).resolve().parent.parent / "configs" / "openclaw_local_agent.yaml" + data = yaml.safe_load(cfg_path.read_text()) + assert set(data) == {"openclaw_agent"} inner = data["openclaw_agent"]["responses_api_agents"]["openclaw_agent"] assert inner["entrypoint"] == "app.py" assert inner["concurrency"] == 32 assert inner["command"] == "openclaw" + assert inner["resources_server"] == {"type": "resources_servers", "name": "???"} diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index 218bc113028..7d4f82ecf22 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -1380,7 +1380,7 @@ async def test_independent_configs_route_prepared_rows_through_environment_run(s { "config_paths": [ "resources_servers/swebench_pro/configs/swebench_pro.yaml", - "responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml", + "responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml", "environment_servers/single_agent_turn_legacy/configs/single_agent_turn_legacy.yaml", ], "single_agent_turn_legacy": { @@ -1390,7 +1390,7 @@ async def test_independent_configs_route_prepared_rows_through_environment_run(s "type": "resources_servers", "name": "swebench_pro_resources_server", }, - "agent_server": {"type": "responses_api_agents", "name": "openclaw_agent_native"}, + "agent_server": {"type": "responses_api_agents", "name": "openclaw_agent"}, } } }, diff --git a/tests/unit_tests/test_add_legacy_agent_environment_servers.py b/tests/unit_tests/test_add_legacy_agent_environment_servers.py index d4606623bc2..84746b468b3 100644 --- a/tests/unit_tests/test_add_legacy_agent_environment_servers.py +++ b/tests/unit_tests/test_add_legacy_agent_environment_servers.py @@ -114,19 +114,19 @@ def test_native_capable_overlay_keeps_inherited_resources(tmp_path: Path, agent_ def test_native_openclaw_template_composes_with_one_environment(tmp_path: Path) -> None: config = tmp_path / "agent.yaml" - default = SCRIPT.parents[1] / "responses_api_agents/openclaw_agent/configs/openclaw_agent_native.yaml" + default = SCRIPT.parents[1] / "responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml" config.write_text(default.read_text()) composition = tmp_path / "run.yaml" composition.write_text( "policy_model_name: test-model\n" - + _server_fronting("openclaw_agent_native", name="native_environment", server_type="single_agent_turn_legacy") + + _server_fronting("openclaw_agent", name="native_environment", server_type="single_agent_turn_legacy") ) before = config.read_text() assert migration.main([str(config)]) == 0 assert config.read_text() == before resolved = _parse(config, composition, strict=True) - assert _environment_servers_by_agent(resolved) == {"openclaw_agent_native": ["native_environment"]} - assert resolved.openclaw_agent_native.responses_api_agents.openclaw_agent.resources_server is None + assert _environment_servers_by_agent(resolved) == {"openclaw_agent": ["native_environment"]} + assert resolved.openclaw_agent.responses_api_agents.openclaw_agent.resources_server is None def test_hermes_overlay_keeps_its_inherited_resources_binding(tmp_path: Path) -> None: From fa5ae88e186964c31b6ff070dec473561eb4528e Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Thu, 8 Oct 2026 10:51:03 -0700 Subject: [PATCH 40/44] test(openclaw): enforce session-based local CLI routing Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/README.md | 7 ++- .../configs/openclaw_agent.yaml | 3 +- .../openclaw_agent/tests/test_sandbox.py | 46 ++++++++++++++++++- 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index 7b7ac815a91..f29660db1bf 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -94,8 +94,11 @@ The host installer settings above apply only to local CLI calls, with installati until the first call. Native sessions use their separate in-sandbox installer and the exact configured `openclaw_version`; host environment overrides do not change that runtime. -Use `configs/openclaw_agent.yaml` for EnvironmentServer sessions and -`configs/openclaw_local_agent.yaml` for the local CLI settings above. +`configs/openclaw_agent.yaml` supports both paths: an agent session selects sandbox +execution; no agent session selects the local CLI. Invalid or closed sessions are +rejected, never retried on the host. `configs/openclaw_local_agent.yaml` is an +alternative local provider/Resources example, not a required routing switch. +Local `/run` requires a Resources binding; direct `/v1/responses` does not. ## Native EnvironmentServer sessions diff --git a/responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml b/responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml index 5a777e39cf9..49ae3788f0c 100644 --- a/responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml +++ b/responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml @@ -1,7 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# EnvironmentServer sessions. Use openclaw_local_agent.yaml for host CLI execution. +# Agent sessions run in a sandbox; unseeded requests run the local CLI. +# openclaw_local_agent.yaml provides an alternative local provider/Resources example. openclaw_agent: responses_api_agents: openclaw_agent: diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index 7d4f82ecf22..c6999b24fb2 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -18,7 +18,7 @@ from nemo_gym.agent_utils.supervisor_client import parse_cleanup_receipt from nemo_gym.base_responses_api_agent import AgentCloseSessionRequest, AgentSeedSessionRequest from nemo_gym.episode_types import EpisodeId, TaskId -from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming +from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming, NeMoGymResponseOutputMessage from nemo_gym.server_utils import ServerClient from responses_api_agents.openclaw_agent.app import OpenClawAgent, OpenClawAgentConfig, _unique_usage_messages @@ -414,6 +414,50 @@ def test_no_session_keeps_existing_local_path(setup): sandbox.launch.assert_not_awaited() +@pytest.mark.parametrize("path", ["/v1/responses", "/ng-rollout/local-smoke/v1/responses"]) +def test_default_config_unseeded_responses_run_local_cli(setup, path: str) -> None: + agent, sandbox = setup + config = OmegaConf.load(Path(__file__).parents[1] / "configs/openclaw_agent.yaml") + config.policy_model_name = "local-model" + agent.config = OpenClawAgentConfig.model_validate( + OmegaConf.to_container(config.openclaw_agent.responses_api_agents.openclaw_agent, resolve=True) + | {"host": "localhost", "port": 8001, "name": "openclaw"} + ) + assert agent.config.resources_server is None + assert agent.config.sandbox_provider is None + message = NeMoGymResponseOutputMessage( + id="msg-local", + role="assistant", + type="message", + status="completed", + content=[{"type": "output_text", "text": "local result", "annotations": []}], + ) + with ( + patch.object( + agent, + "_run_openclaw", + AsyncMock( + return_value=( + [message], + {"input_tokens": 3, "output_tokens": 2}, + "local-model", + ) + ), + ) as local, + patch("responses_api_agents.openclaw_agent.app.create_provider") as provider, + TestClient(agent.setup_webserver()) as client, + ): + response = client.post(path, json={"input": "task", "temperature": 0.7}) + assert response.status_code == 200, response.text + assert response.json()["output"][0]["content"][0]["text"] == "local result" + assert response.json()["usage"]["total_tokens"] == 5 + local.assert_awaited_once() + assert local.call_args.args == ("task", "") + assert local.call_args.kwargs["rollout_id"] == ("local-smoke" if "ng-rollout" in path else None) + provider.assert_not_called() + sandbox.launch.assert_not_awaited() + + async def activate(agent, sandbox): request = Request( {"type": "http", "headers": [], "session": {}, "path_params": {"rollout_id": "openclaw-smoke-a2"}} From 0ecc9b09736c9d0f5353fff4aa6f49a623ba72c6 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Thu, 8 Oct 2026 15:13:50 -0700 Subject: [PATCH 41/44] refactor(openclaw): remove redundant fixed-model example Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/README.md | 29 +++++----------- .../configs/openclaw_agent.yaml | 1 - .../configs/openclaw_local_agent.yaml | 34 ------------------- .../openclaw_agent/tests/test_app.py | 10 ------ 4 files changed, 8 insertions(+), 66 deletions(-) delete mode 100644 responses_api_agents/openclaw_agent/configs/openclaw_local_agent.yaml diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index f29660db1bf..2e7d636495d 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -22,27 +22,15 @@ gym eval run --no-serve --agent openclaw_math_agent \ --output openclaw_rollout.jsonl --limit 3 ``` -## Model id +## Model configuration -OpenClaw drops the leading `/` to form the upstream id, -so we include an extra prefix, such as for `nvidia/...` ids: +The default config binds `model_server` to Gym's `policy_model` and uses +`${policy_model_name}`. The adapter generates the OpenClaw provider configuration +for both local CLI calls and native sessions. Configure the endpoint and API key +on the Gym model server. -```yaml -model: nvinf/nvidia/meta/llama-3.3-70b-instruct -openclaw_config: - models: - providers: - nvinf: - api: openai-completions - baseUrl: ${policy_base_url} - apiKey: ${policy_api_key} - models: - - {id: nvidia/meta/llama-3.3-70b-instruct, name: nvidia/meta/llama-3.3-70b-instruct, api: openai-completions} -``` - -Alternatively, set `model_server` to a Gym model server and set `model` to its served model id. The -agent creates the OpenClaw provider entry automatically. Without `model_server`, the existing -provider configuration is unchanged. +Existing local callers may still supply their own OpenClaw provider configuration +when `model_server` is unset. ## Config fields @@ -96,8 +84,7 @@ configured `openclaw_version`; host environment overrides do not change that run `configs/openclaw_agent.yaml` supports both paths: an agent session selects sandbox execution; no agent session selects the local CLI. Invalid or closed sessions are -rejected, never retried on the host. `configs/openclaw_local_agent.yaml` is an -alternative local provider/Resources example, not a required routing switch. +rejected, never retried on the host. Local `/run` requires a Resources binding; direct `/v1/responses` does not. diff --git a/responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml b/responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml index 49ae3788f0c..1039ba39d59 100644 --- a/responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml +++ b/responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 # Agent sessions run in a sandbox; unseeded requests run the local CLI. -# openclaw_local_agent.yaml provides an alternative local provider/Resources example. openclaw_agent: responses_api_agents: openclaw_agent: diff --git a/responses_api_agents/openclaw_agent/configs/openclaw_local_agent.yaml b/responses_api_agents/openclaw_agent/configs/openclaw_local_agent.yaml deleted file mode 100644 index 652fac50f91..00000000000 --- a/responses_api_agents/openclaw_agent/configs/openclaw_local_agent.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Host CLI example. Use openclaw_agent.yaml for EnvironmentServer sessions. -openclaw_agent: - responses_api_agents: - openclaw_agent: - entrypoint: app.py - resources_server: - type: resources_servers - name: ??? - concurrency: 32 - command: openclaw - model: nvinf/nvidia/meta/llama-3.3-70b-instruct - workspace_root: outputs/openclaw_agent/workspaces - openclaw_agent_id: main - thinking: "off" - system_prompt: null - setup_timeout: 900 - timeout: 900 - extra_args: [] - env: {} - openclaw_config: - models: - providers: - nvinf: - api: openai-completions - baseUrl: https://integrate.api.nvidia.com/v1 - apiKey: ${policy_api_key} - models: - - id: nvidia/meta/llama-3.3-70b-instruct - name: nvidia/meta/llama-3.3-70b-instruct - api: openai-completions - openclaw_version: 2026.6.11 diff --git a/responses_api_agents/openclaw_agent/tests/test_app.py b/responses_api_agents/openclaw_agent/tests/test_app.py index 951d6def673..f5afd95b392 100644 --- a/responses_api_agents/openclaw_agent/tests/test_app.py +++ b/responses_api_agents/openclaw_agent/tests/test_app.py @@ -966,13 +966,3 @@ def test_default_config_selects_environment_sessions(self) -> None: assert inner["resources_server"] is None assert inner["model_server"] == {"type": "responses_api_models", "name": "policy_model"} assert inner["model"] == "${policy_model_name}" - - def test_local_config_yaml_parses(self) -> None: - cfg_path = Path(__file__).resolve().parent.parent / "configs" / "openclaw_local_agent.yaml" - data = yaml.safe_load(cfg_path.read_text()) - assert set(data) == {"openclaw_agent"} - inner = data["openclaw_agent"]["responses_api_agents"]["openclaw_agent"] - assert inner["entrypoint"] == "app.py" - assert inner["concurrency"] == 32 - assert inner["command"] == "openclaw" - assert inner["resources_server"] == {"type": "resources_servers", "name": "???"} From e01e9a09f7bf8c4431df1b1d3c212808e1a33159 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Thu, 8 Oct 2026 16:17:18 -0700 Subject: [PATCH 42/44] fix(openclaw): preserve setup errors with absent output Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/app.py | 4 ++-- .../openclaw_agent/tests/test_sandbox.py | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index 2de819cfc76..1115109ad04 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -544,7 +544,7 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Open if prepared.return_code != 0 or prepared.error_type: raise RuntimeError( f"OpenClaw sandbox preparation failed (exit {prepared.return_code}, error={prepared.error_type}): " - f"stderr={prepared.stderr[-16000:]}; stdout={prepared.stdout[-16000:]}" + f"stderr={(prepared.stderr or '')[-16000:]}; stdout={(prepared.stdout or '')[-16000:]}" ) prepared_directory = True # Install only the agent runtime in the existing task sandbox. @@ -561,7 +561,7 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Open raise RuntimeError( f"OpenClaw runtime setup failed (exit {installed.return_code}, error={installed.error_type}): " f"bash {directory}/{installer} {runtime} {self.config.openclaw_version}\n" - f"stderr={installed.stderr[-16000:]}; stdout={installed.stdout[-16000:]}" + f"stderr={(installed.stderr or '')[-16000:]}; stdout={(installed.stdout or '')[-16000:]}" ) await sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), f"{directory}/sandbox_runner.py") except BaseException as error: diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index c6999b24fb2..34639623079 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -19,6 +19,7 @@ from nemo_gym.base_responses_api_agent import AgentCloseSessionRequest, AgentSeedSessionRequest from nemo_gym.episode_types import EpisodeId, TaskId from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming, NeMoGymResponseOutputMessage +from nemo_gym.sandbox import SandboxExecResult from nemo_gym.server_utils import ServerClient from responses_api_agents.openclaw_agent.app import OpenClawAgent, OpenClawAgentConfig, _unique_usage_messages @@ -189,7 +190,11 @@ def setup(): @pytest.mark.parametrize("stage", ["prepare", "install"]) @pytest.mark.parametrize("error_type", ["timeout", "sandbox"]) @pytest.mark.parametrize("owned", [False, True]) -async def test_provider_error_type_blocks_setup_even_with_zero_exit(setup, stage, error_type, owned): +@pytest.mark.parametrize( + ("stdout", "stderr"), + [("bootstrap output", "provider failed"), (None, "provider failed"), ("bootstrap output", None), (None, None)], +) +async def test_provider_error_type_blocks_setup_even_with_zero_exit(setup, stage, error_type, owned, stdout, stderr): agent, sandbox = setup request = Request({"type": "http", "session": {}}) body = seed() @@ -199,14 +204,17 @@ async def test_provider_error_type_blocks_setup_even_with_zero_exit(setup, stage body = body.model_copy(update={"sandbox_access": None}) sandbox.start = AsyncMock() ok = SimpleNamespace(return_code=0, error_type=None, stdout="", stderr="") - error = SimpleNamespace(return_code=0, error_type=error_type, stdout="bootstrap output", stderr="provider failed") + error = SandboxExecResult(return_code=0, error_type=error_type, stdout=stdout, stderr=stderr) sandbox.exec.side_effect = ([ok] if owned else []) + ([error, ok] if stage == "prepare" else [ok, error, ok]) with patch("responses_api_agents.openclaw_agent.app.AsyncSandbox", return_value=sandbox) as sandbox_class: sandbox_class.connect = AsyncMock(return_value=sandbox) with pytest.raises(RuntimeError, match=f"error={error_type}") as failed: await agent.seed_agent_session(request, body) - assert "bootstrap output" in str(failed.value) - assert "provider failed" in str(failed.value) + assert "exit 0" in str(failed.value) + if stdout is not None: + assert stdout in str(failed.value) + if stderr is not None: + assert stderr in str(failed.value) assert not request.session sandbox.launch.assert_not_awaited() if owned: From 01218cdaf35ec6f2c780d69d9dfebf3c4ca2a52e Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Fri, 9 Oct 2026 10:05:05 -0700 Subject: [PATCH 43/44] fix(openclaw): align sandbox terminology and request contract Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/README.md | 35 ++++--- responses_api_agents/openclaw_agent/app.py | 99 +++++++++---------- .../configs/openclaw_agent.yaml | 2 +- .../install_openclaw_runtime.sh | 18 ++-- .../openclaw_agent/tests/test_app.py | 2 + .../tests/test_install_runtime.py | 2 +- .../openclaw_agent/tests/test_sandbox.py | 38 ++++--- 7 files changed, 104 insertions(+), 92 deletions(-) diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index 2e7d636495d..265d65a6b4f 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -8,7 +8,7 @@ Minimal, meant to be extended, and currently eval-only. ## Quick start -OpenClaw is auto-installed on the first local CLI invocation. Starting an agent server for native +OpenClaw is auto-installed on the first local CLI invocation. Starting an agent server for sandbox sessions does not install or execute OpenClaw on the agent-server host. Make sure `env.yaml` is also set. @@ -26,12 +26,18 @@ gym eval run --no-serve --agent openclaw_math_agent \ The default config binds `model_server` to Gym's `policy_model` and uses `${policy_model_name}`. The adapter generates the OpenClaw provider configuration -for both local CLI calls and native sessions. Configure the endpoint and API key +for both local CLI calls and sandbox sessions. Configure the endpoint and API key on the Gym model server. Existing local callers may still supply their own OpenClaw provider configuration when `model_server` is unset. +Compared with the previous shipped YAML, the fixed `nvinf` provider/model block +is removed. `workspace_root`, `openclaw_agent_id`, `thinking`, `system_prompt`, +`extra_args`, `env`, and `concurrency` now use class defaults unless overridden. +`resources_server` is now `null`: local `/run` needs an explicit Resources binding. +Session install/close timeouts default to 600/60 seconds. + ## Config fields - `concurrency`: max simultaneous `run()` calls @@ -56,9 +62,8 @@ when `model_server` is unset. are rejected); overridden by the `OPENCLAW_VERSION` env var, and falls back to `setup_openclaw.DEFAULT_OPENCLAW_VERSION`. An already-installed `openclaw` is only reused when `openclaw --version` reports exactly the resolved version; anything else - is reinstalled, so changing the override or the config pin takes effect on the next - startup. After an install, the launcher selected on `PATH` must report the requested - version, or startup fails. + is reinstalled on first local use after a restart. After an install, the launcher + selected on `PATH` must report the requested version, or that local request fails. Note: releases ≥ 2026.9.0 store session history in SQLite instead of JSONL, which the agent's transcript reader does not support yet — tool results and interrupted sessions are not captured, so stick to 2026.6.11 until that lands. @@ -79,7 +84,9 @@ reused when it reports exactly the requested Node version. Incompatible runtimes are replaced, not bypassed silently. The host installer settings above apply only to local CLI calls, with installation deferred -until the first call. Native sessions use their separate in-sandbox installer and the exact +until the first call rather than startup. Concurrent callers share one background-thread +attempt; it does not block the event loop, and failed installs can be retried. +Sandbox sessions use their separate in-sandbox installer and the exact configured `openclaw_version`; host environment overrides do not change that runtime. `configs/openclaw_agent.yaml` supports both paths: an agent session selects sandbox @@ -88,9 +95,9 @@ rejected, never retried on the host. Local `/run` requires a Resources binding; direct `/v1/responses` does not. -## Native EnvironmentServer sessions +## EnvironmentServer sandbox sessions -The native path runs OpenClaw and its file/process tools inside the task sandbox created by +The sandbox path runs OpenClaw and its file/process tools inside the task sandbox created by Resources. The agent server borrows `SandboxAccess`, installs the pinned runtime, launches one invocation in `SandboxAccess.workdir`, and disconnects after confirmed cleanup. Resources retains ownership of task preparation, verification, and sandbox destruction. @@ -104,7 +111,7 @@ config_paths: - responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml - environment_servers/single_agent_turn_legacy/configs/single_agent_turn_legacy.yaml -# Route existing prepared flat rows through native session orchestration. +# Route existing prepared flat rows through sandbox session orchestration. environment_routing_mode: legacy environment_server_name: single_agent_turn_legacy @@ -145,7 +152,7 @@ calls the rollout-scoped Responses endpoint, closes the agent, verifies, and clo The model server address must be reachable from inside the task sandbox. Its `/ng-rollout/.../v1` route is embedded in OpenClaw's Chat Completions provider configuration to retain model-call linkage. -The native adapter sets OpenClaw's provider `timeoutSeconds` to `model_timeout_seconds` (600 seconds +The sandbox adapter sets OpenClaw's provider `timeoutSeconds` to `model_timeout_seconds` (600 seconds by default). This raises the upstream 120-second idle watchdog so buffered reasoning responses can finish. For example, set `openclaw_agent.responses_api_agents.openclaw_agent.model_timeout_seconds` to `1200` for a 20-minute model-call limit. Increase the separate `timeout` and EnvironmentServer episode @@ -172,11 +179,11 @@ private Node binary; task Node/Python and the global library search path are not Per-session HOME, config, caches, prompt, transcript, and supervisor output are isolated under `/tmp/nemo-gym-openclaw-sessions/`. OpenClaw reads the generated config directly without onboarding or creating bootstrap files in the task repository. The task's PATH and dependencies are preserved. -The native adapter enables `read`, `write`, `edit`, `exec`, and `process`; subagent/channel/plugin +The sandbox adapter enables `read`, `write`, `edit`, `exec`, and `process`; subagent/channel/plugin execution and required external HTTP/MCP tools are unsupported. OpenClaw's internal `gateway` exec host refers to the embedded CLI process inside the borrowed sandbox. -Native request support is deliberately explicit: +Sandbox request support is deliberately explicit: - One activation per session; one worker per agent server. Concurrent sessions are independent. Identical activation requests join the running task or replay its result/error. A different request @@ -191,11 +198,11 @@ Native request support is deliberately explicit: - The request `model`, when supplied, must match the configured model. - `max_output_tokens`, `temperature`, `top_p`, reasoning overrides, tool-selection controls, output schemas, history replay, and other unsupported Responses options are rejected before activation. - The native config also rejects `max_output_tokens`: model metadata does not prove an effective + The sandbox config also rejects `max_output_tokens`: model metadata does not prove an effective inference limit. Configure sampling and per-call limits on the Gym model server and inspect the effective captured requests. An episode-wide token budget is not implemented. - Custom command, environment, OpenClaw config, Node path, extra arguments, and agent-ID overrides - are rejected for native sessions. Existing callers without an agent-session cookie retain the + are rejected for sandbox sessions. Existing callers without an agent-session cookie retain the local CLI behavior and configuration. Both paths now include request `instructions` in the same configured-system, request-instructions, input-system order. diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index 1115109ad04..df29e73dd64 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -396,14 +396,14 @@ def _sandbox_prepare_command(workdir: str, directory: str, runtime: str) -> str: command -v python3 >/dev/null 2>&1 || set -- "$@" python3 command -v bash >/dev/null 2>&1 || set -- "$@" bash if [ "$#" -gt 0 ]; then - [ "$(id -u)" = 0 ] || { echo "Native OpenClaw requires $*: preinstall these tools or use a root image." >&2; exit 1; } + [ "$(id -u)" = 0 ] || { echo "OpenClaw sandbox execution requires $*: preinstall these tools or use a root image." >&2; exit 1; } if command -v apk >/dev/null 2>&1; then apk add --no-cache "$@" elif command -v apt-get >/dev/null 2>&1; then apt-get update DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "$@" else - echo "Native OpenClaw requires $*: preinstall these tools (automatic installation requires apt-get or apk)." >&2 + echo "OpenClaw sandbox execution requires $*: preinstall these tools (automatic installation requires apt-get or apk)." >&2 exit 1 fi fi @@ -493,21 +493,28 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Open if not isinstance(workdir, str) or not workdir.startswith("/") or workdir in ("/", "/tmp"): raise HTTPException(422, "OpenClaw workdir must be absolute and separate from /tmp runtime storage") if any(access.required for access in self.effective_tool_accesses(body)): - raise HTTPException(422, "Native OpenClaw supports its own sandbox tools, not required HTTP/MCP tools") + raise HTTPException( + 422, "OpenClaw sandbox execution supports its own sandbox tools, not required HTTP/MCP tools" + ) if self.config.model_server is None: - raise HTTPException(422, "Native OpenClaw requires a sandbox-reachable Gym model_server") + raise HTTPException(422, "OpenClaw sandbox execution requires a sandbox-reachable Gym model_server") if not self.config.openclaw_version or not re.fullmatch( r"\d+\.\d+\.\d+(?:-\d+)?", self.config.openclaw_version ): - raise HTTPException(422, "Native OpenClaw requires an exact openclaw_version, for example 2026.6.11") + raise HTTPException( + 422, "OpenClaw sandbox execution requires an exact openclaw_version, for example 2026.6.11" + ) if self.config.openclaw_config or self.config.max_output_tokens is not None or self.config.node_bin_dir: raise HTTPException( - 422, "Native OpenClaw does not support openclaw_config, max_output_tokens, or node_bin_dir overrides" + 422, + "OpenClaw sandbox execution does not support openclaw_config, max_output_tokens, or node_bin_dir overrides", ) if self.config.openclaw_agent_id != "main": - raise HTTPException(422, "Native OpenClaw currently requires openclaw_agent_id=main") + raise HTTPException(422, "OpenClaw sandbox execution currently requires openclaw_agent_id=main") if self.config.command != "openclaw" or self.config.extra_args or self.config.env: - raise HTTPException(422, "Native OpenClaw does not support command, extra_args, or env overrides") + raise HTTPException( + 422, "OpenClaw sandbox execution does not support command, extra_args, or env overrides" + ) provider = create_provider(resolve_provider_config(provider_ref, get_global_config_dict())) if owns_sandbox: @@ -599,57 +606,39 @@ def _prompt_input(self, body: NeMoGymResponseCreateParamsNonStreaming) -> tuple[ def _sandbox_input(self, body: NeMoGymResponseCreateParamsNonStreaming) -> tuple[str, str]: """Validate and normalize input before consuming the session's activation.""" - unsupported = ( - "include", - "store", - "service_tier", - "prompt_cache_key", - "prompt_cache_retention", - "safety_identifier", - "stream_options", - "user", - "max_output_tokens", - "temperature", - "top_p", - "reasoning", - "max_tool_calls", - "previous_response_id", - "prompt", - "text", - "context_management", - "conversation", - "moderation", - "top_logprobs", - "truncation", - ) if body.model is not None and body.model != self.config.model: - raise HTTPException(422, "Native OpenClaw model must match the configured model") + raise HTTPException(422, "OpenClaw sandbox execution model must match the configured model") extras = set(body.model_extra or {}) if extras: - raise HTTPException(422, f"Native OpenClaw does not support extra request fields: {sorted(extras)}") - values = body.model_dump(mode="json") - for name in unsupported: - if values.get(name) is not None: - raise HTTPException(422, f"Native OpenClaw does not support request field {name}") - if body.tools or body.tool_choice != "auto" or not body.parallel_tool_calls or body.background: - raise HTTPException(422, "OpenClaw owns tool selection and execution policy") - if (body.metadata or {}).get("chat_template_kwargs") is not None: - raise HTTPException(422, "Configure chat_template_kwargs on the Gym model server for OpenClaw") + raise HTTPException( + 422, f"OpenClaw sandbox execution does not support extra request fields: {sorted(extras)}" + ) + supported = {"input", "instructions", "model"} + for name, field in type(body).model_fields.items(): + if name in supported: + continue + value = getattr(body, name) + if name in ("stream", "background") and value is False: + continue + if value != field.get_default(call_default_factory=True): + raise HTTPException(422, f"OpenClaw sandbox execution does not support request field {name}") items = ( [NeMoGymEasyInputMessage(role="user", content=body.input)] if isinstance(body.input, str) else body.input ) roles = [getattr(item, "role", None) for item in items] if roles not in (["user"], ["system", "user"]): - raise HTTPException(422, "Native OpenClaw accepts one text user prompt with an optional system message") + raise HTTPException( + 422, "OpenClaw sandbox execution accepts one text user prompt with an optional system message" + ) for item in items: if not isinstance(item.content, str) and any( (part.get("type") if isinstance(part, dict) else getattr(part, "type", None)) != "input_text" for part in item.content ): - raise HTTPException(422, "Native OpenClaw only supports text input") + raise HTTPException(422, "OpenClaw sandbox execution only supports text input") prompt, system = self._prompt_input(body) if not prompt.strip(): - raise HTTPException(422, "Native OpenClaw requires a nonempty user prompt") + raise HTTPException(422, "OpenClaw sandbox execution requires a nonempty user prompt") return prompt, system async def _sandbox_response( @@ -662,13 +651,13 @@ async def _sandbox_response( ) -> NeMoGymResponse: # The pinned CLI reads configuration directly; onboarding would create # workspace bootstrap files and unrelated provider/channel state. - native_config = self._build_openclaw_config({}, state.request.episode_id.capture_key) + session_config = self._build_openclaw_config({}, state.request.episode_id.capture_key) # OpenClaw conservatively disables stream usage for custom origins. Gym # supports it; request the final usage chunk instead of retaining zero counters. - native_config["models"]["providers"]["nemo"]["models"][0]["compat"] = { + session_config["models"]["providers"]["nemo"]["models"][0]["compat"] = { "supportsUsageInStreaming": True, } - native_config.update( + session_config.update( { "agents": { "defaults": { @@ -707,7 +696,7 @@ async def _sandbox_response( str(self.config.timeout), ] payload = { - "config": native_config, + "config": session_config, "directory": state.session.session_dir, "command": command, "prompt": f"{system}\n\n{prompt}" if system else prompt, @@ -840,7 +829,7 @@ def count(name: str) -> int: output_tokens += count("output") if not assistants: # The legacy envelope parser defaults missing counters to zero. - # Native totals retain valid subtotals without coercing malformed cache values. + # Harness totals retain valid subtotals without coercing malformed cache values. cache_read = raw_usage.get("cacheRead") cached_tokens = cache_read if type(cache_read) is int and cache_read > 0 else None input_count, output_count = raw_usage.get("input"), raw_usage.get("output") @@ -906,7 +895,7 @@ def count(name: str) -> int: transcript_available=bool(events), model_ref=self.config.model_server, ) - # Native sessions enable only local file/process tools, so there is + # Sandbox sessions enable only local file/process tools, so there is # exactly one invocation; child sessions cannot be spawned. state.observations.gaps = [ gap for gap in state.observations.gaps if gap.code != "subagent_hierarchy_unavailable" @@ -914,7 +903,7 @@ def count(name: str) -> int: state.observations.records[0].status = status state.observations.gaps.extend(gaps) except Exception: - LOG.exception("Could not build native OpenClaw observations") + LOG.exception("Could not build OpenClaw observations") state.observations = AgentObservationBundle( source=OPENCLAW_OBSERVATION_SOURCE, gaps=[ObservationGap(code="observation_capture_failed"), *gaps], @@ -1100,7 +1089,7 @@ async def _run_openclaw( ] = None, ) -> tuple[list[Any], dict[str, int], str]: """setup and run agent. returns (output_items, usage, model_name).""" - # Local callers still get automatic installation. Native sessions never + # Local callers still get automatic installation. Sandbox sessions never # enter this path, so their host does not need OpenClaw, Node, or npm. if self._local_setup_task is None: self._local_setup_task = asyncio.create_task( @@ -1402,15 +1391,15 @@ def collect_output(value: list[Any]) -> None: async def run(self, request: Request, body: OpenClawAgentRunRequest) -> OpenClawAgentVerifyResponse: if self._agent_session_id_from_request(request) is not None: - raise HTTPException(409, "Native OpenClaw sessions must use EnvironmentServer /run") + raise HTTPException(409, "OpenClaw sandbox sessions must use EnvironmentServer /run") try: if isinstance(request.session, Mapping) and _SANDBOX_SESSION_KEY in request.session: - raise HTTPException(409, "Native OpenClaw sessions must use EnvironmentServer /run") + raise HTTPException(409, "OpenClaw sandbox sessions must use EnvironmentServer /run") except (AssertionError, AttributeError): pass if self.config.resources_server is None: raise HTTPException( - 422, "Direct OpenClaw /run requires resources_server; use EnvironmentServer /run for native sessions" + 422, "Direct OpenClaw /run requires resources_server; use EnvironmentServer /run for sandbox sessions" ) async with self.sem: cookies = request.cookies diff --git a/responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml b/responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml index 1039ba39d59..8f4b288d035 100644 --- a/responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml +++ b/responses_api_agents/openclaw_agent/configs/openclaw_agent.yaml @@ -13,6 +13,6 @@ openclaw_agent: openclaw_version: 2026.6.11 timeout: 900 model_timeout_seconds: 600 - sandbox_install_timeout_seconds: 900 + sandbox_install_timeout_seconds: 600 session_close_timeout_seconds: 60 session_close_retry_window_seconds: 300 diff --git a/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh b/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh index 3c183a89cd5..9d32056f53e 100644 --- a/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh +++ b/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh @@ -8,18 +8,18 @@ runtime=$1 openclaw_version=$2 node_version=22.19.0 if [ "$(uname -s)" != Linux ]; then - echo 'Native OpenClaw requires a Linux sandbox.' >&2 + echo 'OpenClaw sandbox execution requires a Linux sandbox.' >&2 exit 1 fi if ! command -v python3 >/dev/null 2>&1; then - echo 'Native OpenClaw requires Python >=3.8 in the task image for descendant supervision.' >&2 + echo 'OpenClaw sandbox execution requires Python >=3.8 in the task image for descendant supervision.' >&2 exit 1 fi -python3 -c 'import sys; assert sys.version_info >= (3, 8), "Native OpenClaw requires Python >=3.8"' +python3 -c 'import sys; assert sys.version_info >= (3, 8), "OpenClaw sandbox execution requires Python >=3.8"' case "$(uname -m)" in x86_64) arch=x64 ;; aarch64) arch=arm64 ;; - *) echo 'Native OpenClaw supports Linux x86_64/aarch64 only.' >&2; exit 1 ;; + *) echo 'OpenClaw sandbox execution supports Linux x86_64/aarch64 only.' >&2; exit 1 ;; esac platform="linux-${arch}" node_dist="https://nodejs.org/dist/v${node_version}" @@ -27,19 +27,19 @@ if getconf GNU_LIBC_VERSION >/dev/null 2>&1; then : elif [[ "$(ldd --version 2>&1 || true)" == *musl* ]]; then if [ "$arch" != x64 ]; then - echo "Native OpenClaw's pinned Node ${node_version} musl build supports x86_64 only" >&2 + echo "OpenClaw sandbox execution's pinned Node ${node_version} musl build supports x86_64 only" >&2 exit 1 fi platform=linux-x64-musl node_dist="https://unofficial-builds.nodejs.org/download/release/v${node_version}" else - echo 'Native OpenClaw requires glibc or musl; could not identify the sandbox libc' >&2 + echo 'OpenClaw sandbox execution requires glibc or musl; could not identify the sandbox libc' >&2 exit 1 fi install_packages() { if [ "$(id -u)" -ne 0 ]; then - echo "Native OpenClaw requires $*: preinstall these packages (automatic installation requires root)." >&2 + echo "OpenClaw sandbox execution requires $*: preinstall these packages (automatic installation requires root)." >&2 exit 1 fi if command -v apk >/dev/null 2>&1; then @@ -48,7 +48,7 @@ install_packages() { apt-get update DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "$@" else - echo "Native OpenClaw requires $*: preinstall these packages (automatic installation requires apt-get or apk)." >&2 + echo "OpenClaw sandbox execution requires $*: preinstall these packages (automatic installation requires apt-get or apk)." >&2 exit 1 fi } @@ -91,7 +91,7 @@ if [ ! -s /etc/ssl/certs/ca-certificates.crt ] && [ ! -s /etc/pki/tls/certs/ca-b missing+=(ca-certificates) fi if [ "${#missing[@]}" -gt 0 ]; then - echo "Native OpenClaw prerequisites missing: ${missing[*]}. Preinstall curl, ca-certificates, tar, gzip, coreutils, gawk." >&2 + echo "OpenClaw sandbox execution prerequisites missing: ${missing[*]}. Preinstall curl, ca-certificates, tar, gzip, coreutils, gawk." >&2 install_packages curl ca-certificates tar gzip coreutils gawk fi if [ "$platform" = linux-x64-musl ] && ! python3 -c 'import ctypes; ctypes.CDLL("libstdc++.so.6")' 2>/dev/null; then diff --git a/responses_api_agents/openclaw_agent/tests/test_app.py b/responses_api_agents/openclaw_agent/tests/test_app.py index f5afd95b392..ced8f5b5b45 100644 --- a/responses_api_agents/openclaw_agent/tests/test_app.py +++ b/responses_api_agents/openclaw_agent/tests/test_app.py @@ -966,3 +966,5 @@ def test_default_config_selects_environment_sessions(self) -> None: assert inner["resources_server"] is None assert inner["model_server"] == {"type": "responses_api_models", "name": "policy_model"} assert inner["model"] == "${policy_model_name}" + assert inner["sandbox_install_timeout_seconds"] == 600 + assert inner["session_close_timeout_seconds"] == 60 diff --git a/responses_api_agents/openclaw_agent/tests/test_install_runtime.py b/responses_api_agents/openclaw_agent/tests/test_install_runtime.py index 5b7a1610c5a..e6469c39465 100644 --- a/responses_api_agents/openclaw_agent/tests/test_install_runtime.py +++ b/responses_api_agents/openclaw_agent/tests/test_install_runtime.py @@ -11,7 +11,7 @@ INSTALLER = Path(__file__).parents[1] / "install_openclaw_runtime.sh" -pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Native OpenClaw requires Linux") +pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="OpenClaw sandbox execution requires Linux") @pytest.fixture diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index 34639623079..d831885eea0 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -176,7 +176,7 @@ def setup(): module = "responses_api_agents.openclaw_agent.app" with ( patch( - f"{module}.ensure_openclaw", side_effect=AssertionError("native sessions must not install host OpenClaw") + f"{module}.ensure_openclaw", side_effect=AssertionError("sandbox sessions must not install host OpenClaw") ), patch(f"{module}.resolve_provider_config"), patch(f"{module}.get_global_config_dict", return_value={}), @@ -231,7 +231,7 @@ def close_body(session_id): @pytest.mark.parametrize("model_timeout_seconds", [600, 1200]) -def test_http_native_flow_runs_openclaw_in_borrowed_sandbox(setup, model_timeout_seconds): +def test_http_sandbox_flow_runs_openclaw_in_borrowed_sandbox(setup, model_timeout_seconds): agent, sandbox = setup agent.config.model_timeout_seconds = model_timeout_seconds with patch.object(agent, "_run_openclaw", AsyncMock(side_effect=AssertionError("host OpenClaw must not run"))): @@ -412,6 +412,20 @@ def test_rejected_request_does_not_consume_activation(setup): sandbox.launch.assert_awaited_once() +def test_request_allowlist_rejects_future_fields_but_accepts_defaults(setup): + class ExtendedRequest(NeMoGymResponseCreateParamsNonStreaming): + future_control: str | None = None + + agent, sandbox = setup + assert agent._sandbox_input(ExtendedRequest(input="task", stream=False, background=False)) == ("task", "") + with pytest.raises(HTTPException, match="future_control") as error: + agent._sandbox_input(ExtendedRequest(input="task", future_control="unsupported")) + assert error.value.status_code == 422 + with pytest.raises(HTTPException, match="metadata"): + agent._sandbox_input(ExtendedRequest(input="task", metadata={"ignored": "value"})) + sandbox.launch.assert_not_awaited() + + def test_no_session_keeps_existing_local_path(setup): agent, sandbox = setup with patch.object(agent, "_create_response", AsyncMock(side_effect=RuntimeError("legacy path reached"))) as legacy: @@ -544,7 +558,7 @@ def test_runtime_failures_cannot_become_successful_responses(setup, failure, sto assert invocation["conversation"][-1]["content"][0]["text"] == "Fixed" -async def test_local_and_native_prompts_include_the_same_instructions(setup): +async def test_local_and_sandbox_prompts_include_the_same_instructions(setup): agent, _ = setup agent.config.system_prompt = "configured system" body = NeMoGymResponseCreateParamsNonStreaming( @@ -559,7 +573,7 @@ async def test_local_and_native_prompts_include_the_same_instructions(setup): @pytest.mark.parametrize("marker", [None, "expired"]) -async def test_obsolete_native_cookie_never_falls_back_to_host(setup, marker): +async def test_obsolete_sandbox_cookie_never_falls_back_to_host(setup, marker): agent, sandbox = setup request = Request({"type": "http", "session": {"nemo_gym_openclaw_sandbox_session": marker}}) with patch.object(agent, "_create_response", AsyncMock(side_effect=AssertionError("host fallback"))): @@ -928,7 +942,7 @@ async def test_usage_sums_cache_writes_and_failed_calls(setup): (2, {"cacheRead": 3}, 5, 20), ], ) -def test_native_usage_details_preserve_positive_counts_and_treat_defaulted_zero_as_unknown( +def test_sandbox_usage_details_preserve_positive_counts_and_treat_defaulted_zero_as_unknown( setup, first_cache, second_cache, expected_cache, expected_input ): agent, sandbox = setup @@ -959,7 +973,7 @@ def test_native_usage_details_preserve_positive_counts_and_treat_defaulted_zero_ @pytest.mark.parametrize("second_usage", [None, "unavailable"]) -def test_missing_call_usage_keeps_native_cache_aggregate_unknown(setup, second_usage): +def test_missing_call_usage_keeps_sandbox_cache_aggregate_unknown(setup, second_usage): agent, sandbox = setup transcript = [json.loads(line) for line in events().splitlines()] transcript[2]["message"]["usage"] = second_usage @@ -1133,7 +1147,7 @@ async def test_two_sessions_keep_workspaces_model_routes_and_observations_separa @pytest.mark.parametrize("marker", [None, 7, "expired-session"]) -async def test_invalid_or_expired_native_cookie_never_uses_local_execution(setup, marker): +async def test_invalid_or_expired_sandbox_cookie_never_uses_local_execution(setup, marker): agent, sandbox = setup request = Request( { @@ -1149,7 +1163,7 @@ async def test_invalid_or_expired_native_cookie_never_uses_local_execution(setup sandbox.launch.assert_not_awaited() -def test_native_close_cookie_blocks_legacy_run_with_configured_resources(setup): +def test_sandbox_close_cookie_blocks_legacy_run_with_configured_resources(setup): from nemo_gym.config_types import ResourcesServerRef agent, sandbox = setup @@ -1214,7 +1228,7 @@ async def test_invalid_envelope_usage_does_not_discard_valid_transcript(setup): ({"cacheRead": 5}, 5), ], ) -async def test_native_envelope_fallback_preserves_known_and_unknown_cache_details(setup, cache, expected_cache): +async def test_sandbox_envelope_fallback_preserves_known_and_unknown_cache_details(setup, cache, expected_cache): agent, sandbox = setup request, session_id, task = await activate(agent, sandbox) await task @@ -1455,7 +1469,7 @@ async def test_independent_configs_route_prepared_rows_through_environment_run(s offline=True, ) ) - # The benchmark's legacy allowlist predates this native harness. Keep its + # The benchmark's legacy allowlist predates this sandbox harness. Keep its # definition untouched and explicitly opt in, as the CLI smoke does. monkeypatch.setenv("NEMO_GYM_ALLOW_UNSUPPORTED_PAIRING", "1") if failed: @@ -1609,7 +1623,7 @@ async def test_failed_seed_cleanup_retains_state_until_explicit_retry(setup): @pytest.mark.parametrize("marker", [None, [], {}, 7]) -async def test_seed_rejects_malformed_native_cookie_before_connecting(setup, marker): +async def test_seed_rejects_malformed_sandbox_cookie_before_connecting(setup, marker): agent, sandbox = setup request = Request({"type": "http", "session": {"agent_session_id": marker}}) with pytest.raises(HTTPException, match="Invalid agent session marker"): @@ -1639,7 +1653,7 @@ async def test_stale_cookie_cannot_recreate_session_or_close_receipt_after_full_ @pytest.mark.parametrize("owned", [False, True]) -def test_sandbox_source_controls_ownership_and_native_routing(setup, owned): +def test_sandbox_source_controls_ownership_and_sandbox_routing(setup, owned): agent, sandbox = setup agent.config.sandbox_provider = "agent-provider" agent.config.sandbox_config = {"image": "test-image", "workdir": "/agent-workspace"} From 0ba7a55c7dab00208ff4ba4f0d3e086feef60e28 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Fri, 9 Oct 2026 13:47:37 -0700 Subject: [PATCH 44/44] fix(openclaw): clarify metadata controls and retry failed setup Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/README.md | 5 ++- responses_api_agents/openclaw_agent/app.py | 9 +++++ .../openclaw_agent/tests/test_app.py | 33 +++++++++++++++++++ .../openclaw_agent/tests/test_sandbox.py | 22 +++++++++++++ 4 files changed, 68 insertions(+), 1 deletion(-) diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index 265d65a6b4f..029857d02af 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -197,7 +197,10 @@ Sandbox request support is deliberately explicit: and prepended to OpenClaw's user prompt. The harness retains its own system prompt. - The request `model`, when supplied, must match the configured model. - `max_output_tokens`, `temperature`, `top_p`, reasoning overrides, tool-selection controls, output - schemas, history replay, and other unsupported Responses options are rejected before activation. + schemas, history replay, request `metadata`, and other unsupported Responses options are rejected before activation. + This includes `metadata.extra_body.seed` added by `num_repeats_add_seed`; disable that option + for sandbox sessions. Request `metadata.chat_template_kwargs` is also rejected; configure it + on the Gym model server instead. The sandbox config also rejects `max_output_tokens`: model metadata does not prove an effective inference limit. Configure sampling and per-call limits on the Gym model server and inspect the effective captured requests. An episode-wide token budget is not implemented. diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index df29e73dd64..0f732435e88 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -613,6 +613,8 @@ def _sandbox_input(self, body: NeMoGymResponseCreateParamsNonStreaming) -> tuple raise HTTPException( 422, f"OpenClaw sandbox execution does not support extra request fields: {sorted(extras)}" ) + if "chat_template_kwargs" in (body.metadata or {}): + raise HTTPException(422, "Configure chat_template_kwargs on the Gym model server for OpenClaw") supported = {"input", "instructions", "model"} for name, field in type(body).model_fields.items(): if name in supported: @@ -1079,6 +1081,12 @@ def _on_sigterm(signum, frame) -> None: except ValueError: pass # signal handlers need the main thread; fall back to timeout-only salvage + def _clear_failed_local_setup(self, task: asyncio.Task[None]) -> None: + # All shielded waiters may have gone away before the installer fails. + failed = task.cancelled() or task.exception() is not None + if failed and self._local_setup_task is task: + self._local_setup_task = None + async def _run_openclaw( self, instruction: str, @@ -1095,6 +1103,7 @@ async def _run_openclaw( self._local_setup_task = asyncio.create_task( asyncio.to_thread(ensure_openclaw, self.config.openclaw_version, node_bin_dir=self.config.node_bin_dir) ) + self._local_setup_task.add_done_callback(self._clear_failed_local_setup) setup_task = self._local_setup_task try: # Share one installation across concurrent local calls. Cancelling diff --git a/responses_api_agents/openclaw_agent/tests/test_app.py b/responses_api_agents/openclaw_agent/tests/test_app.py index ced8f5b5b45..afe216cb022 100644 --- a/responses_api_agents/openclaw_agent/tests/test_app.py +++ b/responses_api_agents/openclaw_agent/tests/test_app.py @@ -18,6 +18,7 @@ import os import signal import sys +import threading from pathlib import Path from typing import Optional from unittest.mock import AsyncMock, MagicMock, patch @@ -127,6 +128,38 @@ async def test_lazy_local_install_uses_configured_node_runtime(tmp_path: Path) - install.assert_called_once_with("2026.6.11", node_bin_dir="/opt/task-node/bin") +async def test_failed_install_retries_after_all_waiters_cancel() -> None: + started, release = threading.Event(), threading.Event() + + def fail_install(*args, **kwargs) -> None: + started.set() + assert release.wait(3) + raise RuntimeError("install failed after cancellation") + + agent = _make_agent() + with ( + patch("responses_api_agents.openclaw_agent.app.ensure_openclaw", side_effect=fail_install) as install, + patch.object(agent, "_workspace_root", side_effect=RuntimeError("install completed")), + ): + waiter = asyncio.create_task(agent._run_openclaw("task", None)) + try: + async with asyncio.timeout(2): + while not started.is_set(): + await asyncio.sleep(0.01) + setup = agent._local_setup_task + waiter.cancel() + with pytest.raises(asyncio.CancelledError): + await waiter + finally: + release.set() + with pytest.raises(RuntimeError, match="after cancellation"): + await setup + install.side_effect = None + with pytest.raises(RuntimeError, match="install completed"): + await agent._run_openclaw("task", None) + assert install.call_count == 2 + + class TestExtractInstruction: def test_user_only(self) -> None: user, system = _extract_instruction([NeMoGymEasyInputMessage(role="user", content="hello")]) diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index d831885eea0..99682902bf3 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -426,6 +426,28 @@ class ExtendedRequest(NeMoGymResponseCreateParamsNonStreaming): sandbox.launch.assert_not_awaited() +@pytest.mark.parametrize( + "metadata,hint", + [ + ({"extra_body": json.dumps({"seed": 1})}, "request field metadata"), + ({"chat_template_kwargs": json.dumps({"enable_thinking": True})}, "Gym model server"), + ], +) +def test_metadata_rejection_preserves_activation_and_configuration_hint(setup, metadata, hint): + agent, sandbox = setup + with TestClient(agent.setup_webserver()) as client: + created = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")) + created.raise_for_status() + path = "/ng-rollout/openclaw-smoke-a2/v1/responses" + rejected = client.post(path, json={"input": "task", "metadata": metadata}) + assert rejected.status_code == 422 + assert hint in rejected.json()["detail"] + sandbox.launch.assert_not_awaited() + client.post(path, json={"input": "task"}).raise_for_status() + sandbox.launch.assert_awaited_once() + client.post("/v1/agent_sessions/close", json=close_body(created.json()["agent_session_id"])).raise_for_status() + + def test_no_session_keeps_existing_local_path(setup): agent, sandbox = setup with patch.object(agent, "_create_response", AsyncMock(side_effect=RuntimeError("legacy path reached"))) as legacy: