From 6747429a8a56efa94d080d4bbbd7ec77f40226f4 Mon Sep 17 00:00:00 2001 From: Frederic Lafrance Date: Mon, 19 Oct 2026 17:28:34 +0200 Subject: [PATCH 1/7] Add sandbox execution to Claude Code Signed-off-by: Frederic Lafrance --- .../claude_code_agent/README.md | 41 ++ responses_api_agents/claude_code_agent/app.py | 20 +- .../install_claude_runtime.py | 44 ++ .../claude_code_agent/sandbox.py | 382 ++++++++++++++++++ .../claude_code_agent/sandbox_runner.py | 18 + .../claude_code_agent/tests/test_sandbox.py | 285 +++++++++++++ 6 files changed, 787 insertions(+), 3 deletions(-) create mode 100644 responses_api_agents/claude_code_agent/install_claude_runtime.py create mode 100644 responses_api_agents/claude_code_agent/sandbox.py create mode 100644 responses_api_agents/claude_code_agent/sandbox_runner.py create mode 100644 responses_api_agents/claude_code_agent/tests/test_sandbox.py diff --git a/responses_api_agents/claude_code_agent/README.md b/responses_api_agents/claude_code_agent/README.md index fb43c615a37..bd4ee3a8505 100644 --- a/responses_api_agents/claude_code_agent/README.md +++ b/responses_api_agents/claude_code_agent/README.md @@ -209,3 +209,44 @@ The skills path is resolved like `input_jsonl_fpath` (relative paths check the w The served id is not a declared terminal id on `/run`; capture still corroborates it against the transcript. Without capture, thinking stays folded and unmatched tool calls remain omitted. - `turns_used` counts assistant messages, not tool calls. + +## Sandbox execution + +The same `claude_code_agent` supports standard agent sessions. An EnvironmentServer +seeds `/v1/agent_sessions`, sends one text activation to the rollout-scoped +`/v1/responses`, and closes `/v1/agent_sessions/close` to collect observations. +Retries of the same activation reuse its result; a different activation requires a +new session. Unseeded requests retain the existing local behavior. + +For an agent-owned sandbox, add these fields to the agent config: + +```yaml +claude_code_version: "2.1.108" +sandbox_provider: sandbox # top-level provider config, shared with other harnesses +sandbox_config: + image: ubuntu:24.04 + workdir: /app +sandbox_install_timeout_seconds: 600 +session_close_timeout_seconds: 60 +``` + +The image must include `python3`, `tar`, CA certificates, and libraries required by +the native Claude binary. Seed downloads the exact Linux x64/arm64 release and +checks its published SHA-256. For offline images, set `sandbox_executable_path` to +an existing native binary **inside the sandbox**; seed copies it into the private +session directory and checks its version. Configure `model_server` as above, or +`anthropic_base_url` and `anthropic_api_key`; the endpoint must be reachable from +the sandbox. `resources_server` is needed only for the legacy agent `/run` path. + +For borrowed execution, the seed request supplies `sandbox_access` with a direct +provider connection descriptor and an absolute task `workdir`. Resources prepares +the task directory; Claude Code only stages its own runtime and settings outside +it. Required remote tool accesses are rejected: this adapter runs Claude Code's +native tools. Optional static `mcp_config` and `settings` JSON files are staged +from the agent server, with the same command options as local execution. + +The shared `SandboxSession` supervisor stops the harness before reading its +stream and native transcripts. Close returns the observation bundle, disconnects +borrowed sandboxes, and stops agent-owned sandboxes. The task directory remains +available to the borrowed sandbox's owner for verification. `timeout` bounds +harness execution; setup and close use the limits above. diff --git a/responses_api_agents/claude_code_agent/app.py b/responses_api_agents/claude_code_agent/app.py index 8f7b7be955e..02d53e8dd38 100644 --- a/responses_api_agents/claude_code_agent/app.py +++ b/responses_api_agents/claude_code_agent/app.py @@ -28,7 +28,7 @@ from typing import Any, Callable, Optional from uuid import uuid4 -from fastapi import Request +from fastapi import HTTPException, Request from pydantic import ConfigDict, Field, PrivateAttr from nemo_gym.base_resources_server import NEMO_GYM_MCP_METADATA_KEY, BaseRunRequest, BaseVerifyResponse @@ -53,6 +53,7 @@ from nemo_gym.server_utils import apply_rollout_prefix, get_response_json, raise_for_status from nemo_gym.skills import stage_skills from responses_api_agents.claude_code_agent.observability import extract_claude_code_observations +from responses_api_agents.claude_code_agent.sandbox import ClaudeCodeSandboxMixin from responses_api_agents.claude_code_agent.setup_claude_code import ensure_claude_code @@ -293,7 +294,7 @@ def _extract_instruction(body_input) -> tuple[str, Optional[str]]: class ClaudeCodeAgentConfig(BaseResponsesAPIAgentConfig): - resources_server: ResourcesServerRef + resources_server: Optional[ResourcesServerRef] = None # When model_server is set, ANTHROPIC_BASE_URL is resolved from the Gym model # server's URL (requires the server to expose POST /v1/messages). # When None, anthropic_base_url is used directly. @@ -317,6 +318,13 @@ class ClaudeCodeAgentConfig(BaseResponsesAPIAgentConfig): mcp_config: Optional[str] = None settings: Optional[str] = None + # Used only when an episode does not supply a borrowed SandboxAccess. + sandbox_provider: str | None = None + sandbox_config: dict[str, Any] = Field(default_factory=dict) + sandbox_executable_path: str | None = None + sandbox_install_timeout_seconds: float = Field(default=600, gt=0, allow_inf_nan=False) + session_close_timeout_seconds: float = Field(default=60, gt=0, allow_inf_nan=False) + class ClaudeCodeAgentRunRequest(BaseRunRequest): model_config = ConfigDict(extra="allow") @@ -331,7 +339,7 @@ class ClaudeCodeAgentVerifyResponse(BaseVerifyResponse): ) -class ClaudeCodeAgent(SimpleResponsesAPIAgent): +class ClaudeCodeAgent(ClaudeCodeSandboxMixin, SimpleResponsesAPIAgent): config: ClaudeCodeAgentConfig sem: Semaphore = None _static_mcp_config: Optional[dict[str, Any]] = PrivateAttr(default=None) @@ -706,6 +714,8 @@ async def responses( request: Request, body: NeMoGymResponseCreateParamsNonStreaming = Body(), ) -> NeMoGymResponse: + if self._agent_session_id_from_request(request) is not None: + return await self._sandbox_responses(request, body) return await self._create_response(body, rollout_id=request.path_params.get("rollout_id")) async def _create_episode( @@ -754,6 +764,10 @@ def collect(config_dir: Path, run_metadata: dict[str, Any]) -> None: return AgentEpisode(response=response, observations=observations) async def run(self, request: Request, body: ClaudeCodeAgentRunRequest) -> ClaudeCodeAgentVerifyResponse: + if self._agent_session_id_from_request(request) is not None: + raise HTTPException(409, "Claude Code sandbox sessions must use EnvironmentServer /run") + if self.config.resources_server is None: + raise HTTPException(422, "Local Claude Code /run requires resources_server") async with self.sem: cookies = request.cookies diff --git a/responses_api_agents/claude_code_agent/install_claude_runtime.py b/responses_api_agents/claude_code_agent/install_claude_runtime.py new file mode 100644 index 00000000000..4267a067b8a --- /dev/null +++ b/responses_api_agents/claude_code_agent/install_claude_runtime.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Install a pinned native Claude executable inside a sandbox, using stdlib Python.""" + +import hashlib +import json +import platform +import shutil +import subprocess +import sys +import urllib.request +from pathlib import Path + + +def install(version: str, destination: Path, source: str, expected: str) -> None: + """Copy a supplied binary or download the native release and verify its checksum.""" + if source: + shutil.copyfile(source, destination) + else: + architecture = {"x86_64": "x64", "aarch64": "arm64"}.get(platform.machine()) + if platform.system() != "Linux" or architecture is None: + raise RuntimeError("Claude sandbox installation requires Linux x64 or arm64") + target = "linux-" + architecture + if platform.libc_ver()[0] != "glibc" and Path("/etc/alpine-release").exists(): + target += "-musl" + base = "https://downloads.claude.ai/claude-code-releases/" + version + with urllib.request.urlopen(base + "/manifest.json", timeout=60) as response: + manifest = json.load(response) + expected = expected or manifest["platforms"][target]["checksum"] + with urllib.request.urlopen(base + "/" + target + "/claude", timeout=120) as response: + with destination.open("wb") as output: + shutil.copyfileobj(response, output) + if expected: + with destination.open("rb") as binary: + actual = hashlib.file_digest(binary, "sha256").hexdigest() + if actual != expected: + destination.unlink() + raise ValueError("Claude executable SHA-256 mismatch") + destination.chmod(0o700) + subprocess.run([str(destination), "--version"], check=True, timeout=30) + + +if __name__ == "__main__": + install(sys.argv[1], Path(sys.argv[2]), sys.argv[3], sys.argv[4]) diff --git a/responses_api_agents/claude_code_agent/sandbox.py b/responses_api_agents/claude_code_agent/sandbox.py new file mode 100644 index 00000000000..dbcaca91c26 --- /dev/null +++ b/responses_api_agents/claude_code_agent/sandbox.py @@ -0,0 +1,382 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Claude Code sessions with owned or borrowed sandboxes and native tools.""" + +import asyncio +import json +import logging +import re +import tarfile +import tempfile +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from shlex import quote +from time import time +from typing import Any +from uuid import uuid4 + +from fastapi import Body, HTTPException, Request + +from nemo_gym.agent_utils.sandbox_session import SandboxCommand, SandboxSession +from nemo_gym.base_responses_api_agent import ( + AgentCloseSessionResponse, + AgentSeedSessionRequest, + AgentSessionSetupError, + AgentSessionState, +) +from nemo_gym.openai_utils import ( + NeMoGymResponse, + NeMoGymResponseCreateParamsNonStreaming, + NeMoGymResponseInputTokensDetails, + NeMoGymResponseOutputTokensDetails, + NeMoGymResponseUsage, +) +from nemo_gym.rollout_observability import AgentObservationBundle, ObservationGap +from nemo_gym.sandbox import SandboxExecResult, SandboxSpec, create_provider +from nemo_gym.sandbox.api import AsyncSandbox +from nemo_gym.sandbox.config import resolve_provider_config +from nemo_gym.sandbox.utils import upload_text +from responses_api_agents.claude_code_agent.observability import extract_claude_code_observations + + +LOG = logging.getLogger(__name__) + + +def extract_prompt( + body: NeMoGymResponseCreateParamsNonStreaming, *, model: str | None = None +) -> tuple[str, str | None]: + """Validate one text activation without silently dropping conversation history.""" + allowed = {"input", "instructions", "model", "metadata"} + for name, info in type(body).model_fields.items(): + if name not in allowed and getattr(body, name) != info.get_default(call_default_factory=True): + if name in {"stream", "background"} and getattr(body, name) is False: + continue + raise ValueError(f"Sandboxed Claude does not support request field {name}") + if body.model not in (None, "", "dummy_model", model): + raise ValueError("Sandboxed Claude model is selected by agent and model-server configuration") + if body.model_extra: + raise ValueError(f"Sandboxed Claude does not support extra request fields: {sorted(body.model_extra)}") + if body.tools or body.tool_choice != "auto" or not body.parallel_tool_calls or body.background: + raise ValueError("Sandboxed Claude owns tool selection and execution policy") + raw = body.model_dump(mode="json")["input"] + if isinstance(raw, str): + return raw, body.instructions + users = [] + systems = [] + for item in raw: + role = item.get("role") + if item.get("type", "message") != "message" or role not in {"user", "system", "developer"}: + raise ValueError("Sandboxed Claude requires a single user message with optional system instructions") + content = item.get("content") + if isinstance(content, list): + if any(part.get("type") not in {"input_text", "text"} for part in content): + raise ValueError("Sandboxed Claude accepts text input only") + content = "".join(part["text"] for part in content) + if not isinstance(content, str): + raise ValueError("Sandboxed Claude accepts text input only") + if role == "user": + users.append(content) + elif users: + raise ValueError("System instructions must precede the user turn") + else: + systems.append(content) + if len(users) != 1: + raise ValueError("Sandboxed Claude requires exactly one user message") + system = "\n\n".join(systems + ([body.instructions] if body.instructions else [])) or None + return users[0], system + + +@dataclass +class ClaudeCodeSandboxedState(AgentSessionState): + session: SandboxSession[str] + executable: str = "" + body: NeMoGymResponseCreateParamsNonStreaming | None = None + execution: asyncio.Task[NeMoGymResponse] | None = None + observations: AgentObservationBundle | None = None + + +class ClaudeCodeSandboxMixin: + """Add standard episode sessions while leaving unseeded local calls unchanged.""" + + async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> ClaudeCodeSandboxedState: + """Install the harness without modifying a borrowed task workspace.""" + if any(access.required for access in self.effective_tool_accesses(body)): + raise HTTPException( + 422, "Claude Code sandbox sessions use native tools; required remote tools are unsupported" + ) + version = self.config.claude_code_version + if not version or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): + raise HTTPException(422, "Sandboxed Claude Code requires an exact claude_code_version") + access = body.sandbox_access + owns = access is None + if owns and self.config.sandbox_provider is None: + raise HTTPException(422, "Provide SandboxAccess or configure sandbox_provider and sandbox_config") + spec = SandboxSpec(**{"workdir": "/app", **self.config.sandbox_config}) if owns else None + workdir = access.workdir if access else spec.workdir + path = PurePosixPath(workdir or "") + if ( + not path.is_absolute() + or ".." in path.parts + or "\x00" in workdir + or path in (PurePosixPath("/"), PurePosixPath("/tmp")) + or str(path).startswith("/tmp/nemo-gym-claude-") + ): + raise HTTPException(422, "Claude sandbox workdir must be absolute and separate from session storage") + if self.config.timeout <= 0: + raise HTTPException(422, "Claude sandbox execution timeout must be positive") + provider = create_provider( + resolve_provider_config( + access.connection.provider_config_ref if access else self.config.sandbox_provider, + self.server_client.global_config_dict, + ) + ) + if owns: + sandbox = AsyncSandbox(provider) + else: + try: + sandbox = await AsyncSandbox.connect(access.connection.descriptor, provider=provider) + except BaseException: + await provider.aclose() + raise + directory = "/tmp/nemo-gym-claude-" + uuid4().hex + session = SandboxSession( + sandbox=sandbox, session_dir=directory, workdir=workdir, harness="claude_code", owns_sandbox=owns + ) + state = ClaudeCodeSandboxedState(request=body, session=session) + try: + async with asyncio.timeout(self.config.sandbox_install_timeout_seconds): + if owns: + await sandbox.start(spec) + if workdir: + result = await sandbox.exec("mkdir -p -- " + quote(workdir), cwd="/", timeout_s=30) + self._check_claude_setup(result) + validate = ( + "from pathlib import Path; import sys; " + "workdir,session=[Path(p).resolve() for p in sys.argv[1:]]; " + "assert workdir.is_dir(), 'Claude sandbox workdir is missing'; " + "assert workdir != session and workdir not in session.parents and session not in workdir.parents, " + "'Claude session storage overlaps task workdir'" + ) + result = await sandbox.exec( + f"python3 -I -c {quote(validate)} {quote(workdir)} {quote(directory)} && " + f"umask 077 && mkdir -p {quote(directory)}/config {quote(directory)}/runtime", + cwd="/", + timeout_s=30, + ) + self._check_claude_setup(result) + await sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), directory + "/sandbox_runner.py") + await sandbox.upload(Path(__file__).with_name("install_claude_runtime.py"), directory + "/install.py") + source = self.config.sandbox_executable_path or "" + executable = directory + "/runtime/claude" + command = " ".join( + quote(value) + for value in [ + "python3", + directory + "/install.py", + version, + executable, + source, + "", + ] + ) + result = await sandbox.exec(command, timeout_s=self.config.sandbox_install_timeout_seconds) + if result.return_code or result.error_type: + raise RuntimeError( + f"Claude runtime installation failed: command={command!r}, exit={result.return_code}, " + f"error={result.error_type}; stderr={result.stderr}; stdout={result.stdout}" + ) + installed_version = (result.stdout or "").strip() + if not installed_version.startswith(version + " "): + raise RuntimeError(f"Claude version mismatch: expected {version}, got {installed_version}") + state.executable = executable + return state + except BaseException as error: + try: + await session.close(timeout=self.config.session_close_timeout_seconds) + except BaseException: + raise AgentSessionSetupError(state, error=error) from error + raise + + async def _close_agent_session_state(self, state: AgentSessionState) -> AgentCloseSessionResponse: + """Return observations after the shared supervisor confirms cleanup.""" + assert isinstance(state, ClaudeCodeSandboxedState) + await state.session.close(timeout=self.config.session_close_timeout_seconds) + if state.execution and not state.execution.done(): + state.execution.cancel() + try: + await state.execution + except asyncio.CancelledError: + pass + return AgentCloseSessionResponse( + agent_session_id=state.request.agent_session_id, + agent_observations=state.observations, + ) + + @staticmethod + def _check_claude_setup(result: SandboxExecResult) -> None: + if result.return_code != 0 or result.error_type: + raise RuntimeError(f"Claude sandbox setup failed: {result.error_type}: {result.stderr or result.stdout}") + + def _execution_input(self, body: NeMoGymResponseCreateParamsNonStreaming) -> tuple[str, str | None]: + prompt, system = extract_prompt(body, model=self.config.model) + system = "\n\n".join(part for part in (self.config.system_prompt, system) if part) or None + return prompt, system + + async def _execute( + self, + state: ClaudeCodeSandboxedState, + body: NeMoGymResponseCreateParamsNonStreaming, + execution_input: tuple[str, str | None], + ) -> NeMoGymResponse: + from responses_api_agents.claude_code_agent.app import _invocation_outcome, parse_stream_json + + session = state.session + directory = session.session_dir + prompt, system = execution_input + base_url = self._resolve_call_base_url(state.request.episode_id.capture_key) + model = self.config.model if base_url else self.config.model.split("/")[-1] + + async def stage() -> SandboxCommand: + env = { + "CLAUDE_CONFIG_DIR": directory + "/config", + "IS_SANDBOX": "1", + "ANTHROPIC_API_KEY": self.config.anthropic_api_key or "EMPTY", + "ANTHROPIC_MODEL": model, + "ANTHROPIC_DEFAULT_HAIKU_MODEL": model, + "ANTHROPIC_DEFAULT_SONNET_MODEL": model, + "ANTHROPIC_DEFAULT_OPUS_MODEL": model, + "CLAUDE_CODE_SUBAGENT_MODEL": model, + "DISABLE_AUTOUPDATER": "1", + "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", + } + if base_url: + env["ANTHROPIC_BASE_URL"] = base_url + env["ANTHROPIC_AUTH_TOKEN"] = self.config.anthropic_api_key or "local" + await upload_text( + session.sandbox, + path=directory + "/config/settings.json", + text=json.dumps(self._build_settings()), + ) + mcp_config = None + if self.config.mcp_config: + mcp_config = directory + "/mcp.json" + await upload_text(session.sandbox, path=mcp_config, text=json.dumps(self._get_static_mcp_config())) + argv = self._build_command(model, prompt, system, mcp_config=mcp_config) + argv[0] = state.executable + await upload_text( + session.sandbox, path=directory + "/launch.json", text=json.dumps({"argv": argv, "env": env}) + ) + return SandboxCommand( + argv=["python3", directory + "/sandbox_runner.py", directory + "/launch.json"], + python="python3", + ) + + parsed = None + outcome = None + + def classify(metadata: dict[str, Any]) -> tuple[str, str | None]: + if session.cleanup and session.cleanup["timed_out"]: + return "incomplete", "timeout" + return _invocation_outcome(metadata, (session.cleanup or {}).get("return_code")) + + async def collect() -> str: + nonlocal parsed, outcome + raw = await session.read_output_log() + parsed = parse_stream_json(raw, fold_thinking=not self._token_id_capture_enabled()) + _, run_metadata = parsed + outcome = classify(run_metadata) + status, reason = outcome + try: + result = await session.sandbox.exec( + "tar -czf " + + quote(directory + "/transcripts.tar.gz") + + " -C " + + quote(directory + "/config") + + " projects", + timeout_s=30, + ) + if result.return_code or result.error_type: + raise RuntimeError("Claude native transcripts are unavailable") + with tempfile.TemporaryDirectory(prefix="gym-claude-observations-") as temporary: + local = Path(temporary) + await session.sandbox.download(directory + "/transcripts.tar.gz", local / "transcripts.tar.gz") + with tarfile.open(local / "transcripts.tar.gz") as archive: + archive.extractall(local, filter="data") + state.observations = extract_claude_code_observations( + local, + model_ref=self.config.model_server, + root_status=status, + root_duration_ms=run_metadata.get("duration_ms"), + root_error_type=reason, + compaction_attempts=run_metadata.get("compaction_attempts"), + ) + except Exception: + LOG.warning("Could not collect Claude native observations", exc_info=True) + state.observations = AgentObservationBundle( + source="claude_code", gaps=[ObservationGap(code="agent_transcript_unavailable")] + ) + return raw + + async with self.sem: + raw = await session.execute( + stage_activation=stage, + collect=collect, + timeout=self.config.timeout, + close_timeout=self.config.session_close_timeout_seconds, + ) + if session.closing: + raise asyncio.CancelledError + items, usage = parsed if parsed is not None else parse_stream_json(raw) + metadata = { + "runtime_version": self.config.claude_code_version, + "timed_out": "true" if session.cleanup and session.cleanup["timed_out"] else "false", + } + status, reason = outcome or classify(usage) + if reason: + metadata["stop_reason"] = reason + return NeMoGymResponse( + id=(usage.get("terminal_response_id") or "resp_" + uuid4().hex) + if self._token_id_capture_enabled() + else "resp_" + uuid4().hex, + created_at=int(time()), + model=model, + object="response", + output=items, + tools=body.tools, + tool_choice=body.tool_choice, + parallel_tool_calls=body.parallel_tool_calls, + usage=NeMoGymResponseUsage( + input_tokens=usage.get("input_tokens", 0), + output_tokens=usage.get("output_tokens", 0), + total_tokens=usage.get("input_tokens", 0) + usage.get("output_tokens", 0), + input_tokens_details=NeMoGymResponseInputTokensDetails(cached_tokens=0), + output_tokens_details=NeMoGymResponseOutputTokensDetails(reasoning_tokens=0), + ), + status=status, + metadata=metadata, + ) + + async def _sandbox_responses( + self, request: Request, body: NeMoGymResponseCreateParamsNonStreaming = Body() + ) -> NeMoGymResponse: + session_id = self._agent_session_id_from_request(request) + if not session_id: + raise HTTPException(400, "Seed a sandboxed agent session first") + async with self._locked_agent_session(session_id) as record: + if record.closing or not isinstance(record.state, ClaudeCodeSandboxedState): + raise HTTPException(409, "Agent session is closed") + state = record.state + rollout_id = getattr(request, "path_params", {}).get("rollout_id") + if rollout_id != state.request.episode_id.capture_key: + raise HTTPException(409, "Claude activation rollout does not match its session") + if state.body is not None and state.body != body: + raise HTTPException(409, "A different activation was already submitted") + if state.execution is None: + try: + execution_input = self._execution_input(body) + except (ValueError, TypeError) as error: + raise HTTPException(422, str(error)) from error + state.body = body.model_copy(deep=True) + state.execution = asyncio.create_task(self._execute(state, body, execution_input)) + execution = state.execution + return (await asyncio.shield(execution)).model_copy(deep=True) diff --git a/responses_api_agents/claude_code_agent/sandbox_runner.py b/responses_api_agents/claude_code_agent/sandbox_runner.py new file mode 100644 index 00000000000..7bc39b68ed7 --- /dev/null +++ b/responses_api_agents/claude_code_agent/sandbox_runner.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Launch Claude Code under the shared process supervisor.""" + +import json +import os +import sys +from pathlib import Path + + +def main() -> None: + payload = json.loads(Path(sys.argv[1]).read_text()) + os.environ.update(payload["env"]) + os.execvpe(payload["argv"][0], payload["argv"], os.environ) + + +if __name__ == "__main__": + main() diff --git a/responses_api_agents/claude_code_agent/tests/test_sandbox.py b/responses_api_agents/claude_code_agent/tests/test_sandbox.py new file mode 100644 index 00000000000..c04b4166fea --- /dev/null +++ b/responses_api_agents/claude_code_agent/tests/test_sandbox.py @@ -0,0 +1,285 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi import HTTPException, Request + +from nemo_gym.base_responses_api_agent import AGENT_SESSION_COOKIE_KEY, AgentSeedSessionRequest +from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming +from nemo_gym.rollout_observability import AgentObservationBundle +from nemo_gym.server_utils import ServerClient +from responses_api_agents.claude_code_agent import sandbox as adapter +from responses_api_agents.claude_code_agent.app import ClaudeCodeAgent, ClaudeCodeAgentConfig +from responses_api_agents.claude_code_agent.install_claude_runtime import install + + +def seed(*, borrowed=True, workdir="/app"): + return AgentSeedSessionRequest( + agent_session_id="session", + episode_id={"rollout_id": "rollout"}, + task_id={"taskset": "fixture", "task_id": "task"}, + sandbox_access={ + "connection": {"provider_config_ref": "sandbox", "descriptor": {"id": "borrowed"}}, + "workdir": workdir, + } + if borrowed + else None, + ) + + +def agent(**kwargs): + config = ClaudeCodeAgentConfig( + name="claude", + host="127.0.0.1", + port=8080, + entrypoint="app.py", + claude_code_version="2.1.108", + **kwargs, + ) + with ( + patch("responses_api_agents.claude_code_agent.app.ensure_claude_code"), + patch("responses_api_agents.claude_code_agent.app.subprocess.run"), + ): + return ClaudeCodeAgent( + config=config, server_client=MagicMock(spec=ServerClient, global_config_dict={"sandbox": {"local": {}}}) + ) + + +def result(stdout="", *, code=0): + return SimpleNamespace(stdout=stdout, stderr="diagnostic" if code else "", return_code=code, error_type=None) + + +@pytest.mark.parametrize("input", ["Question", [{"role": "user", "content": "Question"}]]) +def test_text_prompt(input): + assert adapter.extract_prompt(NeMoGymResponseCreateParamsNonStreaming(input=input, instructions="System")) == ( + "Question", + "System", + ) + + +@pytest.mark.parametrize( + "input", + [ + [{"role": "user", "content": "a"}, {"role": "user", "content": "b"}], + [{"role": "assistant", "content": "answer"}], + [{"role": "user", "content": [{"type": "input_image", "image_url": "https://example.org/a.png"}]}], + [{"role": "user", "content": "a"}, {"role": "system", "content": "b"}], + ], +) +def test_reject_input_that_would_be_silently_dropped(input): + with pytest.raises(ValueError): + adapter.extract_prompt(NeMoGymResponseCreateParamsNonStreaming(input=input)) + + +def test_system_and_developer_instructions(): + body = NeMoGymResponseCreateParamsNonStreaming( + input=[ + {"role": "system", "content": "First"}, + {"role": "developer", "content": "Second"}, + {"role": "user", "content": "Question"}, + ], + instructions="Third", + stream=False, + background=False, + ) + assert adapter.extract_prompt(body) == ("Question", "First\n\nSecond\n\nThird") + + +@pytest.mark.parametrize("kwargs", [{"max_output_tokens": 10}, {"temperature": 0.5}, {"model": "other"}]) +def test_reject_unsupported_request_controls(kwargs): + with pytest.raises(ValueError): + adapter.extract_prompt(NeMoGymResponseCreateParamsNonStreaming(input="Question", **kwargs)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("borrowed", [True, False]) +async def test_seed_installs_inside_sandbox_and_preserves_ownership(monkeypatch, borrowed): + remote = SimpleNamespace( + exec=AsyncMock(side_effect=([result()] if not borrowed else []) + [result(), result("2.1.108 (Claude Code)")]), + upload=AsyncMock(), + start=AsyncMock(), + stop=AsyncMock(), + disconnect=AsyncMock(), + ) + constructor = MagicMock(return_value=remote) + constructor.connect = AsyncMock(return_value=remote) + monkeypatch.setattr(adapter, "AsyncSandbox", constructor) + provider = SimpleNamespace(aclose=AsyncMock()) + monkeypatch.setattr(adapter, "create_provider", lambda config: provider) + instance = agent(sandbox_provider="sandbox", sandbox_config={"workdir": "/app"}) + state = await instance._seed_agent_session_state(seed(borrowed=borrowed)) + commands = [call.args[0] for call in remote.exec.await_args_list] + assert "install.py" in commands[-1] and "2.1.108" in commands[-1] + assert len(remote.upload.await_args_list) == 2 + assert state.executable.startswith(state.session.session_dir) + assert state.session.owns_sandbox is not borrowed + assert remote.start.await_count == (0 if borrowed else 1) + if borrowed: + constructor.connect.assert_awaited_once_with({"id": "borrowed"}, provider=provider) + assert not any(command.startswith("mkdir -p -- /app") for command in commands) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("workdir", ["/", "/tmp", "relative", "/app/../tmp", "/tmp/nemo-gym-claude-x"]) +async def test_bad_workdir_rejected_before_connect(monkeypatch, workdir): + connector = AsyncMock() + monkeypatch.setattr(adapter.AsyncSandbox, "connect", connector) + with pytest.raises(HTTPException, match="separate from session storage"): + await agent()._seed_agent_session_state(seed(workdir=workdir)) + connector.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_failed_seed_releases_and_failed_release_retains_state(monkeypatch): + remote = SimpleNamespace(exec=AsyncMock(return_value=result(code=1)), upload=AsyncMock()) + monkeypatch.setattr(adapter.AsyncSandbox, "connect", AsyncMock(return_value=remote)) + monkeypatch.setattr(adapter, "create_provider", lambda config: object()) + close = AsyncMock() + monkeypatch.setattr(adapter.SandboxSession, "close", close) + with pytest.raises(RuntimeError, match="diagnostic"): + await agent()._seed_agent_session_state(seed()) + close.assert_awaited_once() + close.side_effect = RuntimeError("release failed") + with pytest.raises(adapter.AgentSessionSetupError) as error: + await agent()._seed_agent_session_state(seed()) + assert error.value.state.request == seed() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "timed_out,code,status,reason", + [ + (False, 0, "completed", None), + (False, 7, "failed", "process_exit_7"), + (True, -15, "incomplete", "timeout"), + ], +) +async def test_supervised_output_uses_cleanup_outcome_and_shared_builders( + monkeypatch, timed_out, code, status, reason +): + files = {} + + async def upload(remote, *, path, text): + files[path] = text + + monkeypatch.setattr(adapter, "upload_text", upload) + instance = agent(anthropic_base_url="https://model.example", max_turns=4, thinking="adaptive") + observed = {} + raw = "\n".join( + map( + json.dumps, + [ + {"type": "assistant", "message": {"id": "served", "content": [{"type": "text", "text": "partial"}]}}, + { + "type": "result", + "subtype": "success", + "is_error": False, + "usage": {"input_tokens": 10, "output_tokens": 2}, + }, + ], + ) + ) + + class Session: + session_dir = "/tmp/session" + sandbox = SimpleNamespace(exec=AsyncMock(return_value=result(code=1))) + cleanup = {"timed_out": timed_out, "return_code": code, "cleanup_confirmed": True} + closing = False + read_output_log = AsyncMock(return_value=raw) + + async def execute(self, **kwargs): + command = await kwargs["stage_activation"]() + observed.update(kwargs, command=command) + return await kwargs["collect"]() + + state = adapter.ClaudeCodeSandboxedState(request=seed(), session=Session(), executable="/runtime/claude") + body = NeMoGymResponseCreateParamsNonStreaming(input="Question", instructions="System") + response = await instance._execute(state, body, instance._execution_input(body)) + launch = json.loads(files["/tmp/session/launch.json"]) + assert launch["argv"] == [ + "/runtime/claude", + *instance._build_command(instance.config.model, "Question", "System")[1:], + ] + assert launch["env"]["ANTHROPIC_BASE_URL"] == "https://model.example" + assert "HOME" not in launch["env"] + assert launch["env"]["CLAUDE_CONFIG_DIR"] == "/tmp/session/config" + assert observed["command"].python == "python3" + assert observed["timeout"] == instance.config.timeout + assert response.output[0].content[0].text == "partial" + assert response.status == status + assert response.metadata.get("stop_reason") == reason + assert response.usage.total_tokens == 12 + assert state.observations.gaps[0].code == "agent_transcript_unavailable" + + +@pytest.mark.asyncio +async def test_close_waits_for_cleanup_and_returns_observations(): + instance = agent() + session = SimpleNamespace(close=AsyncMock(side_effect=RuntimeError("unconfirmed"))) + observations = AgentObservationBundle(source="claude_code") + state = adapter.ClaudeCodeSandboxedState(request=seed(), session=session, observations=observations) + with pytest.raises(RuntimeError, match="unconfirmed"): + await instance._close_agent_session_state(state) + session.close.side_effect = None + response = await instance._close_agent_session_state(state) + assert response.agent_observations == observations + + +def request(session_id="session", rollout="rollout"): + return Request( + {"type": "http", "session": {AGENT_SESSION_COOKIE_KEY: session_id}, "path_params": {"rollout_id": rollout}} + ) + + +@pytest.mark.asyncio +async def test_retry_shares_execution_and_rejects_changed_body(monkeypatch): + instance = agent() + state = adapter.ClaudeCodeSandboxedState(request=seed(), session=SimpleNamespace()) + async with instance._locked_agent_session("session") as record: + record.state = state + response = MagicMock() + response.model_copy.return_value = response + execute = AsyncMock(return_value=response) + monkeypatch.setattr(ClaudeCodeAgent, "_execute", execute) + body = NeMoGymResponseCreateParamsNonStreaming(input="Question") + assert await instance.responses(request(), body) is response + assert await instance.responses(request(), body) is response + assert execute.await_count == 1 + with pytest.raises(HTTPException) as error: + await instance.responses(request(), NeMoGymResponseCreateParamsNonStreaming(input="Changed")) + assert error.value.status_code == 409 + with pytest.raises(HTTPException): + await instance.responses(request(rollout="wrong"), body) + + +@pytest.mark.asyncio +async def test_invalid_session_cannot_fall_back_to_local(monkeypatch): + instance = agent() + local = AsyncMock() + monkeypatch.setattr(ClaudeCodeAgent, "_create_response", local) + with pytest.raises(HTTPException) as error: + await instance.responses(request("unknown"), NeMoGymResponseCreateParamsNonStreaming(input="Question")) + assert error.value.status_code == 409 + local.assert_not_awaited() + + +def test_offline_installer_checks_checksum(tmp_path, monkeypatch): + import hashlib + + source = tmp_path / "source" + source.write_bytes(b"native executable") + destination = tmp_path / "claude" + run = MagicMock() + monkeypatch.setattr("responses_api_agents.claude_code_agent.install_claude_runtime.subprocess.run", run) + checksum = hashlib.sha256(source.read_bytes()).hexdigest() + install("2.1.108", destination, str(source), checksum) + assert destination.read_bytes() == source.read_bytes() + assert destination.stat().st_mode & 0o777 == 0o700 + run.assert_called_once_with([str(destination), "--version"], check=True, timeout=30) + with pytest.raises(ValueError, match="SHA-256 mismatch"): + install("2.1.108", destination, str(source), "incorrect") + assert not destination.exists() From bbcc6dec3e12d2835d497e4765677379755744b3 Mon Sep 17 00:00:00 2001 From: Frederic Lafrance Date: Mon, 19 Oct 2026 17:52:55 +0200 Subject: [PATCH 2/7] Align Claude Code sandbox sessions with canonical harness adapters Signed-off-by: Frederic Lafrance --- .../claude_code_agent/README.md | 41 +- responses_api_agents/claude_code_agent/app.py | 583 ++++++++++-------- .../configs/claude_code_agent.yaml | 15 +- .../install_claude_runtime.py | 119 +++- .../claude_code_agent/parsing.py | 260 ++++++++ .../claude_code_agent/sandbox.py | 515 +++++----------- .../claude_code_agent/sandbox_runner.py | 44 +- .../claude_code_agent/tests/test_app.py | 11 +- .../tests/test_install_claude_runtime.py | 99 +++ .../claude_code_agent/tests/test_sandbox.py | 381 +++++++----- .../tests/test_sandbox_runner.py | 73 +++ 11 files changed, 1346 insertions(+), 795 deletions(-) create mode 100644 responses_api_agents/claude_code_agent/parsing.py create mode 100644 responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py create mode 100644 responses_api_agents/claude_code_agent/tests/test_sandbox_runner.py diff --git a/responses_api_agents/claude_code_agent/README.md b/responses_api_agents/claude_code_agent/README.md index bd4ee3a8505..e3e9b07602a 100644 --- a/responses_api_agents/claude_code_agent/README.md +++ b/responses_api_agents/claude_code_agent/README.md @@ -230,23 +230,46 @@ sandbox_install_timeout_seconds: 600 session_close_timeout_seconds: 60 ``` -The image must include `python3`, `tar`, CA certificates, and libraries required by -the native Claude binary. Seed downloads the exact Linux x64/arm64 release and -checks its published SHA-256. For offline images, set `sandbox_executable_path` to -an existing native binary **inside the sandbox**; seed copies it into the private -session directory and checks its version. Configure `model_server` as above, or -`anthropic_base_url` and `anthropic_api_key`; the endpoint must be reachable from -the sandbox. `resources_server` is needed only for the legacy agent `/run` path. +The image must include Linux Python **3.8+**, `tar`, CA certificates, and libraries +required by the native Claude binary. Seed installs the exact x64/arm64 release in +`/tmp/nemo-gym-claude-runtime-`, separate from per-session state. A file +lock serializes installers; later seeds reuse a binary after checksum and version +health checks. Closing a borrowed session removes its private state and retains +the shared runtime for subsequent sessions. + +Downloads are checked against the release manifest's SHA-256 to detect corruption. +For an independently trusted checksum, set `sandbox_executable_sha256`. Offline +images can set `sandbox_executable_path` to a native binary **inside the sandbox**; +that option requires `sandbox_executable_sha256`. The installer verifies the hash +and version before atomically updating the shared runtime. + +Configure `model_server` as above, or `anthropic_base_url` and `anthropic_api_key`; +the endpoint must be reachable from the sandbox. `resources_server` is needed only +for the legacy agent `/run` path. Sandbox-only servers install no host CLI. For +local execution, installation happens on the first local invocation; concurrent +calls share setup, failures propagate, and a later call can retry. For borrowed execution, the seed request supplies `sandbox_access` with a direct provider connection descriptor and an absolute task `workdir`. Resources prepares the task directory; Claude Code only stages its own runtime and settings outside it. Required remote tool accesses are rejected: this adapter runs Claude Code's native tools. Optional static `mcp_config` and `settings` JSON files are staged -from the agent server, with the same command options as local execution. +from the agent server, with the same command options as local execution. A +`skills.path` in the merged server config is copied into the session's +`CLAUDE_CONFIG_DIR/skills`, including supporting files; skills disable `--bare` +through the same command builder used locally. The legacy `/run` skills reference +still takes precedence over configured skills for that invocation. The collector's +run-level skills option remains unsupported for materialized task rows. + +Prompt conversion preserves the existing local contract in both modes: the last +user message is the prompt, and a leading system message is appended to +`system_prompt`. Request `instructions` and developer messages retain the local +behavior of being ignored. One session runs one activation; it does not resume a +previous Claude conversation. The shared `SandboxSession` supervisor stops the harness before reading its -stream and native transcripts. Close returns the observation bundle, disconnects +dedicated stdout stream and native transcripts. The runner keeps `events.jsonl`, +`stderr.log`, and `runtime.json` separate from supervisor diagnostics. Close returns the observation bundle, disconnects borrowed sandboxes, and stops agent-owned sandboxes. The task directory remains available to the borrowed sandbox's owner for verification. `timeout` bounds harness execution; setup and close use the limits above. diff --git a/responses_api_agents/claude_code_agent/app.py b/responses_api_agents/claude_code_agent/app.py index 02d53e8dd38..a16fc322db8 100644 --- a/responses_api_agents/claude_code_agent/app.py +++ b/responses_api_agents/claude_code_agent/app.py @@ -18,12 +18,14 @@ import json import logging import os +import re import shutil -import subprocess import tempfile from asyncio import Semaphore +from collections.abc import Mapping from contextlib import suppress -from pathlib import Path +from pathlib import Path, PurePosixPath +from shlex import quote from time import monotonic, time from typing import Any, Callable, Optional from uuid import uuid4 @@ -31,268 +33,51 @@ from fastapi import HTTPException, Request from pydantic import ConfigDict, Field, PrivateAttr +from nemo_gym.agent_utils.sandbox_session import SandboxSession from nemo_gym.base_resources_server import NEMO_GYM_MCP_METADATA_KEY, BaseRunRequest, BaseVerifyResponse -from nemo_gym.base_responses_api_agent import BaseResponsesAPIAgentConfig, Body, SimpleResponsesAPIAgent +from nemo_gym.base_responses_api_agent import ( + AgentCloseSessionResponse, + AgentSeedSessionRequest, + AgentSessionSetupError, + AgentSessionState, + BaseResponsesAPIAgentConfig, + Body, + SimpleResponsesAPIAgent, +) from nemo_gym.config_types import ModelServerRef, ResourcesServerRef from nemo_gym.global_config import SKILLS_REF_KEY_NAME, get_first_server_config_dict from nemo_gym.openai_utils import ( NeMoGymEasyInputMessage, - NeMoGymFunctionCallOutput, NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming, - NeMoGymResponseFunctionToolCall, NeMoGymResponseInputTokensDetails, NeMoGymResponseOutputMessage, NeMoGymResponseOutputText, NeMoGymResponseOutputTokensDetails, - NeMoGymResponseReasoningItem, NeMoGymResponseUsage, - NeMoGymSummary, ) -from nemo_gym.rollout_observability import AgentEpisode, AgentObservationBundle, ObservationGap +from nemo_gym.rollout_observability import ( + AgentEpisode, + AgentObservationBundle, + ObservationGap, + SandboxObservation, +) +from nemo_gym.sandbox import AsyncSandbox, SandboxExecResult, SandboxSpec, create_provider +from nemo_gym.sandbox.config import resolve_provider_config from nemo_gym.server_utils import apply_rollout_prefix, get_response_json, raise_for_status from nemo_gym.skills import stage_skills from responses_api_agents.claude_code_agent.observability import extract_claude_code_observations -from responses_api_agents.claude_code_agent.sandbox import ClaudeCodeSandboxMixin +from responses_api_agents.claude_code_agent.parsing import _extract_instruction, _invocation_outcome, parse_stream_json +from responses_api_agents.claude_code_agent.sandbox import ( + ClaudeCodeSandboxSession, + parse_artifact_observations, +) from responses_api_agents.claude_code_agent.setup_claude_code import ensure_claude_code LOG = logging.getLogger(__name__) -def _extract_text(content: list[Any]) -> str: - return "".join(b.get("text", "") for b in content if isinstance(b, dict) and b.get("type") == "text") - - -def _extract_thinking(content: list[Any]) -> str: - parts = [] - for b in content: - if not isinstance(b, dict): - continue - if b.get("type") in ("thinking", "reasoning"): - parts.append(b.get("thinking") or b.get("text") or "") - return "\n".join(p for p in parts if p) - - -def parse_stream_json(stdout: str, *, fold_thinking: bool = True) -> tuple[list[Any], dict]: - """Convert claude -p --output-format=stream-json stdout into (output_items, usage). - - ``fold_thinking=True`` (eval default) prefixes each assistant message with the model's thinking as - ``...`` text. Under training-token capture that must be off: Gym attributes the finished - transcript to a captured model call by hashing the model-authored text, and the captured call's message - holds the bare text (reasoning travels as a separate item). Thinking is then emitted as a ``reasoning`` - item, which the fingerprint deliberately ignores. Capture output contains only the main conversation, - with pending tool calls retained as incomplete. The last main-thread assistant message's served ``id`` is - returned in internal metadata as ``terminal_response_id`` and becomes ``response.id`` under capture. - It is not a declared terminal id on the ``/run`` result; other attribution witnesses still corroborate it. - """ - raw_events: list[dict] = [] - for line in stdout.splitlines(): - line = line.strip() - if not line: - continue - try: - raw_events.append(json.loads(line)) - except json.JSONDecodeError: - continue - - output_items: list[Any] = [] - pending_calls: dict[str, NeMoGymResponseFunctionToolCall] = {} - buffered_think: str | None = None - total_input = 0 - total_output = 0 - num_turns: Optional[int] = None - result_metadata: dict[str, Any] = {} - compacting_sessions: set[str] = set() - compaction_attempts: list[dict[str, str]] = [] - terminal_response_id: Optional[str] = None - - for event in raw_events: - etype = event.get("type") - - if etype == "result": - usage = event.get("usage") or {} - total_input += int(usage.get("input_tokens") or 0) - total_output += int(usage.get("output_tokens") or 0) - # Claude Code's authoritative turn counter (what --max-turns bounds). - if event.get("num_turns") is not None: - num_turns = int(event["num_turns"]) - if isinstance(event.get("subtype"), str): - result_metadata["subtype"] = event["subtype"] - if isinstance(event.get("is_error"), bool): - result_metadata["is_error"] = event["is_error"] - duration_ms = event.get("duration_ms") - if isinstance(duration_ms, (int, float)) and not isinstance(duration_ms, bool) and duration_ms >= 0: - result_metadata["duration_ms"] = float(duration_ms) - - elif etype == "assistant": - message = event.get("message", {}) - content = message.get("content") or [] - usage = message.get("usage") or {} - total_input += int(usage.get("input_tokens") or 0) - total_output += int(usage.get("output_tokens") or 0) - - # Subagent frames belong to a different captured chain. Keep their usage accounting, - # but never let their text or tool ids testify to the main conversation's terminal call. - if not fold_thinking and event.get("parent_tool_use_id"): - continue - - if not isinstance(content, list): - content = [] - - served_id = message.get("id") - if not event.get("parent_tool_use_id") and isinstance(served_id, str) and served_id: - terminal_response_id = served_id - - think = _extract_thinking(content) - if think and fold_thinking: - buffered_think = (buffered_think + "\n" + think) if buffered_think else think - elif think: - output_items.append( - NeMoGymResponseReasoningItem( - id=f"rs_{uuid4().hex}", - summary=[NeMoGymSummary(text=think, type="summary_text")], - type="reasoning", - ) - ) - - text = _extract_text(content) - if text: - if buffered_think: - text = f"\n{buffered_think}\n\n\n{text}" - buffered_think = None - output_items.append( - NeMoGymResponseOutputMessage( - id=f"msg-{len(output_items)}", - content=[NeMoGymResponseOutputText(type="output_text", text=text, annotations=[])], - role="assistant", - status="completed", - type="message", - ) - ) - - for block in content: - if not isinstance(block, dict) or block.get("type") != "tool_use": - continue - call_id = block.get("id") or f"call-{uuid4().hex[:8]}" - input_data = block.get("input") or {} - arguments = json.dumps(input_data) if isinstance(input_data, dict) else str(input_data) - call = NeMoGymResponseFunctionToolCall( - arguments=arguments, - call_id=call_id, - name=block.get("name", ""), - type="function_call", - id=call_id, - status="incomplete", - ) - pending_calls[call_id] = call - if not fold_thinking: - # Preserve model order even when tools finish out of order or never finish. - output_items.append(call) - - elif etype == "user": - if not fold_thinking and event.get("parent_tool_use_id"): - continue - message = event.get("message", {}) - content = message.get("content") or [] - if not isinstance(content, list): - continue - - for block in content: - if not isinstance(block, dict) or block.get("type") != "tool_result": - continue - tool_id = block.get("tool_use_id", "") - call = pending_calls.pop(tool_id, None) - if call: - call.status = "completed" - if fold_thinking: - output_items.append(call) - result_content = block.get("content") or "" - if isinstance(result_content, list): - result_text = _extract_text(result_content) - else: - result_text = str(result_content) - output_items.append( - NeMoGymFunctionCallOutput( - type="function_call_output", - call_id=tool_id, - output=result_text, - status="completed", - ) - ) - - elif etype == "system" and event.get("subtype") == "status": - session_id = event.get("session_id") - if not isinstance(session_id, str) or not session_id: - continue - if event.get("status") == "compacting": - compacting_sessions.add(session_id) - continue - compact_result = event.get("compact_result") - if compact_result in {"failed", "success"}: - if compact_result == "failed": - compaction_attempts.append({"invocation_id": session_id, "outcome": "failed"}) - compacting_sessions.discard(session_id) - - compaction_attempts.extend( - {"invocation_id": session_id, "outcome": "unknown"} for session_id in compacting_sessions - ) - metadata: dict = {"input_tokens": total_input, "output_tokens": total_output} - if terminal_response_id: - metadata["terminal_response_id"] = terminal_response_id - if num_turns is not None: - metadata["num_turns"] = num_turns - if compaction_attempts: - metadata["compaction_attempts"] = compaction_attempts - metadata.update(result_metadata) - return output_items, metadata - - -def _invocation_outcome(metadata: dict[str, Any], returncode: int | None) -> tuple[str, str | None]: - subtype = metadata.get("subtype") - if subtype == "error_max_turns": - return "incomplete", subtype - if metadata.get("is_error") is True or (isinstance(subtype, str) and subtype.startswith("error_")): - return "failed", subtype if isinstance(subtype, str) else "agent_error" - if returncode not in (0, None): - return "failed", f"process_exit_{returncode}" - if subtype == "success": - return "completed", None - return "incomplete", "result_missing" - - -def _extract_instruction(body_input) -> tuple[str, Optional[str]]: - """Return (user_message, system_message) from a responses body input list.""" - items = list(body_input) - system_message: Optional[str] = None - - if items: - first = items[0] - role = getattr(first, "role", None) or (first.get("role") if isinstance(first, dict) else None) - if role == "system": - content = getattr(first, "content", None) or (first.get("content") if isinstance(first, dict) else None) - if isinstance(content, list): - content = "".join( - (p.get("text", "") if isinstance(p, dict) else getattr(p, "text", "")) for p in content - ) - system_message = content or "" - items = items[1:] - - user_message = "" - for item in reversed(items): - role = getattr(item, "role", None) or (item.get("role") if isinstance(item, dict) else None) - if role == "user": - content = getattr(item, "content", None) or (item.get("content") if isinstance(item, dict) else None) - if isinstance(content, list): - content = "".join( - (p.get("text", "") if isinstance(p, dict) else getattr(p, "text", "")) for p in content - ) - user_message = content or "" - break - - return user_message, system_message - - class ClaudeCodeAgentConfig(BaseResponsesAPIAgentConfig): resources_server: Optional[ResourcesServerRef] = None # When model_server is set, ANTHROPIC_BASE_URL is resolved from the Gym model @@ -322,6 +107,7 @@ class ClaudeCodeAgentConfig(BaseResponsesAPIAgentConfig): sandbox_provider: str | None = None sandbox_config: dict[str, Any] = Field(default_factory=dict) sandbox_executable_path: str | None = None + sandbox_executable_sha256: str | None = Field(default=None, pattern=r"^[a-fA-F0-9]{64}$") sandbox_install_timeout_seconds: float = Field(default=600, gt=0, allow_inf_nan=False) session_close_timeout_seconds: float = Field(default=60, gt=0, allow_inf_nan=False) @@ -339,20 +125,35 @@ class ClaudeCodeAgentVerifyResponse(BaseVerifyResponse): ) -class ClaudeCodeAgent(ClaudeCodeSandboxMixin, SimpleResponsesAPIAgent): +class ClaudeCodeAgent(SimpleResponsesAPIAgent): config: ClaudeCodeAgentConfig sem: Semaphore = None + _local_setup_task: asyncio.Task[None] | None = PrivateAttr(default=None) _static_mcp_config: Optional[dict[str, Any]] = PrivateAttr(default=None) model_config = ConfigDict(arbitrary_types_allowed=True) def model_post_init(self, __context: Any) -> None: self.sem = Semaphore(self.config.concurrency) - ensure_claude_code(self.config.claude_code_version) + + async def _ensure_local_runtime(self) -> None: + if self._local_setup_task is None: + self._local_setup_task = asyncio.create_task( + asyncio.to_thread(ensure_claude_code, self.config.claude_code_version) + ) + self._local_setup_task.add_done_callback(self._clear_failed_local_setup) + task = self._local_setup_task try: - ver = subprocess.run(["claude", "--version"], capture_output=True, text=True, timeout=10).stdout.strip() - LOG.warning("claude-code version: %s", ver or "(unknown)") - except Exception as exc: - LOG.warning("could not determine claude-code version: %s", exc) + await asyncio.shield(task) + except asyncio.CancelledError: + raise + except Exception: + if self._local_setup_task is task: + self._local_setup_task = None + raise + + def _clear_failed_local_setup(self, task: asyncio.Task[None]) -> None: + if (task.cancelled() or task.exception() is not None) and self._local_setup_task is task: + self._local_setup_task = None def _resolve_base_url(self) -> str: if self.config.model_server: @@ -487,6 +288,7 @@ async def _run_claude_code( When ``rollout_id`` is set and a model server is configured, the per-rollout capture prefix is applied to ANTHROPIC_BASE_URL so the CLI's streaming /v1/messages calls correlate to this rollout. """ + await self._ensure_local_runtime() base_url = self._resolve_call_base_url(rollout_id) # Keep full model name for local/custom endpoints; strip provider prefix for real Anthropic API. model = self.config.model if base_url else self.config.model.split("/")[-1] @@ -653,19 +455,41 @@ async def _create_response( if isinstance(body.input, str): body.input = [NeMoGymEasyInputMessage(role="user", content=body.input)] - user_message, input_system = _extract_instruction(body.input) - system_parts = [p for p in [self.config.system_prompt, input_system] if p] - system_prompt = "\n\n".join(system_parts) if system_parts else None + user_message, system_prompt = self._conversation_input(body) output_items, model_name, run_metadata = await self._run_claude_code( user_message, system_prompt=system_prompt, mcp_config=mcp_config, - skills_path=skills_path, + skills_path=skills_path or self._configured_skills_path(), rollout_id=rollout_id, observation_collector=observation_collector, ) + return self._response_from_output(body, output_items, model_name, run_metadata) + + def _conversation_input(self, body: NeMoGymResponseCreateParamsNonStreaming) -> tuple[str, str | None]: + """Use the established local last-user/leading-system prompt contract in both modes.""" + items = ( + [NeMoGymEasyInputMessage(role="user", content=body.input)] if isinstance(body.input, str) else body.input + ) + prompt, input_system = _extract_instruction(items) + system = "\n\n".join(part for part in (self.config.system_prompt, input_system) if part) or None + return prompt, system + + def _configured_skills_path(self) -> str | None: + skills = getattr(self.server_client, "global_config_dict", {}).get("skills") + return skills.get("path") if isinstance(skills, Mapping) else None + + def _response_from_output( + self, + body: NeMoGymResponseCreateParamsNonStreaming, + output_items: list[Any], + model_name: str, + run_metadata: dict[str, Any], + *, + status: str | None = None, + ) -> NeMoGymResponse: if not any( getattr(item, "type", None) == "message" and getattr(item, "role", None) == "assistant" for item in output_items @@ -692,6 +516,7 @@ async def _create_response( else f"resp_{uuid4().hex}" ) return NeMoGymResponse( + **({"status": status} if status is not None else {}), id=response_id, created_at=int(time()), model=model_name, @@ -709,13 +534,263 @@ async def _create_response( ), ) + async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> ClaudeCodeSandboxSession: + """Install/reuse the harness runtime before the first activation.""" + if any(access.required for access in self.effective_tool_accesses(body)): + raise HTTPException( + 422, "Claude sandbox execution uses native tools; required remote tools are unsupported" + ) + version = self.config.claude_code_version + if not version or not re.fullmatch(r"\d+\.\d+\.\d+", version): + raise HTTPException(422, "Claude sandbox execution requires an exact claude_code_version") + if self.config.sandbox_executable_path and not self.config.sandbox_executable_sha256: + raise HTTPException(422, "sandbox_executable_path requires sandbox_executable_sha256") + access = body.sandbox_access + owns_sandbox = access is None + if owns_sandbox and not self.config.sandbox_provider: + raise HTTPException(422, "Claude requires sandbox_access or a configured sandbox_provider") + spec = SandboxSpec(**{"workdir": "/app", **self.config.sandbox_config}) if owns_sandbox else None + workdir = access.workdir if access else spec.workdir + path = PurePosixPath(workdir or "") + if ( + not path.is_absolute() + or ".." in path.parts + or "\x00" in workdir + or path in (PurePosixPath("/"), PurePosixPath("/tmp")) + or str(path).startswith(("/tmp/nemo-gym-claude-sessions", "/tmp/nemo-gym-claude-runtime-")) + ): + raise HTTPException( + 422, "Claude sandbox workdir must be absolute and separate from runtime/session storage" + ) + if self.config.timeout <= 0: + raise HTTPException(422, "Claude sandbox execution timeout must be positive") + provider_ref = access.connection.provider_config_ref if access else self.config.sandbox_provider + provider = create_provider(resolve_provider_config(provider_ref, self.server_client.global_config_dict)) + if owns_sandbox: + sandbox = AsyncSandbox(provider) + else: + try: + sandbox = await AsyncSandbox.connect(access.connection.descriptor, provider=provider) + except BaseException: + await provider.aclose() + raise + directory = f"/tmp/nemo-gym-claude-sessions/{uuid4().hex}" + runtime = f"/tmp/nemo-gym-claude-runtime-{version}" + state = ClaudeCodeSandboxSession( + request=body, + runtime=runtime, + session=SandboxSession( + sandbox=sandbox, + session_dir=directory, + workdir=workdir, + harness="Claude Code", + owns_sandbox=owns_sandbox, + ), + ) + try: + async with asyncio.timeout(self.config.sandbox_install_timeout_seconds): + if owns_sandbox: + await sandbox.start(spec) + result = await sandbox.exec(f"mkdir -p -- {quote(workdir)}", cwd="/", timeout_s=30) + self._check_sandbox_setup(result) + result = await sandbox.exec( + f"test -d {quote(workdir)} && umask 077 && mkdir -p {quote(directory)}/config", + cwd="/", + timeout_s=30, + ) + self._check_sandbox_setup(result) + for name in ("sandbox_runner.py", "install_claude_runtime.py"): + await sandbox.upload(Path(__file__).with_name(name), f"{directory}/{name}") + command = [ + "python3", + "-I", + f"{directory}/install_claude_runtime.py", + "--version", + version, + "--runtime", + runtime, + ] + if self.config.sandbox_executable_path: + command += ["--source", self.config.sandbox_executable_path] + if self.config.sandbox_executable_sha256: + command += ["--sha256", self.config.sandbox_executable_sha256] + installed = await sandbox.exec( + " ".join(map(quote, command)), cwd=workdir, timeout_s=self.config.sandbox_install_timeout_seconds + ) + self._check_sandbox_setup(installed) + return state + except BaseException as error: + try: + await state.close(self.config.session_close_timeout_seconds) + except BaseException: + raise AgentSessionSetupError(state, error=error) from error + raise + + @staticmethod + def _check_sandbox_setup(result: SandboxExecResult) -> None: + if result.return_code != 0 or result.error_type: + details = "\n".join(part for part in (result.stderr, result.stdout) if part) + raise RuntimeError( + f"Claude sandbox setup failed (exit={result.return_code}, error={result.error_type}): {details[-16000:]}" + ) + + async def _close_agent_session_state(self, state: AgentSessionState) -> AgentCloseSessionResponse: + """Confirm cleanup and return a bundle even when no activation ran.""" + assert isinstance(state, ClaudeCodeSandboxSession) + await state.close(self.config.session_close_timeout_seconds) + return AgentCloseSessionResponse( + agent_session_id=state.request.agent_session_id, + agent_observations=state.observations + or AgentObservationBundle( + source="claude_code", + gaps=[ObservationGap(code="agent_activation_interrupted")], + ), + ) + + def _validate_sandbox_request(self, body: NeMoGymResponseCreateParamsNonStreaming) -> None: + supported = {"input", "instructions", "model", "metadata"} + for name, field in type(body).model_fields.items(): + value = getattr(body, name) + if name in supported or (name in {"stream", "background"} and value is False): + continue + if value != field.get_default(call_default_factory=True): + raise HTTPException(422, f"Claude sandbox execution does not support request field {name}") + models = {None, "", self.config.model} + if self.config.model_server: + models.add(self.config.model_server.name) + if body.model not in models or body.model_extra: + raise HTTPException(422, "Claude sandbox model is selected by agent/model-server configuration") + if not isinstance(body.input, str): + for item in body.input: + if getattr(item, "role", None) not in {"user", "system", "developer", "assistant"}: + raise HTTPException(422, "Claude sandbox input requires text messages") + if not isinstance(item.content, str) and any( + (part.get("type") if isinstance(part, dict) else part.type) != "input_text" + for part in item.content + ): + raise HTTPException(422, "Claude sandbox input requires text messages") + + async def _sandbox_response( + self, + state: ClaudeCodeSandboxSession, + body: NeMoGymResponseCreateParamsNonStreaming, + *, + prompt: str, + system: str | None, + ) -> NeMoGymResponse: + directory = state.session.session_dir + base_url = self._resolve_call_base_url(state.request.episode_id.capture_key) + model = self.config.model if base_url else self.config.model.split("/")[-1] + env = { + "CLAUDE_CONFIG_DIR": directory + "/config", + "IS_SANDBOX": "1", + "ANTHROPIC_API_KEY": self.config.anthropic_api_key or "EMPTY", + "ANTHROPIC_MODEL": model, + "ANTHROPIC_DEFAULT_HAIKU_MODEL": model, + "ANTHROPIC_DEFAULT_SONNET_MODEL": model, + "ANTHROPIC_DEFAULT_OPUS_MODEL": model, + "CLAUDE_CODE_SUBAGENT_MODEL": model, + "DISABLE_AUTOUPDATER": "1", + "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", + } + if base_url: + env.update(ANTHROPIC_BASE_URL=base_url, ANTHROPIC_AUTH_TOKEN=self.config.anthropic_api_key or "local") + artifacts = None + failure = None + try: + # Close can fence the staging phase just as it fences the execution phase. + async with self.sem: + mcp_config = directory + "/mcp.json" if self.config.mcp_config else None + skills_path = self._configured_skills_path() + command = self._build_command( + model, prompt, system, mcp_config=mcp_config, skills_active=bool(skills_path) + ) + command[0] = state.runtime + "/claude" + artifacts = await state.execute( + { + "directory": directory, + "command": command, + "cwd": state.session.workdir, + "env": env, + "settings": self._build_settings(), + "mcp_config": self._get_static_mcp_config() if mcp_config else None, + "skills_path": skills_path, + }, + timeout=self.config.timeout, + close_timeout=self.config.session_close_timeout_seconds, + ) + except BaseException as error: + failure = error + artifacts = state.session.artifacts + items, run_metadata = ([], {}) + if artifacts is not None: + items, run_metadata = await asyncio.to_thread( + parse_stream_json, + artifacts.events, + fold_thinking=not self._token_id_capture_enabled(), + ) + cleanup = state.session.cleanup + if cleanup and cleanup["timed_out"]: + status, reason = "incomplete", "timeout" + elif isinstance(failure, asyncio.CancelledError): + status, reason = "incomplete", "cancelled" + elif failure is not None: + status, reason = "failed", "sandbox_error" + else: + status, reason = _invocation_outcome(run_metadata, cleanup["return_code"] if cleanup else None) + if artifacts is not None: + state.observations = await asyncio.to_thread( + parse_artifact_observations, + artifacts, + model_ref=self.config.model_server, + status=status, + error_type=reason, + run_metadata=run_metadata, + ) + else: + state.observations = AgentObservationBundle( + source="claude_code", gaps=[ObservationGap(code="agent_artifact_unavailable")] + ) + if state.runtime_info is None: + state.observations.gaps.append(ObservationGap(code="runtime_info_unavailable")) + if cleanup: + access = state.request.sandbox_access + state.observations.records.append( + SandboxObservation( + role="agent", + provider=access.connection.provider_config_ref if access else self.config.sandbox_provider, + outcome="timeout" if cleanup["timed_out"] else "failed" if status == "failed" else "completed", + exit_code=cleanup["return_code"], + error_type=reason, + ) + ) + if failure is not None: + raise failure + return self._response_from_output(body, items, model, run_metadata, status=status) + async def responses( self, request: Request, body: NeMoGymResponseCreateParamsNonStreaming = Body(), ) -> NeMoGymResponse: - if self._agent_session_id_from_request(request) is not None: - return await self._sandbox_responses(request, body) + session_id = self._agent_session_id_from_request(request) + if session_id is not None: + state = self._require_agent_session(session_id) + assert isinstance(state, ClaudeCodeSandboxSession) + if request.path_params.get("rollout_id") != state.request.episode_id.capture_key: + raise HTTPException(409, "Claude activation does not match the seeded session and rollout") + if state.session.closing: + raise HTTPException(409, "Claude session is closing") + self._validate_sandbox_request(body) + prompt, system = self._conversation_input(body) + if state.task is None: + state.activation_request = body.model_copy(deep=True) + state.task = asyncio.create_task( + self._sandbox_response(state, state.activation_request, prompt=prompt, system=system) + ) + elif body != state.activation_request: + raise HTTPException(409, "Claude sandbox sessions support one activation; retry the same request") + return (await asyncio.shield(state.task)).model_copy(deep=True) return await self._create_response(body, rollout_id=request.path_params.get("rollout_id")) async def _create_episode( diff --git a/responses_api_agents/claude_code_agent/configs/claude_code_agent.yaml b/responses_api_agents/claude_code_agent/configs/claude_code_agent.yaml index 5e3140bea77..a2707b38033 100644 --- a/responses_api_agents/claude_code_agent/configs/claude_code_agent.yaml +++ b/responses_api_agents/claude_code_agent/configs/claude_code_agent.yaml @@ -2,9 +2,8 @@ claude_code_agent: responses_api_agents: claude_code_agent: entrypoint: app.py - resources_server: - type: resources_servers - name: ??? + resources_server: null # set a ResourcesServerRef for the legacy /run endpoint + model_server: null # Gym model-server ref, or use anthropic_base_url directly concurrency: 32 model: claude-sonnet-4-6 anthropic_api_key: ${anthropic_api_key} @@ -14,8 +13,16 @@ claude_code_agent: system_prompt: null allowed_tools: null disallowed_tools: null - claude_code_version: null + claude_code_version: null # sandbox sessions require an exact version, e.g. "2.1.108" # Runtime capabilities. Defaults keep the runtime isolated/reproducible. bare: true # set false to auto-discover skills, hooks, plugins, MCP, memory, CLAUDE.md mcp_config: null # path to an MCP server config file (--mcp-config); works even with bare: true settings: null # path to a settings JSON layered into the per-run CLAUDE_CONFIG_DIR + # Sandbox sessions install/reuse a versioned runtime inside the task sandbox. + # The optional local runtime installs lazily on the first unseeded call. + sandbox_provider: null # top-level provider ref; unused with borrowed SandboxAccess + sandbox_config: {} # SandboxSpec for owned execution, e.g. {image: ubuntu:24.04, workdir: /app} + sandbox_executable_path: null # optional native binary already inside the sandbox + sandbox_executable_sha256: null # required trusted SHA-256 for a preinstalled binary + sandbox_install_timeout_seconds: 600 + session_close_timeout_seconds: 60 diff --git a/responses_api_agents/claude_code_agent/install_claude_runtime.py b/responses_api_agents/claude_code_agent/install_claude_runtime.py index 4267a067b8a..ec126fb802a 100644 --- a/responses_api_agents/claude_code_agent/install_claude_runtime.py +++ b/responses_api_agents/claude_code_agent/install_claude_runtime.py @@ -1,44 +1,105 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Install a pinned native Claude executable inside a sandbox, using stdlib Python.""" +"""Install/reuse a versioned native Claude runtime with Linux Python 3.8+ stdlib.""" +import argparse +import fcntl import hashlib import json +import os import platform +import re import shutil import subprocess -import sys +import tempfile import urllib.request from pathlib import Path -def install(version: str, destination: Path, source: str, expected: str) -> None: - """Copy a supplied binary or download the native release and verify its checksum.""" - if source: - shutil.copyfile(source, destination) - else: - architecture = {"x86_64": "x64", "aarch64": "arm64"}.get(platform.machine()) - if platform.system() != "Linux" or architecture is None: - raise RuntimeError("Claude sandbox installation requires Linux x64 or arm64") - target = "linux-" + architecture - if platform.libc_ver()[0] != "glibc" and Path("/etc/alpine-release").exists(): - target += "-musl" - base = "https://downloads.claude.ai/claude-code-releases/" + version - with urllib.request.urlopen(base + "/manifest.json", timeout=60) as response: - manifest = json.load(response) - expected = expected or manifest["platforms"][target]["checksum"] - with urllib.request.urlopen(base + "/" + target + "/claude", timeout=120) as response: - with destination.open("wb") as output: - shutil.copyfileobj(response, output) - if expected: - with destination.open("rb") as binary: - actual = hashlib.file_digest(binary, "sha256").hexdigest() - if actual != expected: - destination.unlink() - raise ValueError("Claude executable SHA-256 mismatch") - destination.chmod(0o700) - subprocess.run([str(destination), "--version"], check=True, timeout=30) +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as binary: + for chunk in iter(lambda: binary.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _version(executable: Path, version: str) -> str: + actual = subprocess.run( + [str(executable), "--version"], + check=True, + capture_output=True, + text=True, + errors="replace", + timeout=30, + ).stdout.strip() + if not actual.startswith(version + " "): + raise ValueError("Claude version mismatch: expected %s, got %s" % (version, actual)) + return actual + + +def install(version: str, runtime: Path, *, source: str = "", expected: str = "") -> str: + """Reuse a healthy verified binary, or atomically install it under a per-runtime lock. + + Downloads use the release manifest for corruption detection unless the caller supplies + a trusted SHA-256. Offline source binaries require that caller-supplied checksum. + """ + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): + raise ValueError("Claude runtime requires an exact version") + if source and not expected: + raise ValueError("A preinstalled Claude binary requires a trusted SHA-256") + expected = expected.lower() + runtime.parent.mkdir(parents=True, exist_ok=True) + with Path(str(runtime) + ".lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + runtime.mkdir(mode=0o700, exist_ok=True) + executable = runtime / "claude" + receipt = runtime / "installation.json" + try: + recorded = json.loads(receipt.read_text()) + checksum = expected or recorded["sha256"] + if recorded["version"] == version and _sha256(executable) == checksum: + return _version(executable, version) + except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError): + pass + fd, name = tempfile.mkstemp(prefix=".claude-", dir=runtime) + os.close(fd) + temporary = Path(name) + try: + if source: + shutil.copyfile(source, temporary) + else: + architecture = {"x86_64": "x64", "aarch64": "arm64"}.get(platform.machine()) + if platform.system() != "Linux" or architecture is None: + raise RuntimeError("Claude sandbox installation requires Linux x64 or arm64") + target = "linux-" + architecture + if platform.libc_ver()[0] != "glibc" and Path("/etc/alpine-release").exists(): + target += "-musl" + base = "https://downloads.claude.ai/claude-code-releases/" + version + if not expected: + with urllib.request.urlopen(base + "/manifest.json", timeout=60) as response: + manifest = json.load(response) + expected = manifest["platforms"][target]["checksum"].lower() + with urllib.request.urlopen(base + "/" + target + "/claude", timeout=120) as response: + with temporary.open("wb") as output: + shutil.copyfileobj(response, output) + checksum = _sha256(temporary) + if checksum != expected: + raise ValueError("Claude executable SHA-256 mismatch") + temporary.chmod(0o700) + actual = _version(temporary, version) + temporary.replace(executable) + receipt.write_text(json.dumps({"version": version, "sha256": checksum})) + return actual + finally: + temporary.unlink(missing_ok=True) if __name__ == "__main__": - install(sys.argv[1], Path(sys.argv[2]), sys.argv[3], sys.argv[4]) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", required=True) + parser.add_argument("--runtime", type=Path, required=True) + parser.add_argument("--source", default="") + parser.add_argument("--sha256", default="") + args = parser.parse_args() + print(install(args.version, args.runtime, source=args.source, expected=args.sha256)) diff --git a/responses_api_agents/claude_code_agent/parsing.py b/responses_api_agents/claude_code_agent/parsing.py new file mode 100644 index 00000000000..6a6d4b4de58 --- /dev/null +++ b/responses_api_agents/claude_code_agent/parsing.py @@ -0,0 +1,260 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from typing import Any, Optional +from uuid import uuid4 + +from nemo_gym.openai_utils import ( + NeMoGymFunctionCallOutput, + NeMoGymResponseFunctionToolCall, + NeMoGymResponseOutputMessage, + NeMoGymResponseOutputText, + NeMoGymResponseReasoningItem, + NeMoGymSummary, +) + + +def _extract_text(content: list[Any]) -> str: + return "".join(b.get("text", "") for b in content if isinstance(b, dict) and b.get("type") == "text") + + +def _extract_thinking(content: list[Any]) -> str: + parts = [] + for b in content: + if not isinstance(b, dict): + continue + if b.get("type") in ("thinking", "reasoning"): + parts.append(b.get("thinking") or b.get("text") or "") + return "\n".join(p for p in parts if p) + + +def parse_stream_json(stdout: str, *, fold_thinking: bool = True) -> tuple[list[Any], dict]: + """Convert claude -p --output-format=stream-json stdout into (output_items, usage). + + ``fold_thinking=True`` (eval default) prefixes each assistant message with the model's thinking as + ``...`` text. Under training-token capture that must be off: Gym attributes the finished + transcript to a captured model call by hashing the model-authored text, and the captured call's message + holds the bare text (reasoning travels as a separate item). Thinking is then emitted as a ``reasoning`` + item, which the fingerprint deliberately ignores. Capture output contains only the main conversation, + with pending tool calls retained as incomplete. The last main-thread assistant message's served ``id`` is + returned in internal metadata as ``terminal_response_id`` and becomes ``response.id`` under capture. + It is not a declared terminal id on the ``/run`` result; other attribution witnesses still corroborate it. + """ + raw_events: list[dict] = [] + for line in stdout.splitlines(): + line = line.strip() + if not line: + continue + try: + raw_events.append(json.loads(line)) + except json.JSONDecodeError: + continue + + output_items: list[Any] = [] + pending_calls: dict[str, NeMoGymResponseFunctionToolCall] = {} + buffered_think: str | None = None + total_input = 0 + total_output = 0 + num_turns: Optional[int] = None + result_metadata: dict[str, Any] = {} + compacting_sessions: set[str] = set() + compaction_attempts: list[dict[str, str]] = [] + terminal_response_id: Optional[str] = None + + for event in raw_events: + etype = event.get("type") + + if etype == "result": + usage = event.get("usage") or {} + total_input += int(usage.get("input_tokens") or 0) + total_output += int(usage.get("output_tokens") or 0) + # Claude Code's authoritative turn counter (what --max-turns bounds). + if event.get("num_turns") is not None: + num_turns = int(event["num_turns"]) + if isinstance(event.get("subtype"), str): + result_metadata["subtype"] = event["subtype"] + if isinstance(event.get("is_error"), bool): + result_metadata["is_error"] = event["is_error"] + duration_ms = event.get("duration_ms") + if isinstance(duration_ms, (int, float)) and not isinstance(duration_ms, bool) and duration_ms >= 0: + result_metadata["duration_ms"] = float(duration_ms) + + elif etype == "assistant": + message = event.get("message", {}) + content = message.get("content") or [] + usage = message.get("usage") or {} + total_input += int(usage.get("input_tokens") or 0) + total_output += int(usage.get("output_tokens") or 0) + + # Subagent frames belong to a different captured chain. Keep their usage accounting, + # but never let their text or tool ids testify to the main conversation's terminal call. + if not fold_thinking and event.get("parent_tool_use_id"): + continue + + if not isinstance(content, list): + content = [] + + served_id = message.get("id") + if not event.get("parent_tool_use_id") and isinstance(served_id, str) and served_id: + terminal_response_id = served_id + + think = _extract_thinking(content) + if think and fold_thinking: + buffered_think = (buffered_think + "\n" + think) if buffered_think else think + elif think: + output_items.append( + NeMoGymResponseReasoningItem( + id=f"rs_{uuid4().hex}", + summary=[NeMoGymSummary(text=think, type="summary_text")], + type="reasoning", + ) + ) + + text = _extract_text(content) + if text: + if buffered_think: + text = f"\n{buffered_think}\n\n\n{text}" + buffered_think = None + output_items.append( + NeMoGymResponseOutputMessage( + id=f"msg-{len(output_items)}", + content=[NeMoGymResponseOutputText(type="output_text", text=text, annotations=[])], + role="assistant", + status="completed", + type="message", + ) + ) + + for block in content: + if not isinstance(block, dict) or block.get("type") != "tool_use": + continue + call_id = block.get("id") or f"call-{uuid4().hex[:8]}" + input_data = block.get("input") or {} + arguments = json.dumps(input_data) if isinstance(input_data, dict) else str(input_data) + call = NeMoGymResponseFunctionToolCall( + arguments=arguments, + call_id=call_id, + name=block.get("name", ""), + type="function_call", + id=call_id, + status="incomplete", + ) + pending_calls[call_id] = call + if not fold_thinking: + # Preserve model order even when tools finish out of order or never finish. + output_items.append(call) + + elif etype == "user": + if not fold_thinking and event.get("parent_tool_use_id"): + continue + message = event.get("message", {}) + content = message.get("content") or [] + if not isinstance(content, list): + continue + + for block in content: + if not isinstance(block, dict) or block.get("type") != "tool_result": + continue + tool_id = block.get("tool_use_id", "") + call = pending_calls.pop(tool_id, None) + if call: + call.status = "completed" + if fold_thinking: + output_items.append(call) + result_content = block.get("content") or "" + if isinstance(result_content, list): + result_text = _extract_text(result_content) + else: + result_text = str(result_content) + output_items.append( + NeMoGymFunctionCallOutput( + type="function_call_output", + call_id=tool_id, + output=result_text, + status="completed", + ) + ) + + elif etype == "system" and event.get("subtype") == "status": + session_id = event.get("session_id") + if not isinstance(session_id, str) or not session_id: + continue + if event.get("status") == "compacting": + compacting_sessions.add(session_id) + continue + compact_result = event.get("compact_result") + if compact_result in {"failed", "success"}: + if compact_result == "failed": + compaction_attempts.append({"invocation_id": session_id, "outcome": "failed"}) + compacting_sessions.discard(session_id) + + compaction_attempts.extend( + {"invocation_id": session_id, "outcome": "unknown"} for session_id in compacting_sessions + ) + metadata: dict = {"input_tokens": total_input, "output_tokens": total_output} + if terminal_response_id: + metadata["terminal_response_id"] = terminal_response_id + if num_turns is not None: + metadata["num_turns"] = num_turns + if compaction_attempts: + metadata["compaction_attempts"] = compaction_attempts + metadata.update(result_metadata) + return output_items, metadata + + +def _invocation_outcome(metadata: dict[str, Any], returncode: int | None) -> tuple[str, str | None]: + subtype = metadata.get("subtype") + if subtype == "error_max_turns": + return "incomplete", subtype + if metadata.get("is_error") is True or (isinstance(subtype, str) and subtype.startswith("error_")): + return "failed", subtype if isinstance(subtype, str) else "agent_error" + if returncode not in (0, None): + return "failed", f"process_exit_{returncode}" + if subtype == "success": + return "completed", None + return "incomplete", "result_missing" + + +def _extract_instruction(body_input) -> tuple[str, Optional[str]]: + """Return (user_message, system_message) from a responses body input list.""" + items = list(body_input) + system_message: Optional[str] = None + + if items: + first = items[0] + role = getattr(first, "role", None) or (first.get("role") if isinstance(first, dict) else None) + if role == "system": + content = getattr(first, "content", None) or (first.get("content") if isinstance(first, dict) else None) + if isinstance(content, list): + content = "".join( + (p.get("text", "") if isinstance(p, dict) else getattr(p, "text", "")) for p in content + ) + system_message = content or "" + items = items[1:] + + user_message = "" + for item in reversed(items): + role = getattr(item, "role", None) or (item.get("role") if isinstance(item, dict) else None) + if role == "user": + content = getattr(item, "content", None) or (item.get("content") if isinstance(item, dict) else None) + if isinstance(content, list): + content = "".join( + (p.get("text", "") if isinstance(p, dict) else getattr(p, "text", "")) for p in content + ) + user_message = content or "" + break + + return user_message, system_message diff --git a/responses_api_agents/claude_code_agent/sandbox.py b/responses_api_agents/claude_code_agent/sandbox.py index dbcaca91c26..22d906465eb 100644 --- a/responses_api_agents/claude_code_agent/sandbox.py +++ b/responses_api_agents/claude_code_agent/sandbox.py @@ -1,382 +1,203 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Claude Code sessions with owned or borrowed sandboxes and native tools.""" +"""Claude Code artifacts and request state over the shared sandbox lifecycle.""" import asyncio +import io import json import logging -import re import tarfile import tempfile from dataclasses import dataclass -from pathlib import Path, PurePosixPath +from pathlib import Path from shlex import quote -from time import time from typing import Any -from uuid import uuid4 -from fastapi import Body, HTTPException, Request +from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError from nemo_gym.agent_utils.sandbox_session import SandboxCommand, SandboxSession -from nemo_gym.base_responses_api_agent import ( - AgentCloseSessionResponse, - AgentSeedSessionRequest, - AgentSessionSetupError, - AgentSessionState, -) -from nemo_gym.openai_utils import ( - NeMoGymResponse, - NeMoGymResponseCreateParamsNonStreaming, - NeMoGymResponseInputTokensDetails, - NeMoGymResponseOutputTokensDetails, - NeMoGymResponseUsage, -) +from nemo_gym.base_responses_api_agent import AgentSessionState +from nemo_gym.config_types import ModelServerRef +from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming from nemo_gym.rollout_observability import AgentObservationBundle, ObservationGap -from nemo_gym.sandbox import SandboxExecResult, SandboxSpec, create_provider -from nemo_gym.sandbox.api import AsyncSandbox -from nemo_gym.sandbox.config import resolve_provider_config -from nemo_gym.sandbox.utils import upload_text +from nemo_gym.sandbox.utils import read_text, upload_text +from nemo_gym.skills import stage_skills from responses_api_agents.claude_code_agent.observability import extract_claude_code_observations LOG = logging.getLogger(__name__) -def extract_prompt( - body: NeMoGymResponseCreateParamsNonStreaming, *, model: str | None = None -) -> tuple[str, str | None]: - """Validate one text activation without silently dropping conversation history.""" - allowed = {"input", "instructions", "model", "metadata"} - for name, info in type(body).model_fields.items(): - if name not in allowed and getattr(body, name) != info.get_default(call_default_factory=True): - if name in {"stream", "background"} and getattr(body, name) is False: - continue - raise ValueError(f"Sandboxed Claude does not support request field {name}") - if body.model not in (None, "", "dummy_model", model): - raise ValueError("Sandboxed Claude model is selected by agent and model-server configuration") - if body.model_extra: - raise ValueError(f"Sandboxed Claude does not support extra request fields: {sorted(body.model_extra)}") - if body.tools or body.tool_choice != "auto" or not body.parallel_tool_calls or body.background: - raise ValueError("Sandboxed Claude owns tool selection and execution policy") - raw = body.model_dump(mode="json")["input"] - if isinstance(raw, str): - return raw, body.instructions - users = [] - systems = [] - for item in raw: - role = item.get("role") - if item.get("type", "message") != "message" or role not in {"user", "system", "developer"}: - raise ValueError("Sandboxed Claude requires a single user message with optional system instructions") - content = item.get("content") - if isinstance(content, list): - if any(part.get("type") not in {"input_text", "text"} for part in content): - raise ValueError("Sandboxed Claude accepts text input only") - content = "".join(part["text"] for part in content) - if not isinstance(content, str): - raise ValueError("Sandboxed Claude accepts text input only") - if role == "user": - users.append(content) - elif users: - raise ValueError("System instructions must precede the user turn") - else: - systems.append(content) - if len(users) != 1: - raise ValueError("Sandboxed Claude requires exactly one user message") - system = "\n\n".join(systems + ([body.instructions] if body.instructions else [])) or None - return users[0], system +class HarnessProcessInfo(BaseModel): + """Optional runner identity, independent of the supervisor cleanup receipt.""" + model_config = ConfigDict(extra="forbid", strict=True) + hostname: str + pid: int + python: str | None = None -@dataclass -class ClaudeCodeSandboxedState(AgentSessionState): - session: SandboxSession[str] - executable: str = "" - body: NeMoGymResponseCreateParamsNonStreaming | None = None - execution: asyncio.Task[NeMoGymResponse] | None = None - observations: AgentObservationBundle | None = None +def parse_runtime_info(payload: object) -> HarnessProcessInfo | None: + """Read runner diagnostics without discarding an otherwise valid transcript.""" + try: + return HarnessProcessInfo.model_validate(payload) + except ValidationError: + LOG.warning("Claude runtime metadata is missing or malformed") + return None -class ClaudeCodeSandboxMixin: - """Add standard episode sessions while leaving unseeded local calls unchanged.""" - async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> ClaudeCodeSandboxedState: - """Install the harness without modifying a borrowed task workspace.""" - if any(access.required for access in self.effective_tool_accesses(body)): - raise HTTPException( - 422, "Claude Code sandbox sessions use native tools; required remote tools are unsupported" - ) - version = self.config.claude_code_version - if not version or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): - raise HTTPException(422, "Sandboxed Claude Code requires an exact claude_code_version") - access = body.sandbox_access - owns = access is None - if owns and self.config.sandbox_provider is None: - raise HTTPException(422, "Provide SandboxAccess or configure sandbox_provider and sandbox_config") - spec = SandboxSpec(**{"workdir": "/app", **self.config.sandbox_config}) if owns else None - workdir = access.workdir if access else spec.workdir - path = PurePosixPath(workdir or "") - if ( - not path.is_absolute() - or ".." in path.parts - or "\x00" in workdir - or path in (PurePosixPath("/"), PurePosixPath("/tmp")) - or str(path).startswith("/tmp/nemo-gym-claude-") - ): - raise HTTPException(422, "Claude sandbox workdir must be absolute and separate from session storage") - if self.config.timeout <= 0: - raise HTTPException(422, "Claude sandbox execution timeout must be positive") - provider = create_provider( - resolve_provider_config( - access.connection.provider_config_ref if access else self.config.sandbox_provider, - self.server_client.global_config_dict, +@dataclass +class ClaudeCodeArtifacts: + """Native output copied before the sandbox connection is released.""" + + events: str + transcripts: bytes | None = None + + +def parse_artifact_observations( + artifacts: ClaudeCodeArtifacts, + *, + model_ref: ModelServerRef | None, + status: str, + error_type: str | None, + run_metadata: dict[str, Any], +) -> AgentObservationBundle: + """Extract native observations off the event loop after capture completes.""" + try: + if artifacts.transcripts is None: + raise ValueError("Claude native transcripts are unavailable") + with tempfile.TemporaryDirectory(prefix="gym-claude-observations-") as temporary: + path = Path(temporary) + with tarfile.open(fileobj=io.BytesIO(artifacts.transcripts)) as archive: + archive.extractall(path, filter="data") + observations = extract_claude_code_observations( + path, + model_ref=model_ref, + root_status=status, + root_duration_ms=run_metadata.get("duration_ms"), + root_error_type=error_type, + compaction_attempts=run_metadata.get("compaction_attempts"), ) + except Exception: + LOG.warning("Could not parse Claude native observations", exc_info=True) + observations = AgentObservationBundle( + source="claude_code", gaps=[ObservationGap(code="agent_transcript_unavailable")] ) - if owns: - sandbox = AsyncSandbox(provider) - else: - try: - sandbox = await AsyncSandbox.connect(access.connection.descriptor, provider=provider) - except BaseException: - await provider.aclose() - raise - directory = "/tmp/nemo-gym-claude-" + uuid4().hex - session = SandboxSession( - sandbox=sandbox, session_dir=directory, workdir=workdir, harness="claude_code", owns_sandbox=owns - ) - state = ClaudeCodeSandboxedState(request=body, session=session) - try: - async with asyncio.timeout(self.config.sandbox_install_timeout_seconds): - if owns: - await sandbox.start(spec) - if workdir: - result = await sandbox.exec("mkdir -p -- " + quote(workdir), cwd="/", timeout_s=30) - self._check_claude_setup(result) - validate = ( - "from pathlib import Path; import sys; " - "workdir,session=[Path(p).resolve() for p in sys.argv[1:]]; " - "assert workdir.is_dir(), 'Claude sandbox workdir is missing'; " - "assert workdir != session and workdir not in session.parents and session not in workdir.parents, " - "'Claude session storage overlaps task workdir'" - ) - result = await sandbox.exec( - f"python3 -I -c {quote(validate)} {quote(workdir)} {quote(directory)} && " - f"umask 077 && mkdir -p {quote(directory)}/config {quote(directory)}/runtime", - cwd="/", - timeout_s=30, - ) - self._check_claude_setup(result) - await sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), directory + "/sandbox_runner.py") - await sandbox.upload(Path(__file__).with_name("install_claude_runtime.py"), directory + "/install.py") - source = self.config.sandbox_executable_path or "" - executable = directory + "/runtime/claude" - command = " ".join( - quote(value) - for value in [ - "python3", - directory + "/install.py", - version, - executable, - source, - "", - ] - ) - result = await sandbox.exec(command, timeout_s=self.config.sandbox_install_timeout_seconds) - if result.return_code or result.error_type: - raise RuntimeError( - f"Claude runtime installation failed: command={command!r}, exit={result.return_code}, " - f"error={result.error_type}; stderr={result.stderr}; stdout={result.stdout}" - ) - installed_version = (result.stdout or "").strip() - if not installed_version.startswith(version + " "): - raise RuntimeError(f"Claude version mismatch: expected {version}, got {installed_version}") - state.executable = executable - return state - except BaseException as error: - try: - await session.close(timeout=self.config.session_close_timeout_seconds) - except BaseException: - raise AgentSessionSetupError(state, error=error) from error - raise - - async def _close_agent_session_state(self, state: AgentSessionState) -> AgentCloseSessionResponse: - """Return observations after the shared supervisor confirms cleanup.""" - assert isinstance(state, ClaudeCodeSandboxedState) - await state.session.close(timeout=self.config.session_close_timeout_seconds) - if state.execution and not state.execution.done(): - state.execution.cancel() - try: - await state.execution - except asyncio.CancelledError: - pass - return AgentCloseSessionResponse( - agent_session_id=state.request.agent_session_id, - agent_observations=state.observations, - ) - - @staticmethod - def _check_claude_setup(result: SandboxExecResult) -> None: - if result.return_code != 0 or result.error_type: - raise RuntimeError(f"Claude sandbox setup failed: {result.error_type}: {result.stderr or result.stdout}") - - def _execution_input(self, body: NeMoGymResponseCreateParamsNonStreaming) -> tuple[str, str | None]: - prompt, system = extract_prompt(body, model=self.config.model) - system = "\n\n".join(part for part in (self.config.system_prompt, system) if part) or None - return prompt, system - - async def _execute( - self, - state: ClaudeCodeSandboxedState, - body: NeMoGymResponseCreateParamsNonStreaming, - execution_input: tuple[str, str | None], - ) -> NeMoGymResponse: - from responses_api_agents.claude_code_agent.app import _invocation_outcome, parse_stream_json - - session = state.session - directory = session.session_dir - prompt, system = execution_input - base_url = self._resolve_call_base_url(state.request.episode_id.capture_key) - model = self.config.model if base_url else self.config.model.split("/")[-1] + if model_ref is None: + observations.gaps.append(ObservationGap(code="model_call_ownership_unavailable")) + return observations - async def stage() -> SandboxCommand: - env = { - "CLAUDE_CONFIG_DIR": directory + "/config", - "IS_SANDBOX": "1", - "ANTHROPIC_API_KEY": self.config.anthropic_api_key or "EMPTY", - "ANTHROPIC_MODEL": model, - "ANTHROPIC_DEFAULT_HAIKU_MODEL": model, - "ANTHROPIC_DEFAULT_SONNET_MODEL": model, - "ANTHROPIC_DEFAULT_OPUS_MODEL": model, - "CLAUDE_CODE_SUBAGENT_MODEL": model, - "DISABLE_AUTOUPDATER": "1", - "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", - } - if base_url: - env["ANTHROPIC_BASE_URL"] = base_url - env["ANTHROPIC_AUTH_TOKEN"] = self.config.anthropic_api_key or "local" - await upload_text( - session.sandbox, - path=directory + "/config/settings.json", - text=json.dumps(self._build_settings()), - ) - mcp_config = None - if self.config.mcp_config: - mcp_config = directory + "/mcp.json" - await upload_text(session.sandbox, path=mcp_config, text=json.dumps(self._get_static_mcp_config())) - argv = self._build_command(model, prompt, system, mcp_config=mcp_config) - argv[0] = state.executable - await upload_text( - session.sandbox, path=directory + "/launch.json", text=json.dumps({"argv": argv, "env": env}) - ) - return SandboxCommand( - argv=["python3", directory + "/sandbox_runner.py", directory + "/launch.json"], - python="python3", - ) - - parsed = None - outcome = None - def classify(metadata: dict[str, Any]) -> tuple[str, str | None]: - if session.cleanup and session.cleanup["timed_out"]: - return "incomplete", "timeout" - return _invocation_outcome(metadata, (session.cleanup or {}).get("return_code")) - - async def collect() -> str: - nonlocal parsed, outcome - raw = await session.read_output_log() - parsed = parse_stream_json(raw, fold_thinking=not self._token_id_capture_enabled()) - _, run_metadata = parsed - outcome = classify(run_metadata) - status, reason = outcome +@dataclass +class ClaudeCodeSandboxSession(AgentSessionState): + """Adapter state and capture hooks; SandboxSession owns supervision and release.""" + + session: SandboxSession[ClaudeCodeArtifacts] + runtime: str + task: asyncio.Task[NeMoGymResponse] | None = None + activation_request: NeMoGymResponseCreateParamsNonStreaming | None = None + runtime_info: HarnessProcessInfo | None = None + observations: AgentObservationBundle | None = None + stderr: str = "" + + async def upload_json(self, name: str, payload: JsonValue) -> None: + """Stage runner input using the shared file-transfer utility.""" + await upload_text(self.session.sandbox, path=f"{self.session.session_dir}/{name}", text=json.dumps(payload)) + + async def read_text(self, name: str) -> str: + """Read a session artifact using file transfer.""" + return await read_text(self.session.sandbox, path=f"{self.session.session_dir}/{name}") + + async def stage_skills(self, path: str) -> None: + """Upload a private copy of configured skills, including supporting files.""" + with tempfile.TemporaryDirectory(prefix="gym-claude-skills-") as temporary: + skills = Path(temporary) / "skills" + await asyncio.to_thread(stage_skills, path, skills) + for source in sorted(skills.rglob("*")): + if source.is_file(): + target = f"{self.session.session_dir}/config/skills/{source.relative_to(skills).as_posix()}" + await self.session.sandbox.upload(source, target) + + async def close(self, timeout: float) -> None: + """Capture before cancelling the activation and bound the wait for its completion.""" + await self.session.close(timeout=timeout) + if self.task is not None and not self.task.done() and not self.task.cancelling(): + self.task.cancel() + if self.task is not None: try: - result = await session.sandbox.exec( - "tar -czf " - + quote(directory + "/transcripts.tar.gz") - + " -C " - + quote(directory + "/config") - + " projects", - timeout_s=30, - ) - if result.return_code or result.error_type: - raise RuntimeError("Claude native transcripts are unavailable") - with tempfile.TemporaryDirectory(prefix="gym-claude-observations-") as temporary: - local = Path(temporary) - await session.sandbox.download(directory + "/transcripts.tar.gz", local / "transcripts.tar.gz") - with tarfile.open(local / "transcripts.tar.gz") as archive: - archive.extractall(local, filter="data") - state.observations = extract_claude_code_observations( - local, - model_ref=self.config.model_server, - root_status=status, - root_duration_ms=run_metadata.get("duration_ms"), - root_error_type=reason, - compaction_attempts=run_metadata.get("compaction_attempts"), - ) + await asyncio.wait_for(asyncio.shield(self.task), timeout=timeout) + except asyncio.CancelledError: + if not self.task.cancelled(): + raise except Exception: - LOG.warning("Could not collect Claude native observations", exc_info=True) - state.observations = AgentObservationBundle( - source="claude_code", gaps=[ObservationGap(code="agent_transcript_unavailable")] - ) - return raw - - async with self.sem: - raw = await session.execute( - stage_activation=stage, - collect=collect, - timeout=self.config.timeout, - close_timeout=self.config.session_close_timeout_seconds, - ) - if session.closing: + if not self.task.done(): + raise + # The agent route replays the activation error independently of close. + + async def execute( + self, payload: dict[str, JsonValue], *, timeout: float, close_timeout: float + ) -> ClaudeCodeArtifacts: + """Execute through the common lifecycle, retaining artifacts on failure.""" + artifacts = await self.session.execute( + stage_activation=lambda: self.stage_activation(payload), + collect=self.collect_artifacts, + timeout=timeout, + close_timeout=close_timeout, + ) + if self.session.closing: raise asyncio.CancelledError - items, usage = parsed if parsed is not None else parse_stream_json(raw) - metadata = { - "runtime_version": self.config.claude_code_version, - "timed_out": "true" if session.cleanup and session.cleanup["timed_out"] else "false", - } - status, reason = outcome or classify(usage) - if reason: - metadata["stop_reason"] = reason - return NeMoGymResponse( - id=(usage.get("terminal_response_id") or "resp_" + uuid4().hex) - if self._token_id_capture_enabled() - else "resp_" + uuid4().hex, - created_at=int(time()), - model=model, - object="response", - output=items, - tools=body.tools, - tool_choice=body.tool_choice, - parallel_tool_calls=body.parallel_tool_calls, - usage=NeMoGymResponseUsage( - input_tokens=usage.get("input_tokens", 0), - output_tokens=usage.get("output_tokens", 0), - total_tokens=usage.get("input_tokens", 0) + usage.get("output_tokens", 0), - input_tokens_details=NeMoGymResponseInputTokensDetails(cached_tokens=0), - output_tokens_details=NeMoGymResponseOutputTokensDetails(reasoning_tokens=0), - ), - status=status, - metadata=metadata, + return artifacts + + async def stage_activation(self, payload: dict[str, JsonValue]) -> SandboxCommand: + """Stage the invocation and return the runner command.""" + payload = dict(payload) + await self.upload_json("config/settings.json", payload.pop("settings")) + mcp_config = payload.pop("mcp_config") + if mcp_config is not None: + await self.upload_json("mcp.json", mcp_config) + skills_path = payload.pop("skills_path") + if skills_path: + await self.stage_skills(skills_path) + await self.upload_json("input.json", payload) + return SandboxCommand( + python="python3", + argv=[ + "python3", + "-I", + f"{self.session.session_dir}/sandbox_runner.py", + f"{self.session.session_dir}/input.json", + ], ) - async def _sandbox_responses( - self, request: Request, body: NeMoGymResponseCreateParamsNonStreaming = Body() - ) -> NeMoGymResponse: - session_id = self._agent_session_id_from_request(request) - if not session_id: - raise HTTPException(400, "Seed a sandboxed agent session first") - async with self._locked_agent_session(session_id) as record: - if record.closing or not isinstance(record.state, ClaudeCodeSandboxedState): - raise HTTPException(409, "Agent session is closed") - state = record.state - rollout_id = getattr(request, "path_params", {}).get("rollout_id") - if rollout_id != state.request.episode_id.capture_key: - raise HTTPException(409, "Claude activation rollout does not match its session") - if state.body is not None and state.body != body: - raise HTTPException(409, "A different activation was already submitted") - if state.execution is None: - try: - execution_input = self._execution_input(body) - except (ValueError, TypeError) as error: - raise HTTPException(422, str(error)) from error - state.body = body.model_copy(deep=True) - state.execution = asyncio.create_task(self._execute(state, body, execution_input)) - execution = state.execution - return (await asyncio.shield(execution)).model_copy(deep=True) + async def collect_artifacts(self) -> ClaudeCodeArtifacts: + """Copy dedicated stdout and transcripts after confirmed harness termination.""" + try: + self.stderr = (await self.read_text("stderr.log"))[-16000:] + except Exception: + LOG.warning("Claude stderr log unavailable", exc_info=True) + try: + events = await self.read_text("events.jsonl") + except Exception as error: + logs = await self.session.read_output_log() + raise RuntimeError(f"Claude runner returned no events: {(self.stderr or logs)[-16000:]}") from error + try: + runtime = json.loads(await self.read_text("runtime.json")) + except Exception: + runtime = None + self.runtime_info = parse_runtime_info(runtime) + transcripts = None + directory = self.session.session_dir + try: + result = await self.session.sandbox.exec( + f"tar -czf {quote(directory + '/transcripts.tar.gz')} -C {quote(directory + '/config')} projects", + timeout_s=30, + ) + if result.return_code or result.error_type: + raise RuntimeError("Claude native transcripts are unavailable") + with tempfile.TemporaryDirectory(prefix="gym-claude-capture-") as temporary: + local = Path(temporary) / "transcripts.tar.gz" + await self.session.sandbox.download(directory + "/transcripts.tar.gz", local) + transcripts = await asyncio.to_thread(local.read_bytes) + except Exception: + LOG.warning("Could not collect Claude native transcripts", exc_info=True) + return ClaudeCodeArtifacts(events=events, transcripts=transcripts) diff --git a/responses_api_agents/claude_code_agent/sandbox_runner.py b/responses_api_agents/claude_code_agent/sandbox_runner.py index 7bc39b68ed7..2fcee755966 100644 --- a/responses_api_agents/claude_code_agent/sandbox_runner.py +++ b/responses_api_agents/claude_code_agent/sandbox_runner.py @@ -1,18 +1,50 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Launch Claude Code under the shared process supervisor.""" +"""Capture Claude stdout independently of stderr and supervisor diagnostics.""" import json import os +import signal +import subprocess import sys from pathlib import Path -def main() -> None: - payload = json.loads(Path(sys.argv[1]).read_text()) - os.environ.update(payload["env"]) - os.execvpe(payload["argv"][0], payload["argv"], os.environ) +def run(params: dict) -> int: + """Keep raw stream-json and partial output; the shared supervisor owns process cleanup.""" + directory = Path(params["directory"]) + (directory / "runtime.json").write_text(json.dumps({"hostname": os.uname().nodename, "pid": os.getpid()})) + stopping = False + + def interrupt(*_): + nonlocal stopping + stopping = True + + signal.signal(signal.SIGTERM, interrupt) + with (directory / "events.jsonl").open("wb") as events, (directory / "stderr.log").open("wb") as stderr: + process = subprocess.Popen( + params["command"], + cwd=params["cwd"], + env={**os.environ, **params["env"]}, + stdin=subprocess.DEVNULL, + stdout=events, + stderr=stderr, + ) + try: + # Regular files avoid waiting for detached task services to close inherited pipes. + while True: + if stopping and process.poll() is None: + process.terminate() + stopping = False + try: + return process.wait(timeout=0.05) + except subprocess.TimeoutExpired: + pass + finally: + if process.poll() is None: + process.kill() + process.wait() if __name__ == "__main__": - main() + raise SystemExit(run(json.loads(Path(sys.argv[1]).read_text()))) diff --git a/responses_api_agents/claude_code_agent/tests/test_app.py b/responses_api_agents/claude_code_agent/tests/test_app.py index de8f2c3a47c..240616facbd 100644 --- a/responses_api_agents/claude_code_agent/tests/test_app.py +++ b/responses_api_agents/claude_code_agent/tests/test_app.py @@ -70,11 +70,14 @@ def _config(**kwargs) -> ClaudeCodeAgentConfig: ) +@pytest.fixture(autouse=True) +def no_host_install(monkeypatch): + # These tests exercise invocation behavior without installing the external CLI. + monkeypatch.setattr("responses_api_agents.claude_code_agent.app.ensure_claude_code", lambda version: None) + + def _make_agent(**kwargs) -> ClaudeCodeAgent: - # Patch only the Claude Code installation check. - # The real model initialization still configures private attributes and the semaphore. - with patch("responses_api_agents.claude_code_agent.app.ensure_claude_code"): - return ClaudeCodeAgent(config=_config(**kwargs), server_client=MagicMock(spec=ServerClient)) + return ClaudeCodeAgent(config=_config(**kwargs), server_client=MagicMock(spec=ServerClient)) def _event(type_: str, **kwargs) -> str: diff --git a/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py b/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py new file mode 100644 index 00000000000..823390cb5f5 --- /dev/null +++ b/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import ast +import hashlib +import io +import json +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from responses_api_agents.claude_code_agent import install_claude_runtime as installer + + +@pytest.fixture +def binary(tmp_path): + source = tmp_path / "source" + source.write_text('#!/bin/sh\nprintf "2.1.108 (Claude Code)\\n"\n') + return source, hashlib.sha256(source.read_bytes()).hexdigest() + + +def test_offline_install_verifies_hash_and_reuses_healthy_cache(tmp_path, binary): + source, checksum = binary + runtime = tmp_path / "runtime" + assert installer.install("2.1.108", runtime, source=str(source), expected=checksum) == "2.1.108 (Claude Code)" + executable = runtime / "claude" + original = executable.stat().st_ino + source.unlink() + assert installer.install("2.1.108", runtime, source=str(source), expected=checksum) == "2.1.108 (Claude Code)" + assert executable.stat().st_ino == original + assert executable.stat().st_mode & 0o777 == 0o700 + + +def test_corrupted_cache_is_replaced_and_bad_source_cannot_replace_it(tmp_path, binary): + source, checksum = binary + runtime = tmp_path / "runtime" + installer.install("2.1.108", runtime, source=str(source), expected=checksum) + (runtime / "claude").write_bytes(b"corrupt") + installer.install("2.1.108", runtime, source=str(source), expected=checksum) + assert (runtime / "claude").read_bytes() == source.read_bytes() + source.write_bytes(b"bad replacement") + with pytest.raises(ValueError, match="SHA-256 mismatch"): + installer.install("2.1.108", runtime, source=str(source), expected="0" * 64) + assert hashlib.sha256((runtime / "claude").read_bytes()).hexdigest() == checksum + assert not list(runtime.glob(".claude-*")) + + +def test_concurrent_installers_share_one_verified_runtime(tmp_path, binary, monkeypatch): + source, checksum = binary + copy = MagicMock(wraps=installer.shutil.copyfile) + monkeypatch.setattr(installer.shutil, "copyfile", copy) + + def install(): + return installer.install("2.1.108", tmp_path / "runtime", source=str(source), expected=checksum) + + with ThreadPoolExecutor(max_workers=4) as pool: + versions = list(pool.map(lambda _: install(), range(4))) + assert versions == ["2.1.108 (Claude Code)"] * 4 + assert copy.call_count == 1 + + +def test_preinstalled_binary_needs_trusted_checksum_and_matching_version(tmp_path, binary): + source, checksum = binary + with pytest.raises(ValueError, match="trusted SHA-256"): + installer.install("2.1.108", tmp_path / "runtime", source=str(source)) + with pytest.raises(ValueError, match="version mismatch"): + installer.install("2.1.109", tmp_path / "runtime", source=str(source), expected=checksum) + assert not (tmp_path / "runtime" / "claude").exists() + + +def test_download_uses_manifest_checksum_and_cached_receipt(tmp_path, binary, monkeypatch): + source, checksum = binary + content = source.read_bytes() + monkeypatch.setattr(installer.platform, "system", lambda: "Linux") + monkeypatch.setattr(installer.platform, "machine", lambda: "x86_64") + monkeypatch.setattr(installer.platform, "libc_ver", lambda: ("glibc", "2.35")) + + def fetch(url, timeout): + assert url.startswith("https://downloads.claude.ai/claude-code-releases/2.1.108/") + if url.endswith("manifest.json"): + return io.BytesIO(json.dumps({"platforms": {"linux-x64": {"checksum": checksum}}}).encode()) + assert url.endswith("linux-x64/claude") + return io.BytesIO(content) + + download = MagicMock(side_effect=fetch) + monkeypatch.setattr(installer.urllib.request, "urlopen", download) + installer.install("2.1.108", tmp_path / "runtime") + assert download.call_count == 2 + installer.install("2.1.108", tmp_path / "runtime") + assert download.call_count == 2 + + +def test_installer_and_runner_accept_python38_syntax(): + for filename in ("install_claude_runtime.py", "sandbox_runner.py"): + path = Path(installer.__file__).with_name(filename) + ast.parse(path.read_text(), feature_version=(3, 8)) + assert "file_digest" not in Path(installer.__file__).read_text() diff --git a/responses_api_agents/claude_code_agent/tests/test_sandbox.py b/responses_api_agents/claude_code_agent/tests/test_sandbox.py index c04b4166fea..0b57eec413f 100644 --- a/responses_api_agents/claude_code_agent/tests/test_sandbox.py +++ b/responses_api_agents/claude_code_agent/tests/test_sandbox.py @@ -1,20 +1,20 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import asyncio import json +import threading from types import SimpleNamespace -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import HTTPException, Request from nemo_gym.base_responses_api_agent import AGENT_SESSION_COOKIE_KEY, AgentSeedSessionRequest from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming -from nemo_gym.rollout_observability import AgentObservationBundle from nemo_gym.server_utils import ServerClient +from responses_api_agents.claude_code_agent import app from responses_api_agents.claude_code_agent import sandbox as adapter -from responses_api_agents.claude_code_agent.app import ClaudeCodeAgent, ClaudeCodeAgentConfig -from responses_api_agents.claude_code_agent.install_claude_runtime import install def seed(*, borrowed=True, workdir="/app"): @@ -32,90 +32,35 @@ def seed(*, borrowed=True, workdir="/app"): def agent(**kwargs): - config = ClaudeCodeAgentConfig( - name="claude", - host="127.0.0.1", - port=8080, - entrypoint="app.py", - claude_code_version="2.1.108", - **kwargs, + return app.ClaudeCodeAgent( + config=app.ClaudeCodeAgentConfig( + name="claude", host="127.0.0.1", port=8080, entrypoint="app.py", claude_code_version="2.1.108", **kwargs + ), + server_client=MagicMock(spec=ServerClient, global_config_dict={"sandbox": {"local": {}}}), ) - with ( - patch("responses_api_agents.claude_code_agent.app.ensure_claude_code"), - patch("responses_api_agents.claude_code_agent.app.subprocess.run"), - ): - return ClaudeCodeAgent( - config=config, server_client=MagicMock(spec=ServerClient, global_config_dict={"sandbox": {"local": {}}}) - ) def result(stdout="", *, code=0): return SimpleNamespace(stdout=stdout, stderr="diagnostic" if code else "", return_code=code, error_type=None) -@pytest.mark.parametrize("input", ["Question", [{"role": "user", "content": "Question"}]]) -def test_text_prompt(input): - assert adapter.extract_prompt(NeMoGymResponseCreateParamsNonStreaming(input=input, instructions="System")) == ( - "Question", - "System", - ) - - -@pytest.mark.parametrize( - "input", - [ - [{"role": "user", "content": "a"}, {"role": "user", "content": "b"}], - [{"role": "assistant", "content": "answer"}], - [{"role": "user", "content": [{"type": "input_image", "image_url": "https://example.org/a.png"}]}], - [{"role": "user", "content": "a"}, {"role": "system", "content": "b"}], - ], -) -def test_reject_input_that_would_be_silently_dropped(input): - with pytest.raises(ValueError): - adapter.extract_prompt(NeMoGymResponseCreateParamsNonStreaming(input=input)) - - -def test_system_and_developer_instructions(): - body = NeMoGymResponseCreateParamsNonStreaming( - input=[ - {"role": "system", "content": "First"}, - {"role": "developer", "content": "Second"}, - {"role": "user", "content": "Question"}, - ], - instructions="Third", - stream=False, - background=False, - ) - assert adapter.extract_prompt(body) == ("Question", "First\n\nSecond\n\nThird") - - -@pytest.mark.parametrize("kwargs", [{"max_output_tokens": 10}, {"temperature": 0.5}, {"model": "other"}]) -def test_reject_unsupported_request_controls(kwargs): - with pytest.raises(ValueError): - adapter.extract_prompt(NeMoGymResponseCreateParamsNonStreaming(input="Question", **kwargs)) - - @pytest.mark.asyncio @pytest.mark.parametrize("borrowed", [True, False]) async def test_seed_installs_inside_sandbox_and_preserves_ownership(monkeypatch, borrowed): - remote = SimpleNamespace( - exec=AsyncMock(side_effect=([result()] if not borrowed else []) + [result(), result("2.1.108 (Claude Code)")]), - upload=AsyncMock(), - start=AsyncMock(), - stop=AsyncMock(), - disconnect=AsyncMock(), - ) + remote = SimpleNamespace(exec=AsyncMock(return_value=result()), upload=AsyncMock(), start=AsyncMock()) constructor = MagicMock(return_value=remote) constructor.connect = AsyncMock(return_value=remote) - monkeypatch.setattr(adapter, "AsyncSandbox", constructor) + monkeypatch.setattr(app, "AsyncSandbox", constructor) provider = SimpleNamespace(aclose=AsyncMock()) - monkeypatch.setattr(adapter, "create_provider", lambda config: provider) + monkeypatch.setattr(app, "create_provider", lambda config: provider) instance = agent(sandbox_provider="sandbox", sandbox_config={"workdir": "/app"}) state = await instance._seed_agent_session_state(seed(borrowed=borrowed)) commands = [call.args[0] for call in remote.exec.await_args_list] - assert "install.py" in commands[-1] and "2.1.108" in commands[-1] + assert "install_claude_runtime.py" in commands[-1] and "--version 2.1.108" in commands[-1] + assert f"--runtime {state.runtime}" in commands[-1] + assert state.runtime == "/tmp/nemo-gym-claude-runtime-2.1.108" + assert state.runtime not in state.session.session_dir assert len(remote.upload.await_args_list) == 2 - assert state.executable.startswith(state.session.session_dir) assert state.session.owns_sandbox is not borrowed assert remote.start.await_count == (0 if borrowed else 1) if borrowed: @@ -124,51 +69,58 @@ async def test_seed_installs_inside_sandbox_and_preserves_ownership(monkeypatch, @pytest.mark.asyncio -@pytest.mark.parametrize("workdir", ["/", "/tmp", "relative", "/app/../tmp", "/tmp/nemo-gym-claude-x"]) +@pytest.mark.parametrize("workdir", ["/", "/tmp", "relative", "/app/../tmp", "/tmp/nemo-gym-claude-runtime-2.1.108"]) async def test_bad_workdir_rejected_before_connect(monkeypatch, workdir): connector = AsyncMock() - monkeypatch.setattr(adapter.AsyncSandbox, "connect", connector) - with pytest.raises(HTTPException, match="separate from session storage"): + monkeypatch.setattr(app.AsyncSandbox, "connect", connector) + with pytest.raises(HTTPException, match="separate from runtime/session storage"): await agent()._seed_agent_session_state(seed(workdir=workdir)) connector.assert_not_awaited() +@pytest.mark.asyncio +async def test_seed_passes_trusted_binary_checksum_to_installer(monkeypatch): + remote = SimpleNamespace(exec=AsyncMock(return_value=result()), upload=AsyncMock()) + monkeypatch.setattr(app.AsyncSandbox, "connect", AsyncMock(return_value=remote)) + monkeypatch.setattr(app, "create_provider", lambda config: object()) + checksum = "a" * 64 + state = await agent( + sandbox_executable_path="/opt/claude", sandbox_executable_sha256=checksum + )._seed_agent_session_state(seed()) + command = remote.exec.await_args.args[0] + assert "--source /opt/claude" in command + assert "--sha256 " + checksum in command + assert "--runtime " + state.runtime in command + + +@pytest.mark.asyncio +async def test_preinstalled_binary_requires_checksum(): + with pytest.raises(HTTPException, match="sandbox_executable_sha256"): + await agent(sandbox_executable_path="/usr/bin/claude")._seed_agent_session_state(seed()) + + @pytest.mark.asyncio async def test_failed_seed_releases_and_failed_release_retains_state(monkeypatch): remote = SimpleNamespace(exec=AsyncMock(return_value=result(code=1)), upload=AsyncMock()) - monkeypatch.setattr(adapter.AsyncSandbox, "connect", AsyncMock(return_value=remote)) - monkeypatch.setattr(adapter, "create_provider", lambda config: object()) + monkeypatch.setattr(app.AsyncSandbox, "connect", AsyncMock(return_value=remote)) + monkeypatch.setattr(app, "create_provider", lambda config: object()) close = AsyncMock() - monkeypatch.setattr(adapter.SandboxSession, "close", close) + monkeypatch.setattr(adapter.ClaudeCodeSandboxSession, "close", close) with pytest.raises(RuntimeError, match="diagnostic"): await agent()._seed_agent_session_state(seed()) close.assert_awaited_once() close.side_effect = RuntimeError("release failed") - with pytest.raises(adapter.AgentSessionSetupError) as error: + with pytest.raises(app.AgentSessionSetupError) as error: await agent()._seed_agent_session_state(seed()) assert error.value.state.request == seed() @pytest.mark.asyncio @pytest.mark.parametrize( - "timed_out,code,status,reason", - [ - (False, 0, "completed", None), - (False, 7, "failed", "process_exit_7"), - (True, -15, "incomplete", "timeout"), - ], + "timed_out,code,status", [(False, 0, "completed"), (False, 7, "failed"), (True, -15, "incomplete")] ) -async def test_supervised_output_uses_cleanup_outcome_and_shared_builders( - monkeypatch, timed_out, code, status, reason -): - files = {} - - async def upload(remote, *, path, text): - files[path] = text - - monkeypatch.setattr(adapter, "upload_text", upload) +async def test_native_events_outcome_and_shared_command_options(monkeypatch, timed_out, code, status): instance = agent(anthropic_base_url="https://model.example", max_turns=4, thinking="adaptive") - observed = {} raw = "\n".join( map( json.dumps, @@ -183,50 +135,98 @@ async def upload(remote, *, path, text): ], ) ) - - class Session: - session_dir = "/tmp/session" - sandbox = SimpleNamespace(exec=AsyncMock(return_value=result(code=1))) - cleanup = {"timed_out": timed_out, "return_code": code, "cleanup_confirmed": True} - closing = False - read_output_log = AsyncMock(return_value=raw) - - async def execute(self, **kwargs): - command = await kwargs["stage_activation"]() - observed.update(kwargs, command=command) - return await kwargs["collect"]() - - state = adapter.ClaudeCodeSandboxedState(request=seed(), session=Session(), executable="/runtime/claude") - body = NeMoGymResponseCreateParamsNonStreaming(input="Question", instructions="System") - response = await instance._execute(state, body, instance._execution_input(body)) - launch = json.loads(files["/tmp/session/launch.json"]) - assert launch["argv"] == [ + state = adapter.ClaudeCodeSandboxSession( + request=seed(), + runtime="/runtime", + session=SimpleNamespace( + session_dir="/tmp/session", + workdir="/app", + cleanup={"timed_out": timed_out, "return_code": code, "cleanup_confirmed": True}, + artifacts=None, + ), + ) + execute = AsyncMock(return_value=adapter.ClaudeCodeArtifacts(raw)) + monkeypatch.setattr(adapter.ClaudeCodeSandboxSession, "execute", execute) + body = NeMoGymResponseCreateParamsNonStreaming( + input=[{"role": "system", "content": "System"}, {"role": "user", "content": "Question"}] + ) + response = await instance._sandbox_response(state, body, prompt="Question", system="System") + payload = execute.await_args.args[0] + assert payload["command"] == [ "/runtime/claude", *instance._build_command(instance.config.model, "Question", "System")[1:], ] - assert launch["env"]["ANTHROPIC_BASE_URL"] == "https://model.example" - assert "HOME" not in launch["env"] - assert launch["env"]["CLAUDE_CONFIG_DIR"] == "/tmp/session/config" - assert observed["command"].python == "python3" - assert observed["timeout"] == instance.config.timeout + assert payload["env"]["ANTHROPIC_BASE_URL"] == "https://model.example" + assert "HOME" not in payload["env"] assert response.output[0].content[0].text == "partial" assert response.status == status - assert response.metadata.get("stop_reason") == reason assert response.usage.total_tokens == 12 - assert state.observations.gaps[0].code == "agent_transcript_unavailable" + assert response.metadata is None # no adapter-specific runtime policy fields + assert "model_call_ownership_unavailable" in [gap.code for gap in state.observations.gaps] + + +@pytest.mark.asyncio +async def test_artifact_capture_reads_separate_stdout_and_process_info(monkeypatch): + state = adapter.ClaudeCodeSandboxSession( + request=seed(), + runtime="/runtime", + session=SimpleNamespace( + session_dir="/tmp/session", + sandbox=SimpleNamespace(exec=AsyncMock(return_value=result(code=1))), + read_output_log=AsyncMock(return_value='{"type":"result","subtype":"error_fake"}'), + ), + ) + read = AsyncMock( + side_effect=lambda name: { + "events.jsonl": '{"type":"result","subtype":"success"}\n', + "stderr.log": "x" * 20000, + "runtime.json": '{"hostname":"worker","pid":123}', + }[name] + ) + monkeypatch.setattr(adapter.ClaudeCodeSandboxSession, "read_text", read) + artifacts = await state.collect_artifacts() + assert artifacts.events == '{"type":"result","subtype":"success"}\n' + assert state.runtime_info.pid == 123 + assert len(state.stderr) == 16000 + state.session.read_output_log.assert_not_awaited() @pytest.mark.asyncio -async def test_close_waits_for_cleanup_and_returns_observations(): +async def test_close_returns_gap_without_activation_and_honors_cleanup_error(): instance = agent() session = SimpleNamespace(close=AsyncMock(side_effect=RuntimeError("unconfirmed"))) - observations = AgentObservationBundle(source="claude_code") - state = adapter.ClaudeCodeSandboxedState(request=seed(), session=session, observations=observations) + state = adapter.ClaudeCodeSandboxSession(request=seed(), runtime="/runtime", session=session) with pytest.raises(RuntimeError, match="unconfirmed"): await instance._close_agent_session_state(state) session.close.side_effect = None response = await instance._close_agent_session_state(state) - assert response.agent_observations == observations + assert response.agent_observations.gaps[0].code == "agent_activation_interrupted" + + +@pytest.mark.asyncio +async def test_close_bounds_a_stubborn_activation(): + started = asyncio.Event() + release = asyncio.Event() + + async def activation(): + started.set() + try: + await asyncio.Future() + except asyncio.CancelledError: + await release.wait() + + task = asyncio.create_task(activation()) + await started.wait() + state = adapter.ClaudeCodeSandboxSession( + request=seed(), runtime="/runtime", session=SimpleNamespace(close=AsyncMock()), task=task + ) + try: + with pytest.raises(TimeoutError): + await state.close(0.01) + assert task.cancelling() + finally: + release.set() + await task def request(session_id="session", rollout="rollout"): @@ -238,13 +238,15 @@ def request(session_id="session", rollout="rollout"): @pytest.mark.asyncio async def test_retry_shares_execution_and_rejects_changed_body(monkeypatch): instance = agent() - state = adapter.ClaudeCodeSandboxedState(request=seed(), session=SimpleNamespace()) + state = adapter.ClaudeCodeSandboxSession( + request=seed(), runtime="/runtime", session=SimpleNamespace(closing=False) + ) async with instance._locked_agent_session("session") as record: record.state = state response = MagicMock() response.model_copy.return_value = response execute = AsyncMock(return_value=response) - monkeypatch.setattr(ClaudeCodeAgent, "_execute", execute) + monkeypatch.setattr(app.ClaudeCodeAgent, "_sandbox_response", execute) body = NeMoGymResponseCreateParamsNonStreaming(input="Question") assert await instance.responses(request(), body) is response assert await instance.responses(request(), body) is response @@ -260,26 +262,121 @@ async def test_retry_shares_execution_and_rejects_changed_body(monkeypatch): async def test_invalid_session_cannot_fall_back_to_local(monkeypatch): instance = agent() local = AsyncMock() - monkeypatch.setattr(ClaudeCodeAgent, "_create_response", local) + monkeypatch.setattr(app.ClaudeCodeAgent, "_create_response", local) with pytest.raises(HTTPException) as error: await instance.responses(request("unknown"), NeMoGymResponseCreateParamsNonStreaming(input="Question")) assert error.value.status_code == 409 local.assert_not_awaited() -def test_offline_installer_checks_checksum(tmp_path, monkeypatch): - import hashlib - - source = tmp_path / "source" - source.write_bytes(b"native executable") - destination = tmp_path / "claude" - run = MagicMock() - monkeypatch.setattr("responses_api_agents.claude_code_agent.install_claude_runtime.subprocess.run", run) - checksum = hashlib.sha256(source.read_bytes()).hexdigest() - install("2.1.108", destination, str(source), checksum) - assert destination.read_bytes() == source.read_bytes() - assert destination.stat().st_mode & 0o777 == 0o700 - run.assert_called_once_with([str(destination), "--version"], check=True, timeout=30) - with pytest.raises(ValueError, match="SHA-256 mismatch"): - install("2.1.108", destination, str(source), "incorrect") - assert not destination.exists() +@pytest.mark.parametrize( + "input,expected", + [ + ("Question", ("Question", "Configured")), + ([{"role": "user", "content": "old"}, {"role": "user", "content": "new"}], ("new", "Configured")), + ( + [{"role": "developer", "content": "Ignored"}, {"role": "user", "content": "Question"}], + ("Question", "Configured"), + ), + ( + [{"role": "system", "content": "System"}, {"role": "user", "content": "Question"}], + ("Question", "Configured\n\nSystem"), + ), + ], +) +def test_prompt_conversion_preserves_local_contract_for_both_modes(input, expected): + instance = agent(system_prompt="Configured") + body = NeMoGymResponseCreateParamsNonStreaming(input=input, instructions="Legacy ignores request instructions") + instance._validate_sandbox_request(body) + assert instance._conversation_input(body) == expected + + +@pytest.mark.asyncio +async def test_empty_sandbox_output_uses_local_padding(monkeypatch): + instance = agent() + state = adapter.ClaudeCodeSandboxSession( + request=seed(), + runtime="/runtime", + session=SimpleNamespace( + session_dir="/tmp/session", workdir="/app", artifacts=None, cleanup={"timed_out": False, "return_code": 0} + ), + ) + monkeypatch.setattr( + adapter.ClaudeCodeSandboxSession, "execute", AsyncMock(return_value=adapter.ClaudeCodeArtifacts("")) + ) + body = NeMoGymResponseCreateParamsNonStreaming(input="Question") + response = await instance._sandbox_response(state, body, prompt="Question", system=None) + assert response.output[0].content[0].text == "" + assert response.output[0].role == "assistant" + assert response.status == "incomplete" + + +@pytest.mark.asyncio +async def test_skills_are_staged_before_launch_and_disable_bare(monkeypatch, tmp_path): + skill = tmp_path / "skills" / "example" + skill.mkdir(parents=True) + (skill / "SKILL.md").write_text("---\nname: example\n---\nUse the helper.\n") + (skill / "helper.sh").write_text("echo helper\n") + files = {} + + async def upload(source, target): + files[target] = source.read_bytes() + + remote = SimpleNamespace(upload=upload) + state = adapter.ClaudeCodeSandboxSession( + request=seed(), runtime="/runtime", session=SimpleNamespace(session_dir="/tmp/session", sandbox=remote) + ) + payload = { + "directory": "/tmp/session", + "command": ["/runtime/claude"], + "cwd": "/app", + "env": {}, + "settings": {}, + "mcp_config": None, + "skills_path": str(tmp_path / "skills"), + } + staged = AsyncMock() + monkeypatch.setattr(adapter.ClaudeCodeSandboxSession, "upload_json", staged) + command = await state.stage_activation(payload) + assert files["/tmp/session/config/skills/example/helper.sh"] == b"echo helper\n" + assert "SKILL.md" in " ".join(files) + assert command.python == "python3" + runner_payload = staged.await_args.args[1] + assert "skills_path" not in runner_payload + instance = agent() + instance.server_client.global_config_dict["skills"] = {"path": str(tmp_path / "skills")} + assert instance._configured_skills_path() == str(tmp_path / "skills") + assert "--bare" not in instance._build_command("model", "prompt", skills_active=True) + + +@pytest.mark.asyncio +async def test_host_install_is_lazy_shared_and_retryable(monkeypatch): + entered = threading.Event() + release = threading.Event() + + def install(version): + entered.set() + release.wait(timeout=5) + + installer = MagicMock(side_effect=install) + monkeypatch.setattr(app, "ensure_claude_code", installer) + instance = agent() + installer.assert_not_called() + waiters = [asyncio.create_task(instance._ensure_local_runtime()) for _ in range(2)] + try: + assert await asyncio.to_thread(entered.wait, 2) + waiters[0].cancel() + with pytest.raises(asyncio.CancelledError): + await waiters[0] + assert not instance._local_setup_task.cancelled() + finally: + release.set() + await asyncio.gather(*waiters, return_exceptions=True) + assert installer.call_count == 1 + instance = agent() + installer.side_effect = RuntimeError("install failed") + with pytest.raises(RuntimeError, match="install failed"): + await instance._ensure_local_runtime() + installer.side_effect = None + await instance._ensure_local_runtime() + assert installer.call_count == 3 diff --git a/responses_api_agents/claude_code_agent/tests/test_sandbox_runner.py b/responses_api_agents/claude_code_agent/tests/test_sandbox_runner.py new file mode 100644 index 00000000000..ecf56e904c6 --- /dev/null +++ b/responses_api_agents/claude_code_agent/tests/test_sandbox_runner.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from responses_api_agents.claude_code_agent import sandbox_runner +from responses_api_agents.claude_code_agent.parsing import _invocation_outcome, parse_stream_json + + +def runner_input(tmp_path, code): + child = tmp_path / "child.py" + child.write_text(code) + payload = {"directory": str(tmp_path), "command": [sys.executable, str(child)], "cwd": str(tmp_path), "env": {}} + params = tmp_path / "input.json" + params.write_text(json.dumps(payload)) + return [sys.executable, str(Path(sandbox_runner.__file__)), str(params)] + + +def test_large_json_stdout_cannot_be_corrupted_by_interleaved_stderr(tmp_path): + command = runner_input( + tmp_path, + """import json,os +text="x"*300000 +line=json.dumps({"type":"assistant","message":{"content":[{"type":"text","text":text}]}}).encode()+b"\\n" +os.write(1,line[:150000]) +os.write(2,b'{"type":"result","subtype":"error_fake"}\\n'+b'diagnostic\\xff\\n') +os.write(1,line[150000:]) +os.write(1,b'{"type":"result","subtype":"success","is_error":false}\\n') +""", + ) + result = subprocess.run(command, capture_output=True, timeout=10) + assert result.returncode == 0, result.stderr + raw = (tmp_path / "events.jsonl").read_text() + assert "error_fake" not in raw + output, metadata = parse_stream_json(raw) + assert output[0].content[0].text == "x" * 300000 + assert _invocation_outcome(metadata, result.returncode) == ("completed", None) + assert b"diagnostic\xff" in (tmp_path / "stderr.log").read_bytes() + runtime = json.loads((tmp_path / "runtime.json").read_text()) + assert runtime["hostname"] and runtime["pid"] > 0 + + +@pytest.mark.skipif(sys.platform == "win32", reason="uses POSIX termination") +def test_termination_preserves_partial_events(tmp_path): + command = runner_input( + tmp_path, + """import os,time +os.write(1,b'{"type":"assistant","message":{"content":[{"type":"text","text":"partial"}]}}\\n') +time.sleep(60) +""", + ) + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + deadline = time.monotonic() + 5 + while not (tmp_path / "events.jsonl").exists() or (tmp_path / "events.jsonl").stat().st_size == 0: + assert time.monotonic() < deadline + time.sleep(0.01) + process.terminate() + process.communicate(timeout=5) + output, metadata = parse_stream_json((tmp_path / "events.jsonl").read_text()) + assert output[0].content[0].text == "partial" + assert "subtype" not in metadata + assert process.returncode != 0 + finally: + if process.poll() is None: + process.kill() + process.communicate(timeout=5) From 27707353edc3d3c73fb98a99b60b1c214c702130 Mon Sep 17 00:00:00 2001 From: Frederic Lafrance Date: Mon, 19 Oct 2026 18:13:09 +0200 Subject: [PATCH 3/7] Stage Claude runtimes from the Gym host and preserve local request behavior Signed-off-by: Frederic Lafrance --- .../claude_code_agent/README.md | 48 +++-- responses_api_agents/claude_code_agent/app.py | 46 +++-- .../configs/claude_code_agent.yaml | 3 +- .../install_claude_runtime.py | 96 ++++----- .../claude_code_agent/runtime.py | 195 ++++++++++++++++++ .../claude_code_agent/tests/test_app.py | 15 ++ .../tests/test_install_claude_runtime.py | 39 ++-- .../claude_code_agent/tests/test_runtime.py | 183 ++++++++++++++++ .../claude_code_agent/tests/test_sandbox.py | 49 ++++- 9 files changed, 557 insertions(+), 117 deletions(-) create mode 100644 responses_api_agents/claude_code_agent/runtime.py create mode 100644 responses_api_agents/claude_code_agent/tests/test_runtime.py diff --git a/responses_api_agents/claude_code_agent/README.md b/responses_api_agents/claude_code_agent/README.md index e3e9b07602a..a2eb7fd6d9b 100644 --- a/responses_api_agents/claude_code_agent/README.md +++ b/responses_api_agents/claude_code_agent/README.md @@ -230,18 +230,26 @@ sandbox_install_timeout_seconds: 600 session_close_timeout_seconds: 60 ``` -The image must include Linux Python **3.8+**, `tar`, CA certificates, and libraries -required by the native Claude binary. Seed installs the exact x64/arm64 release in -`/tmp/nemo-gym-claude-runtime-`, separate from per-session state. A file -lock serializes installers; later seeds reuse a binary after checksum and version -health checks. Closing a borrowed session removes its private state and retains -the shared runtime for subsequent sessions. - -Downloads are checked against the release manifest's SHA-256 to detect corruption. -For an independently trusted checksum, set `sandbox_executable_sha256`. Offline -images can set `sandbox_executable_path` to a native binary **inside the sandbox**; -that option requires `sandbox_executable_sha256`. The installer verifies the hash -and version before atomically updating the shared runtime. +The image must include Linux Python **3.8+**, `tar`, and libraries required by the +native Claude binary. Seed probes the sandbox's architecture and libc, downloads +and caches the matching Linux x64/arm64 release on the **Gym host**, then compresses +and uploads it in bounded pieces through the sandbox provider's file transfer API. +The sandbox needs no internet access for runtime setup. The Gym host needs HTTPS +access to the release service, or a supplied local binary as described below. + +The sandbox installer decompresses the transfer, verifies the native bytes, and +atomically installs them in `/tmp/nemo-gym-claude-runtime-`, separate from +session state. A file lock serializes installation. Later seeds verify the cached +checksum and version before skipping the binary upload. Closing a borrowed session +removes its private state and retains the shared runtime for subsequent sessions. + +Host downloads are checked against the release manifest's SHA-256 to detect +corruption. For an independently trusted checksum, set +`sandbox_executable_sha256`. An offline Gym host can supply +`sandbox_local_executable_path` pointing to a matching Linux binary on the host; +alternatively, `sandbox_executable_path` names a binary already **inside the +sandbox**. These two options are mutually exclusive and require +`sandbox_executable_sha256`. Configure `model_server` as above, or `anthropic_base_url` and `anthropic_api_key`; the endpoint must be reachable from the sandbox. `resources_server` is needed only @@ -257,15 +265,17 @@ native tools. Optional static `mcp_config` and `settings` JSON files are staged from the agent server, with the same command options as local execution. A `skills.path` in the merged server config is copied into the session's `CLAUDE_CONFIG_DIR/skills`, including supporting files; skills disable `--bare` -through the same command builder used locally. The legacy `/run` skills reference -still takes precedence over configured skills for that invocation. The collector's +through the same command builder used locally. This fallback is sandbox-only: +local `/run` uses only its explicit `skills_ref`, and unseeded `/v1/responses` does +not pick up configured skills. The collector's run-level skills option remains unsupported for materialized task rows. -Prompt conversion preserves the existing local contract in both modes: the last -user message is the prompt, and a leading system message is appended to -`system_prompt`. Request `instructions` and developer messages retain the local -behavior of being ignored. One session runs one activation; it does not resume a -previous Claude conversation. +Sandbox requests accept one user text prompt (a string or a single user message), +optionally preceded by one system message. Assistant/developer messages, multiple +user turns, and request `instructions` are rejected with HTTP 422 so history or +instructions cannot be silently dropped. Use the leading system message or +`system_prompt` for instructions. One session runs one activation and does not +resume a previous Claude conversation. Local prompt conversion is unchanged. The shared `SandboxSession` supervisor stops the harness before reading its dedicated stdout stream and native transcripts. The runner keeps `events.jsonl`, diff --git a/responses_api_agents/claude_code_agent/app.py b/responses_api_agents/claude_code_agent/app.py index a16fc322db8..6d7c2b73108 100644 --- a/responses_api_agents/claude_code_agent/app.py +++ b/responses_api_agents/claude_code_agent/app.py @@ -68,6 +68,7 @@ from nemo_gym.skills import stage_skills from responses_api_agents.claude_code_agent.observability import extract_claude_code_observations from responses_api_agents.claude_code_agent.parsing import _extract_instruction, _invocation_outcome, parse_stream_json +from responses_api_agents.claude_code_agent.runtime import install_sandbox_runtime from responses_api_agents.claude_code_agent.sandbox import ( ClaudeCodeSandboxSession, parse_artifact_observations, @@ -107,6 +108,7 @@ class ClaudeCodeAgentConfig(BaseResponsesAPIAgentConfig): sandbox_provider: str | None = None sandbox_config: dict[str, Any] = Field(default_factory=dict) sandbox_executable_path: str | None = None + sandbox_local_executable_path: str | None = None sandbox_executable_sha256: str | None = Field(default=None, pattern=r"^[a-fA-F0-9]{64}$") sandbox_install_timeout_seconds: float = Field(default=600, gt=0, allow_inf_nan=False) session_close_timeout_seconds: float = Field(default=60, gt=0, allow_inf_nan=False) @@ -461,7 +463,7 @@ async def _create_response( user_message, system_prompt=system_prompt, mcp_config=mcp_config, - skills_path=skills_path or self._configured_skills_path(), + skills_path=skills_path, rollout_id=rollout_id, observation_collector=observation_collector, ) @@ -543,8 +545,12 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Clau version = self.config.claude_code_version if not version or not re.fullmatch(r"\d+\.\d+\.\d+", version): raise HTTPException(422, "Claude sandbox execution requires an exact claude_code_version") - if self.config.sandbox_executable_path and not self.config.sandbox_executable_sha256: - raise HTTPException(422, "sandbox_executable_path requires sandbox_executable_sha256") + if self.config.sandbox_executable_path and self.config.sandbox_local_executable_path: + raise HTTPException(422, "Choose either sandbox_executable_path or sandbox_local_executable_path") + if ( + self.config.sandbox_executable_path or self.config.sandbox_local_executable_path + ) and not self.config.sandbox_executable_sha256: + raise HTTPException(422, "A supplied Claude binary requires sandbox_executable_sha256") access = body.sandbox_access owns_sandbox = access is None if owns_sandbox and not self.config.sandbox_provider: @@ -601,23 +607,16 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Clau self._check_sandbox_setup(result) for name in ("sandbox_runner.py", "install_claude_runtime.py"): await sandbox.upload(Path(__file__).with_name(name), f"{directory}/{name}") - command = [ - "python3", - "-I", - f"{directory}/install_claude_runtime.py", - "--version", - version, - "--runtime", - runtime, - ] - if self.config.sandbox_executable_path: - command += ["--source", self.config.sandbox_executable_path] - if self.config.sandbox_executable_sha256: - command += ["--sha256", self.config.sandbox_executable_sha256] - installed = await sandbox.exec( - " ".join(map(quote, command)), cwd=workdir, timeout_s=self.config.sandbox_install_timeout_seconds + await install_sandbox_runtime( + sandbox, + version=version, + directory=directory, + runtime=runtime, + timeout=self.config.sandbox_install_timeout_seconds, + source=self.config.sandbox_executable_path, + local_source=self.config.sandbox_local_executable_path, + expected=self.config.sandbox_executable_sha256, ) - self._check_sandbox_setup(installed) return state except BaseException as error: try: @@ -648,7 +647,7 @@ async def _close_agent_session_state(self, state: AgentSessionState) -> AgentClo ) def _validate_sandbox_request(self, body: NeMoGymResponseCreateParamsNonStreaming) -> None: - supported = {"input", "instructions", "model", "metadata"} + supported = {"input", "model", "metadata"} for name, field in type(body).model_fields.items(): value = getattr(body, name) if name in supported or (name in {"stream", "background"} and value is False): @@ -661,9 +660,12 @@ def _validate_sandbox_request(self, body: NeMoGymResponseCreateParamsNonStreamin if body.model not in models or body.model_extra: raise HTTPException(422, "Claude sandbox model is selected by agent/model-server configuration") if not isinstance(body.input, str): + roles = [getattr(item, "role", None) for item in body.input] + if roles not in (["user"], ["system", "user"]): + raise HTTPException( + 422, "Claude sandbox execution accepts one user message with an optional leading system message" + ) for item in body.input: - if getattr(item, "role", None) not in {"user", "system", "developer", "assistant"}: - raise HTTPException(422, "Claude sandbox input requires text messages") if not isinstance(item.content, str) and any( (part.get("type") if isinstance(part, dict) else part.type) != "input_text" for part in item.content diff --git a/responses_api_agents/claude_code_agent/configs/claude_code_agent.yaml b/responses_api_agents/claude_code_agent/configs/claude_code_agent.yaml index a2707b38033..ac22cadac38 100644 --- a/responses_api_agents/claude_code_agent/configs/claude_code_agent.yaml +++ b/responses_api_agents/claude_code_agent/configs/claude_code_agent.yaml @@ -22,7 +22,8 @@ claude_code_agent: # The optional local runtime installs lazily on the first unseeded call. sandbox_provider: null # top-level provider ref; unused with borrowed SandboxAccess sandbox_config: {} # SandboxSpec for owned execution, e.g. {image: ubuntu:24.04, workdir: /app} + sandbox_local_executable_path: null # optional native Linux binary on the Gym host sandbox_executable_path: null # optional native binary already inside the sandbox - sandbox_executable_sha256: null # required trusted SHA-256 for a preinstalled binary + sandbox_executable_sha256: null # required trusted SHA-256 for either supplied binary option sandbox_install_timeout_seconds: 600 session_close_timeout_seconds: 60 diff --git a/responses_api_agents/claude_code_agent/install_claude_runtime.py b/responses_api_agents/claude_code_agent/install_claude_runtime.py index ec126fb802a..fc8e6e6d234 100644 --- a/responses_api_agents/claude_code_agent/install_claude_runtime.py +++ b/responses_api_agents/claude_code_agent/install_claude_runtime.py @@ -1,22 +1,22 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Install/reuse a versioned native Claude runtime with Linux Python 3.8+ stdlib.""" +"""Verify/install transferred Claude binaries with Linux Python 3.8+ stdlib; no network access.""" import argparse import fcntl +import gzip import hashlib -import json import os -import platform import re import shutil import subprocess import tempfile -import urllib.request +from contextlib import contextmanager from pathlib import Path -def _sha256(path: Path) -> str: +def sha256_file(path: Path) -> str: + """Hash a native binary without requiring Python 3.11's file_digest.""" digest = hashlib.sha256() with path.open("rb") as binary: for chunk in iter(lambda: binary.read(1024 * 1024), b""): @@ -38,58 +38,54 @@ def _version(executable: Path, version: str) -> str: return actual -def install(version: str, runtime: Path, *, source: str = "", expected: str = "") -> str: - """Reuse a healthy verified binary, or atomically install it under a per-runtime lock. - - Downloads use the release manifest for corruption detection unless the caller supplies - a trusted SHA-256. Offline source binaries require that caller-supplied checksum. - """ - if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): - raise ValueError("Claude runtime requires an exact version") - if source and not expected: - raise ValueError("A preinstalled Claude binary requires a trusted SHA-256") - expected = expected.lower() +@contextmanager +def _runtime_lock(runtime: Path): runtime.parent.mkdir(parents=True, exist_ok=True) with Path(str(runtime) + ".lock").open("a") as lock: fcntl.flock(lock, fcntl.LOCK_EX) - runtime.mkdir(mode=0o700, exist_ok=True) - executable = runtime / "claude" - receipt = runtime / "installation.json" + yield + + +def _cached_version(version: str, runtime: Path, expected: str) -> str: + if sha256_file(runtime / "claude") != expected.lower(): + raise ValueError("Claude executable SHA-256 mismatch") + return _version(runtime / "claude", version) + + +def check_runtime(version: str, runtime: Path, *, expected: str) -> str: + """Check bytes and health before the host decides whether an upload is needed.""" + with _runtime_lock(runtime): + return _cached_version(version, runtime, expected) + + +def install(version: str, runtime: Path, *, source: str = "", expected: str = "", compressed: bool = False) -> str: + """Reuse a verified binary or atomically install the supplied bytes under a lock.""" + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): + raise ValueError("Claude runtime requires an exact version") + if not expected: + raise ValueError("A transferred Claude binary requires a trusted SHA-256") + with _runtime_lock(runtime): try: - recorded = json.loads(receipt.read_text()) - checksum = expected or recorded["sha256"] - if recorded["version"] == version and _sha256(executable) == checksum: - return _version(executable, version) - except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError): + return _cached_version(version, runtime, expected) + except (OSError, ValueError, subprocess.SubprocessError): pass + if not source: + raise ValueError("Claude runtime is absent; provide a transferred source binary") + runtime.mkdir(mode=0o700, exist_ok=True) fd, name = tempfile.mkstemp(prefix=".claude-", dir=runtime) os.close(fd) temporary = Path(name) try: - if source: - shutil.copyfile(source, temporary) + if compressed: + with gzip.open(source, "rb") as input_file, temporary.open("wb") as output: + shutil.copyfileobj(input_file, output) else: - architecture = {"x86_64": "x64", "aarch64": "arm64"}.get(platform.machine()) - if platform.system() != "Linux" or architecture is None: - raise RuntimeError("Claude sandbox installation requires Linux x64 or arm64") - target = "linux-" + architecture - if platform.libc_ver()[0] != "glibc" and Path("/etc/alpine-release").exists(): - target += "-musl" - base = "https://downloads.claude.ai/claude-code-releases/" + version - if not expected: - with urllib.request.urlopen(base + "/manifest.json", timeout=60) as response: - manifest = json.load(response) - expected = manifest["platforms"][target]["checksum"].lower() - with urllib.request.urlopen(base + "/" + target + "/claude", timeout=120) as response: - with temporary.open("wb") as output: - shutil.copyfileobj(response, output) - checksum = _sha256(temporary) - if checksum != expected: + shutil.copyfile(source, temporary) + if sha256_file(temporary) != expected.lower(): raise ValueError("Claude executable SHA-256 mismatch") temporary.chmod(0o700) actual = _version(temporary, version) - temporary.replace(executable) - receipt.write_text(json.dumps({"version": version, "sha256": checksum})) + temporary.replace(runtime / "claude") return actual finally: temporary.unlink(missing_ok=True) @@ -100,6 +96,14 @@ def install(version: str, runtime: Path, *, source: str = "", expected: str = "" parser.add_argument("--version", required=True) parser.add_argument("--runtime", type=Path, required=True) parser.add_argument("--source", default="") - parser.add_argument("--sha256", default="") + parser.add_argument("--sha256", required=True) + parser.add_argument("--check-only", action="store_true") + parser.add_argument("--gzip", action="store_true", help="Source is a gzip-compressed native binary") args = parser.parse_args() - print(install(args.version, args.runtime, source=args.source, expected=args.sha256)) + if args.check_only: + try: + print(check_runtime(args.version, args.runtime, expected=args.sha256)) + except (OSError, ValueError, subprocess.SubprocessError): + raise SystemExit(3) + else: + print(install(args.version, args.runtime, source=args.source, expected=args.sha256, compressed=args.gzip)) diff --git a/responses_api_agents/claude_code_agent/runtime.py b/responses_api_agents/claude_code_agent/runtime.py new file mode 100644 index 00000000000..fe1b166a4ba --- /dev/null +++ b/responses_api_agents/claude_code_agent/runtime.py @@ -0,0 +1,195 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Cache native Linux binaries on the Gym host and transfer them to offline sandboxes.""" + +import asyncio +import gzip +import json +import re +import shutil +from pathlib import Path +from shlex import quote +from tempfile import TemporaryDirectory, gettempdir +from uuid import uuid4 + +from aiohttp import ClientTimeout + +from nemo_gym.sandbox import AsyncSandbox, SandboxExecResult +from nemo_gym.server_utils import request +from responses_api_agents.claude_code_agent.install_claude_runtime import sha256_file + + +CACHE_DIR = Path(gettempdir()) / "nemo-gym-claude-assets" +_DOWNLOAD_LOCK = asyncio.Lock() + + +def _target(probe: str) -> str: + lines = probe.strip().splitlines() + if len(lines) != 3 or lines[0] != "Linux": + raise ValueError(f"Unsupported Claude sandbox platform: {probe!r}") + _, architecture, libc = lines + architectures = {"x86_64": "x64", "aarch64": "arm64"} + if architecture not in architectures or libc not in {"gnu", "musl"}: + raise ValueError(f"Unsupported Claude sandbox platform: {architecture}/{libc}") + return "linux-" + architectures[architecture] + ("-musl" if libc == "musl" else "") + + +async def cached_binary(version: str, target: str, *, expected: str | None = None) -> tuple[Path, str]: + """Download on the host once per version/platform and verify the cached bytes.""" + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version) or target not in { + "linux-x64", + "linux-arm64", + "linux-x64-musl", + "linux-arm64-musl", + }: + raise ValueError("Claude runtime requires an exact version and supported Linux target") + async with _DOWNLOAD_LOCK: + directory = CACHE_DIR / version + directory.mkdir(parents=True, exist_ok=True) + base = "https://downloads.claude.ai/claude-code-releases/" + version + if expected is None: + manifest_path = directory / "manifest.json" + try: + manifest = json.loads(manifest_path.read_text()) + expected = manifest["platforms"][target]["checksum"] + except (OSError, ValueError, KeyError, TypeError): + async with await request("GET", base + "/manifest.json", timeout=ClientTimeout(total=60)) as response: + response.raise_for_status() + manifest = await response.json() + expected = manifest["platforms"][target]["checksum"] + temporary = directory / ("manifest-" + uuid4().hex) + try: + temporary.write_text(json.dumps(manifest)) + temporary.replace(manifest_path) + finally: + temporary.unlink(missing_ok=True) + checksum = expected.lower() + path = directory / target / "claude" + if path.is_file() and await asyncio.to_thread(sha256_file, path) == checksum: + return path, checksum + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(".claude-" + uuid4().hex) + try: + async with await request( + "GET", base + "/" + target + "/claude", timeout=ClientTimeout(total=180) + ) as response: + response.raise_for_status() + with temporary.open("wb") as output: + async for chunk in response.content.iter_chunked(1024 * 1024): + await asyncio.to_thread(output.write, chunk) + if await asyncio.to_thread(sha256_file, temporary) != checksum: + raise ValueError("Claude executable SHA-256 mismatch") + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + return path, checksum + + +def _compress(source: Path, destination: Path) -> None: + with source.open("rb") as input_file, gzip.open(destination, "wb", compresslevel=6) as output: + shutil.copyfileobj(input_file, output) + + +async def compressed_binary(source: Path, checksum: str) -> Path: + """Cache a smaller transfer artifact while retaining verification of the native bytes.""" + async with _DOWNLOAD_LOCK: + directory = CACHE_DIR / "compressed" + directory.mkdir(parents=True, exist_ok=True) + path = directory / (checksum + ".gz") + receipt = path.with_suffix(".sha256") + if ( + path.is_file() + and receipt.is_file() + and await asyncio.to_thread(sha256_file, path) == receipt.read_text().strip() + ): + return path + temporary = directory / (".gzip-" + uuid4().hex) + try: + await asyncio.to_thread(_compress, source, temporary) + digest = await asyncio.to_thread(sha256_file, temporary) + temporary.replace(path) + receipt.write_text(digest) + finally: + temporary.unlink(missing_ok=True) + return path + + +def _check(result: SandboxExecResult) -> None: + if result.return_code != 0 or result.error_type: + details = "\n".join(part for part in (result.stderr, result.stdout) if part) + raise RuntimeError( + f"Claude runtime setup failed (exit={result.return_code}, error={result.error_type}): {details[-16000:]}" + ) + + +async def _upload_archive(sandbox: AsyncSandbox, archive: Path, directory: str, timeout: float) -> str: + """Keep each upload below sandbox proxy request-size limits.""" + destination = directory + "/claude-upload.gz" + with TemporaryDirectory(prefix="claude-upload-") as staging: + + def split() -> list[Path]: + parts = [] + with archive.open("rb") as source: + while data := source.read(1024 * 1024): + path = Path(staging) / f"claude-upload-{len(parts):04d}.part" + path.write_bytes(data) + parts.append(path) + return parts + + parts = await asyncio.to_thread(split) + semaphore = asyncio.Semaphore(4) + + async def upload(part: Path) -> None: + async with semaphore: + await sandbox.upload(part, directory + "/" + part.name) + + async with asyncio.TaskGroup() as tasks: + for part in parts: + tasks.create_task(upload(part)) + sources = " ".join(quote(directory + "/" + part.name) for part in parts) + result = await sandbox.exec(f"cat {sources} > {quote(destination)} && rm -f {sources}", timeout_s=timeout) + _check(result) + return destination + + +async def install_sandbox_runtime( + sandbox: AsyncSandbox, + *, + version: str, + directory: str, + runtime: str, + timeout: float, + source: str | None = None, + local_source: str | None = None, + expected: str | None = None, +) -> None: + """Reuse a verified runtime or upload host-cached bytes; never fetch from the sandbox.""" + if (source or local_source) and not expected: + raise ValueError("A supplied Claude binary requires a trusted SHA-256") + command = ["python3", "-I", directory + "/install_claude_runtime.py", "--version", version, "--runtime", runtime] + if source: + command += ["--source", source, "--sha256", expected] + else: + if local_source: + local = Path(local_source).expanduser() + checksum = await asyncio.to_thread(sha256_file, local) + if checksum != expected.lower(): + raise ValueError("Local Claude executable SHA-256 mismatch") + else: + probe = await sandbox.exec( + "uname -s && uname -m && if ls /lib/ld-musl-*.so.1 >/dev/null 2>&1; then echo musl; else echo gnu; fi", + timeout_s=30, + ) + _check(probe) + local, checksum = await cached_binary(version, _target(probe.stdout or ""), expected=expected) + command += ["--sha256", checksum] + cached = await sandbox.exec(" ".join(map(quote, command + ["--check-only"])), timeout_s=timeout) + if cached.return_code == 0 and not cached.error_type: + return + if cached.error_type or cached.return_code != 3: + _check(cached) + archive = await compressed_binary(local, checksum) + destination = await _upload_archive(sandbox, archive, directory, timeout) + command += ["--source", destination, "--gzip"] + installed = await sandbox.exec(" ".join(map(quote, command)), timeout_s=timeout) + _check(installed) diff --git a/responses_api_agents/claude_code_agent/tests/test_app.py b/responses_api_agents/claude_code_agent/tests/test_app.py index 240616facbd..556354c5e52 100644 --- a/responses_api_agents/claude_code_agent/tests/test_app.py +++ b/responses_api_agents/claude_code_agent/tests/test_app.py @@ -316,6 +316,7 @@ def test_skills_ref_path_forwarded(self) -> None: def test_no_skills_ref_forwards_none(self) -> None: agent = _make_agent() + agent.server_client.global_config_dict = {"skills": {"path": "must-not-be-staged"}} run_claude_code = AsyncMock( return_value=([], "claude-sonnet-4-6", {"status": "incomplete", "error_type": "result_missing"}) ) @@ -1269,3 +1270,17 @@ def serve(call_id: str, content: list[dict], *, child: bool = False) -> None: assert attribution.model_call_id == "parent", attribution.reason assert attribution.method == "response_id" assert "corroborated_by=item_id+content_cumulative" in attribution.reason + + +@pytest.mark.asyncio +async def test_unseeded_responses_do_not_pick_up_configured_skills(monkeypatch): + agent = _make_agent() + agent.server_client.global_config_dict = {"skills": {"path": "must-not-be-staged"}} + run = AsyncMock(return_value=([], "model", {})) + monkeypatch.setattr(ClaudeCodeAgent, "_run_claude_code", run) + request = Request({"type": "http", "path_params": {}}) + await agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="Question")) + assert run.await_args.kwargs["skills_path"] is None + assert "--bare" in agent._build_command( + "model", "Question", skills_active=bool(run.await_args.kwargs["skills_path"]) + ) diff --git a/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py b/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py index 823390cb5f5..7cd0e8ecb97 100644 --- a/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py +++ b/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py @@ -3,8 +3,6 @@ import ast import hashlib -import io -import json from concurrent.futures import ThreadPoolExecutor from pathlib import Path from unittest.mock import MagicMock @@ -70,30 +68,21 @@ def test_preinstalled_binary_needs_trusted_checksum_and_matching_version(tmp_pat assert not (tmp_path / "runtime" / "claude").exists() -def test_download_uses_manifest_checksum_and_cached_receipt(tmp_path, binary, monkeypatch): - source, checksum = binary - content = source.read_bytes() - monkeypatch.setattr(installer.platform, "system", lambda: "Linux") - monkeypatch.setattr(installer.platform, "machine", lambda: "x86_64") - monkeypatch.setattr(installer.platform, "libc_ver", lambda: ("glibc", "2.35")) - - def fetch(url, timeout): - assert url.startswith("https://downloads.claude.ai/claude-code-releases/2.1.108/") - if url.endswith("manifest.json"): - return io.BytesIO(json.dumps({"platforms": {"linux-x64": {"checksum": checksum}}}).encode()) - assert url.endswith("linux-x64/claude") - return io.BytesIO(content) - - download = MagicMock(side_effect=fetch) - monkeypatch.setattr(installer.urllib.request, "urlopen", download) - installer.install("2.1.108", tmp_path / "runtime") - assert download.call_count == 2 - installer.install("2.1.108", tmp_path / "runtime") - assert download.call_count == 2 - - def test_installer_and_runner_accept_python38_syntax(): for filename in ("install_claude_runtime.py", "sandbox_runner.py"): path = Path(installer.__file__).with_name(filename) ast.parse(path.read_text(), feature_version=(3, 8)) - assert "file_digest" not in Path(installer.__file__).read_text() + tree = ast.parse(Path(installer.__file__).read_text()) + assert not any(isinstance(n, ast.Attribute) and n.attr == "file_digest" for n in ast.walk(tree)) + + +def test_installer_verifies_decompressed_native_bytes(tmp_path, binary): + import gzip + + source, checksum = binary + compressed = tmp_path / "claude.gz" + compressed.write_bytes(gzip.compress(source.read_bytes())) + runtime = tmp_path / "runtime" + installer.install("2.1.108", runtime, source=str(compressed), expected=checksum, compressed=True) + assert (runtime / "claude").read_bytes() == source.read_bytes() + assert installer.check_runtime("2.1.108", runtime, expected=checksum) == "2.1.108 (Claude Code)" diff --git a/responses_api_agents/claude_code_agent/tests/test_runtime.py b/responses_api_agents/claude_code_agent/tests/test_runtime.py new file mode 100644 index 00000000000..f12a55d48a5 --- /dev/null +++ b/responses_api_agents/claude_code_agent/tests/test_runtime.py @@ -0,0 +1,183 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +import gzip +import hashlib +import shutil +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from responses_api_agents.claude_code_agent import runtime + + +class Response: + def __init__(self, data=b"", manifest=None): + self.data = data + self.manifest = manifest + self.content = self + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + pass + + def raise_for_status(self): + pass + + async def json(self): + return self.manifest + + async def iter_chunked(self, size): + yield self.data + + +@pytest.mark.asyncio +async def test_host_downloads_and_verifies_once_across_concurrent_seeds(tmp_path, monkeypatch): + data = b"Linux x64 Claude bytes" + checksum = hashlib.sha256(data).hexdigest() + monkeypatch.setattr(runtime, "CACHE_DIR", tmp_path) + + async def fetch(method, url, **kwargs): + if url.endswith("manifest.json"): + return Response(manifest={"platforms": {"linux-x64": {"checksum": checksum}}}) + assert url.endswith("/2.1.108/linux-x64/claude") + return Response(data=data) + + request = AsyncMock(side_effect=fetch) + monkeypatch.setattr(runtime, "request", request) + artifacts = await asyncio.gather(*(runtime.cached_binary("2.1.108", "linux-x64") for _ in range(4))) + assert len(set(artifacts)) == 1 + path, expected = artifacts[0] + assert path.read_bytes() == data and expected == checksum + assert request.await_count == 2 + path.write_bytes(b"corrupt") + await runtime.cached_binary("2.1.108", "linux-x64") + assert path.read_bytes() == data + assert request.await_count == 3 + + +@pytest.mark.asyncio +async def test_bad_host_download_is_not_cached(tmp_path, monkeypatch): + monkeypatch.setattr(runtime, "CACHE_DIR", tmp_path) + monkeypatch.setattr(runtime, "request", AsyncMock(return_value=Response(data=b"corrupt"))) + with pytest.raises(ValueError, match="SHA-256 mismatch"): + await runtime.cached_binary("2.1.108", "linux-arm64", expected="0" * 64) + assert not list(tmp_path.rglob("claude")) + assert not list(tmp_path.rglob(".claude-*")) + + +def result(stdout="", code=0): + return SimpleNamespace(stdout=stdout, stderr="", return_code=code, error_type=None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cached", [False, True]) +async def test_upload_uses_sandbox_architecture_and_skips_healthy_remote_cache(tmp_path, monkeypatch, cached): + monkeypatch.setattr(runtime, "CACHE_DIR", tmp_path / "cache") + artifact = tmp_path / "claude" + artifact.write_bytes(b"arm-musl-native-binary") + checksum = hashlib.sha256(artifact.read_bytes()).hexdigest() + resolve = AsyncMock(return_value=(artifact, checksum)) + monkeypatch.setattr(runtime, "cached_binary", resolve) + transfer = AsyncMock(return_value="/session/claude-upload.gz") + monkeypatch.setattr(runtime, "_upload_archive", transfer) + sandbox = SimpleNamespace( + exec=AsyncMock(side_effect=[result("Linux\naarch64\nmusl\n"), result(code=0 if cached else 3), result()]), + upload=AsyncMock(), + ) + await runtime.install_sandbox_runtime( + sandbox, version="2.1.108", directory="/session", runtime="/runtime", timeout=60 + ) + resolve.assert_awaited_once_with("2.1.108", "linux-arm64-musl", expected=None) + if cached: + sandbox.upload.assert_not_awaited() + transfer.assert_not_awaited() + assert sandbox.exec.await_count == 2 + else: + assert gzip.decompress(transfer.await_args.args[1].read_bytes()) == artifact.read_bytes() + command = sandbox.exec.await_args.args[0] + assert ( + "--source /session/claude-upload.gz" in command + and "--gzip" in command + and "--sha256 " + checksum in command + ) + assert "http" not in command + + +@pytest.mark.asyncio +async def test_host_local_binary_requires_no_network(tmp_path, monkeypatch): + monkeypatch.setattr(runtime, "CACHE_DIR", tmp_path / "cache") + source = tmp_path / "claude" + source.write_bytes(b"supplied-linux-binary") + checksum = hashlib.sha256(source.read_bytes()).hexdigest() + download = AsyncMock(side_effect=AssertionError("network forbidden")) + monkeypatch.setattr(runtime, "request", download) + transfer = AsyncMock(return_value="/session/claude-upload.gz") + monkeypatch.setattr(runtime, "_upload_archive", transfer) + sandbox = SimpleNamespace(exec=AsyncMock(side_effect=[result(code=3), result()]), upload=AsyncMock()) + await runtime.install_sandbox_runtime( + sandbox, + version="2.1.108", + directory="/session", + runtime="/runtime", + timeout=60, + local_source=str(source), + expected=checksum, + ) + download.assert_not_awaited() + assert gzip.decompress(transfer.await_args.args[1].read_bytes()) == source.read_bytes() + + +def test_worker_installer_has_no_download_dependency(): + from responses_api_agents.claude_code_agent import install_claude_runtime + + text = Path(install_claude_runtime.__file__).read_text() + assert "urllib" not in text and "https://" not in text + + +@pytest.mark.asyncio +async def test_corrupted_compressed_host_cache_is_rebuilt(tmp_path, monkeypatch): + monkeypatch.setattr(runtime, "CACHE_DIR", tmp_path / "cache") + source = tmp_path / "claude" + source.write_bytes(b"native bytes" * 100) + checksum = hashlib.sha256(source.read_bytes()).hexdigest() + compressed = await runtime.compressed_binary(source, checksum) + compressed.write_bytes(b"damaged gzip") + repaired = await runtime.compressed_binary(source, checksum) + assert gzip.decompress(repaired.read_bytes()) == source.read_bytes() + + +@pytest.mark.asyncio +async def test_bounded_upload_reassembles_bytes_and_removes_parts(tmp_path): + archive = tmp_path / "source.gz" + data = bytes(range(256)) * (1024 * 24 + 7) + archive.write_bytes(data) + directory = tmp_path / "remote session" + directory.mkdir() + active = maximum = 0 + sizes = [] + + async def upload(source, destination): + nonlocal active, maximum + active += 1 + maximum = max(maximum, active) + sizes.append(source.stat().st_size) + await asyncio.sleep(0) + shutil.copyfile(source, destination) + active -= 1 + + async def execute(command, **kwargs): + process = await asyncio.create_subprocess_shell(command) + return result(code=await process.wait()) + + sandbox = SimpleNamespace(upload=upload, exec=execute) + destination = await runtime._upload_archive(sandbox, archive, str(directory), 60) + assert Path(destination).read_bytes() == data + assert not list(directory.glob("*.part")) + assert max(sizes) <= 1024 * 1024 and sum(sizes) == len(data) + assert maximum == 4 diff --git a/responses_api_agents/claude_code_agent/tests/test_sandbox.py b/responses_api_agents/claude_code_agent/tests/test_sandbox.py index 0b57eec413f..86bf310a796 100644 --- a/responses_api_agents/claude_code_agent/tests/test_sandbox.py +++ b/responses_api_agents/claude_code_agent/tests/test_sandbox.py @@ -53,11 +53,14 @@ async def test_seed_installs_inside_sandbox_and_preserves_ownership(monkeypatch, monkeypatch.setattr(app, "AsyncSandbox", constructor) provider = SimpleNamespace(aclose=AsyncMock()) monkeypatch.setattr(app, "create_provider", lambda config: provider) + install = AsyncMock() + monkeypatch.setattr(app, "install_sandbox_runtime", install) instance = agent(sandbox_provider="sandbox", sandbox_config={"workdir": "/app"}) state = await instance._seed_agent_session_state(seed(borrowed=borrowed)) commands = [call.args[0] for call in remote.exec.await_args_list] - assert "install_claude_runtime.py" in commands[-1] and "--version 2.1.108" in commands[-1] - assert f"--runtime {state.runtime}" in commands[-1] + assert install.await_args.args == (remote,) + assert install.await_args.kwargs["version"] == "2.1.108" + assert install.await_args.kwargs["runtime"] == state.runtime assert state.runtime == "/tmp/nemo-gym-claude-runtime-2.1.108" assert state.runtime not in state.session.session_dir assert len(remote.upload.await_args_list) == 2 @@ -284,10 +287,9 @@ async def test_invalid_session_cannot_fall_back_to_local(monkeypatch): ), ], ) -def test_prompt_conversion_preserves_local_contract_for_both_modes(input, expected): +def test_local_prompt_conversion_preserves_existing_contract(input, expected): instance = agent(system_prompt="Configured") body = NeMoGymResponseCreateParamsNonStreaming(input=input, instructions="Legacy ignores request instructions") - instance._validate_sandbox_request(body) assert instance._conversation_input(body) == expected @@ -380,3 +382,42 @@ def install(version): installer.side_effect = None await instance._ensure_local_runtime() assert installer.call_count == 3 + + +@pytest.mark.parametrize( + "input", + [ + [], + [{"role": "assistant", "content": "Earlier answer"}], + [{"role": "user", "content": "Earlier"}, {"role": "user", "content": "Latest"}], + [{"role": "developer", "content": "Instructions"}, {"role": "user", "content": "Question"}], + [ + {"role": "user", "content": "Question"}, + {"role": "assistant", "content": "Answer"}, + {"role": "user", "content": "Follow-up"}, + ], + ], +) +def test_sandbox_rejects_history_instead_of_dropping_it(input): + with pytest.raises(HTTPException) as error: + agent()._validate_sandbox_request(NeMoGymResponseCreateParamsNonStreaming(input=input)) + assert error.value.status_code == 422 + + +def test_sandbox_rejects_request_instructions_that_local_mode_ignores(): + with pytest.raises(HTTPException, match="instructions"): + agent()._validate_sandbox_request( + NeMoGymResponseCreateParamsNonStreaming(input="Question", instructions="Use me") + ) + + +@pytest.mark.parametrize( + "input", + [ + "Question", + [{"role": "user", "content": "Question"}], + [{"role": "system", "content": "Instructions"}, {"role": "user", "content": "Question"}], + ], +) +def test_sandbox_accepts_one_text_turn_with_optional_system_message(input): + agent()._validate_sandbox_request(NeMoGymResponseCreateParamsNonStreaming(input=input)) From d77400bbf045731a63f540d6c2f096f363f047a6 Mon Sep 17 00:00:00 2001 From: Frederic Lafrance Date: Mon, 19 Oct 2026 18:38:24 +0200 Subject: [PATCH 4/7] docs(claude-code): clarify local runtime setup timing Signed-off-by: Frederic Lafrance --- responses_api_agents/claude_code_agent/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/responses_api_agents/claude_code_agent/README.md b/responses_api_agents/claude_code_agent/README.md index a2eb7fd6d9b..47f5f2f9436 100644 --- a/responses_api_agents/claude_code_agent/README.md +++ b/responses_api_agents/claude_code_agent/README.md @@ -88,7 +88,8 @@ Claude Code talks to the model via the Anthropic Messages API (`/v1/messages`). By default the agent runs with `--bare`, which skips auto-discovery of hooks, skills, plugins, MCP servers, memory, and CLAUDE.md so each scripted call starts clean and fast; Claude still has access to Bash, file read, and file edit tools. This isolation is the default because it keeps evals reproducible — a rollout depends only on the model, the task input, and the explicit config, not on ambient state of the host. This is the recommended mode for scripted and SDK calls per [Claude docs](https://code.claude.com/docs/en/headless#start-faster-with-bare-mode). The runtime is configurable via `bare`, `mcp_config`, and `settings` (see [Runtime capabilities](#runtime-capabilities)). -Claude Code is auto-installed on first startup via npm or a local Node.js binary if not already on PATH. +For local execution, Claude Code is auto-installed on the first local invocation +via npm or a local Node.js binary if not already on PATH. ## Configuration From d859b9182bbe574783d86c3fd7d64d2b68d18271 Mon Sep 17 00:00:00 2001 From: Frederic Lafrance Date: Mon, 19 Oct 2026 20:43:16 +0200 Subject: [PATCH 5/7] fix(claude-code): recognize native templates during config migration Signed-off-by: Frederic Lafrance --- .../add_legacy_agent_environment_servers.py | 3 +- ...st_add_legacy_agent_environment_servers.py | 33 +++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/scripts/add_legacy_agent_environment_servers.py b/scripts/add_legacy_agent_environment_servers.py index 0de43f79bc3..2b52c980aee 100644 --- a/scripts/add_legacy_agent_environment_servers.py +++ b/scripts/add_legacy_agent_environment_servers.py @@ -92,7 +92,8 @@ def needs_environment_server(instance: dict) -> bool: # A legacy relay would conflict with the Environment Server supplied by composition. # An omitted binding can inherit Resources and must still migrate. if ( - agent_type in {"hermes_agent", "pi_agent", "codex_agent", "openclaw_agent", "opencode_agent"} + agent_type + in {"hermes_agent", "pi_agent", "codex_agent", "openclaw_agent", "opencode_agent", "claude_code_agent"} and resources_server is None ): return False diff --git a/tests/unit_tests/test_add_legacy_agent_environment_servers.py b/tests/unit_tests/test_add_legacy_agent_environment_servers.py index 88df71a7dd7..261a2043b90 100644 --- a/tests/unit_tests/test_add_legacy_agent_environment_servers.py +++ b/tests/unit_tests/test_add_legacy_agent_environment_servers.py @@ -74,7 +74,16 @@ def test_migrates_a_config_outside_the_repository(tmp_path: Path) -> None: @pytest.mark.parametrize("resources", [None, {"type": "resources_servers", "name": "my_resources"}]) @pytest.mark.parametrize( - "agent_type", ["hermes_agent", "pi_agent", "codex_agent", "openclaw_agent", "opencode_agent", "osworld_agent"] + "agent_type", + [ + "hermes_agent", + "pi_agent", + "codex_agent", + "openclaw_agent", + "opencode_agent", + "claude_code_agent", + "osworld_agent", + ], ) def test_migration_respects_native_session_templates( tmp_path: Path, resources: dict[str, str] | None, agent_type: str @@ -97,7 +106,9 @@ def test_migration_respects_native_session_templates( assert _environment_servers(yaml.safe_load(config.read_text())) == {"my_agent": ["my_environment_server"]} -@pytest.mark.parametrize("agent_type", ["hermes_agent", "pi_agent", "codex_agent", "openclaw_agent", "opencode_agent"]) +@pytest.mark.parametrize( + "agent_type", ["hermes_agent", "pi_agent", "codex_agent", "openclaw_agent", "opencode_agent", "claude_code_agent"] +) def test_native_capable_overlay_keeps_inherited_resources(tmp_path: Path, agent_type: str) -> None: base = tmp_path / "base.yaml" base.write_text(AGENT_CONFIG.replace("simple_agent", agent_type)) @@ -149,6 +160,24 @@ def test_default_codex_composes_with_exactly_one_native_environment(tmp_path: Pa assert resolved.codex_agent.responses_api_agents.codex_agent.resources_server is None +def test_default_claude_composes_with_exactly_one_native_environment(tmp_path: Path) -> None: + config = tmp_path / "agent.yaml" + default = SCRIPT.parents[1] / "responses_api_agents/claude_code_agent/configs/claude_code_agent.yaml" + config.write_text(default.read_text()) + composition = tmp_path / "run.yaml" + composition.write_text( + "anthropic_api_key: test-key\n" + + _server_fronting("claude_code_agent", name="native_environment", server_type="single_agent_turn") + ) + before = config.read_text() + + assert migration.main([str(config)]) == 0 + assert config.read_text() == before + resolved = _parse(config, composition, strict=True) + assert _environment_servers_by_agent(resolved) == {"claude_code_agent": ["native_environment"]} + assert resolved.claude_code_agent.responses_api_agents.claude_code_agent.resources_server is None + + def test_opencode_session_template_composes_with_one_environment(tmp_path: Path) -> None: config = tmp_path / "agent.yaml" default = SCRIPT.parents[1] / "responses_api_agents/opencode_agent/configs/opencode_agent.yaml" From ef18b3af5a5886d89310e3a814e7c50f07a21871 Mon Sep 17 00:00:00 2001 From: Frederic Lafrance Date: Tue, 20 Oct 2026 20:14:07 +0200 Subject: [PATCH 6/7] fix(claude-code): retain close observations and honor task timeouts Signed-off-by: Frederic Lafrance --- .../claude_code_agent/README.md | 6 + responses_api_agents/claude_code_agent/app.py | 7 +- .../claude_code_agent/sandbox.py | 11 +- .../claude_code_agent/tests/test_sandbox.py | 103 +++++++++++++++++- 4 files changed, 121 insertions(+), 6 deletions(-) diff --git a/responses_api_agents/claude_code_agent/README.md b/responses_api_agents/claude_code_agent/README.md index 47f5f2f9436..1897b996269 100644 --- a/responses_api_agents/claude_code_agent/README.md +++ b/responses_api_agents/claude_code_agent/README.md @@ -278,6 +278,12 @@ instructions cannot be silently dropped. Use the leading system message or `system_prompt` for instructions. One session runs one activation and does not resume a previous Claude conversation. Local prompt conversion is unchanged. +Sandbox requests may set `metadata.agent_timeout_sec` to a positive number of +seconds, encoded as a string. This bounds the activation's model and tool work, +capped by the configured `timeout`; missing or invalid values use that ceiling. +Other metadata keys are rejected with HTTP 422. Setup and cleanup keep their +separate limits. + The shared `SandboxSession` supervisor stops the harness before reading its dedicated stdout stream and native transcripts. The runner keeps `events.jsonl`, `stderr.log`, and `runtime.json` separate from supervisor diagnostics. Close returns the observation bundle, disconnects diff --git a/responses_api_agents/claude_code_agent/app.py b/responses_api_agents/claude_code_agent/app.py index 6d7c2b73108..e24708dd1e1 100644 --- a/responses_api_agents/claude_code_agent/app.py +++ b/responses_api_agents/claude_code_agent/app.py @@ -34,6 +34,7 @@ from pydantic import ConfigDict, Field, PrivateAttr from nemo_gym.agent_utils.sandbox_session import SandboxSession +from nemo_gym.agent_utils.task_timeout import AGENT_TIMEOUT_METADATA_KEY, resolve_agent_timeout from nemo_gym.base_resources_server import NEMO_GYM_MCP_METADATA_KEY, BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import ( AgentCloseSessionResponse, @@ -647,11 +648,13 @@ async def _close_agent_session_state(self, state: AgentSessionState) -> AgentClo ) def _validate_sandbox_request(self, body: NeMoGymResponseCreateParamsNonStreaming) -> None: - supported = {"input", "model", "metadata"} + supported = {"input", "model"} for name, field in type(body).model_fields.items(): value = getattr(body, name) if name in supported or (name in {"stream", "background"} and value is False): continue + if name == "metadata" and value is not None and set(value) <= {AGENT_TIMEOUT_METADATA_KEY}: + continue if value != field.get_default(call_default_factory=True): raise HTTPException(422, f"Claude sandbox execution does not support request field {name}") models = {None, "", self.config.model} @@ -718,7 +721,7 @@ async def _sandbox_response( "mcp_config": self._get_static_mcp_config() if mcp_config else None, "skills_path": skills_path, }, - timeout=self.config.timeout, + timeout=resolve_agent_timeout(body.metadata, ceiling=self.config.timeout), close_timeout=self.config.session_close_timeout_seconds, ) except BaseException as error: diff --git a/responses_api_agents/claude_code_agent/sandbox.py b/responses_api_agents/claude_code_agent/sandbox.py index 22d906465eb..6adbe3d5983 100644 --- a/responses_api_agents/claude_code_agent/sandbox.py +++ b/responses_api_agents/claude_code_agent/sandbox.py @@ -119,9 +119,16 @@ async def stage_skills(self, path: str) -> None: await self.session.sandbox.upload(source, target) async def close(self, timeout: float) -> None: - """Capture before cancelling the activation and bound the wait for its completion.""" + """Join captured-artifact parsing before returning observations, within a bound.""" await self.session.close(timeout=timeout) - if self.task is not None and not self.task.done() and not self.task.cancelling(): + # Captured artifacts are parsed by the activation after execute() returns. + # Cancelling that step would discard evidence that close has already retained. + if ( + self.session.artifacts is None + and self.task is not None + and not self.task.done() + and not self.task.cancelling() + ): self.task.cancel() if self.task is not None: try: diff --git a/responses_api_agents/claude_code_agent/tests/test_sandbox.py b/responses_api_agents/claude_code_agent/tests/test_sandbox.py index 86bf310a796..0ed289429b1 100644 --- a/responses_api_agents/claude_code_agent/tests/test_sandbox.py +++ b/responses_api_agents/claude_code_agent/tests/test_sandbox.py @@ -2,7 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 import asyncio +import io import json +import tarfile import threading from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -197,7 +199,7 @@ async def test_artifact_capture_reads_separate_stdout_and_process_info(monkeypat @pytest.mark.asyncio async def test_close_returns_gap_without_activation_and_honors_cleanup_error(): instance = agent() - session = SimpleNamespace(close=AsyncMock(side_effect=RuntimeError("unconfirmed"))) + session = SimpleNamespace(close=AsyncMock(side_effect=RuntimeError("unconfirmed")), artifacts=None) state = adapter.ClaudeCodeSandboxSession(request=seed(), runtime="/runtime", session=session) with pytest.raises(RuntimeError, match="unconfirmed"): await instance._close_agent_session_state(state) @@ -221,7 +223,7 @@ async def activation(): task = asyncio.create_task(activation()) await started.wait() state = adapter.ClaudeCodeSandboxSession( - request=seed(), runtime="/runtime", session=SimpleNamespace(close=AsyncMock()), task=task + request=seed(), runtime="/runtime", session=SimpleNamespace(close=AsyncMock(), artifacts=None), task=task ) try: with pytest.raises(TimeoutError): @@ -232,6 +234,103 @@ async def activation(): await task +@pytest.mark.asyncio +@pytest.mark.parametrize("parser_name", ["parse_stream_json", "parse_artifact_observations"]) +async def test_close_joins_captured_observations_during_parsing(monkeypatch, parser_name): + entered = threading.Event() + release = threading.Event() + original = getattr(app, parser_name) + + def paused_parser(*args, **kwargs): + entered.set() + assert release.wait(timeout=5) + return original(*args, **kwargs) + + monkeypatch.setattr(app, parser_name, paused_parser) + native = json.dumps( + { + "type": "assistant", + "sessionId": "captured", + "uuid": "answer", + "timestamp": "2026-10-20T10:00:00Z", + "message": {"role": "assistant", "id": "served", "content": [{"type": "text", "text": "Answer"}]}, + } + ).encode() + archive_bytes = io.BytesIO() + with tarfile.open(fileobj=archive_bytes, mode="w:gz") as archive: + info = tarfile.TarInfo("projects/work/captured.jsonl") + info.size = len(native) + archive.addfile(info, io.BytesIO(native)) + artifacts = adapter.ClaudeCodeArtifacts( + '{"type":"result","subtype":"success","is_error":false}', archive_bytes.getvalue() + ) + state = adapter.ClaudeCodeSandboxSession( + request=seed(), + runtime="/runtime", + session=SimpleNamespace( + session_dir="/tmp/session", + workdir="/app", + artifacts=artifacts, + cleanup={"timed_out": False, "return_code": 0, "cleanup_confirmed": True}, + close=AsyncMock(), + ), + ) + monkeypatch.setattr(adapter.ClaudeCodeSandboxSession, "execute", AsyncMock(return_value=artifacts)) + instance = agent() + body = NeMoGymResponseCreateParamsNonStreaming(input="Question") + state.task = asyncio.create_task(instance._sandbox_response(state, body, prompt="Question", system=None)) + close = None + try: + assert await asyncio.to_thread(entered.wait, 5) + close = asyncio.create_task(instance._close_agent_session_state(state)) + with pytest.raises(TimeoutError): + await asyncio.wait_for(asyncio.shield(close), timeout=0.02) + assert not state.task.cancelling() + release.set() + receipt = await asyncio.wait_for(close, timeout=5) + assert any(record.kind == "agent_invocation" for record in receipt.agent_observations.records) + assert any(record.kind == "sandbox" for record in receipt.agent_observations.records) + assert "agent_activation_interrupted" not in [gap.code for gap in receipt.agent_observations.gaps] + finally: + release.set() + await asyncio.gather(state.task, *([close] if close else []), return_exceptions=True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "metadata,expected", + [ + (None, 300), + ({}, 300), + ({"agent_timeout_sec": "12.5"}, 12.5), + ({"agent_timeout_sec": "900"}, 300), + ({"agent_timeout_sec": "0"}, 300), + ({"agent_timeout_sec": "NaN"}, 300), + ({"agent_timeout_sec": "bad"}, 300), + ], +) +async def test_sandbox_honors_task_timeout_and_server_ceiling(monkeypatch, metadata, expected): + instance = agent(timeout=300) + state = adapter.ClaudeCodeSandboxSession( + request=seed(), + runtime="/runtime", + session=SimpleNamespace(session_dir="/tmp/session", workdir="/app", cleanup=None), + ) + execute = AsyncMock(return_value=adapter.ClaudeCodeArtifacts("")) + monkeypatch.setattr(adapter.ClaudeCodeSandboxSession, "execute", execute) + body = NeMoGymResponseCreateParamsNonStreaming(input="Question", metadata=metadata) + instance._validate_sandbox_request(body) + await instance._sandbox_response(state, body, prompt="Question", system=None) + assert execute.await_args.kwargs["timeout"] == expected + + +@pytest.mark.parametrize("metadata", [{"other": "value"}, {"agent_timeout_sec": "10", "other": "value"}]) +def test_sandbox_rejects_unsupported_metadata(metadata): + with pytest.raises(HTTPException) as error: + agent()._validate_sandbox_request(NeMoGymResponseCreateParamsNonStreaming(input="Question", metadata=metadata)) + assert error.value.status_code == 422 + + def request(session_id="session", rollout="rollout"): return Request( {"type": "http", "session": {AGENT_SESSION_COOKIE_KEY: session_id}, "path_params": {"rollout_id": rollout}} From 29859b3824fadabd5103ec0bd540e8c903f99ee5 Mon Sep 17 00:00:00 2001 From: Frederic Lafrance Date: Wed, 21 Oct 2026 08:14:01 +0200 Subject: [PATCH 7/7] fix(claude-code): secure and bound sandbox runtime staging Signed-off-by: Frederic Lafrance --- .../claude_code_agent/README.md | 23 ++- responses_api_agents/claude_code_agent/app.py | 17 +- .../claude_code_agent/runtime.py | 118 +++++++++----- .../claude_code_agent/tests/test_runtime.py | 152 +++++++++++++++++- .../claude_code_agent/tests/test_sandbox.py | 52 +++++- 5 files changed, 315 insertions(+), 47 deletions(-) diff --git a/responses_api_agents/claude_code_agent/README.md b/responses_api_agents/claude_code_agent/README.md index 1897b996269..5383b05b449 100644 --- a/responses_api_agents/claude_code_agent/README.md +++ b/responses_api_agents/claude_code_agent/README.md @@ -240,12 +240,17 @@ access to the release service, or a supplied local binary as described below. The sandbox installer decompresses the transfer, verifies the native bytes, and atomically installs them in `/tmp/nemo-gym-claude-runtime-`, separate from -session state. A file lock serializes installation. Later seeds verify the cached -checksum and version before skipping the binary upload. Closing a borrowed session +session state. A file lock serializes installation. Later seeds in the **same +sandbox** verify the cached checksum and version before skipping the binary +upload; a fresh sandbox still needs the transfer. Closing a borrowed session removes its private state and retains the shared runtime for subsequent sessions. -Host downloads are checked against the release manifest's SHA-256 to detect -corruption. For an independently trusted checksum, set +Host downloads use the current user's owned, mode-0700 +`~/.cache/nemo_gym/claude` directory and are checked against the release manifest's +SHA-256. Verified file identities are memoized in each worker; edits or replacements +trigger verification again. Transfers stream from the compressed cache with at +most four 1 MiB pieces staged or uploading per worker, shared across sessions. +For an independently trusted checksum, set `sandbox_executable_sha256`. An offline Gym host can supply `sandbox_local_executable_path` pointing to a matching Linux binary on the host; alternatively, `sandbox_executable_path` names a binary already **inside the @@ -253,11 +258,19 @@ sandbox**. These two options are mutually exclusive and require `sandbox_executable_sha256`. Configure `model_server` as above, or `anthropic_base_url` and `anthropic_api_key`; -the endpoint must be reachable from the sandbox. `resources_server` is needed only +the endpoint must be reachable from the sandbox. With `model_server`, only +placeholder Anthropic credentials enter the sandbox; the configured direct +Anthropic credential stays on the host. `resources_server` is needed only for the legacy agent `/run` path. Sandbox-only servers install no host CLI. For local execution, installation happens on the first local invocation; concurrent calls share setup, failures propagate, and a later call can retry. +The default YAML selects native composition rather than acting as an unbound +legacy swap source. To use `--resources-server X --agent-type claude_code_agent` +with legacy swap configurations, explicitly restore +`resources_server: {type: resources_servers, name: "???"}` on the agent. Existing +resource-bound recipes retain their bindings. + For borrowed execution, the seed request supplies `sandbox_access` with a direct provider connection descriptor and an absolute task `workdir`. Resources prepares the task directory; Claude Code only stages its own runtime and settings outside diff --git a/responses_api_agents/claude_code_agent/app.py b/responses_api_agents/claude_code_agent/app.py index e24708dd1e1..6f492cf5cb3 100644 --- a/responses_api_agents/claude_code_agent/app.py +++ b/responses_api_agents/claude_code_agent/app.py @@ -689,7 +689,7 @@ async def _sandbox_response( env = { "CLAUDE_CONFIG_DIR": directory + "/config", "IS_SANDBOX": "1", - "ANTHROPIC_API_KEY": self.config.anthropic_api_key or "EMPTY", + "ANTHROPIC_API_KEY": "gym" if self.config.model_server else self.config.anthropic_api_key or "EMPTY", "ANTHROPIC_MODEL": model, "ANTHROPIC_DEFAULT_HAIKU_MODEL": model, "ANTHROPIC_DEFAULT_SONNET_MODEL": model, @@ -699,7 +699,10 @@ async def _sandbox_response( "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", } if base_url: - env.update(ANTHROPIC_BASE_URL=base_url, ANTHROPIC_AUTH_TOKEN=self.config.anthropic_api_key or "local") + env.update( + ANTHROPIC_BASE_URL=base_url, + ANTHROPIC_AUTH_TOKEN="gym" if self.config.model_server else self.config.anthropic_api_key or "local", + ) artifacts = None failure = None try: @@ -764,7 +767,15 @@ async def _sandbox_response( SandboxObservation( role="agent", provider=access.connection.provider_config_ref if access else self.config.sandbox_provider, - outcome="timeout" if cleanup["timed_out"] else "failed" if status == "failed" else "completed", + outcome=( + "timeout" + if cleanup["timed_out"] + else "cancelled" + if reason == "cancelled" + else "failed" + if status == "failed" + else "completed" + ), exit_code=cleanup["return_code"], error_type=reason, ) diff --git a/responses_api_agents/claude_code_agent/runtime.py b/responses_api_agents/claude_code_agent/runtime.py index fe1b166a4ba..a196648daf6 100644 --- a/responses_api_agents/claude_code_agent/runtime.py +++ b/responses_api_agents/claude_code_agent/runtime.py @@ -5,11 +5,13 @@ import asyncio import gzip import json +import os import re import shutil +import stat from pathlib import Path from shlex import quote -from tempfile import TemporaryDirectory, gettempdir +from tempfile import TemporaryDirectory from uuid import uuid4 from aiohttp import ClientTimeout @@ -19,8 +21,38 @@ from responses_api_agents.claude_code_agent.install_claude_runtime import sha256_file -CACHE_DIR = Path(gettempdir()) / "nemo-gym-claude-assets" -_DOWNLOAD_LOCK = asyncio.Lock() +CACHE_DIR = Path.home() / ".cache" / "nemo_gym" / "claude" +_CACHE_LOCKS: dict[Path, asyncio.Lock] = {} +_HASH_LOCKS: dict[Path, asyncio.Lock] = {} +_VERIFIED: dict[Path, tuple[tuple[int, ...], str]] = {} +_UPLOAD_SLOTS = asyncio.Semaphore(4) + + +def _private_directory(path: Path) -> None: + path.mkdir(mode=0o700, parents=True, exist_ok=True) + info = path.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o022: + raise PermissionError(f"Claude cache must be owned by the current user without shared write access: {path}") + path.chmod(0o700) + + +def _fingerprint(path: Path) -> tuple[int, ...]: + info = path.stat() + return (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns) + + +async def _verified_sha256(path: Path) -> str: + # Rehash replacements or edits, but share verification across warm concurrent seeds. + async with _HASH_LOCKS.setdefault(path, asyncio.Lock()): + identity = _fingerprint(path) + cached = _VERIFIED.get(path) + if cached is not None and cached[0] == identity: + return cached[1] + digest = await asyncio.to_thread(sha256_file, path) + if _fingerprint(path) != identity: + raise ValueError(f"Claude artifact changed during checksum verification: {path}") + _VERIFIED[path] = (identity, digest) + return digest def _target(probe: str) -> str: @@ -43,9 +75,10 @@ async def cached_binary(version: str, target: str, *, expected: str | None = Non "linux-arm64-musl", }: raise ValueError("Claude runtime requires an exact version and supported Linux target") - async with _DOWNLOAD_LOCK: - directory = CACHE_DIR / version - directory.mkdir(parents=True, exist_ok=True) + _private_directory(CACHE_DIR) + directory = CACHE_DIR / version + async with _CACHE_LOCKS.setdefault(directory, asyncio.Lock()): + _private_directory(directory) base = "https://downloads.claude.ai/claude-code-releases/" + version if expected is None: manifest_path = directory / "manifest.json" @@ -64,10 +97,12 @@ async def cached_binary(version: str, target: str, *, expected: str | None = Non finally: temporary.unlink(missing_ok=True) checksum = expected.lower() + if not re.fullmatch(r"[0-9a-f]{64}", checksum): + raise ValueError("Claude executable requires a SHA-256 checksum") path = directory / target / "claude" - if path.is_file() and await asyncio.to_thread(sha256_file, path) == checksum: + _private_directory(path.parent) + if path.is_file() and await _verified_sha256(path) == checksum: return path, checksum - path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(".claude-" + uuid4().hex) try: async with await request( @@ -77,10 +112,13 @@ async def cached_binary(version: str, target: str, *, expected: str | None = Non with temporary.open("wb") as output: async for chunk in response.content.iter_chunked(1024 * 1024): await asyncio.to_thread(output.write, chunk) - if await asyncio.to_thread(sha256_file, temporary) != checksum: + if await _verified_sha256(temporary) != checksum: raise ValueError("Claude executable SHA-256 mismatch") temporary.replace(path) + _VERIFIED[path] = (_fingerprint(path), checksum) finally: + _VERIFIED.pop(temporary, None) + _HASH_LOCKS.pop(temporary, None) temporary.unlink(missing_ok=True) return path, checksum @@ -92,24 +130,24 @@ def _compress(source: Path, destination: Path) -> None: async def compressed_binary(source: Path, checksum: str) -> Path: """Cache a smaller transfer artifact while retaining verification of the native bytes.""" - async with _DOWNLOAD_LOCK: - directory = CACHE_DIR / "compressed" - directory.mkdir(parents=True, exist_ok=True) - path = directory / (checksum + ".gz") + _private_directory(CACHE_DIR) + directory = CACHE_DIR / "compressed" + _private_directory(directory) + path = directory / (checksum + ".gz") + async with _CACHE_LOCKS.setdefault(path, asyncio.Lock()): receipt = path.with_suffix(".sha256") - if ( - path.is_file() - and receipt.is_file() - and await asyncio.to_thread(sha256_file, path) == receipt.read_text().strip() - ): + if path.is_file() and receipt.is_file() and await _verified_sha256(path) == receipt.read_text().strip(): return path temporary = directory / (".gzip-" + uuid4().hex) try: await asyncio.to_thread(_compress, source, temporary) - digest = await asyncio.to_thread(sha256_file, temporary) + digest = await _verified_sha256(temporary) temporary.replace(path) + _VERIFIED[path] = (_fingerprint(path), digest) receipt.write_text(digest) finally: + _VERIFIED.pop(temporary, None) + _HASH_LOCKS.pop(temporary, None) temporary.unlink(missing_ok=True) return path @@ -123,29 +161,37 @@ def _check(result: SandboxExecResult) -> None: async def _upload_archive(sandbox: AsyncSandbox, archive: Path, directory: str, timeout: float) -> str: - """Keep each upload below sandbox proxy request-size limits.""" + """Stream bounded pieces with a process-wide limit on staged bytes and uploads.""" destination = directory + "/claude-upload.gz" with TemporaryDirectory(prefix="claude-upload-") as staging: - def split() -> list[Path]: - parts = [] - with archive.open("rb") as source: - while data := source.read(1024 * 1024): - path = Path(staging) / f"claude-upload-{len(parts):04d}.part" - path.write_bytes(data) - parts.append(path) - return parts - - parts = await asyncio.to_thread(split) - semaphore = asyncio.Semaphore(4) - async def upload(part: Path) -> None: - async with semaphore: + try: await sandbox.upload(part, directory + "/" + part.name) + finally: + part.unlink(missing_ok=True) + _UPLOAD_SLOTS.release() + parts = [] async with asyncio.TaskGroup() as tasks: - for part in parts: - tasks.create_task(upload(part)) + with archive.open("rb") as source: + while True: + await _UPLOAD_SLOTS.acquire() + try: + data = await asyncio.to_thread(source.read, 1024 * 1024) + if not data: + _UPLOAD_SLOTS.release() + break + part = Path(staging) / f"claude-upload-{len(parts):04d}.part" + await asyncio.to_thread(part.write_bytes, data) + del data + except BaseException: + _UPLOAD_SLOTS.release() + raise + parts.append(part) + tasks.create_task(upload(part)) + if not parts: + raise ValueError("Cannot upload an empty Claude archive") sources = " ".join(quote(directory + "/" + part.name) for part in parts) result = await sandbox.exec(f"cat {sources} > {quote(destination)} && rm -f {sources}", timeout_s=timeout) _check(result) @@ -172,7 +218,7 @@ async def install_sandbox_runtime( else: if local_source: local = Path(local_source).expanduser() - checksum = await asyncio.to_thread(sha256_file, local) + checksum = await _verified_sha256(local) if checksum != expected.lower(): raise ValueError("Local Claude executable SHA-256 mismatch") else: diff --git a/responses_api_agents/claude_code_agent/tests/test_runtime.py b/responses_api_agents/claude_code_agent/tests/test_runtime.py index f12a55d48a5..535a41c1dec 100644 --- a/responses_api_agents/claude_code_agent/tests/test_runtime.py +++ b/responses_api_agents/claude_code_agent/tests/test_runtime.py @@ -4,16 +4,25 @@ import asyncio import gzip import hashlib +import os import shutil from pathlib import Path from types import SimpleNamespace -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock import pytest from responses_api_agents.claude_code_agent import runtime +@pytest.fixture(autouse=True) +def isolated_process_cache(monkeypatch): + monkeypatch.setattr(runtime, "_CACHE_LOCKS", {}) + monkeypatch.setattr(runtime, "_HASH_LOCKS", {}) + monkeypatch.setattr(runtime, "_VERIFIED", {}) + monkeypatch.setattr(runtime, "_UPLOAD_SLOTS", asyncio.Semaphore(4)) + + class Response: def __init__(self, data=b"", manifest=None): self.data = data @@ -180,4 +189,143 @@ async def execute(command, **kwargs): assert Path(destination).read_bytes() == data assert not list(directory.glob("*.part")) assert max(sizes) <= 1024 * 1024 and sum(sizes) == len(data) - assert maximum == 4 + assert 1 <= maximum <= 4 + + +@pytest.mark.asyncio +async def test_warm_seeds_share_hashes_and_detect_same_size_replacements(tmp_path, monkeypatch): + monkeypatch.setattr(runtime, "CACHE_DIR", tmp_path / "cache") + data = b"verified native binary" + checksum = hashlib.sha256(data).hexdigest() + download = AsyncMock(return_value=Response(data=data)) + monkeypatch.setattr(runtime, "request", download) + digest = MagicMock(wraps=runtime.sha256_file) + monkeypatch.setattr(runtime, "sha256_file", digest) + path, _ = await runtime.cached_binary("2.1.108", "linux-x64", expected=checksum) + archive = await runtime.compressed_binary(path, checksum) + initial_hashes = digest.call_count + for _ in range(3): + assert await runtime.cached_binary("2.1.108", "linux-x64", expected=checksum) == (path, checksum) + assert await runtime.compressed_binary(path, checksum) == archive + assert digest.call_count == initial_hashes + identity = path.stat() + replacement = path.with_name("replacement") + replacement.write_bytes(b"x" * len(data)) + os.utime(replacement, ns=(identity.st_atime_ns, identity.st_mtime_ns)) + replacement.replace(path) + await runtime.cached_binary("2.1.108", "linux-x64", expected=checksum) + assert path.read_bytes() == data + assert download.await_count == 2 + assert digest.call_count > initial_hashes + + +@pytest.mark.asyncio +async def test_host_cache_is_private_and_rejects_foreign_ownership(tmp_path, monkeypatch): + cache = tmp_path / "cache" + cache.mkdir(mode=0o755) + monkeypatch.setattr(runtime, "CACHE_DIR", cache) + data = b"native" + checksum = hashlib.sha256(data).hexdigest() + download = AsyncMock(return_value=Response(data=data)) + monkeypatch.setattr(runtime, "request", download) + await runtime.cached_binary("2.1.108", "linux-x64", expected=checksum) + assert all( + path.stat().st_mode & 0o777 == 0o700 for path in (cache, cache / "2.1.108", cache / "2.1.108/linux-x64") + ) + owner = cache.stat().st_uid + monkeypatch.setattr(runtime.os, "getuid", lambda: owner + 1) + download.reset_mock() + with pytest.raises(PermissionError, match="owned by the current user"): + await runtime.cached_binary("2.1.108", "linux-x64", expected=checksum) + download.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_host_cache_rejects_symlink_before_reading_manifest(tmp_path, monkeypatch): + shared = tmp_path / "shared" + shared.mkdir() + cache = tmp_path / "cache" + cache.symlink_to(shared, target_is_directory=True) + monkeypatch.setattr(runtime, "CACHE_DIR", cache) + download = AsyncMock() + monkeypatch.setattr(runtime, "request", download) + with pytest.raises(PermissionError, match="owned by the current user"): + await runtime.cached_binary("2.1.108", "linux-x64") + download.assert_not_awaited() + assert not list(shared.iterdir()) + + +@pytest.mark.asyncio +async def test_host_cache_rejects_previously_shared_write_access(tmp_path, monkeypatch): + cache = tmp_path / "cache" + cache.mkdir() + cache.chmod(0o777) + monkeypatch.setattr(runtime, "CACHE_DIR", cache) + download = AsyncMock() + monkeypatch.setattr(runtime, "request", download) + with pytest.raises(PermissionError, match="shared write access"): + await runtime.cached_binary("2.1.108", "linux-x64") + download.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_concurrent_seeds_bound_staged_chunks_and_uploads_across_sandboxes(tmp_path): + archive = tmp_path / "source.gz" + data = bytes(range(256)) * (1024 * 24 + 7) + archive.write_bytes(data) + release = asyncio.Event() + full = asyncio.Event() + active = maximum = 0 + staging = set() + + async def upload(source, destination): + nonlocal active, maximum + active += 1 + maximum = max(maximum, active) + staging.add(source.parent) + if active == 4: + full.set() + try: + await release.wait() + shutil.copyfile(source, destination) + finally: + active -= 1 + + async def execute(command, **kwargs): + process = await asyncio.create_subprocess_shell(command) + return result(code=await process.wait()) + + sandbox = SimpleNamespace(upload=upload, exec=execute) + directories = [tmp_path / f"remote-{index}" for index in range(3)] + for directory in directories: + directory.mkdir() + tasks = [ + asyncio.create_task(runtime._upload_archive(sandbox, archive, str(directory), 60)) for directory in directories + ] + try: + await asyncio.wait_for(full.wait(), timeout=5) + # Paused transfers keep their source files: a full per-seed split would exceed this bound. + assert ( + sum(path.stat().st_size for directory in staging for path in directory.glob("*.part")) <= 4 * 1024 * 1024 + ) + release.set() + destinations = await asyncio.wait_for(asyncio.gather(*tasks), timeout=10) + assert all(Path(path).read_bytes() == data for path in destinations) + assert maximum == 4 + finally: + release.set() + await asyncio.gather(*tasks, return_exceptions=True) + assert all(not directory.exists() for directory in staging) + + +@pytest.mark.asyncio +async def test_failed_upload_releases_slots_for_later_seeds(tmp_path): + archive = tmp_path / "source.gz" + archive.write_bytes(b"x" * 5 * 1024 * 1024) + upload = AsyncMock(side_effect=RuntimeError("transfer failed")) + with pytest.raises(ExceptionGroup, match="TaskGroup"): + await runtime._upload_archive(SimpleNamespace(upload=upload), archive, "/remote", 60) + assert all(not call.args[0].exists() for call in upload.await_args_list) + sandbox = SimpleNamespace(upload=AsyncMock(), exec=AsyncMock(return_value=result())) + await asyncio.wait_for(runtime._upload_archive(sandbox, archive, "/remote", 60), timeout=5) + assert sandbox.upload.await_count == 5 diff --git a/responses_api_agents/claude_code_agent/tests/test_sandbox.py b/responses_api_agents/claude_code_agent/tests/test_sandbox.py index 0ed289429b1..4554349d2db 100644 --- a/responses_api_agents/claude_code_agent/tests/test_sandbox.py +++ b/responses_api_agents/claude_code_agent/tests/test_sandbox.py @@ -356,8 +356,10 @@ async def test_retry_shares_execution_and_rejects_changed_body(monkeypatch): with pytest.raises(HTTPException) as error: await instance.responses(request(), NeMoGymResponseCreateParamsNonStreaming(input="Changed")) assert error.value.status_code == 409 - with pytest.raises(HTTPException): + with pytest.raises(HTTPException) as error: await instance.responses(request(rollout="wrong"), body) + assert error.value.status_code == 409 + assert execute.await_count == 1 @pytest.mark.asyncio @@ -412,6 +414,54 @@ async def test_empty_sandbox_output_uses_local_padding(monkeypatch): assert response.status == "incomplete" +@pytest.mark.asyncio +@pytest.mark.parametrize("model_server", [None, {"type": "responses_api_models", "name": "policy"}]) +async def test_sandbox_only_receives_direct_credentials_without_gym_model_server(monkeypatch, model_server): + credential = "direct-provider-credential" + instance = agent(model_server=model_server, anthropic_api_key=credential, anthropic_base_url="http://direct") + state = adapter.ClaudeCodeSandboxSession( + request=seed(), + runtime="/runtime", + session=SimpleNamespace(session_dir="/tmp/session", workdir="/app", cleanup=None), + ) + execute = AsyncMock(return_value=adapter.ClaudeCodeArtifacts("")) + monkeypatch.setattr(adapter.ClaudeCodeSandboxSession, "execute", execute) + monkeypatch.setattr(app.ClaudeCodeAgent, "_resolve_call_base_url", lambda *_: "http://model-server") + await instance._sandbox_response( + state, NeMoGymResponseCreateParamsNonStreaming(input="Question"), prompt="Question", system=None + ) + payload = execute.await_args.args[0] + expected = "gym" if model_server else credential + assert payload["env"]["ANTHROPIC_API_KEY"] == expected + assert payload["env"]["ANTHROPIC_AUTH_TOKEN"] == expected + if model_server: + assert credential not in json.dumps(payload) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("timed_out,expected", [(False, "cancelled"), (True, "timeout")]) +async def test_cancelled_activation_reports_sandbox_outcome(monkeypatch, timed_out, expected): + instance = agent() + state = adapter.ClaudeCodeSandboxSession( + request=seed(), + runtime="/runtime", + session=SimpleNamespace( + session_dir="/tmp/session", + workdir="/app", + artifacts=adapter.ClaudeCodeArtifacts(""), + cleanup={"timed_out": timed_out, "return_code": 143}, + ), + ) + monkeypatch.setattr(adapter.ClaudeCodeSandboxSession, "execute", AsyncMock(side_effect=asyncio.CancelledError())) + with pytest.raises(asyncio.CancelledError): + await instance._sandbox_response( + state, NeMoGymResponseCreateParamsNonStreaming(input="Question"), prompt="Question", system=None + ) + record = next(record for record in state.observations.records if record.kind == "sandbox") + assert record.outcome == expected + assert record.exit_code == 143 + + @pytest.mark.asyncio async def test_skills_are_staged_before_launch_and_disable_bare(monkeypatch, tmp_path): skill = tmp_path / "skills" / "example"