From ae45be90cf370d6914881546543e49eca1ef9035 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Mon, 26 Oct 2026 08:54:56 -0700 Subject: [PATCH 01/32] refactor(agent_utils): run the harness process supervisor in bash The supervisor uploaded into task sandboxes now needs only bash, /proc, sleep and mv, so sandboxed harnesses no longer require Python 3.8+ in the task image. It keeps the CLI and receipt contract: run the harness in its own process group, stop it at the deadline or when the stop file appears (TERM, then KILL the group), and confirm no group member runs. Stops now reach the supervisor through stop.request instead of SIGTERM, so the launch shell no longer ignores TERM and the harness keeps the default disposition. Shared receipt types and timeouts move to supervisor_client; SandboxCommand drops its python field. shell_assignments() renders runner inputs as bash source. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Hemil Desai Signed-off-by: yaoyu-33 --- nemo_gym/agent_utils/process_supervisor.py | 227 ------- nemo_gym/agent_utils/process_supervisor.sh | 107 +++ nemo_gym/agent_utils/sandbox_session.py | 38 +- nemo_gym/agent_utils/supervisor_client.py | 57 +- .../claude_code_agent/sandbox.py | 1 - responses_api_agents/codex_agent/sandbox.py | 1 - responses_api_agents/hermes_agent/sandbox.py | 1 - .../hermes_sandboxed_agent/app.py | 16 +- responses_api_agents/kilocode_agent/app.py | 2 +- .../kilocode_agent/sandbox.py | 1 - .../openclaw_agent/sandbox.py | 1 - .../opencode_agent/continuation.py | 2 +- .../opencode_agent/sandbox.py | 1 - responses_api_agents/pi_agent/sandbox.py | 1 - responses_api_agents/stirrup_agent/sandbox.py | 1 - tests/unit_tests/test_local_provider.py | 21 +- tests/unit_tests/test_process_supervisor.py | 629 ++++-------------- tests/unit_tests/test_sandbox_session.py | 44 +- .../test_sandbox_supervisor_bootstrap.py | 112 ++-- .../test_sandbox_supervisor_client.py | 47 +- 20 files changed, 456 insertions(+), 854 deletions(-) delete mode 100644 nemo_gym/agent_utils/process_supervisor.py create mode 100755 nemo_gym/agent_utils/process_supervisor.sh diff --git a/nemo_gym/agent_utils/process_supervisor.py b/nemo_gym/agent_utils/process_supervisor.py deleted file mode 100644 index 63f6bf4f815..00000000000 --- a/nemo_gym/agent_utils/process_supervisor.py +++ /dev/null @@ -1,227 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Standalone Linux process supervisor to upload beside a sandboxed harness. - -Uses only the standard library; Gym need not be installed in the task sandbox. -The receipt confirms termination independently of the harness result. On forced -stop, newly adopted children are also drained, including tools in other sessions. -Detached services adopted before stopping remain for verification. Natural exit -only drains the harness group: orphaned services and unfinished detached tools -cannot be distinguished without cooperation from the harness. -""" - -from __future__ import annotations - -import argparse -import ctypes -import json -import math -import os -import signal -import subprocess -import sys -import time -from pathlib import Path -from typing import TypedDict - - -DEFAULT_CLEANUP_TIMEOUT = 10.0 -CLEANUP_PHASE_COUNT = 3 - - -class CleanupReceipt(TypedDict): - """Outcome after bounded harness termination. - - ``cleanup_confirmed`` means no live harness-group members remain. Forced - stops also drain newly adopted children. Previously adopted detached services - remain until the owner closes the sandbox; this is not a sandbox-empty check. - """ - - cleanup_confirmed: bool - return_code: int | None - error: str | None - timed_out: bool - - -def exec_timeout(*, timeout: float, cleanup_timeout: float = DEFAULT_CLEANUP_TIMEOUT) -> float: - """Reserve TERM grace, direct-process reaping, and process-group reaping. - - The extra 30 seconds cover startup, polling, and receipt I/O beyond those - bounded cleanup phases. Keep their count shared with the controller budget. - """ - return timeout + CLEANUP_PHASE_COUNT * cleanup_timeout + 30 - - -def _child_pids() -> set[int]: - """Read this single-threaded subreaper's current children.""" - return {int(pid) for pid in Path(f"/proc/self/task/{os.getpid()}/children").read_text().split()} - - -def _group_has_live_processes(pgid: int) -> bool: - # killpg(..., 0) also succeeds for zombies parented outside this subreaper. - # They cannot execute or write, and their owner alone can reap them. Any - # unreadable/invalid proc state propagates instead of confirming termination. - for directory in Path("/proc").iterdir(): - if not directory.name.isdigit(): - continue - try: - fields = (directory / "stat").read_text().rsplit(")", 1)[1].split() - except (FileNotFoundError, ProcessLookupError): - continue - if int(fields[2]) == pgid and fields[0] not in {"Z", "X"}: - return True - return False - - -def _reap_process_group(pgid: int, timeout: float, *, preserved_children: set[int] | None = None) -> None: - """Drain the group and, on forced stop, newly adopted tools within one budget.""" - deadline = time.monotonic() + timeout - while True: - try: - os.killpg(pgid, signal.SIGKILL) - except ProcessLookupError: - pass - group_children_pending = True - try: - while os.waitpid(-pgid, os.WNOHANG)[0]: - pass - except ChildProcessError: - group_children_pending = False - pending_children = set() - if preserved_children is not None: - # Killing one generation can orphan more children in other groups. - # Re-read each round rather than taking only one post-stop snapshot. - for pid in _child_pids() - preserved_children: - try: - os.kill(pid, signal.SIGKILL) - os.waitpid(pid, os.WNOHANG) - except ProcessLookupError: - pass - pending_children = _child_pids() - preserved_children - try: - os.killpg(pgid, 0) - group_alive = _group_has_live_processes(pgid) - except ProcessLookupError: - group_alive = False - if not group_alive and not group_children_pending and not pending_children: - return - if time.monotonic() >= deadline: - raise TimeoutError("Harness process group or newly adopted tools have not terminated") - time.sleep(0.01) - - -def _supervise( - command: list[str], - *, - timeout: float, - cleanup_timeout: float = DEFAULT_CLEANUP_TIMEOUT, - stop_path: Path | None = None, -) -> CleanupReceipt: - """Stop the harness and forced-stop tools; preserve earlier detached services.""" - process = None - launch_attempted = False - stopping = False - forced_stop = False - preserved_children = None - receipt: CleanupReceipt = {"cleanup_confirmed": False, "return_code": None, "error": None, "timed_out": False} - - def interrupt(*_: object) -> None: - nonlocal stopping - # A signal between Popen and handle assignment must not lose the child. - stopping = True - - signal.signal(signal.SIGTERM, interrupt) - try: - # TERM can be ignored during interpreter startup. A durable stop marker fences - # that window; later signals are handled without interrupting Popen. - if stopping or (stop_path is not None and stop_path.exists()): - return receipt # The finally block confirms that no harness process was launched. - if sys.platform != "linux": - raise RuntimeError("Sandbox process supervision requires Linux") - # Leave the provider group when possible. An exec launcher may already make us - # its group leader, so setsid cannot protect us from that group's SIGKILL. - # Callers must confirm cleanup before cancelling provider exec, in either case. - if os.getpgrp() != os.getpid(): - os.setsid() - # Reap killed group members ourselves instead of depending on the - # container's PID 1. Otherwise zombies keep killpg(..., 0) succeeding. - if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0: # PR_SET_CHILD_SUBREAPER - raise OSError(ctypes.get_errno(), "Cannot supervise harness process group") - launch_attempted = True - process = subprocess.Popen(command, start_new_session=True) - deadline = time.monotonic() + timeout - while process.poll() is None and not stopping and time.monotonic() < deadline: - time.sleep(0.05) - if process.poll() is None: - forced_stop = True - receipt["timed_out"] = not stopping - # Completed tool calls can leave services already adopted by us. - # Preserve those, but drain children orphaned by this forced stop. - preserved_children = _child_pids() - {process.pid} - # Let the harness checkpoint before the bounded hard cleanup. - process.send_signal(signal.SIGTERM) - try: - process.wait(timeout=cleanup_timeout) - except subprocess.TimeoutExpired: - pass - except Exception as exc: - receipt["error"] = f"Harness launch failed: {exc}" if process is None and launch_attempted else str(exc) - finally: - signal.signal(signal.SIGTERM, signal.SIG_IGN) - try: - if process is not None: - # Any services still in the harness group stop here too. Only - # previously adopted children outside that group are preserved - # on forced stop; audit service-launching task recipes. - # Natural exit has no pre-stop boundary for detached processes. - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - receipt["return_code"] = process.wait(timeout=cleanup_timeout) - _reap_process_group(process.pid, cleanup_timeout, preserved_children=preserved_children) - if forced_stop and preserved_children is None: - raise RuntimeError("Cannot identify pre-stop adopted children") - receipt["cleanup_confirmed"] = True - else: - # Popen can fail after spawning but before returning a handle. - receipt["cleanup_confirmed"] = not launch_attempted - except Exception as exc: - receipt["error"] = f"Harness termination was not confirmed: {exc}" - return receipt - - -def _positive_seconds(value: str) -> float: - seconds = float(value) - if not math.isfinite(seconds) or seconds <= 0: - raise argparse.ArgumentTypeError("timeout must be finite and positive") - return seconds - - -def main() -> int: - """Run COMMAND and atomically write its cleanup receipt outside the task workdir.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--timeout", type=_positive_seconds, required=True) - parser.add_argument("--cleanup-timeout", type=_positive_seconds, default=DEFAULT_CLEANUP_TIMEOUT) - parser.add_argument("--receipt", type=Path, required=True) - parser.add_argument("--stop-file", type=Path) - parser.add_argument("command", nargs=argparse.REMAINDER) - args = parser.parse_args() - command = args.command[1:] if args.command[:1] == ["--"] else args.command - if not command: - parser.error("a harness command is required after --") - receipt = _supervise( - command, - timeout=args.timeout, - cleanup_timeout=args.cleanup_timeout, - stop_path=args.stop_file, - ) - temporary = args.receipt.with_suffix(".tmp") - temporary.write_text(json.dumps(receipt)) - temporary.replace(args.receipt) - return 0 if receipt["cleanup_confirmed"] and receipt["error"] is None else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/nemo_gym/agent_utils/process_supervisor.sh b/nemo_gym/agent_utils/process_supervisor.sh new file mode 100755 index 00000000000..eba1e108a03 --- /dev/null +++ b/nemo_gym/agent_utils/process_supervisor.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Process supervisor uploaded beside a sandboxed harness. Needs only bash, /proc, sleep and mv. +# +# Runs COMMAND in its own process group until it exits, the deadline passes or the stop file +# appears. A stopped harness gets TERM, then the whole group gets KILL. The receipt confirms that +# no group member can still run. Processes that left the group (detached services) are left for +# the sandbox owner. +# +# usage: process_supervisor.sh --timeout SECONDS --cleanup-timeout SECONDS --receipt PATH --stop-file PATH -- COMMAND... + +# Seconds as integer centiseconds, rounded down; fails unless positive. +centiseconds() { + [[ $1 =~ ^([0-9]*)\.?([0-9]*)$ ]] || return 1 + local fraction="${BASH_REMATCH[2]}00" + local value=$((10#0${BASH_REMATCH[1]} * 100 + 10#${fraction:0:2})) + ((value > 0)) && echo "$value" +} + +# Linux uptime in centiseconds, without forking. +now() { + local uptime + read -r uptime _ /dev/null <"$stat" || continue + fields=(${stat##*) }) + ((fields[2] == harness)) && [[ ${fields[0]} != [ZX] ]] && return 0 + done + return 1 +} + +stopped() { ((stopping)) || [[ -e $stop_file ]]; } + +write_receipt() { + printf '{"cleanup_confirmed": %s, "return_code": %s, "error": %s, "timed_out": %s}' \ + "$1" "$return_code" "$2" "$timed_out" >"$receipt.tmp" && mv -f "$receipt.tmp" "$receipt" +} + +while [[ $1 == --* && $1 != -- ]]; do + case $1 in + --timeout) timeout=$(centiseconds "$2") || { echo "--timeout must be finite and positive" >&2; exit 2; } ;; + --cleanup-timeout) cleanup=$(centiseconds "$2") || { echo "--cleanup-timeout must be finite and positive" >&2; exit 2; } ;; + --receipt) receipt=$2 ;; + --stop-file) stop_file=$2 ;; + *) echo "unexpected argument: $1" >&2; exit 2 ;; + esac + shift 2 +done +shift +[[ -n ${timeout-} && -n ${cleanup-} && -n ${receipt-} && -n ${stop_file-} && $# -gt 0 ]] || { + echo "usage: process_supervisor.sh --timeout S --cleanup-timeout S --receipt PATH --stop-file PATH -- COMMAND..." >&2 + exit 2 +} + +return_code=null timed_out=false stopping=0 +trap 'stopping=1' TERM +if stopped; then + write_receipt true null # Nothing was launched. + exit 0 +fi + +# Job control gives the harness its own process group. +set -m +"$@" & +harness=$! +set +m + +now +deadline=$((NOW + timeout)) +while kill -0 "$harness" 2>/dev/null && ! stopped && ((NOW < deadline)); do + sleep 0.05 + now +done +if kill -0 "$harness" 2>/dev/null; then + stopped || timed_out=true + kill -TERM "$harness" 2>/dev/null + grace=$((NOW + cleanup)) + while kill -0 "$harness" 2>/dev/null && ((NOW < grace)); do + sleep 0.05 + now + done +fi +trap '' TERM + +kill -KILL -- "-$harness" 2>/dev/null +wait "$harness" +return_code=$? +now +deadline=$((NOW + cleanup)) +while group_alive; do + kill -KILL -- "-$harness" 2>/dev/null + now + if ((NOW >= deadline)); then + write_receipt false '"Harness termination was not confirmed: its process group still runs"' + exit 1 + fi + sleep 0.01 +done +write_receipt true null diff --git a/nemo_gym/agent_utils/sandbox_session.py b/nemo_gym/agent_utils/sandbox_session.py index 34248652296..4b3cf1b1abd 100644 --- a/nemo_gym/agent_utils/sandbox_session.py +++ b/nemo_gym/agent_utils/sandbox_session.py @@ -4,18 +4,19 @@ import asyncio import logging -from collections.abc import Awaitable, Callable, Coroutine +import re +from collections.abc import Awaitable, Callable, Coroutine, Mapping, Sequence from dataclasses import dataclass, field -from pathlib import Path +from shlex import quote from time import monotonic from typing import cast -from nemo_gym.agent_utils import process_supervisor -from nemo_gym.agent_utils.process_supervisor import CleanupReceipt from nemo_gym.agent_utils.supervisor_client import ( OUTPUT_LOG_FILE, STOP_REQUEST_FILE, SUPERVISOR_FILE, + SUPERVISOR_SCRIPT, + CleanupReceipt, HarnessTerminationError, remove_session_directory, stop_and_confirm_cleanup, @@ -28,6 +29,7 @@ LOG = logging.getLogger(__name__) +_SHELL_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") def _reuse_or_start[T]( @@ -40,10 +42,31 @@ def _reuse_or_start[T]( @dataclass(frozen=True, kw_only=True) class SandboxCommand: - """Harness argv and the installed interpreter used to launch its supervisor.""" + """Harness argv, run under the bash process supervisor.""" argv: list[str] - python: str + + +def shell_assignments(values: Mapping[str, str | Sequence[str] | Mapping[str, str]]) -> str: + """Bash source assigning each value, for sandbox runners that cannot parse JSON. + + A string becomes a scalar and a sequence an array. A mapping becomes an array of + ``KEY=VALUE`` words, ready for ``export`` or ``env``. Every word is shell-quoted. + """ + lines = [] + for name, value in values.items(): + if not _SHELL_NAME.fullmatch(name): + raise ValueError(f"Invalid shell variable name: {name!r}") + if isinstance(value, str): + lines.append(f"{name}={quote(value)}") + continue + if isinstance(value, Mapping): + invalid = [key for key in value if not _SHELL_NAME.fullmatch(key)] + if invalid: + raise ValueError(f"Invalid environment variable names: {invalid!r}") + value = [f"{key}={item}" for key, item in value.items()] + lines.append(f"{name}=({' '.join(quote(item) for item in value)})") + return "".join(f"{line}\n" for line in lines) @dataclass(kw_only=True) @@ -130,7 +153,7 @@ async def read_output_log(self) -> str: async def _stage(self, stage_activation: Callable[[], Awaitable[SandboxCommand]]) -> SandboxCommand: command = await stage_activation() - await self.sandbox.upload(Path(process_supervisor.__file__), f"{self.session_dir}/{SUPERVISOR_FILE}") + await self.sandbox.upload(SUPERVISOR_SCRIPT, f"{self.session_dir}/{SUPERVISOR_FILE}") return command async def execute( @@ -168,7 +191,6 @@ async def execute( command=command.argv, timeout=timeout, cleanup_timeout=cleanup_timeout, - python=command.python, ) self.launch_started = True self._exec_task = asyncio.create_task( diff --git a/nemo_gym/agent_utils/supervisor_client.py b/nemo_gym/agent_utils/supervisor_client.py index af5bbdb2d82..451ca4b1338 100644 --- a/nemo_gym/agent_utils/supervisor_client.py +++ b/nemo_gym/agent_utils/supervisor_client.py @@ -3,17 +3,17 @@ """Controller-side transport for the shared sandbox process supervisor. Session ownership and cancellation live in sandbox_session.py; output parsing stays with the adapter. -Unlike process_supervisor.py, this module is not uploaded to the task sandbox. +Unlike process_supervisor.sh, this module is not uploaded to the task sandbox. Session control files (all paths are relative to session_dir): ===================== ==================== =========================================== Filename Writer When ===================== ==================== =========================================== -process_supervisor.py Session Uploaded before launch +process_supervisor.sh Session Uploaded before launch launch.claim Launch or stop shell Atomically claims launch or fences it supervisor.pid Launch shell After claiming launch, before exec -stop.request Stop shell Before signalling the supervisor +stop.request Stop shell Polled by the supervisor to stop the harness cleanup.json Supervisor or stop After cleanup, or when stop fences launch output.log Launch shell Captures supervisor and harness diagnostics ===================== ==================== =========================================== @@ -21,23 +21,52 @@ import json import logging +from pathlib import Path from shlex import join, quote +from typing import TypedDict from uuid import uuid4 from pydantic import TypeAdapter, ValidationError -from nemo_gym.agent_utils.process_supervisor import CLEANUP_PHASE_COUNT, CleanupReceipt, exec_timeout from nemo_gym.sandbox import AsyncSandbox from nemo_gym.sandbox.utils import read_text -SUPERVISOR_FILE = "process_supervisor.py" +SUPERVISOR_FILE = "process_supervisor.sh" +SUPERVISOR_SCRIPT = Path(__file__).with_name(SUPERVISOR_FILE) LAUNCH_CLAIM_FILE = "launch.claim" SUPERVISOR_PID_FILE = "supervisor.pid" STOP_REQUEST_FILE = "stop.request" CLEANUP_RECEIPT_FILE = "cleanup.json" OUTPUT_LOG_FILE = "output.log" +DEFAULT_CLEANUP_TIMEOUT = 10.0 +CLEANUP_PHASE_COUNT = 3 + + +class CleanupReceipt(TypedDict): + """Outcome after bounded harness termination, as written by process_supervisor.sh. + + ``cleanup_confirmed`` means no live harness-group members remain. Forced + stops also drain the harness's descendants. Services orphaned before the stop + remain until the owner closes the sandbox; this is not a sandbox-empty check. + """ + + cleanup_confirmed: bool + return_code: int | None + error: str | None + timed_out: bool + + +def exec_timeout(*, timeout: float, cleanup_timeout: float = DEFAULT_CLEANUP_TIMEOUT) -> float: + """Reserve TERM grace, direct-process reaping, and process-group reaping. + + The extra 30 seconds cover startup, polling, and receipt I/O beyond those + bounded cleanup phases. Keep their count shared with the controller budget. + """ + return timeout + CLEANUP_PHASE_COUNT * cleanup_timeout + 30 + + _CLEANUP_RECEIPT = TypeAdapter(CleanupReceipt) LOG = logging.getLogger(__name__) @@ -97,23 +126,23 @@ def supervised_launch_command( command: list[str], timeout: float, cleanup_timeout: float, - python: str, ) -> str: """Fence delayed launches and run a harness command under the shared supervisor. - The adapter must install or select the interpreter explicitly. Check that it - can load the supervisor before claiming a launch: a failed bootstrap cannot - write a cleanup receipt. Supervisor and harness diagnostics share output.log. - The supervisor stops the harness group; the sandbox owner stops escaped services. + The supervisor needs only bash. Check that bash can parse it before claiming a + launch: a failed bootstrap cannot write a cleanup receipt. Supervisor and harness + diagnostics share output.log. The supervisor stops the harness group; the sandbox + owner stops escaped services. Stops reach the supervisor through the stop file, + not a signal, so the harness keeps the default TERM disposition. """ supervisor = quote(f"{session_dir}/{SUPERVISOR_FILE}") claim_path = quote(f"{session_dir}/{LAUNCH_CLAIM_FILE}") return ( f"[ -d {quote(session_dir)} ] && [ ! -L {claim_path} ] || exit 0; " - f"{quote(python)} -I {supervisor} --help >/dev/null || exit $?; " - f"trap '' TERM; ln -s launch {claim_path} 2>/dev/null || exit 0; " + f"bash -n {supervisor} || exit $?; " + f"ln -s launch {claim_path} 2>/dev/null || exit 0; " f"echo $$ > {quote(f'{session_dir}/{SUPERVISOR_PID_FILE}')} && " - f"exec {quote(python)} -I {supervisor} " + f"exec bash {supervisor} " f"--timeout {timeout} --cleanup-timeout {cleanup_timeout} " f"--stop-file {quote(f'{session_dir}/{STOP_REQUEST_FILE}')} " f"--receipt {quote(f'{session_dir}/{CLEANUP_RECEIPT_FILE}')} -- {join(command)} " @@ -136,7 +165,6 @@ async def stop_and_confirm_cleanup( receipt = {} _raise_terminal_failure(receipt, harness=harness) if not isinstance(receipt, dict) or receipt.get("cleanup_confirmed") is not True: - pid_path = quote(f"{session_dir}/{SUPERVISOR_PID_FILE}") stop_path = quote(f"{session_dir}/{STOP_REQUEST_FILE}") claim_path = quote(f"{session_dir}/{LAUNCH_CLAIM_FILE}") temporary = quote(f"{receipt_path}.{uuid4().hex}.tmp") @@ -149,7 +177,6 @@ async def stop_and_confirm_cleanup( f"ln -s stop {claim_path} 2>/dev/null || true; " f'if [ "$(readlink {claim_path})" = stop ]; then ' f"printf '%s' {stopped} > {temporary} && mv {temporary} {quote(receipt_path)}; exit $?; fi; " - f'if [ -s {pid_path} ]; then kill -TERM "$(cat {pid_path})" 2>/dev/null || true; fi; ' f"for _ in $(seq 1 {max(1, int(timeout))}); do " f"[ -f {quote(receipt_path)} ] && exit 0; sleep 1; done; exit 1" ) diff --git a/responses_api_agents/claude_code_agent/sandbox.py b/responses_api_agents/claude_code_agent/sandbox.py index 6adbe3d5983..5ea51ad45b3 100644 --- a/responses_api_agents/claude_code_agent/sandbox.py +++ b/responses_api_agents/claude_code_agent/sandbox.py @@ -167,7 +167,6 @@ async def stage_activation(self, payload: dict[str, JsonValue]) -> SandboxComman await self.stage_skills(skills_path) await self.upload_json("input.json", payload) return SandboxCommand( - python="python3", argv=[ "python3", "-I", diff --git a/responses_api_agents/codex_agent/sandbox.py b/responses_api_agents/codex_agent/sandbox.py index d667fe374b6..2959a7b48cc 100644 --- a/responses_api_agents/codex_agent/sandbox.py +++ b/responses_api_agents/codex_agent/sandbox.py @@ -92,7 +92,6 @@ async def stage_activation(self, payload: dict[str, JsonValue]) -> SandboxComman """Stage the invocation and return the Codex worker command.""" await self.upload_json("input.json", payload) return SandboxCommand( - python="python3", argv=[ "python3", "-I", diff --git a/responses_api_agents/hermes_agent/sandbox.py b/responses_api_agents/hermes_agent/sandbox.py index 0550b331e09..060b48ff298 100644 --- a/responses_api_agents/hermes_agent/sandbox.py +++ b/responses_api_agents/hermes_agent/sandbox.py @@ -173,7 +173,6 @@ async def stage_activation(self, payload: dict[str, JsonValue]) -> SandboxComman f"{self.session.session_dir}/input.json", f"{self.session.session_dir}/output.json", ], - python=_SANDBOX_PYTHON, ) async def collect_artifacts(self) -> dict[str, JsonValue]: diff --git a/responses_api_agents/hermes_sandboxed_agent/app.py b/responses_api_agents/hermes_sandboxed_agent/app.py index 2d414a0199f..dcea119a4c1 100644 --- a/responses_api_agents/hermes_sandboxed_agent/app.py +++ b/responses_api_agents/hermes_sandboxed_agent/app.py @@ -15,7 +15,12 @@ from pydantic import ConfigDict, Field from nemo_gym._checkpoint.steps import CHECKPOINT_CLOSE_FOLLOWS_HEADER -from nemo_gym.agent_utils import process_supervisor +from nemo_gym.agent_utils.supervisor_client import ( + DEFAULT_CLEANUP_TIMEOUT, + SUPERVISOR_FILE, + SUPERVISOR_SCRIPT, + exec_timeout, +) from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import BaseResponsesAPIAgentConfig, SimpleResponsesAPIAgent from nemo_gym.config_types import ModelServerRef, ResourcesServerRef @@ -226,16 +231,17 @@ async def _run_in_sandbox(self, sandbox, body, rollout_id): (local / "request.json").write_text(json.dumps(params)) await sandbox.upload(local / "request.json", f"{remote}/request.json") await sandbox.upload(Path(__file__).with_name("runner.py"), f"{remote}/runner.py") - await sandbox.upload(Path(process_supervisor.__file__), f"{remote}/process_supervisor.py") + await sandbox.upload(SUPERVISOR_SCRIPT, f"{remote}/{SUPERVISOR_FILE}") launched = True executed = await sandbox.exec( - f"{quote(self.config.runtime_python)} -I {quote(remote + '/process_supervisor.py')} " - f"--timeout {self.config.sandbox_timeout} --receipt {quote(remote + '/cleanup.json')} -- " + f"bash {quote(f'{remote}/{SUPERVISOR_FILE}')} " + f"--timeout {self.config.sandbox_timeout} --cleanup-timeout {DEFAULT_CLEANUP_TIMEOUT} " + f"--receipt {quote(remote + '/cleanup.json')} --stop-file {quote(remote + '/stop.request')} -- " f"{quote(self.config.runtime_python)} -I {quote(remote + '/runner.py')} {quote(remote + '/request.json')} " f">{quote(remote + '/output.log')} 2>&1", cwd=remote, # The runner owns the model budget; leave time for process cleanup. - timeout_s=process_supervisor.exec_timeout(timeout=self.config.sandbox_timeout), + timeout_s=exec_timeout(timeout=self.config.sandbox_timeout), ) return_code, error_type = executed.return_code, executed.error_type stdout, stderr = executed.stdout or "", executed.stderr or "" diff --git a/responses_api_agents/kilocode_agent/app.py b/responses_api_agents/kilocode_agent/app.py index b1f33070862..23f5bafa9f7 100644 --- a/responses_api_agents/kilocode_agent/app.py +++ b/responses_api_agents/kilocode_agent/app.py @@ -31,8 +31,8 @@ from openai.types.responses import ResponseError from pydantic import ConfigDict, Field, JsonValue, PrivateAttr -from nemo_gym.agent_utils.process_supervisor import CleanupReceipt from nemo_gym.agent_utils.sandbox_session import SandboxSession +from nemo_gym.agent_utils.supervisor_client import CleanupReceipt from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import ( AgentCloseSessionResponse, diff --git a/responses_api_agents/kilocode_agent/sandbox.py b/responses_api_agents/kilocode_agent/sandbox.py index 13f65d0f38a..57c6d539c1f 100644 --- a/responses_api_agents/kilocode_agent/sandbox.py +++ b/responses_api_agents/kilocode_agent/sandbox.py @@ -133,7 +133,6 @@ async def stage_activation(self, *, command: list[str], config: str) -> SandboxC await upload_text(self.session.sandbox, path=f"{directory}/input.json", text=json.dumps({"command": command})) return SandboxCommand( argv=["python3", "-I", f"{directory}/sandbox_runner.py", directory, self.session.workdir], - python="python3", ) async def collect_artifacts(self) -> KiloArtifacts: diff --git a/responses_api_agents/openclaw_agent/sandbox.py b/responses_api_agents/openclaw_agent/sandbox.py index 378cdc1d4fb..9fec3289ebf 100644 --- a/responses_api_agents/openclaw_agent/sandbox.py +++ b/responses_api_agents/openclaw_agent/sandbox.py @@ -103,7 +103,6 @@ async def stage_activation(self, payload: dict[str, JsonValue]) -> SandboxComman await self.upload_json("home/.openclaw/openclaw.json", payload["config"]) await self.upload_json("input.json", payload) return SandboxCommand( - python="python3", argv=[ "python3", "-I", diff --git a/responses_api_agents/opencode_agent/continuation.py b/responses_api_agents/opencode_agent/continuation.py index 7d949d706cc..ca714545f55 100644 --- a/responses_api_agents/opencode_agent/continuation.py +++ b/responses_api_agents/opencode_agent/continuation.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import Any, Literal -from nemo_gym.agent_utils.process_supervisor import CleanupReceipt +from nemo_gym.agent_utils.supervisor_client import CleanupReceipt @dataclass(frozen=True, kw_only=True) diff --git a/responses_api_agents/opencode_agent/sandbox.py b/responses_api_agents/opencode_agent/sandbox.py index cea5377da86..1772bbe4303 100644 --- a/responses_api_agents/opencode_agent/sandbox.py +++ b/responses_api_agents/opencode_agent/sandbox.py @@ -235,7 +235,6 @@ async def stage_activation(self, payload: dict[str, JsonValue]) -> SandboxComman ) await self.upload_json("input.json", payload) return SandboxCommand( - python=self.python_executable, argv=[ self.python_executable, "-I", diff --git a/responses_api_agents/pi_agent/sandbox.py b/responses_api_agents/pi_agent/sandbox.py index 3ef322409c0..78113925726 100644 --- a/responses_api_agents/pi_agent/sandbox.py +++ b/responses_api_agents/pi_agent/sandbox.py @@ -85,7 +85,6 @@ async def stage_activation(self, payload: dict[str, JsonValue]) -> SandboxComman """Stage the invocation and return the Pi worker command.""" await self.upload_json("input.json", payload) return SandboxCommand( - python="python3", argv=[ "python3", "-I", diff --git a/responses_api_agents/stirrup_agent/sandbox.py b/responses_api_agents/stirrup_agent/sandbox.py index 7d4b763e179..54600694edf 100644 --- a/responses_api_agents/stirrup_agent/sandbox.py +++ b/responses_api_agents/stirrup_agent/sandbox.py @@ -85,7 +85,6 @@ async def stage() -> SandboxCommand: await upload_text(self.session.sandbox, path=input_path, text=json.dumps(payload)) return SandboxCommand( argv=[SANDBOX_PYTHON, f"{_PACKAGE_DIR}/sandbox_runner.py", input_path, output_path], - python=SANDBOX_PYTHON, ) async def collect() -> dict[str, Any]: diff --git a/tests/unit_tests/test_local_provider.py b/tests/unit_tests/test_local_provider.py index f900e2706f7..c9f9b78fa97 100644 --- a/tests/unit_tests/test_local_provider.py +++ b/tests/unit_tests/test_local_provider.py @@ -25,7 +25,7 @@ import pytest -from nemo_gym.agent_utils import process_supervisor +from nemo_gym.agent_utils.supervisor_client import SUPERVISOR_SCRIPT, exec_timeout from nemo_gym.sandbox import AsyncSandbox, SandboxSpec, SandboxStatus, create_provider from nemo_gym.sandbox.providers.local import LocalProvider @@ -33,7 +33,7 @@ pytestmark = pytest.mark.sandbox -@pytest.mark.skipif(sys.platform != "linux", reason="Linux subreaper and /proc are required") +@pytest.mark.skipif(sys.platform != "linux", reason="Linux /proc is required") async def test_supervisor_deadline_finishes_before_provider_hard_timeout(tmp_path: Path) -> None: provider = LocalProvider() handle = await provider.create(SandboxSpec(workdir=str(tmp_path))) @@ -41,7 +41,7 @@ async def test_supervisor_deadline_finishes_before_provider_hard_timeout(tmp_pat worker.write_text( "import pathlib, signal, subprocess, sys, time\n" "signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" - "child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'], start_new_session=True)\n" + "child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])\n" "pathlib.Path('child.pid').write_text(str(child.pid))\n" "time.sleep(60)\n" ) @@ -49,15 +49,16 @@ async def test_supervisor_deadline_finishes_before_provider_hard_timeout(tmp_pat command = ( join( [ - sys.executable, - "-I", - process_supervisor.__file__, + "bash", + str(SUPERVISOR_SCRIPT), "--timeout", "1", "--cleanup-timeout", "0.5", "--receipt", str(receipt_path), + "--stop-file", + str(tmp_path / "stop.request"), "--", sys.executable, "-I", @@ -67,18 +68,16 @@ async def test_supervisor_deadline_finishes_before_provider_hard_timeout(tmp_pat + f" >{quote(str(tmp_path / 'output.log'))} 2>&1" ) try: - result = await provider.exec( - handle, command, timeout_s=process_supervisor.exec_timeout(timeout=1, cleanup_timeout=0.5) - ) + result = await provider.exec(handle, command, timeout_s=exec_timeout(timeout=1, cleanup_timeout=0.5)) assert result.error_type is None, result.stderr assert result.return_code == 0 receipt = loads(receipt_path.read_text()) assert receipt["cleanup_confirmed"] is True assert receipt["timed_out"] is True - assert receipt["return_code"] == -signal.SIGKILL + assert receipt["return_code"] == 128 + signal.SIGKILL assert receipt["error"] is None child_pid = int((tmp_path / "child.pid").read_text()) - assert not Path(f"/proc/{child_pid}").exists(), "in-flight detached tool survived" + assert not Path(f"/proc/{child_pid}").exists(), "in-flight tool survived" finally: if (tmp_path / "child.pid").exists(): try: diff --git a/tests/unit_tests/test_process_supervisor.py b/tests/unit_tests/test_process_supervisor.py index 03496b45291..2524d76975d 100644 --- a/tests/unit_tests/test_process_supervisor.py +++ b/tests/unit_tests/test_process_supervisor.py @@ -1,95 +1,150 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Exercise the uploaded script in an isolated interpreter, without Gym imports.""" +"""Exercise the uploaded bash supervisor as the sandbox runs it, without Gym imports.""" import json -import os -import runpy -import signal +import shutil import subprocess import sys import time from pathlib import Path -from types import SimpleNamespace import pytest +from nemo_gym.agent_utils.supervisor_client import SUPERVISOR_SCRIPT -SUPERVISOR = Path(__file__).parents[2] / "nemo_gym/agent_utils/process_supervisor.py" +SUPERVISOR = str(SUPERVISOR_SCRIPT) +linux_only = pytest.mark.skipif(sys.platform != "linux", reason="Linux /proc is required") -@pytest.mark.parametrize("harness", ["pi", "codex", "opencode", "openclaw"]) -def test_capture_error_stops_direct_cli_before_wrapper_exits(tmp_path: Path, harness: str) -> None: - runner = SUPERVISOR.parents[2] / f"responses_api_agents/{harness}_agent/sandbox_runner.py" - driver = """ -import json, os, pathlib, runpy, subprocess, sys -module = runpy.run_path(sys.argv[1]) -children = [] -class InterruptedWait(subprocess.Popen): - interrupted = False - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - children.append(self) - def wait(self, *args, **kwargs): - if not self.interrupted: - self.interrupted = True - raise OSError('capture interrupted') - return super().wait(*args, **kwargs) -subprocess.Popen = InterruptedWait -root = sys.argv[2] -try: - try: - module['run']({'directory': root, 'cwd': root, 'env': {}, 'prompt': 'task', - 'command': [sys.executable, '-c', 'import os,time; os.close(1); time.sleep(60)']}) - except OSError as error: - assert str(error) == 'capture interrupted', error - else: - raise AssertionError('capture failure did not propagate') - assert children and all(child.poll() is not None for child in children), 'CLI is still running' -finally: - for child in children: - if child.poll() is None: - child.kill() - child.wait() -""" - completed = subprocess.run( - [sys.executable, "-I", "-c", driver, str(runner), str(tmp_path)], + +def _supervise(tmp_path: Path, *command: str, timeout: str = "10", **kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + "bash", + SUPERVISOR, + "--timeout", + timeout, + "--cleanup-timeout", + "0.5", + "--receipt", + str(tmp_path / "cleanup.json"), + ] + + ["--stop-file", str(tmp_path / "stop.request"), "--", *command], capture_output=True, text=True, errors="replace", - timeout=10, + timeout=20, + **kwargs, ) - assert completed.returncode == 0, completed.stderr -@pytest.mark.parametrize("timeout, cleanup", [(1, 0.1), (2700, 10), (21600, 100)]) -def test_exec_timeout_reserves_all_cleanup_phases(timeout: float, cleanup: float) -> None: - supervisor = runpy.run_path(str(SUPERVISOR)) - assert supervisor["exec_timeout"](timeout=timeout, cleanup_timeout=cleanup) == ( - timeout + supervisor["CLEANUP_PHASE_COUNT"] * cleanup + 30 - ) +def _receipt(tmp_path: Path) -> dict: + return json.loads((tmp_path / "cleanup.json").read_text()) -@pytest.mark.parametrize("timeout", ["0", "-1", "nan", "inf"]) +@pytest.mark.parametrize("timeout", ["0", ".", "0.000", "-1", "nan", "inf", "1e3", ""]) def test_supervisor_rejects_invalid_deadlines(tmp_path: Path, timeout: str) -> None: + result = _supervise(tmp_path, "true", timeout=timeout) + assert result.returncode == 2 + assert "finite and positive" in result.stderr + assert not (tmp_path / "cleanup.json").exists() + + +def test_supervisor_requires_a_command_before_writing_a_receipt(tmp_path: Path) -> None: + result = _supervise(tmp_path) + assert result.returncode == 2 + assert "usage:" in result.stderr + assert not (tmp_path / "cleanup.json").exists() + + +@linux_only +@pytest.mark.parametrize("command, return_code", [("exit 0", 0), ("exit 7", 7), ("kill -KILL $$", 137)]) +def test_natural_exit_reports_the_harness_status(tmp_path: Path, command: str, return_code: int) -> None: + result = _supervise(tmp_path, "bash", "-c", command) + assert result.returncode == 0, result.stderr + assert _receipt(tmp_path) == { + "cleanup_confirmed": True, + "return_code": return_code, + "error": None, + "timed_out": False, + } + + +@linux_only +def test_runs_with_only_bash_and_core_utilities_on_path(tmp_path: Path) -> None: + # Task images need not ship Python: the supervisor itself uses only these programs. + tools = tmp_path / "bin" + tools.mkdir() + for name in ("bash", "sleep", "mv", "sh"): + (tools / name).symlink_to(shutil.which(name)) + work = tmp_path / "work" + work.mkdir() + result = subprocess.run( + [str(tools / "bash"), SUPERVISOR, "--timeout", "0.3", "--cleanup-timeout", "0.3"] + + ["--receipt", str(work / "cleanup.json"), "--stop-file", str(work / "stop"), "--", "sh", "-c", "sleep 30"], + capture_output=True, + text=True, + env={"PATH": str(tools)}, + timeout=20, + ) + assert result.returncode == 0, result.stderr + receipt = _receipt(work) + assert receipt["cleanup_confirmed"] is True and receipt["timed_out"] is True, receipt + + +@linux_only +def test_existing_stop_file_fences_the_launch(tmp_path: Path) -> None: + stop = tmp_path / "stop.request" + stop.touch() result = subprocess.run( - [sys.executable, "-I", str(SUPERVISOR), "--timeout", timeout, "--receipt", str(tmp_path / "cleanup.json")], + ["bash", SUPERVISOR, "--timeout", "10", "--cleanup-timeout", "1", "--receipt", str(tmp_path / "cleanup.json")] + + ["--stop-file", str(stop), "--", "touch", str(tmp_path / "started")], capture_output=True, text=True, timeout=10, ) - assert result.returncode == 2 - assert "finite and positive" in result.stderr - assert not (tmp_path / "cleanup.json").exists() + assert result.returncode == 0, result.stderr + assert _receipt(tmp_path) == {"cleanup_confirmed": True, "return_code": None, "error": None, "timed_out": False} + assert not (tmp_path / "started").exists() + + +@linux_only +def test_stop_file_stops_the_harness_with_term_first(tmp_path: Path) -> None: + # The controller requests a stop with this file rather than a signal. + stop = tmp_path / "stop.request" + worker = "trap 'echo saved > checkpoint; exit 0' TERM; touch ready; while :; do sleep 0.05; done" + process = subprocess.Popen( + ["bash", SUPERVISOR, "--timeout", "30", "--cleanup-timeout", "2", "--receipt", str(tmp_path / "cleanup.json")] + + ["--stop-file", str(stop), "--", "bash", "-c", worker], + cwd=tmp_path, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + deadline = time.monotonic() + 10 + while not (tmp_path / "ready").exists(): + assert process.poll() is None and time.monotonic() < deadline + time.sleep(0.01) + stop.touch() + _, stderr = process.communicate(timeout=10) + assert process.returncode == 0, stderr + receipt = _receipt(tmp_path) + assert receipt == {"cleanup_confirmed": True, "return_code": 0, "error": None, "timed_out": False} + assert (tmp_path / "checkpoint").read_text() == "saved\n" + finally: + if process.poll() is None: + process.kill() + process.wait() -@pytest.mark.skipif(sys.platform != "linux", reason="Linux owner subreaper and /proc are required") +@linux_only @pytest.mark.parametrize("ending", ["normal", "crash", "timeout", "cancel", "grace", "stubborn", "killed"]) @pytest.mark.parametrize("detached", [False, True]) def test_only_detached_services_survive_harness_group_termination(tmp_path: Path, ending: str, detached: bool) -> None: # The isolated parent emulates the sandbox owner and reaps every test process. - # A completed tool leaves a service adopted before interruption. Only + # A completed tool leaves a service orphaned before interruption. Only # explicitly detached services survive normal and interrupted exits. driver = r""" import ctypes, json, os, pathlib, signal, subprocess, sys, time, urllib.request @@ -124,19 +179,14 @@ def test_only_detached_services_survive_harness_group_termination(tmp_path: Path " assert time.monotonic() < deadline\n" " time.sleep(0.01)\n" "pathlib.Path('ready').touch()\n" - "signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGTERM})\n" "if ending == 'crash': raise SystemExit(7)\n" "if ending == 'killed': os.kill(os.getpid(), signal.SIGKILL)\n" "if ending != 'normal': time.sleep(60)\n") -launcher = [sys.executable, '-I'] timed_out = ending in ('timeout', 'grace', 'stubborn') -if timed_out: - launcher += ['-c', "import os, signal, sys; signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGTERM});" - " os.execv(sys.executable, [sys.executable, '-I', *sys.argv[1:]])"] process = None try: - process = subprocess.Popen([*launcher, supervisor, '--timeout', '0.8' if timed_out else '10', - '--cleanup-timeout', '0.5', '--receipt', str(root / 'cleanup.json'), '--', + process = subprocess.Popen(['bash', supervisor, '--timeout', '1.5' if timed_out else '10', + '--cleanup-timeout', '0.5', '--receipt', str(root / 'cleanup.json'), '--stop-file', str(root / 'stop'), '--', sys.executable, str(worker), ending], cwd=root, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if ending == 'cancel': deadline = time.monotonic() + 5 @@ -149,7 +199,7 @@ def test_only_detached_services_survive_harness_group_termination(tmp_path: Path receipt = json.loads((root / 'cleanup.json').read_text()) assert receipt['cleanup_confirmed'] is True, receipt assert receipt['timed_out'] is timed_out, receipt - expected = {'normal': 0, 'crash': 7, 'grace': 0, 'stubborn': -signal.SIGKILL, 'killed': -signal.SIGKILL}.get(ending, 0) + expected = {'crash': 7, 'stubborn': 137, 'killed': 137}.get(ending, 0) assert receipt['return_code'] == expected, receipt assert receipt['error'] is None, receipt if ending == 'grace': @@ -159,13 +209,17 @@ def test_only_detached_services_survive_harness_group_termination(tmp_path: Path except ProcessLookupError: pass else: - raise AssertionError('supervised process group still present, including zombies') + stats = [p.read_text() for p in pathlib.Path('/proc').glob('[0-9]*/stat')] + group = int((root / 'worker.pid').read_text()) + live = [s for s in stats if int(s.rsplit(')', 1)[1].split()[2]) == group and s.rsplit(')', 1)[1].split()[0] not in 'ZX'] + assert not live, ('supervised process group still running', live) port = int((root / 'port').read_text()) if detached: with urllib.request.urlopen('http://127.0.0.1:%d/' % port, timeout=2) as response: assert response.read() == b'task solved' else: - assert not pathlib.Path('/proc/' + (root / 'service.pid').read_text()).exists() + stat = pathlib.Path('/proc/' + (root / 'service.pid').read_text() + '/stat') + assert not stat.exists() or stat.read_text().rsplit(')', 1)[1].split()[0] in 'ZX' try: urllib.request.urlopen('http://127.0.0.1:%d/' % port, timeout=0.2) except OSError: @@ -186,54 +240,42 @@ def test_only_detached_services_survive_harness_group_termination(tmp_path: Path os.waitpid(-1, 0) except ChildProcessError: pass -if (root / 'service.pid').exists(): - try: - os.kill(int((root / 'service.pid').read_text()), 0) - except ProcessLookupError: - pass - else: - raise AssertionError('Sandbox teardown left the task service alive') """ result = subprocess.run( - [sys.executable, "-I", "-c", driver, str(SUPERVISOR), str(tmp_path), ending, str(detached)], + [sys.executable, "-I", "-c", driver, SUPERVISOR, str(tmp_path), ending, str(detached)], capture_output=True, text=True, errors="replace", - timeout=25, + timeout=40, ) assert result.returncode == 0, result.stderr -@pytest.mark.skipif(sys.platform != "linux", reason="Linux supervisor contract") +@linux_only @pytest.mark.parametrize("cooperative", [True, False]) -@pytest.mark.parametrize("detached", [False, True]) @pytest.mark.parametrize("ending", ["timeout", "cancel"]) def test_foreground_tool_cannot_mutate_task_after_confirmed_receipt( - tmp_path: Path, cooperative: bool, detached: bool, ending: str + tmp_path: Path, cooperative: bool, ending: str ) -> None: - runner = SUPERVISOR.parents[2] / "responses_api_agents/pi_agent/sandbox_runner.py" # A separate owner reaps any escaped process even when this regression fails. driver = r""" import ctypes, json, os, pathlib, signal, subprocess, sys, time assert ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) == 0 -supervisor, runner, root, cooperative = sys.argv[1], sys.argv[2], pathlib.Path(sys.argv[3]), sys.argv[4] == 'True' -detached, ending = sys.argv[5] == 'True', sys.argv[6] +supervisor, root, cooperative, ending = sys.argv[1], pathlib.Path(sys.argv[2]), sys.argv[3] == 'True', sys.argv[4] writer = "import pathlib,time\nwhile not pathlib.Path('release').exists(): time.sleep(.01)\npathlib.Path('mutated').touch()" -# A second detached generation exercises repeated adoption after the tool dies. +# The writer is orphaned when the tool dies, but stays in the harness group. tool = ("import pathlib,subprocess,sys,time\n" - "child=subprocess.Popen([sys.executable,'-c'," + repr(writer) + "],start_new_session=True)\n" + "child=subprocess.Popen([sys.executable,'-c'," + repr(writer) + "])\n" "pathlib.Path('writer.pid').write_text(str(child.pid))\nchild.wait()\n") cli = ("import pathlib,signal,subprocess,sys\n" + ("" if cooperative else "signal.signal(signal.SIGTERM, signal.SIG_IGN)\n") - + "child=subprocess.Popen([sys.executable,'-c'," + repr(tool) + "], start_new_session=" + str(detached) + ")\n" + + "child=subprocess.Popen([sys.executable,'-c'," + repr(tool) + "])\n" + "pathlib.Path('tool.pid').write_text(str(child.pid))\nchild.wait()\n") -params = {'directory': str(root), 'cwd': str(root), 'env': {}, 'prompt': 'task', - 'command': [sys.executable, '-c', cli]} -(root / 'input.json').write_text(json.dumps(params)) process = None try: - process = subprocess.Popen([sys.executable, '-I', supervisor, '--timeout', '1' if ending == 'timeout' else '10', '--cleanup-timeout', '.3', - '--receipt', str(root / 'cleanup.json'), '--', sys.executable, '-I', runner, str(root / 'input.json')], + process = subprocess.Popen(['bash', supervisor, '--timeout', '3' if ending == 'timeout' else '10', + '--cleanup-timeout', '.3', '--receipt', str(root / 'cleanup.json'), '--stop-file', str(root / 'stop'), + '--', sys.executable, '-c', cli], cwd=root, stdout=subprocess.PIPE, stderr=subprocess.PIPE) deadline = time.monotonic() + 5 while not (root / 'writer.pid').exists(): @@ -244,12 +286,12 @@ def test_foreground_tool_cannot_mutate_task_after_confirmed_receipt( assert process.returncode == 0, (stdout, stderr) receipt = json.loads((root / 'cleanup.json').read_text()) assert receipt['cleanup_confirmed'] and receipt['timed_out'] == (ending == 'timeout'), receipt - assert (root / 'tool.pid').exists(), 'foreground tool never started' (root / 'release').touch() time.sleep(.2) assert not (root / 'mutated').exists(), 'foreground tool wrote after termination receipt' - assert not pathlib.Path('/proc/' + (root / 'tool.pid').read_text()).exists(), 'tool was not reaped' - assert not pathlib.Path('/proc/' + (root / 'writer.pid').read_text()).exists(), 'writer was not reaped' + for name in ('tool.pid', 'writer.pid'): + stat = pathlib.Path('/proc/' + (root / name).read_text() + '/stat') + assert not stat.exists() or stat.read_text().rsplit(')', 1)[1].split()[0] in 'ZX', name + ' still runs' finally: if process is not None and process.poll() is None: process.kill(); process.wait(timeout=5) @@ -261,407 +303,10 @@ def test_foreground_tool_cannot_mutate_task_after_confirmed_receipt( except ChildProcessError: pass """ result = subprocess.run( - [ - sys.executable, - "-I", - "-c", - driver, - str(SUPERVISOR), - str(runner), - str(tmp_path), - str(cooperative), - str(detached), - ending, - ], + [sys.executable, "-I", "-c", driver, SUPERVISOR, str(tmp_path), str(cooperative), ending], capture_output=True, text=True, errors="replace", - timeout=15, + timeout=30, ) assert result.returncode == 0, result.stderr - - -@pytest.mark.skipif(sys.platform != "linux", reason="Linux supervisor contract") -def test_missing_binary_preserves_launch_error(tmp_path: Path) -> None: - result = subprocess.run( - [ - sys.executable, - "-I", - str(SUPERVISOR), - "--timeout", - "1", - "--receipt", - str(tmp_path / "cleanup.json"), - "--", - str(tmp_path / "missing-worker"), - ], - capture_output=True, - text=True, - errors="replace", - timeout=5, - ) - receipt = json.loads((tmp_path / "cleanup.json").read_text()) - assert result.returncode == 1 - assert receipt["cleanup_confirmed"] is False - assert receipt["return_code"] is None - assert "Harness launch failed" in receipt["error"] and "missing-worker" in receipt["error"] - - -@pytest.mark.skipif(sys.platform != "linux", reason="Linux supervisor contract") -def test_provider_group_kill_leaves_supervisor_alive_to_reap_worker(tmp_path: Path) -> None: - # Keep the shell as the provider group leader; the supervisor must leave that group. - process = subprocess.Popen( - [ - "sh", - "-c", - '"$@" & wait', - "provider", - sys.executable, - "-I", - str(SUPERVISOR), - # A deadline that fired before the group kill would skip the case under test. - # The test stops the supervisor itself once the group is gone. - "--timeout", - "60", - "--cleanup-timeout", - "0.2", - "--receipt", - str(tmp_path / "cleanup.json"), - "--", - sys.executable, - "-c", - # Publish both PIDs by rename so the test never reads a created but still empty file. - "import json, os, pathlib, time; pathlib.Path('pids.tmp').write_text(json.dumps([os.getppid(), os.getpid()]));" - " os.replace('pids.tmp', 'pids.json'); time.sleep(60)", - ], - cwd=tmp_path, - start_new_session=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - pids = [] - try: - deadline = time.monotonic() + 30 - while not (tmp_path / "pids.json").exists() and time.monotonic() < deadline: - time.sleep(0.01) - # The worker exists, so the supervisor has already left the provider group. - pids = json.loads((tmp_path / "pids.json").read_text()) - supervisor_pid, worker_pid = pids - os.killpg(process.pid, signal.SIGKILL) - process.wait(timeout=10) - # SIGKILL takes effect before any later signal, so a supervisor killed with the group cannot handle this one. - os.kill(supervisor_pid, signal.SIGTERM) - process.communicate(timeout=10) - receipt = json.loads((tmp_path / "cleanup.json").read_text()) - assert receipt["timed_out"] is False - assert receipt["cleanup_confirmed"] is True - assert receipt["error"] is None - with pytest.raises(ProcessLookupError): - os.kill(worker_pid, 0) - finally: - if process.poll() is None: - process.kill() - process.wait(timeout=5) - for pid in reversed(pids): - try: - os.kill(pid, signal.SIGKILL) - except ProcessLookupError: - pass - - -@pytest.mark.skipif(sys.platform != "linux", reason="Linux supervisor contract") -def test_sigterm_during_spawn_does_not_lose_child_handle(tmp_path): - # Deliver SIGTERM after the real child exists but before Popen returns to _supervise(). - # Isolate signal handlers from pytest, and always reap the test child. - driver = """ -import json, os, runpy, signal, subprocess, sys -runner = runpy.run_path(sys.argv[1]) -spawn = subprocess.Popen -children = [] -def interrupted_spawn(*args, **kwargs): - child = spawn(*args, **kwargs) - children.append(child) - os.kill(os.getpid(), signal.SIGTERM) - return child -subprocess.Popen = interrupted_spawn -try: - summary = runner['_supervise']( - [sys.executable, '-c', 'import time; time.sleep(60)'], timeout=5, cleanup_timeout=0.2, - ) - summary['child_alive'] = any(child.poll() is None for child in children) - print(json.dumps(summary)) -finally: - for child in children: - if child.poll() is None: - child.kill() - child.wait() -""" - completed = subprocess.run( - [sys.executable, "-c", driver, str(SUPERVISOR), str(tmp_path)], - capture_output=True, - text=True, - errors="replace", - timeout=10, - check=True, - ) - summary = json.loads(completed.stdout) - assert summary["timed_out"] is False - assert summary["cleanup_confirmed"] is True - assert summary["error"] is None - assert summary["child_alive"] is False - - -@pytest.mark.skipif(sys.platform != "linux", reason="Linux supervisor contract") -def test_lost_spawn_handle_requires_sandbox_teardown(tmp_path): - # A lost handle cannot confirm termination; the owner must destroy the sandbox. - driver = """ -import json, os, runpy, subprocess, sys -runner = runpy.run_path(sys.argv[1]) -spawn = subprocess.Popen -children = [] -def lost_handle(*args, **kwargs): - child = spawn(*args, **kwargs) - children.append(child) - raise OSError('launch handle lost after process creation') -subprocess.Popen = lost_handle -try: - summary = runner['_supervise']( - [sys.executable, '-c', 'import time; time.sleep(60)'], timeout=5, cleanup_timeout=0.2, - ) - try: - os.kill(children[0].pid, 0) - summary['child_alive'] = True - except ProcessLookupError: - summary['child_alive'] = False - print(json.dumps(summary)) -finally: - for child in children: - if child.poll() is None: - child.kill() - child.wait() -""" - completed = subprocess.run( - [sys.executable, "-c", driver, str(SUPERVISOR), str(tmp_path)], - capture_output=True, - text=True, - errors="replace", - timeout=10, - check=True, - ) - summary = json.loads(completed.stdout) - assert summary["return_code"] is None - assert summary["error"] == "Harness launch failed: launch handle lost after process creation" - assert summary["cleanup_confirmed"] is False - assert summary["child_alive"] is True - - -@pytest.mark.skipif(sys.platform != "linux", reason="Linux /proc zombie state") -def test_zombie_group_owned_by_other_parent_is_terminated(tmp_path: Path) -> None: - # Keep our own child unreaped. The helper cannot reap another parent's zombie, - # but it must not confuse that zombie with a process still able to write. - child = subprocess.Popen([sys.executable, "-I", "-c", "pass"], start_new_session=True) - try: - deadline = time.monotonic() + 5 - stat = Path(f"/proc/{child.pid}/stat") - while stat.read_text().rsplit(")", 1)[1].split()[0] != "Z": - assert time.monotonic() < deadline - time.sleep(0.01) - os.killpg(child.pid, 0) - completed = subprocess.run( - [ - sys.executable, - "-I", - "-c", - "import runpy,sys; runpy.run_path(sys.argv[1])['_reap_process_group'](int(sys.argv[2]), 0.1)", - str(SUPERVISOR), - str(child.pid), - ], - capture_output=True, - text=True, - errors="replace", - timeout=5, - ) - assert completed.returncode == 0, completed.stderr - assert stat.read_text().rsplit(")", 1)[1].split()[0] == "Z" - finally: - child.wait(timeout=5) - - -def test_unreadable_group_state_does_not_confirm_termination(monkeypatch: pytest.MonkeyPatch) -> None: - supervisor = runpy.run_path(str(SUPERVISOR)) - monkeypatch.setattr(os, "killpg", lambda *_: None) - monkeypatch.setattr(os, "waitpid", lambda *_: (0, 0)) - monkeypatch.setattr(Path, "iterdir", lambda _: iter([Path("/proc/123")])) - - def unreadable(*_: object) -> str: - raise PermissionError("Cannot read process state") - - monkeypatch.setattr(Path, "read_text", unreadable) - with pytest.raises(PermissionError, match="Cannot read process state"): - supervisor["_reap_process_group"](123, 0.1) - - -@pytest.mark.parametrize("state, alive", [("Z", False), ("X", False), ("S", True)]) -def test_group_scan_ignores_vanished_and_unrelated_processes( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, state: str, alive: bool -) -> None: - scan = runpy.run_path(str(SUPERVISOR))["_group_has_live_processes"] - # A process may disappear between listing /proc and reading its stat file. - for name in ("self", "100", "101", "102"): - (tmp_path / name).mkdir() - (tmp_path / "101/stat").write_text("101 (unrelated) R 1 999 0") - (tmp_path / "102/stat").write_text(f"102 (name with ) parentheses) {state} 1 123 0") - monkeypatch.setitem(scan.__globals__, "Path", lambda _: tmp_path) - - assert scan(123) is alive - - -def test_forced_cleanup_drains_successive_adoptions_but_preserves_existing_services( - monkeypatch: pytest.MonkeyPatch, -) -> None: - reap = runpy.run_path(str(SUPERVISOR))["_reap_process_group"] - children = {101, 202, 204} - killed: list[int] = [] - reaped: list[int] = [] - - def absent_group(*_: object) -> None: - raise ProcessLookupError - - def kill(pid: int, sig: int) -> None: - assert sig == signal.SIGKILL - killed.append(pid) - if pid == 204: - children.remove(pid) - raise ProcessLookupError # Exited before the signal arrived. - - def waitpid(pid: int, options: int) -> tuple[int, int]: - assert options == os.WNOHANG - if pid == -123: - raise ChildProcessError - children.remove(pid) - reaped.append(pid) - if pid == 202: - children.add(303) # Killing this tool adopts another detached generation. - return pid, 0 - - monkeypatch.setitem( - reap.__globals__, "os", SimpleNamespace(killpg=absent_group, kill=kill, waitpid=waitpid, WNOHANG=os.WNOHANG) - ) - monkeypatch.setitem(reap.__globals__, "_child_pids", lambda: set(children)) - - reap(123, 1, preserved_children={101}) - - assert set(killed) == {202, 204, 303} - assert reaped == [202, 303] - assert children == {101} - - -@pytest.mark.parametrize("remaining", ["live_group", "unreaped_group", "adopted_tool"]) -def test_cleanup_deadline_rejects_each_kind_of_unfinished_process( - monkeypatch: pytest.MonkeyPatch, remaining: str -) -> None: - reap = runpy.run_path(str(SUPERVISOR))["_reap_process_group"] - clock = iter([0.0, 0.5, 1.0]) - sleeps: list[float] = [] - - def killpg(*_: object) -> None: - if remaining != "live_group": - raise ProcessLookupError - - def waitpid(pid: int, _: int) -> tuple[int, int]: - if pid == -123 and remaining != "unreaped_group": - raise ChildProcessError - return 0, 0 # Still present after a nonblocking reap attempt. - - monkeypatch.setitem( - reap.__globals__, - "os", - SimpleNamespace(killpg=killpg, kill=lambda *_: None, waitpid=waitpid, WNOHANG=os.WNOHANG), - ) - monkeypatch.setitem(reap.__globals__, "time", SimpleNamespace(monotonic=lambda: next(clock), sleep=sleeps.append)) - monkeypatch.setitem(reap.__globals__, "_group_has_live_processes", lambda _: remaining == "live_group") - monkeypatch.setitem(reap.__globals__, "_child_pids", lambda: {101, 202} if remaining == "adopted_tool" else {101}) - - with pytest.raises(TimeoutError, match="have not terminated"): - reap(123, 1, preserved_children={101}) - assert sleeps == [0.01] - - -@pytest.mark.parametrize("boundary", ["stop_marker", "unsupported_platform", "subreaper_failure"]) -def test_supervisor_does_not_launch_without_startup_ownership( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, boundary: str -) -> None: - supervise = runpy.run_path(str(SUPERVISOR))["_supervise"] - launched: list[list[str]] = [] - stop = tmp_path / "stop" - if boundary == "stop_marker": - stop.touch() - monkeypatch.setitem( - supervise.__globals__, "signal", SimpleNamespace(signal=lambda *_: None, SIGTERM=15, SIG_IGN=1) - ) - monkeypatch.setitem( - supervise.__globals__, - "sys", - SimpleNamespace(platform="darwin" if boundary == "unsupported_platform" else "linux"), - ) - monkeypatch.setitem(supervise.__globals__, "os", SimpleNamespace(getpgrp=lambda: 1, getpid=lambda: 1)) - monkeypatch.setitem( - supervise.__globals__, - "ctypes", - SimpleNamespace( - CDLL=lambda *_, **__: SimpleNamespace(prctl=lambda *_: -1 if boundary == "subreaper_failure" else 0), - get_errno=lambda: 1, - ), - ) - monkeypatch.setitem( - supervise.__globals__, "subprocess", SimpleNamespace(Popen=lambda command, **_: launched.append(command)) - ) - - receipt = supervise(["worker"], timeout=1, stop_path=stop) - - assert launched == [] - assert receipt["cleanup_confirmed"] is True # No process was started. - assert receipt["return_code"] is None - assert receipt["timed_out"] is False - if boundary == "stop_marker": - assert receipt["error"] is None - elif boundary == "unsupported_platform": - assert receipt["error"] == "Sandbox process supervision requires Linux" - else: - assert "Cannot supervise harness process group" in receipt["error"] - - -def test_supervisor_requires_a_command_before_writing_a_receipt( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - main = runpy.run_path(str(SUPERVISOR))["main"] - receipt = tmp_path / "cleanup.json" - monkeypatch.setattr(sys, "argv", [str(SUPERVISOR), "--timeout", "1", "--receipt", str(receipt), "--"]) - - with pytest.raises(SystemExit) as error: - main() - assert error.value.code == 2 - assert "a harness command is required" in capsys.readouterr().err - assert not receipt.exists() - - -@pytest.mark.skipif(sys.platform != "linux", reason="Linux supervisor contract") -def test_missing_pre_stop_snapshot_does_not_confirm_termination(tmp_path: Path) -> None: - driver = """ -import json,runpy,sys -supervise = runpy.run_path(sys.argv[1])['_supervise'] -def unavailable(): - raise OSError('children unavailable') -supervise.__globals__['_child_pids'] = unavailable -receipt = supervise([sys.executable, '-c', 'import time; time.sleep(60)'], timeout=.1, cleanup_timeout=.1) -assert receipt['cleanup_confirmed'] is False, receipt -assert 'Cannot identify pre-stop adopted children' in receipt['error'], receipt -""" - completed = subprocess.run( - [sys.executable, "-I", "-c", driver, str(SUPERVISOR)], - capture_output=True, - text=True, - errors="replace", - timeout=5, - ) - assert completed.returncode == 0, completed.stderr diff --git a/tests/unit_tests/test_sandbox_session.py b/tests/unit_tests/test_sandbox_session.py index be69612776b..f4406543d5c 100644 --- a/tests/unit_tests/test_sandbox_session.py +++ b/tests/unit_tests/test_sandbox_session.py @@ -3,12 +3,14 @@ import asyncio import json +import shutil +import subprocess from unittest.mock import AsyncMock import pytest from nemo_gym.agent_utils import supervisor_client -from nemo_gym.agent_utils.sandbox_session import SandboxCommand, SandboxSession +from nemo_gym.agent_utils.sandbox_session import SandboxCommand, SandboxSession, shell_assignments from nemo_gym.sandbox import AsyncSandbox, SandboxExecResult @@ -32,8 +34,7 @@ def session(receipt_reader): async def execute(session, *, collect=None, stage_activation=None, timeout=1): return await session.execute( - stage_activation=stage_activation - or AsyncMock(return_value=SandboxCommand(argv=["worker"], python="/runtime/python")), + stage_activation=stage_activation or AsyncMock(return_value=SandboxCommand(argv=["worker"])), collect=collect or AsyncMock(return_value="transcript"), timeout=10, close_timeout=timeout, @@ -73,7 +74,7 @@ async def test_interruption_captures_before_transport_cancellation_and_release(s events = [] async def provider_exec(command, **kwargs): - if "trap '' TERM" in command: + if "exec bash " in command: launched.set() try: await asyncio.Future() @@ -192,7 +193,7 @@ async def test_rearm_uses_fresh_launch_files_and_close_removes_the_root(session) first.assert_awaited_once() second.assert_awaited_once() uploads = [c.args[1] for c in session.sandbox.upload.await_args_list] - assert uploads == ["/session/activation-1/process_supervisor.py", "/session/activation-2/process_supervisor.py"] + assert uploads == ["/session/activation-1/process_supervisor.sh", "/session/activation-2/process_supervisor.sh"] await session.close(timeout=1) assert "mv /session /session.closed" in session.sandbox.exec.await_args.args[0] with pytest.raises(RuntimeError, match="closing"): @@ -439,9 +440,9 @@ async def test_nonzero_worker_exit_still_passes_artifacts_to_adapter(session, re async def test_bootstrap_failure_preserves_stderr_and_still_releases(session, receipt_reader): receipt_reader.return_value = json.dumps(RECEIPT | {"return_code": None}) - session.sandbox.exec.return_value = SandboxExecResult("", "/runtime/python: not found", 127) + session.sandbox.exec.return_value = SandboxExecResult("", "sh: 1: bash: not found", 127) collector = AsyncMock(side_effect=FileNotFoundError("no worker output")) - with pytest.raises(RuntimeError, match="sandbox execution failed.*127.*python: not found") as error: + with pytest.raises(RuntimeError, match="sandbox execution failed.*127.*bash: not found") as error: await execute(session, collect=collector) assert isinstance(error.value.__cause__, FileNotFoundError) assert session.cleanup["cleanup_confirmed"] is True @@ -456,13 +457,12 @@ async def test_bootstrap_failure_preserves_stderr_and_still_releases(session, re async def test_session_uploads_supervisor_before_launch(session): from pathlib import Path - from nemo_gym.agent_utils import process_supervisor - from nemo_gym.agent_utils.supervisor_client import SUPERVISOR_FILE + from nemo_gym.agent_utils.supervisor_client import SUPERVISOR_FILE, SUPERVISOR_SCRIPT events = [] async def upload(source, destination): - assert Path(source).read_bytes() == Path(process_supervisor.__file__).read_bytes() + assert Path(source).read_bytes() == SUPERVISOR_SCRIPT.read_bytes() assert destination == f"/session/{SUPERVISOR_FILE}" events.append("upload") @@ -580,3 +580,27 @@ async def collect(): assert session.cleanup == RECEIPT await session.close(timeout=1) assert session.closed + + +@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is required") +def test_shell_assignments_round_trip_through_bash(tmp_path): + tricky = 'it\'s $(touch unintended) `x` "q" \\ \n new line' + script = tmp_path / "params.sh" + script.write_text( + shell_assignments({"cwd": tricky, "command": ["cli", tricky, ""], "env": {"KEY": tricky, "EMPTY": ""}}) + ) + printed = subprocess.run( + ["bash", "-c", 'source "$1"; printf "%s\\0" "$cwd" "${#command[@]}" "${command[@]}" "${env[@]}"', "_", script], + capture_output=True, + text=True, + check=True, + cwd=tmp_path, + ).stdout.split("\0")[:-1] + assert printed == [tricky, "3", "cli", tricky, "", f"KEY={tricky}", "EMPTY="] + assert not (tmp_path / "unintended").exists() + + +@pytest.mark.parametrize("values", [{"bad-name": "x"}, {"env": {"BAD KEY": "x"}}]) +def test_shell_assignments_reject_names_bash_cannot_assign(values): + with pytest.raises(ValueError, match="Invalid"): + shell_assignments(values) diff --git a/tests/unit_tests/test_sandbox_supervisor_bootstrap.py b/tests/unit_tests/test_sandbox_supervisor_bootstrap.py index 015d48b79b8..0d6fcb05ee7 100644 --- a/tests/unit_tests/test_sandbox_supervisor_bootstrap.py +++ b/tests/unit_tests/test_sandbox_supervisor_bootstrap.py @@ -1,21 +1,21 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Failed interpreter/supervisor bootstrap must remain safe to close.""" +"""Failed supervisor bootstrap must remain safe to close.""" import asyncio -import json +import os import shutil -import sys from pathlib import Path +from shlex import quote import pytest -from nemo_gym.agent_utils import process_supervisor from nemo_gym.agent_utils.supervisor_client import ( CLEANUP_RECEIPT_FILE, LAUNCH_CLAIM_FILE, SUPERVISOR_FILE, SUPERVISOR_PID_FILE, + SUPERVISOR_SCRIPT, remove_session_directory, stop_and_confirm_cleanup, supervised_launch_command, @@ -24,52 +24,59 @@ class LocalSandbox: + def __init__(self, path: str | None = None) -> None: + self.env = os.environ | ({"PATH": path} if path is not None else {}) + async def download(self, source: str, destination: Path) -> None: shutil.copyfile(source, destination) async def exec(self, command: str, *, cwd: str | None = None, timeout_s: float = 30) -> SandboxExecResult: process = await asyncio.create_subprocess_exec( - "sh", "-c", command, cwd=cwd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + shutil.which("sh"), + "-c", + command, + cwd=cwd, + env=self.env, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, ) stdout, stderr = await asyncio.wait_for(process.communicate(), timeout_s) return SandboxExecResult(stdout.decode(errors="replace"), stderr.decode(errors="replace"), process.returncode) -@pytest.mark.parametrize("failure", ["missing_python", "broken_python", "missing_supervisor", "broken_supervisor"]) +def _tools(tmp_path: Path, *names: str) -> str: + """A PATH holding only the named programs.""" + directory = tmp_path / "bin" + directory.mkdir() + for name in names: + (directory / name).symlink_to(shutil.which(name)) + return str(directory) + + +@pytest.mark.parametrize("failure", ["missing_bash", "missing_supervisor", "broken_supervisor"]) async def test_bootstrap_failure_preserves_diagnostics_and_allows_close(tmp_path: Path, failure: str) -> None: session_dir = tmp_path / "session" session_dir.mkdir() - python = sys.executable supervisor = session_dir / SUPERVISOR_FILE - shutil.copyfile(process_supervisor.__file__, supervisor) - expected_error = "" - if failure == "missing_python": - python = str(tmp_path / "missing-python") - expected_error = "missing-python" - elif failure == "broken_python": - interpreter = tmp_path / "incompatible-python" - interpreter.write_text("#!/bin/sh\nprintf '%s\\n' 'incompatible interpreter' >&2\nexit 17\n") - interpreter.chmod(0o755) - python = str(interpreter) - expected_error = "incompatible interpreter" + shutil.copyfile(SUPERVISOR_SCRIPT, supervisor) + sandbox = LocalSandbox() + if failure == "missing_bash": + sandbox = LocalSandbox(_tools(tmp_path, "touch")) + expected_error = "bash" elif failure == "missing_supervisor": supervisor.unlink() - expected_error = "process_supervisor.py" + expected_error = SUPERVISOR_FILE else: - supervisor.write_text("def invalid syntax\n") - expected_error = "SyntaxError" - sandbox = LocalSandbox() + supervisor.write_text("if then fi\n") + expected_error = "syntax error" command = supervised_launch_command( session_dir=str(session_dir), command=["touch", str(session_dir / "worker-started")], timeout=1, cleanup_timeout=1, - python=python, ) result = await sandbox.exec(command) assert result.return_code != 0 - if failure == "broken_python": - assert result.return_code == 17 assert expected_error in result.stderr assert not (session_dir / LAUNCH_CLAIM_FILE).is_symlink() assert not (session_dir / SUPERVISOR_PID_FILE).exists() @@ -77,7 +84,7 @@ async def test_bootstrap_failure_preserves_diagnostics_and_allows_close(tmp_path assert not (session_dir / CLEANUP_RECEIPT_FILE).exists() receipt = await stop_and_confirm_cleanup( - sandbox, session_dir=str(session_dir), workdir=None, timeout=1, harness="bootstrap-test" + LocalSandbox(), session_dir=str(session_dir), workdir=None, timeout=1, harness="bootstrap-test" ) assert receipt["cleanup_confirmed"] is True assert receipt["return_code"] is None @@ -85,19 +92,18 @@ async def test_bootstrap_failure_preserves_diagnostics_and_allows_close(tmp_path @pytest.mark.parametrize("removed", [False, True]) -async def test_fenced_delayed_launch_does_not_require_the_runtime(tmp_path: Path, removed: bool) -> None: +async def test_fenced_delayed_launch_does_not_require_bash(tmp_path: Path, removed: bool) -> None: session_dir = tmp_path / "session" session_dir.mkdir() (session_dir / LAUNCH_CLAIM_FILE).symlink_to("stop") if removed: shutil.rmtree(session_dir) - result = await LocalSandbox().exec( + result = await LocalSandbox(_tools(tmp_path, "touch")).exec( supervised_launch_command( session_dir=str(session_dir), command=["touch", str(tmp_path / "worker-started")], timeout=1, cleanup_timeout=1, - python=str(tmp_path / "missing-python"), ) ) assert result.return_code == 0 @@ -112,26 +118,21 @@ async def test_close_during_preflight_fences_later_launch(tmp_path: Path, remove preflight_started = tmp_path / "preflight-started" continue_preflight = tmp_path / "continue-preflight" worker_started = tmp_path / "worker-started" - supervisor = session_dir / SUPERVISOR_FILE - supervisor.write_text( - "import pathlib, sys, time\n" - "if '--help' in sys.argv:\n" - f" pathlib.Path({str(preflight_started)!r}).touch()\n" - f" while not pathlib.Path({str(continue_preflight)!r}).exists():\n" - " time.sleep(0.01)\n" - "else:\n" - f" pathlib.Path({str(worker_started)!r}).touch()\n" + shutil.copyfile(SUPERVISOR_SCRIPT, session_dir / SUPERVISOR_FILE) + # A bash that pauses in the syntax check stands in for a slow sandbox. + fake = tmp_path / "fake-bin" + fake.mkdir() + (fake / "bash").write_text( + "#!/bin/sh\n" + f'if [ "$1" = -n ]; then touch {str(preflight_started)!r}; ' + f"while [ ! -e {str(continue_preflight)!r} ]; do sleep 0.01; done; exit 0; fi\n" + f"touch {str(worker_started)!r}\n" ) - sandbox = LocalSandbox() + (fake / "bash").chmod(0o755) + sandbox = LocalSandbox(f"{fake}:{os.environ['PATH']}") launch = asyncio.create_task( sandbox.exec( - supervised_launch_command( - session_dir=str(session_dir), - command=["worker"], - timeout=1, - cleanup_timeout=1, - python=sys.executable, - ) + supervised_launch_command(session_dir=str(session_dir), command=["worker"], timeout=1, cleanup_timeout=1) ) ) try: @@ -154,31 +155,20 @@ async def test_close_during_preflight_fences_later_launch(tmp_path: Path, remove assert not worker_started.exists() -async def test_bootstrap_quotes_private_paths_and_uses_the_same_interpreter(tmp_path: Path) -> None: +async def test_bootstrap_quotes_private_paths(tmp_path: Path) -> None: session_dir = tmp_path / "session's $(touch unintended) files" session_dir.mkdir() - runtime = tmp_path / "runtime's $(touch unintended) files" - runtime.mkdir() - python = runtime / "python interpreter" - python.symlink_to(sys.executable) supervisor = session_dir / SUPERVISOR_FILE - arguments = session_dir / "arguments.json" - supervisor.write_text( - "import json, pathlib, sys\n" - "if '--help' not in sys.argv:\n" - f" pathlib.Path({str(arguments)!r}).write_text(json.dumps([sys.executable, *sys.argv[1:]]))\n" - ) + arguments = session_dir / "arguments" + supervisor.write_text(f'printf "%s\\n" "$@" > {quote(str(arguments))}\n') command = supervised_launch_command( session_dir=str(session_dir), command=["worker", "$(touch unintended)", "argument's value"], timeout=1, cleanup_timeout=1, - python=str(python), ) result = await LocalSandbox().exec(command, cwd=str(tmp_path)) assert result.return_code == 0, result.stderr - captured = json.loads(arguments.read_text()) - assert captured[0] == str(python) - assert captured[-3:] == ["worker", "$(touch unintended)", "argument's value"] + assert arguments.read_text().splitlines()[-3:] == ["worker", "$(touch unintended)", "argument's value"] assert not (tmp_path / "unintended").exists() assert (session_dir / LAUNCH_CLAIM_FILE).readlink() == Path("launch") diff --git a/tests/unit_tests/test_sandbox_supervisor_client.py b/tests/unit_tests/test_sandbox_supervisor_client.py index 8adadd1fd6f..c8427aa4c5d 100644 --- a/tests/unit_tests/test_sandbox_supervisor_client.py +++ b/tests/unit_tests/test_sandbox_supervisor_client.py @@ -7,19 +7,22 @@ import shutil import sys from pathlib import Path +from shlex import quote from unittest.mock import AsyncMock import pytest -from nemo_gym.agent_utils import process_supervisor from nemo_gym.agent_utils.supervisor_client import ( + CLEANUP_PHASE_COUNT, CLEANUP_RECEIPT_FILE, LAUNCH_CLAIM_FILE, OUTPUT_LOG_FILE, STOP_REQUEST_FILE, SUPERVISOR_FILE, SUPERVISOR_PID_FILE, + SUPERVISOR_SCRIPT, HarnessTerminationError, + exec_timeout, parse_cleanup_receipt, stop_and_confirm_cleanup, supervised_launch_command, @@ -120,7 +123,6 @@ async def test_stop_wins_claim_and_fences_delayed_launch(session, caplog): command=["touch", str(session_dir / "started")], timeout=1, cleanup_timeout=1, - python=sys.executable, ) assert (await sandbox.exec(command)).return_code == 0 assert not (session_dir / SUPERVISOR_PID_FILE).exists() @@ -142,17 +144,9 @@ async def test_missing_receipt_after_launch_is_not_cleanup_confirmation(session) @pytest.mark.skipif(sys.platform != "linux", reason="Linux supervisor contract") -@pytest.mark.parametrize("private_python", [False, True]) -async def test_launch_supervises_harness_with_selected_or_private_python(session, tmp_path, private_python): +async def test_launch_supervises_harness_and_shares_its_output_log(session, tmp_path): sandbox, session_dir = session - runtime = tmp_path / "runtime's files" - runtime.mkdir() - supervisor = session_dir / SUPERVISOR_FILE - shutil.copyfile(process_supervisor.__file__, supervisor) - python = Path(sys.executable) - if private_python: - python = runtime / "python interpreter" - python.symlink_to(sys.executable) + shutil.copyfile(SUPERVISOR_SCRIPT, session_dir / SUPERVISOR_FILE) command = supervised_launch_command( session_dir=str(session_dir), command=[ @@ -163,7 +157,6 @@ async def test_launch_supervises_harness_with_selected_or_private_python(session ], timeout=5, cleanup_timeout=1, - python=str(python), ) launched = await sandbox.exec(command) assert launched.return_code == 0, launched.stderr @@ -174,6 +167,30 @@ async def test_launch_supervises_harness_with_selected_or_private_python(session assert set((session_dir / OUTPUT_LOG_FILE).read_text().splitlines()) == {"$(literal)", "stderr"} +@pytest.mark.skipif(sys.platform != "linux", reason="Linux supervisor contract") +async def test_stop_reaches_running_supervisor_through_the_stop_file(session, tmp_path): + # The stop shell sends no signal, so the harness keeps the default TERM disposition and checkpoints. + sandbox, session_dir = session + shutil.copyfile(SUPERVISOR_SCRIPT, session_dir / SUPERVISOR_FILE) + ready = session_dir / "ready" + worker = f"trap 'echo saved > checkpoint; exit 0' TERM; touch {quote(str(ready))}; while :; do sleep 0.05; done" + command = supervised_launch_command( + session_dir=str(session_dir), command=["bash", "-c", worker], timeout=30, cleanup_timeout=2 + ) + launch = asyncio.create_task(sandbox.exec(command, cwd=str(session_dir))) + try: + async with asyncio.timeout(10): + while not ready.exists(): + await asyncio.sleep(0.01) + receipt = await stop_and_confirm_cleanup( + sandbox, session_dir=str(session_dir), workdir=str(session_dir), timeout=5, harness="test" + ) + finally: + await asyncio.wait_for(launch, timeout=10) + assert receipt == {"cleanup_confirmed": True, "error": None, "return_code": 0, "timed_out": False} + assert (session_dir / "checkpoint").read_text() == "saved\n" + + @pytest.mark.parametrize("confirmed", [False, "true", 1]) async def test_unconfirmed_cleanup_preserves_receipt_for_retry(session, confirmed): sandbox, session_dir = session @@ -230,9 +247,9 @@ async def stop(*args, **kwargs): @pytest.mark.parametrize("timeout,close_timeout", [(1, 0.3), (2700, 30), (21600, 300)]) def test_supervision_timeouts_reserve_cleanup_and_provider_margin(timeout, close_timeout): cleanup_timeout, provider_timeout = supervision_timeouts(timeout=timeout, close_timeout=close_timeout) - assert cleanup_timeout * process_supervisor.CLEANUP_PHASE_COUNT == pytest.approx(close_timeout) + assert cleanup_timeout * CLEANUP_PHASE_COUNT == pytest.approx(close_timeout) assert provider_timeout == pytest.approx(timeout + close_timeout + 30) - assert provider_timeout == process_supervisor.exec_timeout(timeout=timeout, cleanup_timeout=cleanup_timeout) + assert provider_timeout == exec_timeout(timeout=timeout, cleanup_timeout=cleanup_timeout) @pytest.mark.parametrize( From 30681cde7b733dc51485ea59f851c4f0405caa34 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Mon, 26 Oct 2026 08:55:44 -0700 Subject: [PATCH 02/32] feat(agent_utils): add a shared bash harness runner harness_runner.sh replaces per-harness Python runners: it sources the PARAMS written by shell_assignments(), records runtime.json, applies the harness environment and runs the CLI with file-backed stdio. With events set, it stamps each JSON stdout line with the time it was seen, reading a file so a detached tool holding stdout cannot stall it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Hemil Desai Signed-off-by: yaoyu-33 --- nemo_gym/agent_utils/harness_runner.sh | 54 ++++++++++++ nemo_gym/agent_utils/sandbox_session.py | 4 + tests/unit_tests/test_harness_runner.py | 112 ++++++++++++++++++++++++ 3 files changed, 170 insertions(+) create mode 100755 nemo_gym/agent_utils/harness_runner.sh create mode 100644 tests/unit_tests/test_harness_runner.py diff --git a/nemo_gym/agent_utils/harness_runner.sh b/nemo_gym/agent_utils/harness_runner.sh new file mode 100755 index 00000000000..6ac6ffa26f4 --- /dev/null +++ b/nemo_gym/agent_utils/harness_runner.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harness runner uploaded into task sandboxes; runs under process_supervisor.sh with only bash. +# +# usage: harness_runner.sh PARAMS +# +# PARAMS is bash source written by shell_assignments() on the agent server. It sets: +# cwd, command (array), env (array of KEY=VALUE), stdin, stdout, stderr +# and optionally: +# setup bash code run first, before env is applied +# unset_env array of extglob patterns naming inherited variables to drop +# events file receiving each JSON object line of stdout as [seen_at, object] +# Writes runtime.json beside PARAMS. + +shopt -s extglob +source "$1" || exit 1 +printf '{"hostname": "%s", "pid": %d}\n' "$HOSTNAME" "$$" >"$(dirname "$1")/runtime.json" +eval "${setup-}" || exit 1 +for pattern in "${unset_env[@]}"; do + for name in $(compgen -e); do + [[ $name == $pattern ]] && unset "$name" + done +done +((${#env[@]})) && export "${env[@]}" +cd "$cwd" || exit 1 +[[ -n ${events-} ]] || exec "${command[@]}" <"$stdin" >"$stdout" 2>"$stderr" + +# Stamp stdout lines as they appear. A file, unlike a pipe, cannot be held open by a detached tool. +: >"$stdout" +exec 3<"$stdout" 4>"$events" +"${command[@]}" <"$stdin" >>"$stdout" 2>"$stderr" & +cli=$! +trap 'kill -TERM "$cli" 2>/dev/null' TERM +pending="" +stamp() { + local line seen=${EPOCHREALTIME/,/.} + [[ -n $seen ]] || printf -v seen '%(%s)T' -1 + while IFS= read -r line <&3; do + line=$pending$line pending="" + [[ $line == {* ]] && printf '[%s, %s]\n' "$seen" "$line" >&4 + done + pending+=$line +} +while kill -0 "$cli" 2>/dev/null; do + stamp + sleep 0.05 +done +wait "$cli" +status=$? +stamp +[[ $pending == {* ]] && printf '[%s, %s]\n' "${EPOCHREALTIME/,/.}" "$pending" >&4 +exit "$status" diff --git a/nemo_gym/agent_utils/sandbox_session.py b/nemo_gym/agent_utils/sandbox_session.py index 4b3cf1b1abd..89e2c164663 100644 --- a/nemo_gym/agent_utils/sandbox_session.py +++ b/nemo_gym/agent_utils/sandbox_session.py @@ -7,6 +7,7 @@ import re from collections.abc import Awaitable, Callable, Coroutine, Mapping, Sequence from dataclasses import dataclass, field +from pathlib import Path from shlex import quote from time import monotonic from typing import cast @@ -29,6 +30,9 @@ LOG = logging.getLogger(__name__) +# Shared runner uploaded by adapters; reads the bash source written by shell_assignments(). +HARNESS_RUNNER_FILE = "harness_runner.sh" +HARNESS_RUNNER_SCRIPT = Path(__file__).with_name(HARNESS_RUNNER_FILE) _SHELL_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") diff --git a/tests/unit_tests/test_harness_runner.py b/tests/unit_tests/test_harness_runner.py new file mode 100644 index 00000000000..f9c69f8e554 --- /dev/null +++ b/tests/unit_tests/test_harness_runner.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise the uploaded bash harness runner as the sandbox runs it.""" + +import json +import os +import shutil +import signal +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from nemo_gym.agent_utils.sandbox_session import HARNESS_RUNNER_SCRIPT, shell_assignments + + +pytestmark = pytest.mark.skipif(shutil.which("bash") is None, reason="bash is required") + + +def _params(tmp_path: Path, **values) -> Path: + (tmp_path / "prompt.txt").write_text("the task\n") + params = tmp_path / "params.sh" + defaults = { + "cwd": str(tmp_path), + "env": {}, + "stdin": str(tmp_path / "prompt.txt"), + "stdout": str(tmp_path / "stdout.log"), + "stderr": str(tmp_path / "stderr.log"), + } + params.write_text(shell_assignments(defaults | values)) + return params + + +def _run(params: Path, **kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["bash", str(HARNESS_RUNNER_SCRIPT), str(params)], capture_output=True, text=True, timeout=20, **kwargs + ) + + +def test_runs_the_command_with_files_env_and_cwd(tmp_path: Path) -> None: + work = tmp_path / "work's dir" + work.mkdir() + script = 'cat; pwd; echo "$GYM_VALUE|${DROPPED_TOKEN-unset}|${KEPT-unset}"; echo diagnostics >&2; exit 3' + params = _params( + tmp_path, + cwd=str(work), + command=["bash", "-c", script], + env={"GYM_VALUE": "it's $(set)"}, + unset_env=["DROPPED_*"], + ) + result = _run(params, env=os.environ | {"DROPPED_TOKEN": "secret", "KEPT": "yes"}) + assert result.returncode == 3, result.stderr + assert (tmp_path / "stdout.log").read_text() == f"the task\n{work}\nit's $(set)|unset|yes\n" + assert (tmp_path / "stderr.log").read_text() == "diagnostics\n" + runtime = json.loads((tmp_path / "runtime.json").read_text()) + assert set(runtime) == {"hostname", "pid"} + + +def test_setup_runs_before_the_harness_environment_applies(tmp_path: Path) -> None: + params = _params( + tmp_path, + setup=f'echo "$HOME" > {tmp_path}/task-home', + command=["bash", "-c", 'echo "$HOME"'], + env={"HOME": "/private/home"}, + ) + result = _run(params, env=os.environ | {"HOME": "/task/home"}) + assert result.returncode == 0, result.stderr + assert (tmp_path / "task-home").read_text() == "/task/home\n" + assert (tmp_path / "stdout.log").read_text() == "/private/home\n" + + +def test_events_keep_json_objects_with_the_time_they_were_seen(tmp_path: Path) -> None: + script = 'printf \'{"type": "a"}\\nnot json\\n\\n{"type": "b"}\\n{"type": "partial"}\'; sleep 0.2' + params = _params(tmp_path, command=["bash", "-c", script], events=str(tmp_path / "events.jsonl")) + before = time.time() + result = _run(params) + assert result.returncode == 0, result.stderr + events = [json.loads(line) for line in (tmp_path / "events.jsonl").read_text().splitlines()] + assert [event for _, event in events] == [{"type": "a"}, {"type": "b"}, {"type": "partial"}] + assert all(before - 1 <= seen <= time.time() for seen, _ in events) + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux process semantics") +def test_events_mode_forwards_term_and_ignores_a_detached_stdout_holder(tmp_path: Path) -> None: + # A detached tool keeps stdout open; the runner must still finish when the CLI does. + script = ( + "setsid sleep 30 & echo $! > holder.pid; " + "trap 'echo \"{\\\"type\\\": \\\"stopped\\\"}\"; exit 0' TERM; echo '{\"type\": \"ready\"}'; " + "while :; do sleep 0.05; done" + ) + params = _params(tmp_path, command=["bash", "-c", script], events=str(tmp_path / "events.jsonl")) + process = subprocess.Popen(["bash", str(HARNESS_RUNNER_SCRIPT), str(params)], stderr=subprocess.PIPE) + try: + deadline = time.monotonic() + 10 + while "ready" not in ((tmp_path / "events.jsonl").read_text() if (tmp_path / "events.jsonl").exists() else ""): + assert process.poll() is None and time.monotonic() < deadline + time.sleep(0.02) + process.send_signal(signal.SIGTERM) + assert process.wait(timeout=10) == 0, process.stderr.read() + events = [json.loads(line)[1] for line in (tmp_path / "events.jsonl").read_text().splitlines()] + assert events == [{"type": "ready"}, {"type": "stopped"}] + finally: + if process.poll() is None: + process.kill() + if (tmp_path / "holder.pid").exists(): + try: + os.kill(int((tmp_path / "holder.pid").read_text()), signal.SIGKILL) + except ProcessLookupError: + pass From 7aae7456ca1067fb5fd3fdb614272b2597b3f05f Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Mon, 26 Oct 2026 09:18:55 -0700 Subject: [PATCH 03/32] docs(agent_utils): describe the bash supervisor and runner; update hermes tests Point the harness guide, package docs and SandboxSession docstring at process_supervisor.sh and harness_runner.sh, drop the Python-in-sandbox requirement and the subreaper drain the bash supervisor no longer performs. Hermes tests use the bash supervisor. The runner stamps a final unterminated event line with the same fallback time as the others on bash < 5. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Hemil Desai Signed-off-by: yaoyu-33 --- .../agent-server/add-or-update-a-harness.mdx | 87 +++++++++---------- nemo_gym/agent_utils/__init__.py | 8 +- nemo_gym/agent_utils/harness_runner.sh | 11 ++- nemo_gym/agent_utils/sandbox_session.py | 7 +- .../hermes_agent/tests/test_app.py | 2 +- .../hermes_agent/tests/test_sandbox.py | 43 ++------- .../hermes_agent/tests/test_sandbox_runner.py | 8 +- .../hermes_sandboxed_agent/tests/test_app.py | 6 +- 8 files changed, 66 insertions(+), 106 deletions(-) diff --git a/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx b/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx index 4e5c807668d..e0ea053eb2a 100644 --- a/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx +++ b/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx @@ -48,7 +48,7 @@ An adapter can stop at any level below. Each level adds work and enables a speci An adapter supplies the knowledge that is specific to the harness or framework: 1. **Install or load the agent.** For example, a framework adapter lists the framework in its `requirements.txt`, while Pi and Codex install a pinned runtime into the task sandbox at seed. -2. **Configure and invoke it.** Build the harness configuration from the adapter configuration and the request, then call the SDK or start the CLI. For example, Hermes writes a `config.yaml`, and Pi and Codex start a `sandbox_runner.py` shim. +2. **Configure and invoke it.** Build the harness configuration from the adapter configuration and the request, then call the SDK or start the CLI. For example, Hermes writes a `config.yaml`, and Pi and Codex start their CLI through the shared `harness_runner.sh`. 3. **Map the harness's inputs, results, and traces to Gym.** Convert the request into the harness's input, the harness's output into a `NeMoGymResponse`, and its model and tool activity into an `AgentObservationBundle`. 4. **Map Gym connections into the harness's own configuration.** Set the rollout-prefixed Model Server URL as the harness's model base URL, add tool grants to its tool configuration, and run it in `SandboxAccess.workdir`. For example, Hermes adds each MCP grant to the `mcp_servers` section of its configuration. @@ -317,24 +317,25 @@ The [`nemo_gym/agent_utils`](https://github.com/NVIDIA-NeMo/Gym/tree/main/nemo_g ### Code layout -Organize a new adapter under `responses_api_agents//`. The adapter's `sandbox.py` runs on the Agent Server. Its `sandbox_runner.py` shim, a small program that wraps the harness, runs inside the sandbox. +Organize a new adapter under `responses_api_agents//`. The adapter's `sandbox.py` runs on the Agent Server. Inside the sandbox, the shared `harness_runner.sh` starts the harness CLI, or an adapter-specific `sandbox_runner.py` shim wraps a harness that is itself a Python program. | Adapter file | Responsibility | | --- | --- | | `app.py` | Gym endpoints and configuration; request validation and mapping; session hooks; response and observation conversion; dispatch to host or sandbox execution. | | `sandbox.py` | A harness-specific session state class that composes `SandboxSession`. It installs the runtime at seed, stages input in `stage_activation()`, collects artifacts, and holds harness-specific state and diagnostics. | -| `sandbox_runner.py` | Reads the staged input, invokes the harness SDK or CLI, writes the harness's output and optional process diagnostics, and keeps partial output where the harness supports it. Handles `SIGTERM` by stopping the harness and writing that partial output before it exits, because the supervisor sends `SIGTERM` at the deadline and on close. | +| `sandbox_runner.py` (Python harnesses only) | Reads the staged input, invokes the harness SDK, writes the harness's output and optional process diagnostics, and keeps partial output where the harness supports it. Handles `SIGTERM` by stopping the harness and writing that partial output before it exits, because the supervisor sends `SIGTERM` at the deadline and on close. CLI harnesses use the shared `harness_runner.sh` instead, so the task image needs only bash. | | `install__runtime.sh` or an existing harness-specific installer | Installs the pinned harness and its private dependencies, then checks that they run. Keep installation out of the activation path. | | `requirements.txt` | The adapter's isolated environment: `-e nemo-gym[dev,sandbox] @ ../../` plus packages that run on the Agent Server, such as a framework or SDK. Without the `sandbox` extra, an SDK-backed provider such as OpenSandbox cannot import its SDK, and the first sandbox session fails. | | `tests/` | Host dispatch, session setup, request semantics, output parsing, installation, and lifecycle regressions. | -The shared modules have separate responsibilities. All of them run on the Agent Server, except `process_supervisor.py`, which is uploaded into the sandbox. Provider APIs and general file transfer stay in `nemo_gym/sandbox/`. +The shared modules have separate responsibilities. All of them run on the Agent Server, except `process_supervisor.sh` and `harness_runner.sh`, which are uploaded into the sandbox. Provider APIs and general file transfer stay in `nemo_gym/sandbox/`. | Shared module | Responsibility | | --- | --- | -| `agent_utils/sandbox_session.py` | `SandboxSession` owns the sandbox connection, the ownership flag, `workdir`, `session_dir`, the execution and finalization tasks, the termination receipt, artifacts, and retryable close state. It uploads the supervisor before launch. `SandboxCommand` carries the harness argv and the interpreter that runs the supervisor. | +| `agent_utils/sandbox_session.py` | `SandboxSession` owns the sandbox connection, the ownership flag, `workdir`, `session_dir`, the execution and finalization tasks, the termination receipt, artifacts, and retryable close state. It uploads the supervisor before launch. `SandboxCommand` carries the harness argv. `shell_assignments()` renders runner input as bash source. | | `agent_utils/supervisor_client.py` | Builds the launch and stop shell commands, defines control-file names and timeout budgets, validates termination receipts, and retires session directories safely. | -| `agent_utils/process_supervisor.py` | Runs inside the Linux sandbox using only the standard library. It enforces the harness deadline, stops and reaps the harness process group, and writes the termination receipt. The sandbox does not need a Gym installation. | +| `agent_utils/process_supervisor.sh` | Runs inside the Linux sandbox and needs only bash, `/proc`, `sleep`, and `mv`. It enforces the harness deadline, stops the harness process group, confirms that no member still runs, and writes the termination receipt. The sandbox needs neither Gym nor Python. | +| `agent_utils/harness_runner.sh` | Runs inside the sandbox with only bash. It sources the input that `shell_assignments()` wrote, applies the harness environment, runs the CLI with file-backed stdio, and optionally records each JSON line of stdout with the time it was seen. | | `agent_utils/uv_bootstrap.py` | `uv_for_sandbox()` chooses the `uv` binary to upload for sandboxed Python runtime installation. It reuses a compatible host binary or downloads the static musl build for the detected platform. If the platform probe fails, it warns and falls back to the host binary without confirming compatibility. Hermes uses it. | | `agent_utils/task_timeout.py` | `resolve_agent_timeout(metadata, ceiling=...)` returns the harness time budget: a task's positive `metadata.agent_timeout_sec`, which Harbor task rows carry, capped by the configured timeout. A missing or invalid value keeps the configured timeout. Hermes, Pi, Codex, OpenCode, and OpenClaw use it. | | `sandbox/utils.py` | `upload_text()` and `read_text()` transfer UTF-8 text through `AsyncSandbox.upload()` and `download()`. Use those underlying APIs directly for binary files. | @@ -372,13 +373,13 @@ flowchart TB subgraph SB["Task sandbox"] RUNTIME["Pinned harness runtime"] - SUPERVISOR["process_supervisor.py: process lifetime"] - SHIM["sandbox_runner.py: harness shim"] + SUPERVISOR["process_supervisor.sh: process lifetime"] + SHIM["harness_runner.sh or sandbox_runner.py: harness shim"] HARNESS["Harness SDK or CLI and its tools"] CLEANUP["cleanup.json"] OUTPUT["Harness output and optional diagnostics"] SUPERVISOR -->|"Subprocess argv + OS signals"| SHIM - SHIM -->|"Python SDK / CLI subprocess"| HARNESS + SHIM -->|"CLI subprocess or Python SDK"| HARNESS RUNTIME -->|"Provides executable and dependencies"| HARNESS SUPERVISOR -->|"Local file I/O: JSON receipt"| CLEANUP SHIM -->|"Local file I/O: JSON output"| OUTPUT @@ -404,7 +405,7 @@ Keep harness-specific response parsing and model and tool behavior in the adapte **1. Define the session state.** Subclass `AgentSessionState` and compose a `SandboxSession`. Implement two hooks: -- `stage_activation` uploads the activation input and returns `SandboxCommand(argv=..., python=...)`. `python` is the sandbox interpreter that runs the supervisor. Install the runtime earlier, during seed. +- `stage_activation` uploads the activation input and returns `SandboxCommand(argv=...)`. Install the runtime earlier, during seed. - `collect` copies artifacts to the Agent Server after harness termination is confirmed, including partial output from interrupted runs. It may run a bounded snapshot command, but it must not restart the harness. Bound the size of what it downloads: trim large logs inside the sandbox first and keep the tail with a truncation marker. For example, the older `hermes_sandboxed_agent`, which calls the process supervisor directly, keeps the last 1 MiB of its runner log; apply the same bound in `collect`. ```python @@ -414,19 +415,27 @@ class MyHarnessSession(AgentSessionState): task: asyncio.Task[NeMoGymResponse] | None = None activation_request: NeMoGymResponseCreateParamsNonStreaming | None = None - async def stage_activation(self, payload: dict[str, JsonValue]) -> SandboxCommand: - input_path = f"{self.session.session_dir}/input.json" - await upload_text(self.session.sandbox, path=input_path, text=json.dumps(payload)) - return SandboxCommand( - python=RUNTIME_PYTHON, # interpreter installed or validated at seed - argv=[RUNTIME_PYTHON, "-I", f"{self.session.session_dir}/sandbox_runner.py", input_path], + async def stage_activation(self, prompt: str) -> SandboxCommand: + directory = self.session.session_dir + await upload_text(self.session.sandbox, path=f"{directory}/prompt.txt", text=prompt) + params = shell_assignments( + { + "cwd": self.session.workdir, + "command": [f"{RUNTIME}/my-harness", "run", "--json"], + "env": {"MY_HARNESS_BASE_URL": self.base_url}, + "stdin": f"{directory}/prompt.txt", + "stdout": f"{directory}/output.json", + "stderr": f"{directory}/stderr.log", + } ) + await upload_text(self.session.sandbox, path=f"{directory}/params.sh", text=params) + return SandboxCommand(argv=["bash", f"{directory}/{HARNESS_RUNNER_FILE}", f"{directory}/params.sh"]) async def collect(self) -> str: return await read_text(self.session.sandbox, path=f"{self.session.session_dir}/output.json") ``` -**2. Seed the session.** In `_seed_agent_session_state()`, validate tool grants and configuration, then connect to the borrowed sandbox or start an owned one. Create the `SandboxSession` with a `session_dir` that is separate from `workdir`, which is the task repository. Install the runtime and upload `sandbox_runner.py`. If setup fails, close the session; if that close also fails, raise `AgentSessionSetupError` so that a later close can retry. +**2. Seed the session.** In `_seed_agent_session_state()`, validate tool grants and configuration, then connect to the borrowed sandbox or start an owned one. Create the `SandboxSession` with a `session_dir` that is separate from `workdir`, which is the task repository. Install the runtime and upload `HARNESS_RUNNER_SCRIPT`. If setup fails, close the session; if that close also fails, raise `AgentSessionSetupError` so that a later close can retry. ```python state = MyHarnessSession( @@ -440,7 +449,7 @@ state = MyHarnessSession( ), ) try: - await install_runtime(state) # adapter-specific; also uploads sandbox_runner.py + await install_runtime(state) # adapter-specific; also uploads harness_runner.sh except BaseException as error: try: await state.session.close(timeout=self.config.session_close_timeout_seconds) @@ -463,7 +472,7 @@ async with self.sem: # asyncio.Semaphore(self.config.concurrency) ) ``` -`execute()` uploads `process_supervisor.py` into `session_dir` after `stage_activation` returns, so adapters do not upload it themselves. It then launches the harness under the supervisor, confirms harness termination, collects artifacts, and releases the provider exec. Captured artifacts remain in `session.artifacts` even when execution raises. If the supervisor writes a final negative receipt, `execute()` raises `HarnessTerminationError` and does not capture artifacts. A nonzero harness exit code does not raise by itself, because the output can still be useful. Convert artifacts into a response and observations in the adapter. +`execute()` uploads `process_supervisor.sh` into `session_dir` after `stage_activation` returns, so adapters do not upload it themselves. It then launches the harness under the supervisor, confirms harness termination, collects artifacts, and releases the provider exec. Captured artifacts remain in `session.artifacts` even when execution raises. If the supervisor writes a final negative receipt, `execute()` raises `HarnessTerminationError` and does not capture artifacts. A nonzero harness exit code does not raise by itself, because the output can still be useful. Convert artifacts into a response and observations in the adapter. **4. Close the session.** In `_close_agent_session_state()`, call `session.close()` first, then cancel the activation task if it is still running. Close joins any finalization in progress, never captures artifacts twice, and is safe to retry while termination evidence is missing. After a final negative receipt, close skips capture, releases a borrowed connection, and leaves teardown to the Resources Server; an owned sandbox is stopped through the provider. Build observations from `session.artifacts` if activation was interrupted before parsing them. @@ -485,7 +494,7 @@ Adapters normally call `SandboxSession` instead of repeating its launch, stop, c | --- | --- | | `upload_text(sandbox, *, path, text)` and `read_text(sandbox, *, path)` | Transfer UTF-8 text through upload and download APIs without putting file contents into shell commands. `read_text()` replaces invalid UTF-8; the adapter validates the decoded payload. These are module functions in `nemo_gym/sandbox/utils.py`, not `AsyncSandbox` methods. | | `supervised_launch_command()` | Requires an explicit supervisor interpreter and checks that it can load the supervisor before claiming the launch. Builds a quoted command with an atomic launch claim that fences delayed execution. | -| `stop_and_confirm_cleanup()` | Fences a pending launch or asks the supervisor to stop, then requires an explicit termination receipt. Reads the receipt before signaling a stored PID, because the PID may have been reused. Raises `HarnessTerminationError` for a well-formed negative receipt, which is final; missing or malformed evidence raises a retryable error. | +| `stop_and_confirm_cleanup()` | Fences a pending launch or asks the supervisor to stop through `stop.request`, then requires an explicit termination receipt. It sends no signal, so a reused PID cannot be hit. Raises `HarnessTerminationError` for a well-formed negative receipt, which is final; missing or malformed evidence raises a retryable error. | | `remove_session_directory()` | Renames the session directory before deleting it, so its launch fence stays effective. Removes only adapter-owned session files, after the termination outcome is final and artifacts are collected. | | `parse_cleanup_receipt()` | Requires a literal `cleanup_confirmed: true`, and normalizes malformed optional diagnostics with a warning. A fenced launch that never ran has no harness exit code, so missing diagnostics never become a successful exit. | | `SandboxSession.stop_harness()` | Confirms harness termination before provider exec is cancelled. Normal execution and close both call it through shared finalization. | @@ -496,14 +505,14 @@ All control files live under `session_dir`, and their names are defined in `supe | File | Writer and purpose | | --- | --- | -| `process_supervisor.py` | `SandboxSession` uploads it after staging input and before launch. | +| `process_supervisor.sh` | `SandboxSession` uploads it after staging input and before launch. | | `launch.claim` | The launch or stop shell creates it atomically, either to claim execution or to fence a delayed launch. | | `supervisor.pid` | The launch shell records the PID that becomes the supervisor when the shell executes it. | -| `stop.request` | The stop shell writes this cancellation marker before signaling the supervisor. | +| `stop.request` | The stop shell writes this cancellation marker; the supervisor polls for it. A signal is not used, so the harness keeps the default `SIGTERM` disposition. | | `cleanup.json` | The supervisor records here whether the harness process group terminated. If a stop wins the launch claim, the stop shell writes the same receipt shape with `return_code: null` and `timed_out: false`. | | `output.log` | The launch shell redirects supervisor and harness diagnostics here. Task output is a separate, adapter-owned artifact. | -To migrate an adapter that launches the process supervisor itself, such as `hermes_sandboxed_agent`, move launch, termination, collection, and release into `SandboxSession`. Adapters written before [#3961](https://github.com/NVIDIA-NeMo/Gym/pull/3961) also import `process_supervisor` from `nemo_gym.sandbox`, which #3961 moved to `nemo_gym.agent_utils`. Update the imports, hook arguments, control-file assumptions, test doubles, and tests together. The adapter imports these modules at startup, so a stale import also breaks host calls. +To migrate an adapter that launches the process supervisor itself, such as `hermes_sandboxed_agent`, move launch, termination, collection, and release into `SandboxSession`. Receipt types and timeout helpers that used to live in `process_supervisor.py` are now in `nemo_gym.agent_utils.supervisor_client`. Update the imports, hook arguments, control-file assumptions, test doubles, and tests together. The adapter imports these modules at startup, so a stale import also breaks host calls. ## Prepare the runtime during session setup @@ -525,18 +534,18 @@ Run the installation check at every seed, and reuse a compatible runtime if one 4. Mark the runtime as ready only after installation and its health check succeed. Handle partial installations and concurrent setup without exposing an incomplete runtime. On failure, keep the commands, exit codes, and logs. 5. For offline support, provide the complete dependency set, including bootstrap interpreters, package dependencies, and native libraries. Uploading a CLI or ripgrep alone is not enough if the installer still downloads other dependencies. If it does download them, document that the sandbox needs network access to the package index. - -Every supervised harness needs a compatible Python 3 interpreter inside the sandbox, including standalone binaries such as Codex and OpenCode. Pass it explicitly as `SandboxCommand.python`. - + +The supervisor and the shared runner need only bash in the task image. Do not require Python in the sandbox for a CLI harness: check paths, install runtimes, and post-process output with shell commands or on the Agent Server. + -Hermes gets Python from its private runtime. For a binary harness, installation must provision or validate an interpreter for the supervisor. The launch command checks that the interpreter can load the supervisor before it claims the launch. The supervisor uses only the standard library, so the sandbox does not need Gym or the Agent Server's environment. +Hermes gets Python from its private runtime because Hermes itself is a Python program. The launch command checks that bash can parse the supervisor before it claims the launch. The proposed shared installer must not depend on system package managers or root access. Not all existing adapter installers meet this goal yet. Until they migrate, document the supported images and any remaining prerequisites. For example, [Hermes seed](https://github.com/NVIDIA-NeMo/Gym/blob/main/responses_api_agents/hermes_agent/sandbox.py) reuses a private virtual environment keyed to the pinned version, after checking that `run_agent` and `mcp` import. A cold setup uploads the `uv` binary that `uv_for_sandbox()` selects for the sandbox's platform, creates a private Python 3.13 environment, and installs the pinned Hermes package with its MCP extra. Installation still needs network access for Python and the Hermes dependencies. Seed also uploads harness support files into the runtime directory, so Hermes does not yet support offline installation or read-only runtime mounts. -The [shared installation proposal](https://github.com/NVIDIA-NeMo/Gym/pull/3961#issuecomment-6003983236) would add an Agent Server helper, `harness_install.py`, with `ensure_harness_installed(...)`. It would detect the platform, reuse or stage verified artifacts, run the adapter's installer at seed, and return the runtime location, including the supervisor interpreter. Neither the helper nor its package location exists yet. +The [shared installation proposal](https://github.com/NVIDIA-NeMo/Gym/pull/3961#issuecomment-6003983236) would add an Agent Server helper, `harness_install.py`, with `ensure_harness_installed(...)`. It would detect the platform, reuse or stage verified artifacts, run the adapter's installer at seed, and return the runtime location. Neither the helper nor its package location exists yet. A common runtime mount configuration across providers, with a convention such as `/opt/nemo-gym-runtimes`, is also planned. Enroot already supports its own `exec.default_mounts`. Mount defaults must be applied when the sandbox is created, with defined behavior for destination conflicts and unsupported providers. The installer then discovers the mounted runtime at seed. Read-only runtime mounts require separate writable session state. @@ -659,24 +668,11 @@ Serialize close requests. Keep successful close responses for a documented, boun ### Harness termination - -This termination contract landed in [#3902](https://github.com/NVIDIA-NeMo/Gym/pull/3902) ([`process_supervisor.py` at merge commit `8bd25f372`](https://github.com/NVIDIA-NeMo/Gym/blob/8bd25f372f3117a7a24433adfdcb2f250e5063d2/nemo_gym/agent_utils/process_supervisor.py)). Earlier checkouts also kill processes that left the harness process group and treat every unconfirmed receipt as retryable. - - -The supervisor starts the harness in a new process group and enables child-subreaper mode so it can adopt orphaned descendants of the harness. How it stops processes depends on how the harness ends: - -- **Timeout or stop.** Before it signals the harness, the supervisor records the children it has already adopted, typically services or unfinished background jobs left by completed tool calls. It sends `SIGTERM`, waits for the grace period, and kills and reaps the harness process group. It then repeatedly kills and reaps every newly adopted child that is not in that record, including in-flight tool commands in their own sessions and the children they orphan, until none remain or the cleanup deadline passes. The recorded services outside the harness group survive. -- **Natural exit.** The supervisor kills and reaps the harness process group. It has no earlier point at which to separate services from tools, so tool commands that are still running in their own sessions are left alone. +The supervisor starts the harness in a new process group. At the deadline or on a stop request it sends `SIGTERM` to the harness and waits for the grace period. In every case it then kills the harness process group and waits until no member of the group can still run. Processes that left the group, such as tool commands started in their own sessions, are left for the sandbox owner. Shell-tool commands can run in their own sessions, so a harness-group kill alone may not reach them. A background process left by a completed tool, even a plain `server &`, can then survive as a task service. Detachment depends on the harness and launch path: OpenClaw's [pinned child adapter](https://github.com/openclaw/openclaw/blob/e085fa1a3ffd32d0ea6917e1e6fb4ecbffbb77d2/src/process/supervisor/adapters/child.ts#L50-L100) disables detachment when `OPENCLAW_SERVICE_MARKER` is nonblank and can fall back to spawning with `detached: false`. Commands on those paths can remain in the harness group and stop at close. -`cleanup_confirmed: true` means that no live harness-group processes remain and, after a timeout or stop, that newly adopted children were drained. A zombie held by a parent outside the group does not block confirmation, but unreadable process state fails closed. The receipt does not mean the sandbox is idle, and it is not a security boundary. These cases remain open: - -- After a natural exit, unfinished tool commands in other sessions cannot be distinguished from services and can still change task files. -- A process adopted just before the stop is preserved as a service. -- A PID can be reused during the stop window. -- A preserved service can orphan more children during cleanup. -- Descendants that stay parented by a preserved process are not drained, including work an in-flight tool hands to it, such as a command sent to an existing `tmux` server. +`cleanup_confirmed: true` means that no live harness-group processes remain. A zombie does not block confirmation, because it cannot run. The receipt does not mean the sandbox is idle, and it is not a security boundary: a tool command still running in its own session can change task files after the receipt. Services survive until the sandbox owner tears down the sandbox: the Resources Server after verification or episode failure, or the agent when it closes an owned sandbox. @@ -687,7 +683,6 @@ A task service that must survive the harness needs to: - Detach its process group and standard streams, for example with `subprocess.Popen(start_new_session=True)`, stdin from `/dev/null`, and stdout and stderr in task-owned log files. `nohup ... &` alone is not portable across harnesses, because a shell deadline or a process-group stop can still end the service. - Keep its data in task-owned files, not in the agent's session directory, which agent close removes. - Be reachable before the tool call that starts it finishes. -- Be adopted before the stop: let the process that launched it exit before the harness deadline or stop. A service still parented by an in-flight tool is adopted only when that tool exits or is killed during the stop, and is drained with the in-flight tools. Redirect supervisor and harness output to a file, as `supervised_launch_command()` does with `output.log`. A surviving service that inherits the provider's output pipe keeps provider exec open until its hard timeout. @@ -751,9 +746,9 @@ Record evidence conformance and sandbox lifecycle validation separately, because ### Runtime and compatibility - **Execution modes.** Test host calls with no sandbox access or provider, borrowed agent sessions, agent-owned sessions with a configured provider, and rejection of a seed with neither. Assert that host calls never create a sandbox and that sandbox failures never fall back to the host. Test the compatibility `/run` endpoint with and without a Resources Server binding. -- **Installation.** Test cold installation, runtime reuse, failed and partial installation, and unsupported platforms. Include a binary harness on an image without Python, and require setup to provision the supervisor interpreter or reject the image explicitly. Claim offline or read-only mounted-runtime support only after you test those deployments. +- **Installation.** Test cold installation, runtime reuse, failed and partial installation, and unsupported platforms. Include an image without Python: a CLI harness must run there. Claim offline or read-only mounted-runtime support only after you test those deployments. - **Failures and races.** Test setup failures, request mapping, unsupported controls, isolation, retries, cancellation, close races, and termination failures. Include supervisor upload failures, close during upload, interpreter or supervisor bootstrap failures before launch, termination before transport cancellation, artifact capture before teardown, capture failures, and malformed optional diagnostics. Check that a final negative receipt releases a borrowed connection without making the activation gradable. -- **Process termination.** Use real Linux processes. Cover tool commands in the harness group and in their own sessions (`start_new_session=True`), with grandchildren, under a harness that exits on `SIGTERM` and one that must be killed, for both a deadline and a close. A tool still running at the stop must not change the task after a confirmed receipt, and a service started by an earlier, completed tool call must survive. Check that every harness process shares the supervised process group, and that a zombie held by a parent outside the group does not block confirmation. Read the process state from `/proc//stat`, because `os.kill(pid, 0)` also succeeds for a zombie. Run the `sandbox`-marked tests with the local provider, which exposes a surviving process that holds the exec output pipe open. +- **Process termination.** Use real Linux processes. Cover tool commands in the harness group, with grandchildren, under a harness that exits on `SIGTERM` and one that must be killed, for both a deadline and a close. A tool in the harness group must not change the task after a confirmed receipt, and a detached service must survive. Check that every harness process shares the supervised process group, and that a zombie does not block confirmation. Read the process state from `/proc//stat`, because `os.kill(pid, 0)` also succeeds for a zombie. Run the `sandbox`-marked tests with the local provider, which exposes a surviving process that holds the exec output pipe open. - **Test hygiene.** Bound every wait in lifecycle tests, for example with `asyncio.wait_for()`, so that a regression fails instead of hanging. Check that each new regression test fails when you revert the fix it covers. - **Task environment.** Confirm that tool shells keep the task's `HOME` and `XDG_*` directories, and that files a tool writes under `~` are visible to verification. - **Real smoke test.** Run a real model on a real task with tools and verification. Confirm that the harness runs in the sandbox, that the agent closes before verification, and that the sandbox owner cleans it up. Inspect failures, traces, artifacts, and grading coverage. diff --git a/nemo_gym/agent_utils/__init__.py b/nemo_gym/agent_utils/__init__.py index 4a6dfe276a1..deeb45a9987 100644 --- a/nemo_gym/agent_utils/__init__.py +++ b/nemo_gym/agent_utils/__init__.py @@ -8,9 +8,9 @@ implements the shell protocol over session control files. All three run on the agent server. -Only process_supervisor.py is copied from this package into the task sandbox. It -uses the standard library to enforce deadlines, stop and reap the harness process -group, and write its termination receipt. Explicitly detached task services remain -for the sandbox owner to stop. Core sandbox APIs and providers live in +Only process_supervisor.sh and harness_runner.sh are copied from this package into +the task sandbox; they need bash, not Python. The supervisor enforces deadlines, stops +the harness process group and writes its termination receipt. Detached task services +remain for the sandbox owner to stop. Core sandbox APIs and providers live in :mod:`nemo_gym.sandbox`. """ diff --git a/nemo_gym/agent_utils/harness_runner.sh b/nemo_gym/agent_utils/harness_runner.sh index 6ac6ffa26f4..79cb7df3502 100755 --- a/nemo_gym/agent_utils/harness_runner.sh +++ b/nemo_gym/agent_utils/harness_runner.sh @@ -34,12 +34,15 @@ exec 3<"$stdout" 4>"$events" cli=$! trap 'kill -TERM "$cli" 2>/dev/null' TERM pending="" +emit() { [[ $1 == {* ]] && printf '[%s, %s]\n' "$seen" "$1" >&4; } +# Stamp new complete lines, keeping a partial one for the next pass. Bash < 5 has whole seconds. stamp() { - local line seen=${EPOCHREALTIME/,/.} + local line + seen=${EPOCHREALTIME/,/.} [[ -n $seen ]] || printf -v seen '%(%s)T' -1 while IFS= read -r line <&3; do - line=$pending$line pending="" - [[ $line == {* ]] && printf '[%s, %s]\n' "$seen" "$line" >&4 + emit "$pending$line" + pending="" done pending+=$line } @@ -50,5 +53,5 @@ done wait "$cli" status=$? stamp -[[ $pending == {* ]] && printf '[%s, %s]\n' "${EPOCHREALTIME/,/.}" "$pending" >&4 +emit "$pending" exit "$status" diff --git a/nemo_gym/agent_utils/sandbox_session.py b/nemo_gym/agent_utils/sandbox_session.py index 89e2c164663..bce08431686 100644 --- a/nemo_gym/agent_utils/sandbox_session.py +++ b/nemo_gym/agent_utils/sandbox_session.py @@ -90,11 +90,8 @@ class SandboxSession[Artifacts]: receipt skips capture, releases a borrowed connection and leaves teardown to Resources; an owned sandbox falls back to provider stop. Closed means release succeeded. - On timeout or close, the supervisor preserves previously adopted children outside - the harness group and drains newly adopted children, including services orphaned - by in-flight tools during the stop. Preserved children can also be unfinished - background jobs; they remain until the sandbox owner destroys the sandbox. - Natural CLI exit has no adoption cutoff, so detached unfinished tools may remain. + The supervisor stops the harness process group. Processes that left the group + (detached services or tools) remain until the sandbox owner destroys the sandbox. Borrowed sandboxes stay available for Resources verification; owned sandboxes are stopped on close. Unconfirmed harness termination prevents capture and verification. diff --git a/responses_api_agents/hermes_agent/tests/test_app.py b/responses_api_agents/hermes_agent/tests/test_app.py index 7ac19c3965e..ddd728de5d5 100644 --- a/responses_api_agents/hermes_agent/tests/test_app.py +++ b/responses_api_agents/hermes_agent/tests/test_app.py @@ -492,7 +492,7 @@ async def download(self, remote_path, local_path) -> None: response = await hermes.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="fix bug")) runner_input = json.loads(sandbox.uploaded["/session/input.json"]) - assert "/session/process_supervisor.py" in sandbox.uploaded + assert "/session/process_supervisor.sh" in sandbox.uploaded assert runner_input["model_base_url"] == f"http://model-server:1/ng-rollout/{seed.episode_id.capture_key}/v1" assert runner_input["user_message"] == "fix bug" assert response.metadata["harness_execution"] == "sandbox" diff --git a/responses_api_agents/hermes_agent/tests/test_sandbox.py b/responses_api_agents/hermes_agent/tests/test_sandbox.py index e8b253c1ede..e027eda5600 100644 --- a/responses_api_agents/hermes_agent/tests/test_sandbox.py +++ b/responses_api_agents/hermes_agent/tests/test_sandbox.py @@ -18,9 +18,8 @@ from pydantic import ValidationError from environment_servers.single_agent_turn.app import AgentOutcomePolicyConfig, OutcomeAction, agent_outcome_policy -from nemo_gym.agent_utils import process_supervisor from nemo_gym.agent_utils.sandbox_session import SandboxSession -from nemo_gym.agent_utils.supervisor_client import STOP_REQUEST_FILE, SUPERVISOR_FILE, parse_cleanup_receipt +from nemo_gym.agent_utils.supervisor_client import CLEANUP_PHASE_COUNT, STOP_REQUEST_FILE, parse_cleanup_receipt from nemo_gym.base_responses_api_agent import AgentCloseSessionRequest, AgentSeedSessionRequest, _AgentSessionRecord from nemo_gym.episode_types import EpisodeId, TaskId from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming @@ -720,48 +719,18 @@ async def test_launch_claim_without_pid_is_not_proof_of_cleanup(agent, state, lo state.session.sandbox.disconnect.assert_not_awaited() -@pytest.mark.parametrize("stop_timing", ["before-shell", "before-handler"]) -async def test_stop_during_interpreter_startup_closes_without_starting_a_worker( - agent, state, local_runner, monkeypatch, tmp_path, stop_timing -): +async def test_stop_before_launch_closes_without_starting_a_worker(agent, state, local_runner, tmp_path): directory = Path(state.session.session_dir) - ready = tmp_path / "before-handler" - worker_started = tmp_path / "worker-started" - wrapper = tmp_path / "delayed_supervisor.py" - wrapper.write_text( - "import pathlib,runpy,sys,time\n" - f"if '--help' in sys.argv: runpy.run_path({process_supervisor.__file__!r}, run_name='__main__')\n" - f"pathlib.Path({str(ready)!r}).touch()\n" - f"while not pathlib.Path({str(directory / STOP_REQUEST_FILE)!r}).exists(): time.sleep(0.01)\n" - # Keep the interpreter in the pre-handler window while close sends TERM. - "time.sleep(0.15)\n" - f"runner=runpy.run_path({process_supervisor.__file__!r})\n" - "def unexpected_worker(*args, **kwargs):\n" - f" pathlib.Path({str(worker_started)!r}).touch()\n" - " raise AssertionError('Worker must not start after the stop marker')\n" - "runner['subprocess'].Popen=unexpected_worker\n" - "raise SystemExit(runner['main']())\n" - ) - state.session.sandbox.upload.side_effect = lambda source, destination: shutil.copyfile( - wrapper if Path(source).name == SUPERVISOR_FILE else source, destination - ) - if stop_timing == "before-shell": - (directory / STOP_REQUEST_FILE).touch() + (directory / STOP_REQUEST_FILE).touch() state.task = asyncio.create_task( agent._run_sandbox_episode( body=NeMoGymResponseCreateParamsNonStreaming(input="task"), agent_session_id="session", state=state ) ) try: - if stop_timing == "before-handler": - async with asyncio.timeout(3): - while not ready.exists(): - await asyncio.sleep(0.01) - await state.session.stop_harness(timeout=agent.config.session_close_timeout_seconds) with pytest.raises(RuntimeError, match="exited without output"): await state.task assert state.session.cleanup is not None and state.session.cleanup["cleanup_confirmed"] is True - assert not worker_started.exists() assert json.loads((directory / "cleanup.json").read_text()) == { "cleanup_confirmed": True, "error": None, @@ -931,12 +900,10 @@ async def test_session_prompt_and_limits_reach_runner(agent, state, overrides, t await agent._run_sandbox_episode(body=body, agent_session_id="session", state=state) launch_command = state.session.sandbox.exec.await_args_list[0].args[0] launch_args = shlex.split(launch_command) - assert any(arg.endswith("/process_supervisor.py") for arg in launch_args) + assert any(arg.endswith("/process_supervisor.sh") for arg in launch_args) assert float(launch_args[launch_args.index("--timeout") + 1]) == agent.config.sandbox_runner_timeout_seconds cleanup_timeout = float(launch_args[launch_args.index("--cleanup-timeout") + 1]) - assert cleanup_timeout * process_supervisor.CLEANUP_PHASE_COUNT == pytest.approx( - agent.config.session_close_timeout_seconds - ) + assert cleanup_timeout * CLEANUP_PHASE_COUNT == pytest.approx(agent.config.session_close_timeout_seconds) assert state.session.sandbox.exec.await_args_list[0].kwargs["timeout_s"] == pytest.approx( agent.config.sandbox_runner_timeout_seconds + agent.config.session_close_timeout_seconds + 30 ) diff --git a/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py b/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py index 25e093ac26d..c6f56a52b91 100644 --- a/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py @@ -21,8 +21,7 @@ from run_agent import AIAgent from tools.mcp_tool import shutdown_mcp_servers -from nemo_gym.agent_utils import process_supervisor -from nemo_gym.agent_utils.supervisor_client import STOP_REQUEST_FILE +from nemo_gym.agent_utils.supervisor_client import STOP_REQUEST_FILE, SUPERVISOR_SCRIPT from nemo_gym.mcp_auto_exposure import TOKEN_HEADER, maybe_auto_expose from nemo_gym.openai_utils import NeMoGymChatCompletionCreateParamsNonStreaming from nemo_gym.rollout_observability import AgentInvocation @@ -528,9 +527,8 @@ def run_conversation(self, *args, **kwargs): """ process = subprocess.Popen( [ - sys.executable, - "-I", - process_supervisor.__file__, + "bash", + str(SUPERVISOR_SCRIPT), "--timeout", "2" if ending == "deadline" else "10", "--cleanup-timeout", diff --git a/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py b/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py index 67bf716a7de..5d8bb2ae5fd 100644 --- a/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py +++ b/responses_api_agents/hermes_sandboxed_agent/tests/test_app.py @@ -12,7 +12,7 @@ import pytest -from nemo_gym.agent_utils.process_supervisor import DEFAULT_CLEANUP_TIMEOUT +from nemo_gym.agent_utils.supervisor_client import DEFAULT_CLEANUP_TIMEOUT from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming from nemo_gym.sandbox import SandboxExecResult from nemo_gym.server_utils import ServerClient @@ -231,10 +231,10 @@ async def download(remote, local): assert {call.args[0].name for call in sandbox.upload.await_args_list} == { "request.json", "runner.py", - "process_supervisor.py", + "process_supervisor.sh", } command_args = shlex.split(command) - assert any(arg.endswith("/process_supervisor.py") for arg in command_args) + assert any(arg.endswith("/process_supervisor.sh") for arg in command_args) assert float(command_args[command_args.index("--timeout") + 1]) == agent.config.sandbox_timeout assert launch.kwargs["timeout_s"] > agent.config.sandbox_timeout + 3 * DEFAULT_CLEANUP_TIMEOUT assert launch.kwargs["cwd"].startswith("/tmp/nemo-hermes-") From 79d90410c935143662d7268b2e4418e4430a3973 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Mon, 26 Oct 2026 09:28:39 -0700 Subject: [PATCH 04/32] test(hermes_agent): expect stop requests as the stop file, not a signal Co-Authored-By: Claude Opus 5.5 Signed-off-by: Hemil Desai Signed-off-by: yaoyu-33 --- responses_api_agents/hermes_agent/tests/test_app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/responses_api_agents/hermes_agent/tests/test_app.py b/responses_api_agents/hermes_agent/tests/test_app.py index ddd728de5d5..dc8c7d07f9a 100644 --- a/responses_api_agents/hermes_agent/tests/test_app.py +++ b/responses_api_agents/hermes_agent/tests/test_app.py @@ -455,7 +455,7 @@ async def download(remote_path, local_path): Path(local_path).write_text( json.dumps( { - "cleanup_confirmed": commands is None or any("kill -TERM" in cmd for cmd in commands), + "cleanup_confirmed": commands is None or any("touch" in cmd and "stop.request" in cmd for cmd in commands), "error": None, } ), @@ -577,7 +577,7 @@ async def exec( with pytest.raises(TimeoutError, match="exceeded its execution deadline"): await hermes.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="fix bug")) - stop = [command for command in sandbox.commands if "stop.request" in command and "kill -TERM" in command] + stop = [command for command in sandbox.commands if "touch" in command and "stop.request" in command] assert len(stop) == 1 @staticmethod @@ -722,7 +722,7 @@ async def exec( timeout=5, ) - stop = [command for command in sandbox.commands if "stop.request" in command and "kill -TERM" in command] + stop = [command for command in sandbox.commands if "touch" in command and "stop.request" in command] assert len(stop) == 1 assert hermes._session_records["session"].state is None with pytest.raises(asyncio.CancelledError): From 40547ce7c7d04e6cc2b4a7ef1d0e04050ead328d Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Mon, 26 Oct 2026 09:27:46 -0700 Subject: [PATCH 05/32] refactor(kilocode): run the harness without Python in the sandbox Kilo sessions now run through the shared bash harness runner with PARAMS written by shell_assignments(); sandbox_runner.py is gone. A short setup string removes a stale plugin receipt and records the task's HOME/XDG for the task-environment plugin, unset_env drops inherited KILO_*/OPENCODE_*/ OpenAI variables, and the exit code comes from the cleanup receipt. Path checks are POSIX shell, the bootstrap installs only bash, and the installer drops its Python checks. The SQLite snapshot moves host-side: kilo.db and its WAL files are downloaded and backed up locally. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Hemil Desai Signed-off-by: yaoyu-33 --- responses_api_agents/kilocode_agent/README.md | 6 +- .../kilocode_agent/install_kilo_runtime.sh | 6 +- .../kilocode_agent/observability.py | 10 ++ .../kilocode_agent/sandbox.py | 111 ++++++++++++------ .../kilocode_agent/sandbox_runner.py | 69 ----------- .../kilocode_agent/tests/test_bootstrap.py | 64 ++++++---- .../tests/test_exec_transport.py | 49 +++++--- .../tests/test_install_runtime.py | 2 - .../tests/test_lifecycle_contract.py | 5 +- .../tests/test_observability.py | 16 +-- .../kilocode_agent/tests/test_sandbox.py | 66 ++++++++--- .../tests/test_task_environment.py | 15 ++- 12 files changed, 236 insertions(+), 183 deletions(-) delete mode 100644 responses_api_agents/kilocode_agent/sandbox_runner.py diff --git a/responses_api_agents/kilocode_agent/README.md b/responses_api_agents/kilocode_agent/README.md index 5e75ab4baba..32d78882a1a 100644 --- a/responses_api_agents/kilocode_agent/README.md +++ b/responses_api_agents/kilocode_agent/README.md @@ -182,8 +182,8 @@ unchanged. Check task-image support and the benchmark's `allowed_agents` before ### Runtime and ownership The task image must be Linux (x86_64 or aarch64 with glibc; x86_64 only with musl), and the sandbox needs network -access to nodejs.org, the npm registry, and the Gym Model Server. Python 3.8+ and Bash must be -present or, on a root image, are installed with apk or apt-get. Task images need not ship Node: +access to nodejs.org, the npm registry, and the Gym Model Server. Bash must be present or, +on a root image, is installed with apk or apt-get; the image needs no Python. Task images need not ship Node: session setup runs [install_kilo_runtime.sh](install_kilo_runtime.sh), which downloads a checksum-verified Node 22 and installs the pinned `kilo_version` into a private directory under `/tmp/nemo-gym-kilo-sessions//`. Missing download tools (`curl`, CA certificates) are installed @@ -214,7 +214,7 @@ mark them `failed` instead; see Quick start). A killed run or a `length` stop returns an `incomplete` response, except a run that times out before any model call completes: Kilo retries an unreachable model server without reporting it, so that run raises with the end of Kilo's log instead. Observations come from a snapshot of Kilo's -SQLite database (root and child sessions, tool calls with timing and bash exit status, usage), +SQLite database, taken on the agent server after the run (root and child sessions, tool calls with timing and bash exit status, usage), annotated with the root outcome and a sandbox record; if the snapshot is unavailable, a coverage gap is reported. The response metadata records the sandbox runner's hostname and PID and the Kilo version. Token IDs and logprobs are not available. diff --git a/responses_api_agents/kilocode_agent/install_kilo_runtime.sh b/responses_api_agents/kilocode_agent/install_kilo_runtime.sh index 0046b4f987c..beffa1b619d 100644 --- a/responses_api_agents/kilocode_agent/install_kilo_runtime.sh +++ b/responses_api_agents/kilocode_agent/install_kilo_runtime.sh @@ -11,8 +11,6 @@ kilo_version=$2 node_version=22.19.0 [[ "$kilo_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo 'An exact Kilo version is required' >&2; exit 1; } [ "$(uname -s)" = Linux ] || { echo 'Kilo sandbox sessions require Linux' >&2; exit 1; } -# The session runner is a standalone Python script. -python3 -c 'import sys; assert sys.version_info >= (3, 8), "Kilo sandbox sessions require Python >=3.8"' case "$(uname -m)" in x86_64) arch=x64 ;; aarch64) arch=arm64 ;; @@ -53,7 +51,9 @@ if [ ! -s /etc/ssl/certs/ca-certificates.crt ] && [ ! -s /etc/pki/tls/certs/ca-b if [ "$missing" = 1 ]; then install_packages curl ca-certificates tar gzip coreutils gawk fi -if [ "$platform" = linux-x64-musl ] && ! python3 -c 'import ctypes; ctypes.CDLL("libstdc++.so.6")' 2>/dev/null; then +# Look where musl's loader looks by default. +if [ "$platform" = linux-x64-musl ] && [ ! -e /lib/libstdc++.so.6 ] && [ ! -e /usr/local/lib/libstdc++.so.6 ] \ + && [ ! -e /usr/lib/libstdc++.so.6 ]; then install_packages libstdc++ fi diff --git a/responses_api_agents/kilocode_agent/observability.py b/responses_api_agents/kilocode_agent/observability.py index d8d4c628028..85baca792a4 100644 --- a/responses_api_agents/kilocode_agent/observability.py +++ b/responses_api_agents/kilocode_agent/observability.py @@ -17,6 +17,16 @@ from responses_api_agents.opencode_agent.artifacts import parse_opencode_observations +def read_kilo_snapshot( + directory: Path, *, fallback_invocation_id: str +) -> tuple[AgentObservationBundle, NeMoGymResponseUsage | None]: + """Read ``kilo.db`` copied with its WAL files, through one consistent copy that includes WAL contents.""" + with sqlite3.connect(directory / "kilo.db") as source: + with sqlite3.connect(directory / "observations.db") as destination: + source.backup(destination) + return read_kilo_observations(directory / "observations.db", fallback_invocation_id=fallback_invocation_id) + + def read_kilo_observations( database: Path, *, fallback_invocation_id: str ) -> tuple[AgentObservationBundle, NeMoGymResponseUsage | None]: diff --git a/responses_api_agents/kilocode_agent/sandbox.py b/responses_api_agents/kilocode_agent/sandbox.py index 57c6d539c1f..808c5999377 100644 --- a/responses_api_agents/kilocode_agent/sandbox.py +++ b/responses_api_agents/kilocode_agent/sandbox.py @@ -11,12 +11,18 @@ from tempfile import TemporaryDirectory from time import perf_counter -from nemo_gym.agent_utils.sandbox_session import SandboxCommand, SandboxSession +from nemo_gym.agent_utils.sandbox_session import ( + HARNESS_RUNNER_FILE, + HARNESS_RUNNER_SCRIPT, + SandboxCommand, + SandboxSession, + shell_assignments, +) from nemo_gym.base_responses_api_agent import AgentSessionState from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming, NeMoGymResponseUsage from nemo_gym.rollout_observability import AgentObservationBundle from nemo_gym.sandbox.utils import read_text, upload_text -from responses_api_agents.kilocode_agent.observability import read_kilo_observations +from responses_api_agents.kilocode_agent.observability import read_kilo_snapshot LOG = logging.getLogger(__name__) @@ -24,11 +30,35 @@ TASK_ENVIRONMENT_PLUGIN = "task_environment.mjs" +# Runs in the sandbox before Kilo's private overrides: a stale plugin receipt must not count, +# and the plugin's shell.env hook needs the task's own HOME and XDG directories. +_ACTIVATION_SETUP = r"""rm -f -- "$session/task-environment.ready" || exit 1 +mkdir -p -- "$session/home" "$session/data" "$session/config" "$session/cache" || exit 1 +json_string() { + local value=${1//\\/\\\\} + value=${value//\"/\\\"} + printf '"%s"' "$value" +} +home=${HOME:-$(unset HOME; echo ~)} +printf '{"HOME": %s, "XDG_DATA_HOME": %s, "XDG_CONFIG_HOME": %s, "XDG_CACHE_HOME": %s, "XDG_STATE_HOME": %s}\n' \ + "$(json_string "$home")" "$(json_string "${XDG_DATA_HOME:-$home/.local/share}")" \ + "$(json_string "${XDG_CONFIG_HOME:-$home/.config}")" "$(json_string "${XDG_CACHE_HOME:-$home/.cache}")" \ + "$(json_string "${XDG_STATE_HOME:-$home/.local/state}")" >"$session/task-env.json" +""" + +# Resolves symlinks in the longest existing prefix, since the session root may not exist yet. +_RESOLVE = r"""resolve() { + path=$1 rest= + while [ -n "$path" ] && [ ! -d "$path" ]; do rest=/${path##*/}$rest path=${path%/*}; done + path=$(cd -- "${path:-/}" && pwd -P) && printf '%s%s\n' "${path%/}" "$rest" +} +""" + + def _sandbox_prepare_command(workdir: str, directory: str) -> str: - # Bootstrap with POSIX sh; the runtime installer itself requires Bash. + # Bootstrap with POSIX sh; the runtime installer and the harness runner require Bash. bootstrap = """set -eu set -- -command -v python3 >/dev/null 2>&1 || set -- "$@" python3 command -v bash >/dev/null 2>&1 || set -- "$@" bash if [ "$#" -gt 0 ]; then [ "$(id -u)" = 0 ] || { echo "Kilo sandbox sessions require $*: preinstall these tools or use a root image." >&2; exit 1; } @@ -43,17 +73,16 @@ def _sandbox_prepare_command(workdir: str, directory: str) -> str: fi fi """ + root = quote(str(PurePosixPath(directory).parent)) validate_paths = ( - "from pathlib import Path; import sys; " - "workdir,*roots=[Path(p).resolve() for p in sys.argv[1:]]; " - "assert workdir.is_dir(), 'Kilo task workdir is missing'; " - "assert all(workdir != root and workdir not in root.parents and root not in workdir.parents " - "for root in roots), 'Kilo session storage overlaps the task workdir'" - ) - return ( - bootstrap + f"python3 -I -c {quote(validate_paths)} {quote(workdir)} " - f"{quote(str(PurePosixPath(directory).parent))} && mkdir -p -- {quote(directory)}" + f"workdir=$(cd -- {quote(workdir)} && pwd -P) || {{ echo 'Kilo task workdir is missing' >&2; exit 1; }}\n" + f"root=$(resolve {root})\n" + # Each pattern also matches equal paths. + 'case "$root/" in "${workdir%/}"/*) overlap=1 ;; esac\n' + 'case "$workdir/" in "${root%/}"/*) overlap=1 ;; esac\n' + '[ -z "${overlap-}" ] || { echo "Kilo session storage overlaps the task workdir" >&2; exit 1; }\n' ) + return bootstrap + _RESOLVE + validate_paths + f"mkdir -p -- {quote(directory)}" @dataclass(frozen=True, kw_only=True) @@ -121,7 +150,7 @@ async def install_runtime(self, *, version: str, timeout: float) -> None: f"Kilo sandbox installation failed (exit {installed.return_code}, " f"error_type={installed.error_type}): {details[-16000:]}" ) - await sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), f"{directory}/sandbox_runner.py") + await sandbox.upload(HARNESS_RUNNER_SCRIPT, f"{directory}/{HARNESS_RUNNER_FILE}") await sandbox.upload( Path(__file__).with_name(TASK_ENVIRONMENT_PLUGIN), f"{directory}/{TASK_ENVIRONMENT_PLUGIN}" ) @@ -130,10 +159,29 @@ async def stage_activation(self, *, command: list[str], config: str) -> SandboxC """Stage the isolated provider config and CLI argv without altering task files.""" directory = self.session.session_dir await upload_text(self.session.sandbox, path=f"{directory}/kilo.json", text=config) - await upload_text(self.session.sandbox, path=f"{directory}/input.json", text=json.dumps({"command": command})) - return SandboxCommand( - argv=["python3", "-I", f"{directory}/sandbox_runner.py", directory, self.session.workdir], - ) + params = { + "cwd": self.session.workdir, + "command": command, + "env": { + "HOME": f"{directory}/home", + "XDG_DATA_HOME": f"{directory}/data", + "XDG_CONFIG_HOME": f"{directory}/config", + "XDG_CACHE_HOME": f"{directory}/cache", + "KILO_NO_DAEMON": "1", + "KILO_DB": f"{directory}/kilo.db", + "KILO_CONFIG": f"{directory}/kilo.json", + "KILO_DISABLE_PROJECT_CONFIG": "1", + }, + # Do not inherit provider credentials or configuration overrides from the image. + "unset_env": ["KILO_*", "OPENCODE_*", "OPENAI_BASE_URL", "OPENAI_API_KEY"], + "setup": f"session={quote(directory)}\n{_ACTIVATION_SETUP}", + "stdin": "/dev/null", + # File-backed stdout survives supervisor termination. + "stdout": f"{directory}/stdout.jsonl", + "stderr": f"{directory}/stderr.log", + } + await upload_text(self.session.sandbox, path=f"{directory}/params.sh", text=shell_assignments(params)) + return SandboxCommand(argv=["bash", f"{directory}/{HARNESS_RUNNER_FILE}", f"{directory}/params.sh"]) async def collect_artifacts(self) -> KiloArtifacts: """Retain flushed CLI output, including partial events after a supervised stop.""" @@ -148,13 +196,9 @@ async def collect_artifacts(self) -> KiloArtifacts: stderr = await read_text(self.session.sandbox, path=f"{directory}/stderr.log") except Exception: stderr = await self.session.read_output_log() - try: - raw = json.loads(await read_text(self.session.sandbox, path=f"{directory}/exit.json")) - except Exception: - # A killed runner cannot record a CLI exit. Cleanup proof comes from the supervisor. - LOG.warning("Kilo exit diagnostics unavailable; retaining partial CLI output", exc_info=True) - raw = None - exit_code = raw if type(raw) is int else None + # The runner execs Kilo, so the receipt holds Kilo's exit status; a timed-out Kilo has none of its own. + cleanup = self.session.cleanup + exit_code = cleanup["return_code"] if cleanup is not None and not cleanup["timed_out"] else None try: runtime = json.loads(await read_text(self.session.sandbox, path=f"{directory}/runtime.json")) except Exception: @@ -172,18 +216,19 @@ async def collect_artifacts(self) -> KiloArtifacts: try: if self.session.cleanup is None or not self.session.cleanup["cleanup_confirmed"]: raise RuntimeError("Kilo database capture requires confirmed cleanup") - snapshot = await self.session.sandbox.exec( - f"python3 -I {quote(directory + '/sandbox_runner.py')} --snapshot {quote(directory)}", - cwd=self.session.workdir, + # Kilo has stopped, so its database files are quiescent; the WAL holds recent commits. + listed = await self.session.sandbox.exec( + f"cd {quote(directory)} && [ -f kilo.db ] && ls -- kilo.db*", + cwd="/", timeout_s=30, ) - if snapshot.error_type or snapshot.return_code != 0: - raise RuntimeError(f"Kilo database snapshot failed: {snapshot.stderr}") + if listed.error_type or listed.return_code != 0: + raise RuntimeError(f"Kilo database unavailable: {listed.stderr}") with TemporaryDirectory(prefix="nemo-gym-kilo-observations-") as temporary: - database = Path(temporary) / "observations.db" - await self.session.sandbox.download(f"{directory}/observations.db", database) + for name in {"kilo.db", "kilo.db-wal", "kilo.db-shm"} & set((listed.stdout or "").split()): + await self.session.sandbox.download(f"{directory}/{name}", Path(temporary) / name) observations, usage = await asyncio.to_thread( - read_kilo_observations, database, fallback_invocation_id=self.request.agent_session_id + read_kilo_snapshot, Path(temporary), fallback_invocation_id=self.request.agent_session_id ) except Exception: # Partial stdout remains usable even if a killed worker never initialized its database. diff --git a/responses_api_agents/kilocode_agent/sandbox_runner.py b/responses_api_agents/kilocode_agent/sandbox_runner.py deleted file mode 100644 index 7fcf04658f9..00000000000 --- a/responses_api_agents/kilocode_agent/sandbox_runner.py +++ /dev/null @@ -1,69 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Standalone Python 3.8+ runner; Gym stays on the agent-server host.""" - -import json -import os -import sqlite3 -import subprocess -import sys -from pathlib import Path - - -def main() -> int: - """Run Kilo in the task workdir while keeping all adapter files in its session directory.""" - directory = Path(sys.argv[1]) - workdir = sys.argv[2] - (directory / "runtime.json").write_text(json.dumps({"hostname": os.uname().nodename, "pid": os.getpid()})) - payload = json.loads((directory / "input.json").read_text()) - # Never accept an initialization receipt left by an earlier invocation. - (directory / "task-environment.ready").unlink(missing_ok=True) - # Save the task's home and XDG directories for the shell.env hook, before Kilo's private overrides. - task_home = os.environ.get("HOME") or str(Path.home()) - task_env = {"HOME": task_home} - for key, suffix in ( - ("XDG_DATA_HOME", ".local/share"), - ("XDG_CONFIG_HOME", ".config"), - ("XDG_CACHE_HOME", ".cache"), - ("XDG_STATE_HOME", ".local/state"), - ): - task_env[key] = os.environ.get(key) or str(Path(task_home) / suffix) - (directory / "task-env.json").write_text(json.dumps(task_env)) - env = dict(os.environ) - # Do not inherit provider credentials or configuration overrides from the image. - for key in list(env): - if key.startswith(("KILO_", "OPENCODE_")) or key in ("OPENAI_BASE_URL", "OPENAI_API_KEY"): - env.pop(key) - env.update( - HOME=str(directory / "home"), - XDG_DATA_HOME=str(directory / "data"), - XDG_CONFIG_HOME=str(directory / "config"), - XDG_CACHE_HOME=str(directory / "cache"), - KILO_NO_DAEMON="1", - KILO_DB=str(directory / "kilo.db"), - KILO_CONFIG=str(directory / "kilo.json"), - KILO_DISABLE_PROJECT_CONFIG="1", - ) - for name in ("home", "data", "config", "cache"): - (directory / name).mkdir(exist_ok=True) - # File-backed stdout survives supervisor termination; a pipe buffered by this runner would not. - with (directory / "stdout.jsonl").open("wb") as stdout, (directory / "stderr.log").open("wb") as stderr: - result = subprocess.run( - payload["command"], cwd=workdir, env=env, stdin=subprocess.DEVNULL, stdout=stdout, stderr=stderr - ) - (directory / "exit.json").write_text(json.dumps(result.returncode)) - return result.returncode - - -def snapshot(directory: Path) -> None: - """Retain a consistent database, including WAL contents, after supervised cleanup.""" - with sqlite3.connect(f"file:{directory / 'kilo.db'}?mode=ro", uri=True) as source: - with sqlite3.connect(directory / "observations.db") as destination: - source.backup(destination) - - -if __name__ == "__main__": - if sys.argv[1] == "--snapshot": - snapshot(Path(sys.argv[2])) - else: - sys.exit(main()) diff --git a/responses_api_agents/kilocode_agent/tests/test_bootstrap.py b/responses_api_agents/kilocode_agent/tests/test_bootstrap.py index e2c54414494..88ca2b2d4df 100644 --- a/responses_api_agents/kilocode_agent/tests/test_bootstrap.py +++ b/responses_api_agents/kilocode_agent/tests/test_bootstrap.py @@ -5,21 +5,27 @@ import shlex import shutil import subprocess -import sys +from pathlib import Path import pytest from responses_api_agents.kilocode_agent.sandbox import _sandbox_prepare_command +def prepare(command: str, env: dict[str, str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["/bin/sh", "-c", command], env=env, capture_output=True, text=True, errors="replace", timeout=10 + ) + + @pytest.mark.parametrize("package_manager", ["apk", "apt-get"]) -@pytest.mark.parametrize("missing", [(), ("bash",), ("python3",), ("python3", "bash")]) +@pytest.mark.parametrize("missing", [(), ("bash",)]) @pytest.mark.parametrize("failure", [None, "nonroot", "install", "unavailable"]) def test_posix_bootstrap_installs_only_missing_prerequisites(tmp_path, package_manager, missing, failure): - """Exercise the actual preparation through sh without invoking a host package manager.""" + """Exercise the actual preparation through sh, with no Python, without invoking a host package manager.""" bindir = tmp_path / "bin" bindir.mkdir() - binaries = {"python3": sys.executable, "bash": shutil.which("bash"), "mkdir": shutil.which("mkdir")} + binaries = {"bash": shutil.which("bash"), "mkdir": shutil.which("mkdir")} assert all(binaries.values()) for name, executable in binaries.items(): if name not in missing: @@ -33,8 +39,7 @@ def test_posix_bootstrap_installs_only_missing_prerequisites(tmp_path, package_m '#!/bin/sh\nprintf "%s\\n" "$*" >> "$TEST_ROOT/packages.log"\n' + ("exit 100\n" if failure == "install" else "") + 'for name in "$@"; do\n' - + f'case "$name" in\n python3) ln -s {shlex.quote(sys.executable)} "$TEST_ROOT/bin/python3" ;;\n' - + f' bash) ln -s {shlex.quote(binaries["bash"])} "$TEST_ROOT/bin/bash" ;;\n' + + f'case "$name" in\n bash) ln -s {shlex.quote(binaries["bash"])} "$TEST_ROOT/bin/bash" ;;\n' + "esac\ndone\n" ) manager.chmod(0o755) @@ -43,14 +48,9 @@ def test_posix_bootstrap_installs_only_missing_prerequisites(tmp_path, package_m directory = tmp_path / "sessions" / "session with spaces" task = tmp_path / "task" task.mkdir() - command = _sandbox_prepare_command(str(task), str(directory)) - result = subprocess.run( - ["/bin/sh", "-c", command], - env=os.environ | {"PATH": str(bindir), "TEST_ROOT": str(tmp_path)}, - capture_output=True, - text=True, - errors="replace", - timeout=10, + result = prepare( + _sandbox_prepare_command(str(task), str(directory)), + os.environ | {"PATH": str(bindir), "TEST_ROOT": str(tmp_path)}, ) if failure and missing: assert result.returncode == (100 if failure == "install" else 1), result.stderr @@ -70,13 +70,33 @@ def test_posix_bootstrap_installs_only_missing_prerequisites(tmp_path, package_m ) else: assert not (tmp_path / "packages.log").exists() - bash = subprocess.run( - ["/bin/sh", "-c", "bash -c 'printf ready'"], - env=os.environ | {"PATH": str(bindir)}, - capture_output=True, - text=True, - errors="replace", - timeout=10, - ) + bash = prepare("bash -c 'printf ready'", os.environ | {"PATH": str(bindir)}) assert bash.returncode == 0, bash.stderr assert bash.stdout == "ready" + + +@pytest.mark.parametrize("layout", ["inside", "same", "contains", "symlink", "root_workdir", "missing"]) +def test_session_storage_must_not_overlap_the_task_workdir(tmp_path: Path, layout: str) -> None: + task = tmp_path / "task" + task.mkdir() + workdir, directory = task, tmp_path / "sessions" / "id" + if layout == "inside": + directory = task / "sessions" / "id" + elif layout == "same": + directory = task / "id" + elif layout == "contains": + workdir = tmp_path / "sessions" / "task" + workdir.mkdir(parents=True) + elif layout == "symlink": + # Lexically separate, but the session root resolves into the task workdir. + (tmp_path / "link").symlink_to(task) + directory = tmp_path / "link" / "sessions" / "id" + elif layout == "root_workdir": + workdir = Path("/") + else: + workdir = tmp_path / "absent" + result = prepare(_sandbox_prepare_command(str(workdir), str(directory)), dict(os.environ)) + assert result.returncode == 1 + expected = "workdir is missing" if layout == "missing" else "overlaps the task workdir" + assert expected in result.stderr + assert not directory.exists() and not (task / "sessions").exists() diff --git a/responses_api_agents/kilocode_agent/tests/test_exec_transport.py b/responses_api_agents/kilocode_agent/tests/test_exec_transport.py index 937e5649d52..e14050779d0 100644 --- a/responses_api_agents/kilocode_agent/tests/test_exec_transport.py +++ b/responses_api_agents/kilocode_agent/tests/test_exec_transport.py @@ -7,15 +7,17 @@ import os import shutil import signal +import sqlite3 import sys from pathlib import Path import pytest -from nemo_gym.agent_utils.sandbox_session import SandboxSession +from nemo_gym.agent_utils.sandbox_session import HARNESS_RUNNER_FILE, HARNESS_RUNNER_SCRIPT, SandboxSession +from nemo_gym.rollout_observability import AgentInvocation from nemo_gym.sandbox.providers.base import SandboxExecResult -from responses_api_agents.kilocode_agent import sandbox_runner from responses_api_agents.kilocode_agent.sandbox import KiloSandboxSession +from responses_api_agents.kilocode_agent.tests.test_observability import database, family from responses_api_agents.kilocode_agent.tests.test_sandbox import seed @@ -40,7 +42,7 @@ async def disconnect(self): self.disconnected = True async def exec(self, command, *, cwd=None, timeout_s=30): - launch = "--receipt" in command and "process_supervisor.py" in command + launch = "--receipt" in command and "process_supervisor.sh" in command if launch: self.deadline = timeout_s self.delayed_command = command @@ -74,8 +76,8 @@ def make_session(tmp_path): provider_name="local", session=SandboxSession(sandbox=provider, session_dir=str(directory), workdir=str(workdir), harness="KiloCode"), ) - shutil.copyfile(sandbox_runner.__file__, directory / "sandbox_runner.py") - assert not (directory / "process_supervisor.py").exists() + shutil.copyfile(HARNESS_RUNNER_SCRIPT, directory / HARNESS_RUNNER_FILE) + assert not (directory / "process_supervisor.sh").exists() return state, provider, workdir @@ -89,15 +91,17 @@ def activate(state, code, timeout=0.5): @pytest.mark.parametrize("ending", ["natural", "timeout", "cancel", "close"]) -async def test_exec_only_supervision_reaps_detached_child(tmp_path, ending): +async def test_exec_only_supervision_reaps_harness_group(tmp_path, ending): state, provider, workdir = make_session(tmp_path) + # A tool that stays in Kilo's process group; detached services are left to the sandbox owner. code = ( "import subprocess,sys,time,pathlib; " - "p=subprocess.Popen([sys.executable,'-c','import time; time.sleep(60)'],start_new_session=True); " + "p=subprocess.Popen([sys.executable,'-c','import time; time.sleep(60)']); " "pathlib.Path('child.pid').write_text(str(p.pid)); " + ("print('{}')" if ending == "natural" else "time.sleep(60)") ) - deadline = 0.5 if ending == "timeout" else 60 + # Long enough for a loaded host to start the tool before the deadline. + deadline = 3 if ending == "timeout" else 60 task = state.task = asyncio.create_task(activate(state, code, timeout=deadline)) try: async with asyncio.timeout(5): @@ -116,13 +120,12 @@ async def test_exec_only_supervision_reaps_detached_child(tmp_path, ending): assert state.session.cleanup["cleanup_confirmed"] is True if ending == "timeout": assert state.session.cleanup["timed_out"] is True - child_pid = int((workdir / "child.pid").read_text()) - if ending == "natural": - # Natural CLI exit has no adoption cutoff; the sandbox owner tears down detached tools. - os.kill(child_pid, 0) - else: - with pytest.raises(ProcessLookupError): - os.kill(child_pid, 0) + # The orphaned tool may linger as a zombie until init reaps it; it can no longer run. + try: + stat = Path(f"/proc/{(workdir / 'child.pid').read_text()}/stat").read_text() + except FileNotFoundError: + stat = ") X" + assert stat.rpartition(")")[2].split()[0] in ("Z", "X") assert provider.cancelled_launch is False assert provider.deadline > deadline + 3 * 1 await state.close(timeout=3) @@ -141,6 +144,22 @@ async def test_exec_only_supervision_reaps_detached_child(tmp_path, ending): pass +async def test_observations_include_commits_still_in_the_wal(tmp_path): + state, _, _ = make_session(tmp_path) + path = Path(state.session.session_dir) / "kilo.db" + database(path, family()) + # An open connection keeps the latest commit in kilo.db-wal, as a stopped Kilo can leave it. + with sqlite3.connect(path) as connection: + connection.execute("pragma wal_checkpoint(truncate)") + connection.execute("insert into session values ('new-child', 'root', 2)") + connection.commit() + artifacts = await activate(state, "print('{}')", timeout=60) + records = artifacts.observations.records + invocations = {record.invocation_id for record in records if isinstance(record, AgentInvocation)} + assert {"root", "new-child"} <= invocations + await state.close(timeout=3) + + async def test_lost_launch_is_fenced_even_after_directory_retirement(tmp_path): state, provider, workdir = make_session(tmp_path) provider.lost_launch = True diff --git a/responses_api_agents/kilocode_agent/tests/test_install_runtime.py b/responses_api_agents/kilocode_agent/tests/test_install_runtime.py index 58bcd325074..fafcf120204 100644 --- a/responses_api_agents/kilocode_agent/tests/test_install_runtime.py +++ b/responses_api_agents/kilocode_agent/tests/test_install_runtime.py @@ -7,7 +7,6 @@ import os import shutil import subprocess -import sys import tarfile from pathlib import Path @@ -56,7 +55,6 @@ def sandbox(tmp_path: Path) -> tuple[Path, dict[str, str]]: if executable is None: pytest.skip(f"Installer test requires {name}") (bindir / name).symlink_to(executable) - (bindir / "python3").symlink_to(sys.executable) archive = _node_archive() (tmp_path / ARCHIVE).write_bytes(archive) (tmp_path / "SHASUMS256.txt").write_text(f"{hashlib.sha256(archive).hexdigest()} {ARCHIVE}\n") diff --git a/responses_api_agents/kilocode_agent/tests/test_lifecycle_contract.py b/responses_api_agents/kilocode_agent/tests/test_lifecycle_contract.py index 0b1fb1edd76..ddd563d2606 100644 --- a/responses_api_agents/kilocode_agent/tests/test_lifecycle_contract.py +++ b/responses_api_agents/kilocode_agent/tests/test_lifecycle_contract.py @@ -28,13 +28,12 @@ def __init__(self): self.disconnect = AsyncMock() async def run_exec(self, command, **kwargs): - if "--receipt" in command and "process_supervisor.py" in command: + if "--receipt" in command and "process_supervisor.sh" in command: self.events.append("launch") self.files[f"{DIRECTORY}/stdout.jsonl"] = '{"type":"text","part":{"text":"done"}}\n' self.files[f"{DIRECTORY}/stderr.log"] = "" - self.files[f"{DIRECTORY}/exit.json"] = "0" self.files[f"{DIRECTORY}/cleanup.json"] = json.dumps(self.result) - elif "--snapshot" in command: + elif "kilo.db" in command: self.events.append("snapshot") return SandboxExecResult(stdout="", stderr="no database", return_code=1) return OK diff --git a/responses_api_agents/kilocode_agent/tests/test_observability.py b/responses_api_agents/kilocode_agent/tests/test_observability.py index d53102931dd..9364025edf9 100644 --- a/responses_api_agents/kilocode_agent/tests/test_observability.py +++ b/responses_api_agents/kilocode_agent/tests/test_observability.py @@ -3,9 +3,8 @@ """Family coverage, output separation, accounting, and consistent artifact capture.""" import json +import shutil import sqlite3 -import subprocess -import sys from pathlib import Path from unittest.mock import AsyncMock @@ -13,7 +12,7 @@ from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming from nemo_gym.rollout_observability import AgentInvocation, ToolCallObservation -from responses_api_agents.kilocode_agent.observability import read_kilo_observations +from responses_api_agents.kilocode_agent.observability import read_kilo_observations, read_kilo_snapshot from responses_api_agents.kilocode_agent.sandbox import KiloArtifacts from responses_api_agents.kilocode_agent.tests.test_app import _make_model_server_agent from responses_api_agents.kilocode_agent.tests.test_sandbox import state @@ -173,11 +172,12 @@ def test_snapshot_keeps_committed_wal_data(tmp_path: Path) -> None: connection.execute("pragma wal_checkpoint(truncate)") connection.execute("insert into session values ('new-child', 'root', 2)") connection.commit() - runner = Path(__file__).parents[1] / "sandbox_runner.py" - subprocess.run( - [sys.executable, "-I", str(runner), "--snapshot", str(tmp_path)], check=True, capture_output=True - ) - observations, _ = read_kilo_observations(tmp_path / "observations.db", fallback_invocation_id="gym") + # The agent server downloads the database and its WAL, without the shared-memory index. + downloaded = tmp_path / "downloaded" + downloaded.mkdir() + for name in ("kilo.db", "kilo.db-wal"): + shutil.copyfile(tmp_path / name, downloaded / name) + observations, _ = read_kilo_snapshot(downloaded, fallback_invocation_id="gym") assert "new-child" in { record.invocation_id for record in observations.records if isinstance(record, AgentInvocation) } diff --git a/responses_api_agents/kilocode_agent/tests/test_sandbox.py b/responses_api_agents/kilocode_agent/tests/test_sandbox.py index 3352e997a28..f3451ba1cb9 100644 --- a/responses_api_agents/kilocode_agent/tests/test_sandbox.py +++ b/responses_api_agents/kilocode_agent/tests/test_sandbox.py @@ -5,6 +5,7 @@ import asyncio import json import os +import shutil import subprocess import sys import threading @@ -16,7 +17,7 @@ from fastapi.testclient import TestClient from starlette.requests import Request -from nemo_gym.agent_utils.sandbox_session import SandboxSession +from nemo_gym.agent_utils.sandbox_session import HARNESS_RUNNER_FILE, HARNESS_RUNNER_SCRIPT, SandboxSession from nemo_gym.base_responses_api_agent import ( AgentCloseSessionRequest, AgentSeedSessionRequest, @@ -293,11 +294,15 @@ async def test_connection_failure_never_falls_back_to_host() -> None: install.assert_not_called() -async def test_artifacts_keep_partial_output_without_inventing_exit_code() -> None: +@pytest.mark.parametrize("timed_out", [False, True]) +async def test_artifacts_take_the_exit_code_from_cleanup_only_when_kilo_finished(timed_out: bool) -> None: session = state() + session.session.cleanup = receipt(return_code=143 if timed_out else 3, timed_out=timed_out) with patch(f"{TRANSPORT}.read_text", side_effect=["partial", "diagnostic", RuntimeError("not written")]): artifacts = await session.collect_artifacts() - assert artifacts == KiloArtifacts(stdout="partial", stderr="diagnostic", exit_code=None) + assert (artifacts.stdout, artifacts.stderr) == ("partial", "diagnostic") + # The supervisor's TERM status after a timeout is not an exit Kilo chose. + assert artifacts.exit_code == (None if timed_out else 3) async def test_identical_activations_join_and_replay_independent_copies() -> None: @@ -452,7 +457,7 @@ async def test_runner_identity_reaches_response_metadata() -> None: session = state() with patch( f"{TRANSPORT}.read_text", - side_effect=[TEXT, "", "0", json.dumps({"hostname": "sandbox-7", "pid": 42}), "ready\n"], + side_effect=[TEXT, "", json.dumps({"hostname": "sandbox-7", "pid": 42}), "ready\n"], ): artifacts = await session.collect_artifacts() assert (artifacts.runner_hostname, artifacts.runner_pid) == ("sandbox-7", 42) @@ -494,8 +499,27 @@ async def activation() -> None: assert cleaned.is_set() and session.task.cancelled() +class LocalFiles: + """File transfer onto the local filesystem, standing in for the sandbox.""" + + async def upload(self, source: Path, destination: str) -> None: + shutil.copyfile(source, destination) + + +async def stage_locally(directory: Path, workdir: Path, command: list[str]) -> list[str]: + """Stage an activation into ``directory`` as the sandbox would receive it; return the runner argv.""" + session = KiloSandboxSession( + request=seed(workdir=str(workdir)), + session=SandboxSession( + sandbox=LocalFiles(), session_dir=str(directory), workdir=str(workdir), harness="KiloCode" + ), + ) + shutil.copyfile(HARNESS_RUNNER_SCRIPT, directory / HARNESS_RUNNER_FILE) + return (await session.stage_activation(command=command, config="{}")).argv + + @pytest.mark.parametrize("exit_code", [0, 3]) -def test_standalone_runner_executes_in_task_directory(tmp_path: Path, exit_code: int) -> None: +async def test_runner_executes_in_task_directory(tmp_path: Path, exit_code: int) -> None: directory, repo = tmp_path / "session", tmp_path / "repo" directory.mkdir() repo.mkdir() @@ -505,19 +529,17 @@ def test_standalone_runner_executes_in_task_directory(tmp_path: Path, exit_code: "print(json.dumps({'cwd':os.getcwd(),'config':os.environ['KILO_CONFIG']," "'home':os.environ['HOME'],'db':os.environ['KILO_DB']," "'project':os.environ['KILO_DISABLE_PROJECT_CONFIG'],'key':os.getenv('OPENAI_API_KEY')," - "'stdin':sys.stdin.read()})); " + "'kilo':os.getenv('KILO_MODEL'),'kept':os.getenv('KEPT'),'stdin':sys.stdin.read()})); " "sys.stderr.buffer.write(b'diagnostic\\xff'); " f"sys.exit({exit_code})" ) - (directory / "input.json").write_text(json.dumps({"command": [sys.executable, "-c", script]})) - runner = Path(__file__).parents[1] / "sandbox_runner.py" - result = subprocess.run( - [sys.executable, str(runner), str(directory), str(repo)], - env={**os.environ, "OPENAI_API_KEY": "test-credential"}, - input=b"unexpected inherited input", - capture_output=True, - ) - assert result.returncode == exit_code + argv = await stage_locally(directory, repo, [sys.executable, "-c", script]) + assert argv == ["bash", f"{directory}/harness_runner.sh", f"{directory}/params.sh"] + task_home = str(tmp_path / 'task "home" \\') + env = {key: value for key, value in os.environ.items() if not key.startswith("XDG_")} + inherited = {"OPENAI_API_KEY": "test-credential", "KILO_MODEL": "other", "KEPT": "yes", "HOME": task_home} + result = subprocess.run(argv, env=env | inherited, input=b"unexpected inherited input", capture_output=True) + assert result.returncode == exit_code, result.stderr output = json.loads((directory / "stdout.jsonl").read_text()) assert output == { "cwd": str(repo), @@ -526,12 +548,22 @@ def test_standalone_runner_executes_in_task_directory(tmp_path: Path, exit_code: "db": str(directory / "kilo.db"), "project": "1", "key": None, + "kilo": None, + "kept": "yes", "stdin": "", } assert (repo / "patch.txt").read_text() == "changed" assert not (repo / "kilo.json").exists() assert (directory / "stderr.log").read_bytes() == b"diagnostic\xff" - assert json.loads((directory / "exit.json").read_text()) == exit_code + # The plugin's shell.env hook receives the task's directories, recorded before Kilo's overrides. + assert json.loads((directory / "task-env.json").read_text()) == { + "HOME": task_home, + "XDG_DATA_HOME": f"{task_home}/.local/share", + "XDG_CONFIG_HOME": f"{task_home}/.config", + "XDG_CACHE_HOME": f"{task_home}/.cache", + "XDG_STATE_HOME": f"{task_home}/.local/state", + } + assert all((directory / name).is_dir() for name in ("home", "data", "config", "cache")) runtime = json.loads((directory / "runtime.json").read_text()) assert runtime["hostname"] == os.uname().nodename and runtime["pid"] != os.getpid() @@ -611,7 +643,7 @@ async def test_snapshot_requires_confirmed_cleanup_before_any_io() -> None: session = state() session.started_at = 0.0 session.session.cleanup = {**receipt(), "cleanup_confirmed": False} - with patch(f"{TRANSPORT}.read_text", side_effect=["partial", "diagnostic", "0"]): + with patch(f"{TRANSPORT}.read_text", side_effect=["partial", "diagnostic"]): artifacts = await session.collect_artifacts() assert artifacts.stdout == "partial" and artifacts.observations is None session.session.sandbox.exec.assert_not_awaited() diff --git a/responses_api_agents/kilocode_agent/tests/test_task_environment.py b/responses_api_agents/kilocode_agent/tests/test_task_environment.py index 1b8b6e1d608..b568c924478 100644 --- a/responses_api_agents/kilocode_agent/tests/test_task_environment.py +++ b/responses_api_agents/kilocode_agent/tests/test_task_environment.py @@ -11,16 +11,16 @@ import pytest -from responses_api_agents.kilocode_agent import sandbox_runner +from responses_api_agents.kilocode_agent.tests.test_sandbox import stage_locally -PLUGIN = Path(sandbox_runner.__file__).with_name("task_environment.mjs") +PLUGIN = Path(__file__).parents[1] / "task_environment.mjs" TASK_DIRECTORIES = ("HOME", "XDG_DATA_HOME", "XDG_CONFIG_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME") @pytest.mark.skipif(shutil.which("node") is None, reason="Node.js is required for the Kilo plugin") @pytest.mark.parametrize("custom_xdg", [False, True]) -def test_shell_environment_keeps_task_files_after_session_close(tmp_path: Path, custom_xdg: bool) -> None: +async def test_shell_environment_keeps_task_files_after_session_close(tmp_path: Path, custom_xdg: bool) -> None: session, workdir, home = tmp_path / "session", tmp_path / "task", tmp_path / "task-home" for directory in (session, workdir, home): directory.mkdir() @@ -41,14 +41,14 @@ def test_shell_environment_keeps_task_files_after_session_close(tmp_path: Path, console.log(JSON.stringify({cliHome: process.env.HOME, cliConfig: process.env.XDG_CONFIG_HOME, tool: output.env})); """ ) - (session / "input.json").write_text(json.dumps({"command": [shutil.which("node"), str(session / "probe.mjs")]})) + argv = await stage_locally(session, workdir, [shutil.which("node"), str(session / "probe.mjs")]) # No task directories leak in from the agent-server host. env = {key: value for key, value in os.environ.items() if not key.startswith("XDG_")} env["HOME"] = str(home) if custom_xdg: env["XDG_CONFIG_HOME"] = str(tmp_path / "task-config") result = subprocess.run( - [sys.executable, "-I", sandbox_runner.__file__, str(session), str(workdir)], + argv, env=env, capture_output=True, text=True, @@ -98,16 +98,15 @@ def test_invalid_task_environment_cannot_acknowledge_hook_initialization(tmp_pat assert not (tmp_path / "task-environment.ready").exists() -def test_runner_removes_stale_task_environment_receipt_before_launch(tmp_path: Path) -> None: +async def test_runner_removes_stale_task_environment_receipt_before_launch(tmp_path: Path) -> None: session, workdir = tmp_path / "session", tmp_path / "task" session.mkdir() workdir.mkdir() receipt = session / "task-environment.ready" receipt.write_text("ready\n") command = [sys.executable, "-c", "print('Kilo continued without loading the hook')"] - (session / "input.json").write_text(json.dumps({"command": command})) result = subprocess.run( - [sys.executable, "-I", sandbox_runner.__file__, str(session), str(workdir)], + await stage_locally(session, workdir, command), capture_output=True, text=True, timeout=30, From 226d2a7cab5be422c8ae3427d157e9fd7ee9cf67 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Mon, 26 Oct 2026 09:14:39 -0700 Subject: [PATCH 06/32] refactor(claude_code_agent): run the harness without Python in the sandbox Claude sessions now launch through the shared bash harness_runner.sh with params.sh instead of sandbox_runner.py and input.json. The offline runtime installer is a bash script with the same SHA-256, version and lock semantics, so task images need bash and coreutils but no Python. Signed-off-by: Hemil Desai Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- .../claude_code_agent/README.md | 4 +- responses_api_agents/claude_code_agent/app.py | 8 +- .../install_claude_runtime.py | 109 ------------------ .../install_claude_runtime.sh | 60 ++++++++++ .../claude_code_agent/runtime.py | 13 ++- .../claude_code_agent/sandbox.py | 24 ++-- .../claude_code_agent/sandbox_runner.py | 50 -------- .../tests/test_install_claude_runtime.py | 85 +++++++------- .../claude_code_agent/tests/test_runtime.py | 7 +- .../claude_code_agent/tests/test_sandbox.py | 4 +- .../tests/test_sandbox_runner.py | 51 ++++++-- 11 files changed, 180 insertions(+), 235 deletions(-) delete mode 100644 responses_api_agents/claude_code_agent/install_claude_runtime.py create mode 100644 responses_api_agents/claude_code_agent/install_claude_runtime.sh delete mode 100644 responses_api_agents/claude_code_agent/sandbox_runner.py diff --git a/responses_api_agents/claude_code_agent/README.md b/responses_api_agents/claude_code_agent/README.md index 30ce874b8f4..b7929db0459 100644 --- a/responses_api_agents/claude_code_agent/README.md +++ b/responses_api_agents/claude_code_agent/README.md @@ -231,8 +231,8 @@ sandbox_install_timeout_seconds: 600 session_close_timeout_seconds: 60 ``` -The image must include Linux Python **3.8+**, `tar`, and libraries required by the -native Claude binary. Seed probes the sandbox's architecture and libc, downloads +The image must include `bash`, coreutils, `gzip`, `tar`, and libraries required by the +native Claude binary; it needs no Python. Seed probes the sandbox's architecture and libc, downloads and caches the matching Linux x64/arm64 release on the **Gym host**, then compresses and uploads it in bounded pieces through the sandbox provider's file transfer API. The sandbox needs no internet access for runtime setup. The Gym host needs HTTPS diff --git a/responses_api_agents/claude_code_agent/app.py b/responses_api_agents/claude_code_agent/app.py index f533f764be4..268fb86253c 100644 --- a/responses_api_agents/claude_code_agent/app.py +++ b/responses_api_agents/claude_code_agent/app.py @@ -33,7 +33,7 @@ from fastapi import HTTPException, Request from pydantic import ConfigDict, Field, PrivateAttr -from nemo_gym.agent_utils.sandbox_session import SandboxSession +from nemo_gym.agent_utils.sandbox_session import HARNESS_RUNNER_FILE, HARNESS_RUNNER_SCRIPT, SandboxSession from nemo_gym.agent_utils.task_timeout import AGENT_TIMEOUT_METADATA_KEY, resolve_agent_timeout from nemo_gym.base_resources_server import NEMO_GYM_MCP_METADATA_KEY, BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import ( @@ -606,8 +606,9 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Clau timeout_s=30, ) self._check_sandbox_setup(result) - for name in ("sandbox_runner.py", "install_claude_runtime.py"): - await sandbox.upload(Path(__file__).with_name(name), f"{directory}/{name}") + await sandbox.upload(HARNESS_RUNNER_SCRIPT, f"{directory}/{HARNESS_RUNNER_FILE}") + installer = "install_claude_runtime.sh" + await sandbox.upload(Path(__file__).with_name(installer), f"{directory}/{installer}") await install_sandbox_runtime( sandbox, version=version, @@ -717,7 +718,6 @@ async def _sandbox_response( command[0] = state.runtime + "/claude" artifacts = await state.execute( { - "directory": directory, "command": command, "cwd": state.session.workdir, "env": env, diff --git a/responses_api_agents/claude_code_agent/install_claude_runtime.py b/responses_api_agents/claude_code_agent/install_claude_runtime.py deleted file mode 100644 index fc8e6e6d234..00000000000 --- a/responses_api_agents/claude_code_agent/install_claude_runtime.py +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Verify/install transferred Claude binaries with Linux Python 3.8+ stdlib; no network access.""" - -import argparse -import fcntl -import gzip -import hashlib -import os -import re -import shutil -import subprocess -import tempfile -from contextlib import contextmanager -from pathlib import Path - - -def sha256_file(path: Path) -> str: - """Hash a native binary without requiring Python 3.11's file_digest.""" - digest = hashlib.sha256() - with path.open("rb") as binary: - for chunk in iter(lambda: binary.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def _version(executable: Path, version: str) -> str: - actual = subprocess.run( - [str(executable), "--version"], - check=True, - capture_output=True, - text=True, - errors="replace", - timeout=30, - ).stdout.strip() - if not actual.startswith(version + " "): - raise ValueError("Claude version mismatch: expected %s, got %s" % (version, actual)) - return actual - - -@contextmanager -def _runtime_lock(runtime: Path): - runtime.parent.mkdir(parents=True, exist_ok=True) - with Path(str(runtime) + ".lock").open("a") as lock: - fcntl.flock(lock, fcntl.LOCK_EX) - yield - - -def _cached_version(version: str, runtime: Path, expected: str) -> str: - if sha256_file(runtime / "claude") != expected.lower(): - raise ValueError("Claude executable SHA-256 mismatch") - return _version(runtime / "claude", version) - - -def check_runtime(version: str, runtime: Path, *, expected: str) -> str: - """Check bytes and health before the host decides whether an upload is needed.""" - with _runtime_lock(runtime): - return _cached_version(version, runtime, expected) - - -def install(version: str, runtime: Path, *, source: str = "", expected: str = "", compressed: bool = False) -> str: - """Reuse a verified binary or atomically install the supplied bytes under a lock.""" - if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): - raise ValueError("Claude runtime requires an exact version") - if not expected: - raise ValueError("A transferred Claude binary requires a trusted SHA-256") - with _runtime_lock(runtime): - try: - return _cached_version(version, runtime, expected) - except (OSError, ValueError, subprocess.SubprocessError): - pass - if not source: - raise ValueError("Claude runtime is absent; provide a transferred source binary") - runtime.mkdir(mode=0o700, exist_ok=True) - fd, name = tempfile.mkstemp(prefix=".claude-", dir=runtime) - os.close(fd) - temporary = Path(name) - try: - if compressed: - with gzip.open(source, "rb") as input_file, temporary.open("wb") as output: - shutil.copyfileobj(input_file, output) - else: - shutil.copyfile(source, temporary) - if sha256_file(temporary) != expected.lower(): - raise ValueError("Claude executable SHA-256 mismatch") - temporary.chmod(0o700) - actual = _version(temporary, version) - temporary.replace(runtime / "claude") - return actual - finally: - temporary.unlink(missing_ok=True) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--version", required=True) - parser.add_argument("--runtime", type=Path, required=True) - parser.add_argument("--source", default="") - parser.add_argument("--sha256", required=True) - parser.add_argument("--check-only", action="store_true") - parser.add_argument("--gzip", action="store_true", help="Source is a gzip-compressed native binary") - args = parser.parse_args() - if args.check_only: - try: - print(check_runtime(args.version, args.runtime, expected=args.sha256)) - except (OSError, ValueError, subprocess.SubprocessError): - raise SystemExit(3) - else: - print(install(args.version, args.runtime, source=args.source, expected=args.sha256, compressed=args.gzip)) diff --git a/responses_api_agents/claude_code_agent/install_claude_runtime.sh b/responses_api_agents/claude_code_agent/install_claude_runtime.sh new file mode 100644 index 00000000000..51e00d7e0e9 --- /dev/null +++ b/responses_api_agents/claude_code_agent/install_claude_runtime.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Verify or install a transferred Claude binary with bash and coreutils; no network access. +# +# usage: install_claude_runtime.sh --version V --runtime DIR --sha256 HEX [--check-only | --source PATH [--gzip]] +# Prints the version of the verified runtime. --check-only exits 3 when the runtime is absent or unhealthy. + +set -euo pipefail +version="" runtime="" expected="" source="" check_only="" compressed="" temporary="" lock_dir="" +while (($#)); do + case $1 in + --version) version=$2 && shift 2 ;; + --runtime) runtime=$2 && shift 2 ;; + --sha256) expected=${2,,} && shift 2 ;; + --source) source=$2 && shift 2 ;; + --check-only) check_only=1 && shift ;; + --gzip) compressed=1 && shift ;; + *) echo "Unknown argument: $1" >&2 && exit 2 ;; + esac +done +[[ $version =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo "Claude runtime requires an exact version" >&2; exit 2; } +[[ -n $expected ]] || { echo "A transferred Claude binary requires a trusted SHA-256" >&2; exit 2; } + +# Print the version when the executable has the trusted bytes and reports the expected version. +verify() { + local digest actual + [[ -f $1 ]] || return 1 + read -r digest _ < <(sha256sum "$1") + [[ $digest == "$expected" ]] || { echo "Claude executable SHA-256 mismatch" >&2; return 1; } + actual=$(timeout 30 "$1" --version) || return 1 + [[ $actual == "$version "* ]] || { echo "Claude version mismatch: expected $version, got $actual" >&2; return 1; } + echo "$actual" +} + +trap 'rm -f -- "$temporary"; [[ -z $lock_dir ]] || rmdir -- "$lock_dir"' EXIT +mkdir -p -- "$(dirname -- "$runtime")" +# Concurrent sessions in one sandbox share the runtime; only one may replace it. +if command -v flock >/dev/null 2>&1; then + exec 9>>"$runtime.lock" + flock 9 +else + until mkdir -- "$runtime.lock.d" 2>/dev/null; do sleep 0.1; done + lock_dir=$runtime.lock.d +fi + +verify "$runtime/claude" 2>/dev/null && exit 0 +[[ -z $check_only ]] || exit 3 +[[ -n $source ]] || { echo "Claude runtime is absent; provide a transferred source binary" >&2; exit 1; } +mkdir -p -m 700 -- "$runtime" +temporary=$(mktemp "$runtime/.claude-XXXXXX") +if [[ -n $compressed ]]; then + gzip -dc -- "$source" >"$temporary" +else + cat -- "$source" >"$temporary" +fi +chmod 700 "$temporary" +verify "$temporary" +mv -f -- "$temporary" "$runtime/claude" diff --git a/responses_api_agents/claude_code_agent/runtime.py b/responses_api_agents/claude_code_agent/runtime.py index a196648daf6..b9ed3255e1f 100644 --- a/responses_api_agents/claude_code_agent/runtime.py +++ b/responses_api_agents/claude_code_agent/runtime.py @@ -4,6 +4,7 @@ import asyncio import gzip +import hashlib import json import os import re @@ -18,7 +19,6 @@ from nemo_gym.sandbox import AsyncSandbox, SandboxExecResult from nemo_gym.server_utils import request -from responses_api_agents.claude_code_agent.install_claude_runtime import sha256_file CACHE_DIR = Path.home() / ".cache" / "nemo_gym" / "claude" @@ -28,6 +28,15 @@ _UPLOAD_SLOTS = asyncio.Semaphore(4) +def sha256_file(path: Path) -> str: + """Hash a native binary in bounded chunks.""" + digest = hashlib.sha256() + with path.open("rb") as binary: + for chunk in iter(lambda: binary.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + def _private_directory(path: Path) -> None: path.mkdir(mode=0o700, parents=True, exist_ok=True) info = path.lstat() @@ -212,7 +221,7 @@ async def install_sandbox_runtime( """Reuse a verified runtime or upload host-cached bytes; never fetch from the sandbox.""" if (source or local_source) and not expected: raise ValueError("A supplied Claude binary requires a trusted SHA-256") - command = ["python3", "-I", directory + "/install_claude_runtime.py", "--version", version, "--runtime", runtime] + command = ["bash", directory + "/install_claude_runtime.sh", "--version", version, "--runtime", runtime] if source: command += ["--source", source, "--sha256", expected] else: diff --git a/responses_api_agents/claude_code_agent/sandbox.py b/responses_api_agents/claude_code_agent/sandbox.py index 5ea51ad45b3..931a90587f4 100644 --- a/responses_api_agents/claude_code_agent/sandbox.py +++ b/responses_api_agents/claude_code_agent/sandbox.py @@ -15,7 +15,7 @@ from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError -from nemo_gym.agent_utils.sandbox_session import SandboxCommand, SandboxSession +from nemo_gym.agent_utils.sandbox_session import HARNESS_RUNNER_FILE, SandboxCommand, SandboxSession, shell_assignments from nemo_gym.base_responses_api_agent import AgentSessionState from nemo_gym.config_types import ModelServerRef from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming @@ -34,7 +34,6 @@ class HarnessProcessInfo(BaseModel): model_config = ConfigDict(extra="forbid", strict=True) hostname: str pid: int - python: str | None = None def parse_runtime_info(payload: object) -> HarnessProcessInfo | None: @@ -165,15 +164,18 @@ async def stage_activation(self, payload: dict[str, JsonValue]) -> SandboxComman skills_path = payload.pop("skills_path") if skills_path: await self.stage_skills(skills_path) - await self.upload_json("input.json", payload) - return SandboxCommand( - argv=[ - "python3", - "-I", - f"{self.session.session_dir}/sandbox_runner.py", - f"{self.session.session_dir}/input.json", - ], - ) + directory = self.session.session_dir + # Regular files avoid waiting for detached task services to close inherited pipes. + params = { + "cwd": payload["cwd"], + "command": payload["command"], + "env": payload["env"], + "stdin": "/dev/null", + "stdout": f"{directory}/events.jsonl", + "stderr": f"{directory}/stderr.log", + } + await upload_text(self.session.sandbox, path=f"{directory}/params.sh", text=shell_assignments(params)) + return SandboxCommand(argv=["bash", f"{directory}/{HARNESS_RUNNER_FILE}", f"{directory}/params.sh"]) async def collect_artifacts(self) -> ClaudeCodeArtifacts: """Copy dedicated stdout and transcripts after confirmed harness termination.""" diff --git a/responses_api_agents/claude_code_agent/sandbox_runner.py b/responses_api_agents/claude_code_agent/sandbox_runner.py deleted file mode 100644 index 2fcee755966..00000000000 --- a/responses_api_agents/claude_code_agent/sandbox_runner.py +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Capture Claude stdout independently of stderr and supervisor diagnostics.""" - -import json -import os -import signal -import subprocess -import sys -from pathlib import Path - - -def run(params: dict) -> int: - """Keep raw stream-json and partial output; the shared supervisor owns process cleanup.""" - directory = Path(params["directory"]) - (directory / "runtime.json").write_text(json.dumps({"hostname": os.uname().nodename, "pid": os.getpid()})) - stopping = False - - def interrupt(*_): - nonlocal stopping - stopping = True - - signal.signal(signal.SIGTERM, interrupt) - with (directory / "events.jsonl").open("wb") as events, (directory / "stderr.log").open("wb") as stderr: - process = subprocess.Popen( - params["command"], - cwd=params["cwd"], - env={**os.environ, **params["env"]}, - stdin=subprocess.DEVNULL, - stdout=events, - stderr=stderr, - ) - try: - # Regular files avoid waiting for detached task services to close inherited pipes. - while True: - if stopping and process.poll() is None: - process.terminate() - stopping = False - try: - return process.wait(timeout=0.05) - except subprocess.TimeoutExpired: - pass - finally: - if process.poll() is None: - process.kill() - process.wait() - - -if __name__ == "__main__": - raise SystemExit(run(json.loads(Path(sys.argv[1]).read_text()))) diff --git a/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py b/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py index 7cd0e8ecb97..b1ccb242ab3 100644 --- a/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py +++ b/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py @@ -1,32 +1,50 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import ast +import gzip import hashlib +import shutil +import subprocess from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from unittest.mock import MagicMock import pytest -from responses_api_agents.claude_code_agent import install_claude_runtime as installer +from responses_api_agents.claude_code_agent import runtime as host_runtime + + +INSTALLER = Path(host_runtime.__file__).with_name("install_claude_runtime.sh") +pytestmark = pytest.mark.skipif( + shutil.which("bash") is None or shutil.which("sha256sum") is None, reason="bash and sha256sum are required" +) @pytest.fixture def binary(tmp_path): source = tmp_path / "source" - source.write_text('#!/bin/sh\nprintf "2.1.108 (Claude Code)\\n"\n') + source.write_text('#!/bin/sh\necho run >> "$0.calls"\nprintf "2.1.108 (Claude Code)\\n"\n') return source, hashlib.sha256(source.read_bytes()).hexdigest() +def install(runtime: Path, *args: str, version: str = "2.1.108") -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["bash", str(INSTALLER), "--version", version, "--runtime", str(runtime), *args], + capture_output=True, + text=True, + timeout=60, + ) + + def test_offline_install_verifies_hash_and_reuses_healthy_cache(tmp_path, binary): source, checksum = binary runtime = tmp_path / "runtime" - assert installer.install("2.1.108", runtime, source=str(source), expected=checksum) == "2.1.108 (Claude Code)" + first = install(runtime, "--sha256", checksum, "--source", str(source)) + assert (first.returncode, first.stdout) == (0, "2.1.108 (Claude Code)\n"), first.stderr executable = runtime / "claude" original = executable.stat().st_ino source.unlink() - assert installer.install("2.1.108", runtime, source=str(source), expected=checksum) == "2.1.108 (Claude Code)" + second = install(runtime, "--sha256", checksum.upper(), "--source", str(source)) + assert (second.returncode, second.stdout) == (0, "2.1.108 (Claude Code)\n"), second.stderr assert executable.stat().st_ino == original assert executable.stat().st_mode & 0o777 == 0o700 @@ -34,55 +52,44 @@ def test_offline_install_verifies_hash_and_reuses_healthy_cache(tmp_path, binary def test_corrupted_cache_is_replaced_and_bad_source_cannot_replace_it(tmp_path, binary): source, checksum = binary runtime = tmp_path / "runtime" - installer.install("2.1.108", runtime, source=str(source), expected=checksum) + assert install(runtime, "--sha256", checksum, "--source", str(source)).returncode == 0 (runtime / "claude").write_bytes(b"corrupt") - installer.install("2.1.108", runtime, source=str(source), expected=checksum) + assert install(runtime, "--sha256", checksum, "--source", str(source)).returncode == 0 assert (runtime / "claude").read_bytes() == source.read_bytes() source.write_bytes(b"bad replacement") - with pytest.raises(ValueError, match="SHA-256 mismatch"): - installer.install("2.1.108", runtime, source=str(source), expected="0" * 64) + failed = install(runtime, "--sha256", "0" * 64, "--source", str(source)) + assert failed.returncode != 0 and "SHA-256 mismatch" in failed.stderr assert hashlib.sha256((runtime / "claude").read_bytes()).hexdigest() == checksum - assert not list(runtime.glob(".claude-*")) + assert not list(runtime.glob(".claude-??????")) # temporary copies; the stub also logs .calls files -def test_concurrent_installers_share_one_verified_runtime(tmp_path, binary, monkeypatch): +def test_concurrent_installers_share_one_verified_runtime(tmp_path, binary): source, checksum = binary - copy = MagicMock(wraps=installer.shutil.copyfile) - monkeypatch.setattr(installer.shutil, "copyfile", copy) - - def install(): - return installer.install("2.1.108", tmp_path / "runtime", source=str(source), expected=checksum) - + runtime = tmp_path / "runtime" with ThreadPoolExecutor(max_workers=4) as pool: - versions = list(pool.map(lambda _: install(), range(4))) - assert versions == ["2.1.108 (Claude Code)"] * 4 - assert copy.call_count == 1 + results = list(pool.map(lambda _: install(runtime, "--sha256", checksum, "--source", str(source)), range(4))) + assert [(result.returncode, result.stdout) for result in results] == [(0, "2.1.108 (Claude Code)\n")] * 4 + # The stub logs each run under its own path: one temporary copy was made; the others reused the install. + assert len(list(runtime.glob(".claude-*.calls"))) == 1 + assert (runtime / "claude.calls").read_text() == "run\n" * 3 def test_preinstalled_binary_needs_trusted_checksum_and_matching_version(tmp_path, binary): source, checksum = binary - with pytest.raises(ValueError, match="trusted SHA-256"): - installer.install("2.1.108", tmp_path / "runtime", source=str(source)) - with pytest.raises(ValueError, match="version mismatch"): - installer.install("2.1.109", tmp_path / "runtime", source=str(source), expected=checksum) + missing = install(tmp_path / "runtime", "--source", str(source)) + assert missing.returncode != 0 and "trusted SHA-256" in missing.stderr + mismatch = install(tmp_path / "runtime", "--sha256", checksum, "--source", str(source), version="2.1.109") + assert mismatch.returncode != 0 and "version mismatch" in mismatch.stderr assert not (tmp_path / "runtime" / "claude").exists() -def test_installer_and_runner_accept_python38_syntax(): - for filename in ("install_claude_runtime.py", "sandbox_runner.py"): - path = Path(installer.__file__).with_name(filename) - ast.parse(path.read_text(), feature_version=(3, 8)) - tree = ast.parse(Path(installer.__file__).read_text()) - assert not any(isinstance(n, ast.Attribute) and n.attr == "file_digest" for n in ast.walk(tree)) - - -def test_installer_verifies_decompressed_native_bytes(tmp_path, binary): - import gzip - +def test_installer_verifies_decompressed_native_bytes_and_check_only_reports_absence(tmp_path, binary): source, checksum = binary + runtime = tmp_path / "runtime" + assert install(runtime, "--sha256", checksum, "--check-only").returncode == 3 compressed = tmp_path / "claude.gz" compressed.write_bytes(gzip.compress(source.read_bytes())) - runtime = tmp_path / "runtime" - installer.install("2.1.108", runtime, source=str(compressed), expected=checksum, compressed=True) + assert install(runtime, "--sha256", checksum, "--source", str(compressed), "--gzip").returncode == 0 assert (runtime / "claude").read_bytes() == source.read_bytes() - assert installer.check_runtime("2.1.108", runtime, expected=checksum) == "2.1.108 (Claude Code)" + checked = install(runtime, "--sha256", checksum, "--check-only") + assert (checked.returncode, checked.stdout) == (0, "2.1.108 (Claude Code)\n") diff --git a/responses_api_agents/claude_code_agent/tests/test_runtime.py b/responses_api_agents/claude_code_agent/tests/test_runtime.py index 535a41c1dec..22737806c4a 100644 --- a/responses_api_agents/claude_code_agent/tests/test_runtime.py +++ b/responses_api_agents/claude_code_agent/tests/test_runtime.py @@ -110,6 +110,7 @@ async def test_upload_uses_sandbox_architecture_and_skips_healthy_remote_cache(t else: assert gzip.decompress(transfer.await_args.args[1].read_bytes()) == artifact.read_bytes() command = sandbox.exec.await_args.args[0] + assert command.startswith("bash /session/install_claude_runtime.sh ") assert ( "--source /session/claude-upload.gz" in command and "--gzip" in command @@ -143,10 +144,8 @@ async def test_host_local_binary_requires_no_network(tmp_path, monkeypatch): def test_worker_installer_has_no_download_dependency(): - from responses_api_agents.claude_code_agent import install_claude_runtime - - text = Path(install_claude_runtime.__file__).read_text() - assert "urllib" not in text and "https://" not in text + text = Path(runtime.__file__).with_name("install_claude_runtime.sh").read_text() + assert not any(tool in text for tool in ("curl", "wget", "https://", "python")) @pytest.mark.asyncio diff --git a/responses_api_agents/claude_code_agent/tests/test_sandbox.py b/responses_api_agents/claude_code_agent/tests/test_sandbox.py index 050b609050e..12f544e7a0c 100644 --- a/responses_api_agents/claude_code_agent/tests/test_sandbox.py +++ b/responses_api_agents/claude_code_agent/tests/test_sandbox.py @@ -481,7 +481,6 @@ async def upload(source, target): request=seed(), runtime="/runtime", session=SimpleNamespace(session_dir="/tmp/session", sandbox=remote) ) payload = { - "directory": "/tmp/session", "command": ["/runtime/claude"], "cwd": "/app", "env": {}, @@ -494,7 +493,8 @@ async def upload(source, target): command = await state.stage_activation(payload) assert files["/tmp/session/config/skills/example/helper.sh"] == b"echo helper\n" assert "SKILL.md" in " ".join(files) - assert command.python == "python3" + assert command.argv == ["bash", "/tmp/session/harness_runner.sh", "/tmp/session/params.sh"] + assert "/runtime/claude" in files["/tmp/session/params.sh"].decode() runner_payload = staged.await_args.args[1] assert "skills_path" not in runner_payload instance = agent() diff --git a/responses_api_agents/claude_code_agent/tests/test_sandbox_runner.py b/responses_api_agents/claude_code_agent/tests/test_sandbox_runner.py index ecf56e904c6..82294536015 100644 --- a/responses_api_agents/claude_code_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/claude_code_agent/tests/test_sandbox_runner.py @@ -1,36 +1,63 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +"""Run Claude's staged activation with the shared bash runner, as the sandbox does.""" +import asyncio import json +import shutil import subprocess import sys import time from pathlib import Path +from types import SimpleNamespace import pytest -from responses_api_agents.claude_code_agent import sandbox_runner +from nemo_gym.agent_utils.sandbox_session import HARNESS_RUNNER_FILE, HARNESS_RUNNER_SCRIPT +from responses_api_agents.claude_code_agent import sandbox as adapter from responses_api_agents.claude_code_agent.parsing import _invocation_outcome, parse_stream_json +from responses_api_agents.claude_code_agent.tests.test_sandbox import seed -def runner_input(tmp_path, code): +pytestmark = pytest.mark.skipif(shutil.which("bash") is None, reason="bash is required") + + +def runner_command(tmp_path: Path, code: str) -> list[str]: + """Stage a stub CLI through stage_activation into a local session directory.""" + + async def upload(source, target): + Path(target).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + + shutil.copyfile(HARNESS_RUNNER_SCRIPT, tmp_path / HARNESS_RUNNER_FILE) child = tmp_path / "child.py" child.write_text(code) - payload = {"directory": str(tmp_path), "command": [sys.executable, str(child)], "cwd": str(tmp_path), "env": {}} - params = tmp_path / "input.json" - params.write_text(json.dumps(payload)) - return [sys.executable, str(Path(sandbox_runner.__file__)), str(params)] + state = adapter.ClaudeCodeSandboxSession( + request=seed(), + runtime="/runtime", + session=SimpleNamespace(session_dir=str(tmp_path), sandbox=SimpleNamespace(upload=upload)), + ) + payload = { + "command": [sys.executable, str(child)], + "cwd": str(tmp_path), + "env": {"GYM_VALUE": "it's set"}, + "settings": {}, + "mcp_config": None, + "skills_path": None, + } + return asyncio.run(state.stage_activation(payload)).argv def test_large_json_stdout_cannot_be_corrupted_by_interleaved_stderr(tmp_path): - command = runner_input( + command = runner_command( tmp_path, - """import json,os + """import json,os,sys text="x"*300000 line=json.dumps({"type":"assistant","message":{"content":[{"type":"text","text":text}]}}).encode()+b"\\n" os.write(1,line[:150000]) os.write(2,b'{"type":"result","subtype":"error_fake"}\\n'+b'diagnostic\\xff\\n') os.write(1,line[150000:]) +os.write(2,(os.environ["GYM_VALUE"]+"|"+repr(sys.stdin.read())).encode()) os.write(1,b'{"type":"result","subtype":"success","is_error":false}\\n') """, ) @@ -41,14 +68,14 @@ def test_large_json_stdout_cannot_be_corrupted_by_interleaved_stderr(tmp_path): output, metadata = parse_stream_json(raw) assert output[0].content[0].text == "x" * 300000 assert _invocation_outcome(metadata, result.returncode) == ("completed", None) - assert b"diagnostic\xff" in (tmp_path / "stderr.log").read_bytes() - runtime = json.loads((tmp_path / "runtime.json").read_text()) - assert runtime["hostname"] and runtime["pid"] > 0 + assert (tmp_path / "stderr.log").read_bytes().endswith(b"diagnostic\xff\nit's set|''") + runtime = adapter.parse_runtime_info(json.loads((tmp_path / "runtime.json").read_text())) + assert runtime is not None and runtime.hostname and runtime.pid > 0 @pytest.mark.skipif(sys.platform == "win32", reason="uses POSIX termination") def test_termination_preserves_partial_events(tmp_path): - command = runner_input( + command = runner_command( tmp_path, """import os,time os.write(1,b'{"type":"assistant","message":{"content":[{"type":"text","text":"partial"}]}}\\n') From 3fe0155907e9e4ccaeaa17d04973c61588e68b0f Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Mon, 26 Oct 2026 09:27:20 -0700 Subject: [PATCH 07/32] refactor(openclaw_agent): run the harness without Python in the sandbox Launch OpenClaw through the shared bash harness_runner.sh with PARAMS from shell_assignments() and a staged prompt file, replacing sandbox_runner.py. The workdir overlap check is POSIX sh, session setup installs only a missing bash, and the runtime installer drops its Python checks (apk now reports whether musl images have libstdc++; the prepare step already guards the runtime path against the task workdir). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Hemil Desai Signed-off-by: yaoyu-33 --- responses_api_agents/openclaw_agent/README.md | 7 +- responses_api_agents/openclaw_agent/app.py | 12 +- .../install_openclaw_runtime.sh | 15 +- .../openclaw_agent/sandbox.py | 54 ++++--- .../openclaw_agent/sandbox_runner.py | 75 --------- .../openclaw_agent/tests/test_bootstrap.py | 8 +- .../tests/test_exec_transport.py | 25 ++- .../tests/test_install_runtime.py | 20 +-- .../tests/test_pinned_runtime.py | 27 ++-- .../openclaw_agent/tests/test_sandbox.py | 68 +++++--- .../tests/test_sandbox_runner.py | 147 ------------------ 11 files changed, 122 insertions(+), 336 deletions(-) delete mode 100644 responses_api_agents/openclaw_agent/sandbox_runner.py delete mode 100644 responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index 048b69b06e9..82ea57a36fc 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -166,10 +166,9 @@ finish. For example, set `openclaw_agent.responses_api_agents.openclaw_agent.mod to `1200` for a 20-minute model-call limit. Increase the separate `timeout` and EnvironmentServer episode deadline when a task needs more total time; increasing those alone does not raise the model-idle limit. -The installer supports Linux glibc on x86_64/aarch64 and musl on x86_64, with Python 3.8 or later for the -supervisor. Session setup installs missing Python and Bash with apt-get or apk -when running as root; otherwise preinstall Python 3.8+ and Bash. Existing tools -are preserved. Node 22.19.0 and OpenClaw are installed under +The installer supports Linux glibc on x86_64/aarch64 and musl on x86_64. The task image needs no Python: +the supervisor and harness runner are Bash scripts. Session setup installs a missing Bash with apt-get or apk +when running as root; otherwise preinstall Bash. Existing tools are preserved. Node 22.19.0 and OpenClaw are installed under `/tmp/nemo-gym-openclaw-node-22.19.0-`. Node archives are checked against upstream SHA-256 sums; the installed package version is verified before caching and when reusing a cached runtime. A version-scoped `flock` serializes runtime installation and cache validation across session setup diff --git a/responses_api_agents/openclaw_agent/app.py b/responses_api_agents/openclaw_agent/app.py index 1f399624af0..8ee1b9b3f11 100644 --- a/responses_api_agents/openclaw_agent/app.py +++ b/responses_api_agents/openclaw_agent/app.py @@ -35,7 +35,7 @@ from fastapi import HTTPException, Request from pydantic import ConfigDict, Field, PrivateAttr -from nemo_gym.agent_utils.sandbox_session import SandboxSession +from nemo_gym.agent_utils.sandbox_session import HARNESS_RUNNER_FILE, HARNESS_RUNNER_SCRIPT, SandboxSession from nemo_gym.agent_utils.task_timeout import AGENT_TIMEOUT_METADATA_KEY, resolve_agent_timeout from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import ( @@ -392,10 +392,9 @@ def _extract_instruction(body_input) -> tuple[str, Optional[str]]: def _sandbox_prepare_command(workdir: str, directory: str, runtime: str) -> str: - # Bootstrap with POSIX sh; the runtime installer itself requires Bash. + # Bootstrap with POSIX sh; the runtime installer and harness runner require Bash. bootstrap = """set -eu set -- -command -v python3 >/dev/null 2>&1 || set -- "$@" python3 command -v bash >/dev/null 2>&1 || set -- "$@" bash if [ "$#" -gt 0 ]; then [ "$(id -u)" = 0 ] || { echo "OpenClaw sandbox execution requires $*: preinstall these tools or use a root image." >&2; exit 1; } @@ -410,9 +409,7 @@ def _sandbox_prepare_command(workdir: str, directory: str, runtime: str) -> str: fi fi """ - check_paths = shlex.join( - ["python3", "-I", "-c", _SANDBOX_PATH_CHECK, workdir, str(Path(directory).parent), runtime] - ) + check_paths = f"(set -- {shlex.join([workdir, str(Path(directory).parent), runtime])}{_SANDBOX_PATH_CHECK})" return bootstrap + f"{check_paths} && mkdir -p {shlex.quote(directory + '/home/.openclaw')}" @@ -578,7 +575,7 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Open f"bash {directory}/{installer} {runtime} {self.config.openclaw_version}\n" f"stderr={(installed.stderr or '')[-16000:]}; stdout={(installed.stdout or '')[-16000:]}" ) - await sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), f"{directory}/sandbox_runner.py") + await sandbox.upload(HARNESS_RUNNER_SCRIPT, f"{directory}/{HARNESS_RUNNER_FILE}") except BaseException as error: try: if prepared_directory or owns_sandbox: @@ -711,7 +708,6 @@ async def _sandbox_response( ] payload = { "config": session_config, - "directory": state.session.session_dir, "command": command, "prompt": f"{system}\n\n{prompt}" if system else prompt, "cwd": state.session.workdir, diff --git a/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh b/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh index 9d32056f53e..5c6f3be527b 100644 --- a/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh +++ b/responses_api_agents/openclaw_agent/install_openclaw_runtime.sh @@ -11,11 +11,6 @@ if [ "$(uname -s)" != Linux ]; then echo 'OpenClaw sandbox execution requires a Linux sandbox.' >&2 exit 1 fi -if ! command -v python3 >/dev/null 2>&1; then - echo 'OpenClaw sandbox execution requires Python >=3.8 in the task image for descendant supervision.' >&2 - exit 1 -fi -python3 -c 'import sys; assert sys.version_info >= (3, 8), "OpenClaw sandbox execution requires Python >=3.8"' case "$(uname -m)" in x86_64) arch=x64 ;; aarch64) arch=arm64 ;; @@ -53,13 +48,6 @@ install_packages() { fi } -# Never change the task repository or its dependency environment. -python3 - "$runtime" "$PWD" <<'PY' -import pathlib, sys -runtime, workdir = (pathlib.Path(p).resolve() for p in sys.argv[1:]) -if workdir == runtime or workdir in runtime.parents or runtime in workdir.parents: - raise SystemExit('OpenClaw runtime must be outside the task repository') -PY # One sandbox can receive overlapping session setup attempts. Keep the ready # check and all cache writes under a version-scoped advisory lock. if ! command -v flock >/dev/null 2>&1; then @@ -94,7 +82,8 @@ if [ "${#missing[@]}" -gt 0 ]; then echo "OpenClaw sandbox execution prerequisites missing: ${missing[*]}. Preinstall curl, ca-certificates, tar, gzip, coreutils, gawk." >&2 install_packages curl ca-certificates tar gzip coreutils gawk fi -if [ "$platform" = linux-x64-musl ] && ! python3 -c 'import ctypes; ctypes.CDLL("libstdc++.so.6")' 2>/dev/null; then +# musl task images are Alpine-based; apk records whether the C++ runtime Node needs is installed. +if [ "$platform" = linux-x64-musl ] && command -v apk >/dev/null 2>&1 && ! apk info -e libstdc++ >/dev/null; then install_packages libstdc++ fi cd "$runtime" diff --git a/responses_api_agents/openclaw_agent/sandbox.py b/responses_api_agents/openclaw_agent/sandbox.py index 9fec3289ebf..028a7a1c536 100644 --- a/responses_api_agents/openclaw_agent/sandbox.py +++ b/responses_api_agents/openclaw_agent/sandbox.py @@ -9,7 +9,12 @@ from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError -from nemo_gym.agent_utils.sandbox_session import SandboxCommand, SandboxSession +from nemo_gym.agent_utils.sandbox_session import ( + HARNESS_RUNNER_FILE, + SandboxCommand, + SandboxSession, + shell_assignments, +) from nemo_gym.base_responses_api_agent import AgentSessionState from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming from nemo_gym.rollout_observability import AgentObservationBundle @@ -19,17 +24,22 @@ LOG = logging.getLogger(__name__) -# Executed by sandbox Python before uploading any adapter-owned files. Resolve -# symlinks and '..' on the sandbox filesystem, not on the agent-server host. +# POSIX sh run with positional parameters WORKDIR SESSION_ROOT RUNTIME_ROOT before uploading +# any adapter-owned files. Resolve symlinks and '..' on the sandbox filesystem, not on the agent-server +# host; an owned root that does not exist yet resolves through its nearest existing ancestor. _SANDBOX_PATH_CHECK = """ -from pathlib import Path -import sys -workdir, session_root, runtime_root = (Path(value).resolve() for value in sys.argv[1:]) -if not workdir.is_dir(): - raise SystemExit("OpenClaw task workdir is missing or is not a directory: " + str(workdir)) -for owned in (session_root, runtime_root): - if workdir == owned or workdir in owned.parents or owned in workdir.parents: - raise SystemExit("OpenClaw task workdir overlaps adapter-owned storage: " + str(owned)) +[ -d "$1" ] || { echo "OpenClaw task workdir is missing or is not a directory: $1" >&2; exit 1; } +workdir=$(cd "$1" && pwd -P) || exit 1 +shift +for owned in "$@"; do + path=$owned rest= + until [ -d "${path:-/}" ]; do rest=/${path##*/}$rest path=${path%/*}; done + owned=$(cd "${path:-/}" && pwd -P) || exit 1 + owned=${owned%/}$rest + case "$workdir/" in "$owned/"*) overlap=1 ;; *) overlap= ;; esac + case "$owned/" in "$workdir/"*) overlap=1 ;; esac + [ -z "$overlap" ] || { echo "OpenClaw task workdir overlaps adapter-owned storage: $owned" >&2; exit 1; } +done """ @@ -39,7 +49,6 @@ class HarnessProcessInfo(BaseModel): model_config = ConfigDict(extra="forbid", strict=True) hostname: str pid: int - python: str | None = None def parse_runtime_info(payload: object) -> HarnessProcessInfo | None: @@ -100,16 +109,19 @@ async def execute(self, payload: dict[str, JsonValue], *, timeout: float, close_ async def stage_activation(self, payload: dict[str, JsonValue]) -> SandboxCommand: """Stage the invocation and return the OpenClaw worker command.""" + directory = self.session.session_dir await self.upload_json("home/.openclaw/openclaw.json", payload["config"]) - await self.upload_json("input.json", payload) - return SandboxCommand( - argv=[ - "python3", - "-I", - f"{self.session.session_dir}/sandbox_runner.py", - f"{self.session.session_dir}/input.json", - ], - ) + await upload_text(self.session.sandbox, path=f"{directory}/prompt.txt", text=payload["prompt"]) + params = { + "cwd": payload["cwd"], + "command": payload["command"], + "env": payload["env"], + "stdin": "/dev/null", + "stdout": f"{directory}/stdout.log", + "stderr": f"{directory}/stderr.log", + } + await upload_text(self.session.sandbox, path=f"{directory}/params.sh", text=shell_assignments(params)) + return SandboxCommand(argv=["bash", f"{directory}/{HARNESS_RUNNER_FILE}", f"{directory}/params.sh"]) async def collect_artifacts(self) -> dict[str, str]: """Retain transcript and diagnostics before releasing the task sandbox.""" diff --git a/responses_api_agents/openclaw_agent/sandbox_runner.py b/responses_api_agents/openclaw_agent/sandbox_runner.py deleted file mode 100644 index de109c04818..00000000000 --- a/responses_api_agents/openclaw_agent/sandbox_runner.py +++ /dev/null @@ -1,75 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Capture OpenClaw output; the supervisor bounds execution; the sandbox owner stops task services.""" - -from __future__ import annotations - -import json -import os -import signal -import subprocess -import sys -from contextlib import ExitStack -from pathlib import Path -from typing import TypedDict - - -class RunnerInput(TypedDict): - """Harness input, independent of supervisor cleanup and runtime metadata.""" - - directory: str - prompt: str - command: list[str] - cwd: str - env: dict[str, str] - - -def run(params: RunnerInput) -> int: - """Run the CLI with isolated files and wait for it before returning captured output.""" - directory = Path(params["directory"]) - (directory / "runtime.json").write_text(json.dumps({"hostname": os.uname().nodename, "pid": os.getpid()})) - (directory / "prompt.txt").write_text(params["prompt"]) - stopping = False - - def interrupt(*_: object) -> None: - nonlocal stopping - # Defer TERM until Popen returns so a signal cannot lose the child handle. - stopping = True - - signal.signal(signal.SIGTERM, interrupt) - with ExitStack() as stack: - stdout = stack.enter_context((directory / "stdout.log").open("wb")) - stderr = stack.enter_context((directory / "stderr.log").open("wb")) - stdin = subprocess.DEVNULL - process = subprocess.Popen( - params["command"], - cwd=params["cwd"], - env={**os.environ, **params["env"]}, - stdin=stdin, - stdout=stdout, - stderr=stderr, - ) - try: - while True: - if stopping: - process.terminate() - stopping = False - try: - return process.wait(timeout=0.05) - except subprocess.TimeoutExpired: - pass - finally: - # An I/O/capture error must not leave the CLI changing the task. - # Only stop our direct CLI; task services belong to the sandbox owner. - if process.poll() is None: - process.kill() - process.wait() - - -def main() -> None: - params = json.loads(Path(sys.argv[1]).read_text()) - raise SystemExit(run(params)) - - -if __name__ == "__main__": - main() diff --git a/responses_api_agents/openclaw_agent/tests/test_bootstrap.py b/responses_api_agents/openclaw_agent/tests/test_bootstrap.py index df450abd01b..20b1b50bc8f 100644 --- a/responses_api_agents/openclaw_agent/tests/test_bootstrap.py +++ b/responses_api_agents/openclaw_agent/tests/test_bootstrap.py @@ -5,7 +5,6 @@ import shlex import shutil import subprocess -import sys import pytest @@ -13,13 +12,13 @@ @pytest.mark.parametrize("package_manager", ["apk", "apt-get"]) -@pytest.mark.parametrize("missing", [(), ("bash",), ("python3",), ("python3", "bash")]) +@pytest.mark.parametrize("missing", [(), ("bash",)]) @pytest.mark.parametrize("failure", [None, "nonroot", "install", "unavailable"]) def test_posix_bootstrap_installs_only_missing_prerequisites(tmp_path, package_manager, missing, failure): """Exercise the actual preparation through sh without invoking a host package manager.""" bindir = tmp_path / "bin" bindir.mkdir() - binaries = {"python3": sys.executable, "bash": shutil.which("bash"), "mkdir": shutil.which("mkdir")} + binaries = {"bash": shutil.which("bash"), "mkdir": shutil.which("mkdir")} assert all(binaries.values()) for name, executable in binaries.items(): if name not in missing: @@ -33,8 +32,7 @@ def test_posix_bootstrap_installs_only_missing_prerequisites(tmp_path, package_m '#!/bin/sh\nprintf "%s\\n" "$*" >> "$TEST_ROOT/packages.log"\n' + ("exit 100\n" if failure == "install" else "") + 'for name in "$@"; do\n' - + f'case "$name" in\n python3) ln -s {shlex.quote(sys.executable)} "$TEST_ROOT/bin/python3" ;;\n' - + f' bash) ln -s {shlex.quote(binaries["bash"])} "$TEST_ROOT/bin/bash" ;;\n' + + f'case "$name" in\n bash) ln -s {shlex.quote(binaries["bash"])} "$TEST_ROOT/bin/bash" ;;\n' + "esac\ndone\n" ) manager.chmod(0o755) diff --git a/responses_api_agents/openclaw_agent/tests/test_exec_transport.py b/responses_api_agents/openclaw_agent/tests/test_exec_transport.py index 0ce32fff188..9b674086de9 100644 --- a/responses_api_agents/openclaw_agent/tests/test_exec_transport.py +++ b/responses_api_agents/openclaw_agent/tests/test_exec_transport.py @@ -12,9 +12,8 @@ import pytest -from nemo_gym.agent_utils.sandbox_session import SandboxSession +from nemo_gym.agent_utils.sandbox_session import HARNESS_RUNNER_FILE, HARNESS_RUNNER_SCRIPT, SandboxSession from nemo_gym.sandbox.providers.base import SandboxExecResult -from responses_api_agents.openclaw_agent import sandbox_runner from responses_api_agents.openclaw_agent.sandbox import OpenClawSandboxSession from responses_api_agents.openclaw_agent.tests.test_sandbox import seed @@ -40,7 +39,7 @@ async def disconnect(self): self.disconnected = True async def exec(self, command, *, cwd=None, timeout_s=30): - launch = "--receipt" in command and "process_supervisor.py" in command + launch = "--receipt" in command and "process_supervisor.sh" in command if launch: self.deadline = timeout_s self.delayed_command = command @@ -78,21 +77,18 @@ def make_session(tmp_path): ), runtime=str(tmp_path / "runtime"), ) - shutil.copyfile(sandbox_runner.__file__, directory / "sandbox_runner.py") - assert not (directory / "process_supervisor.py").exists() + shutil.copyfile(HARNESS_RUNNER_SCRIPT, directory / HARNESS_RUNNER_FILE) + assert not (directory / "process_supervisor.sh").exists() return state, provider, workdir -def payload(state, code, timeout=0.5): +def payload(state, code): return { - "directory": state.session.session_dir, "prompt": "task", "config": {}, "cwd": state.request.sandbox_access.workdir, "command": [sys.executable, "-c", code], "env": {}, - "timeout": timeout, - "cleanup_timeout": 1, } @@ -112,10 +108,8 @@ async def test_exec_only_supervision_leaves_task_children_for_sandbox_owner(tmp_ "pathlib.Path('ready').touch(); " + {"natural": "print('{}')", "crash": "raise SystemExit(7)"}.get(ending, "time.sleep(60)") ) - deadline = 0.5 if ending == "timeout" else 60 - task = state.task = asyncio.create_task( - state.execute(payload(state, code, timeout=deadline), timeout=deadline, close_timeout=3) - ) + deadline = 3 if ending == "timeout" else 60 + task = state.task = asyncio.create_task(state.execute(payload(state, code), timeout=deadline, close_timeout=3)) try: async with asyncio.timeout(5): while not (workdir / "ready").exists(): @@ -130,6 +124,11 @@ async def test_exec_only_supervision_leaves_task_children_for_sandbox_owner(tmp_ else: await task assert state.session.cleanup["cleanup_confirmed"] is True + if ending == "natural": + assert state.session.artifacts["stdout.log"] == "{}\n" + assert state.runtime_info is not None + if ending == "crash": + assert state.session.cleanup["return_code"] == 7 if ending == "timeout": assert state.session.cleanup["timed_out"] is True child_pid = int((workdir / "child.pid").read_text()) diff --git a/responses_api_agents/openclaw_agent/tests/test_install_runtime.py b/responses_api_agents/openclaw_agent/tests/test_install_runtime.py index e6469c39465..24edb774302 100644 --- a/responses_api_agents/openclaw_agent/tests/test_install_runtime.py +++ b/responses_api_agents/openclaw_agent/tests/test_install_runtime.py @@ -24,7 +24,6 @@ def sandbox(tmp_path: Path) -> tuple[Path, dict[str, str]]: if executable is None: pytest.skip(f"Installer test requires {name}") (bindir / name).symlink_to(executable) - (bindir / "python3").symlink_to(sys.executable) scripts = { "id": '#!/bin/bash\necho "${TEST_UID:-0}"\n', "apt-get": """#!/bin/bash @@ -116,15 +115,6 @@ def test_cached_runtime_needs_no_package_installation(sandbox: tuple[Path, dict[ assert not (root / "download.log").exists() -def test_missing_supervisor_python_explains_task_image_requirement(sandbox): - root, env = sandbox - (root / "bin/python3").unlink() - result = run_installer(root, env) - assert result.returncode == 1 - assert "requires Python >=3.8 in the task image" in result.stderr - assert not (root / "download.log").exists() - - def test_download_failure_identifies_command_and_exit_status(sandbox): root, env = sandbox shutil.copy(root / "curl", root / "bin/curl") @@ -208,17 +198,11 @@ def _use_musl_platform(root: Path, *, architecture: str = "x86_64") -> None: _replace_command(root, "getconf", "exit 1\n") _replace_command(root, "ldd", "echo 'musl libc' >&2\nexit 1\n") _replace_command(root, "uname", f'case "$1" in -s) echo Linux;; -m) echo {architecture};; esac\n') - # The task image has its existing C++ library, but our private Node may need - # a newer version. Other Python calls still exercise the real interpreter. - _replace_command( - root, - "python3", - 'if [[ "$*" == *ctypes.CDLL* ]]; then exit 0; fi\n' + f'exec "{sys.executable}" "$@"\n', - ) + # The task image has its existing C++ library, but our private Node may need a newer version. _replace_command( root, "apk", - 'echo "$*" >> "$TEST_ROOT/packages.log"\ncp "$TEST_ROOT/curl" "$TEST_ROOT/bin/curl"\n', + '[ "$1" != info ] || exit 0\necho "$*" >> "$TEST_ROOT/packages.log"\ncp "$TEST_ROOT/curl" "$TEST_ROOT/bin/curl"\n', ) diff --git a/responses_api_agents/openclaw_agent/tests/test_pinned_runtime.py b/responses_api_agents/openclaw_agent/tests/test_pinned_runtime.py index 3dc7699dbfd..c75ca918cf3 100644 --- a/responses_api_agents/openclaw_agent/tests/test_pinned_runtime.py +++ b/responses_api_agents/openclaw_agent/tests/test_pinned_runtime.py @@ -14,10 +14,10 @@ from aiohttp import web from fastapi import Request -from nemo_gym.agent_utils import process_supervisor +from nemo_gym.agent_utils.sandbox_session import HARNESS_RUNNER_SCRIPT, shell_assignments +from nemo_gym.agent_utils.supervisor_client import SUPERVISOR_SCRIPT from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming -from responses_api_agents.openclaw_agent import sandbox_runner -from responses_api_agents.openclaw_agent.tests.test_sandbox import seed, setup # noqa: F401 +from responses_api_agents.openclaw_agent.tests.test_sandbox import seed, setup, staged # noqa: F401 @pytest.mark.skipif(sys.platform != "linux", reason="Linux process supervision required") @@ -57,7 +57,7 @@ async def hold_model_call(request: web.Request) -> web.Response: state = agent._session_records[body.agent_session_id].state state.session.session_dir = str(directory) await agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="Wait for the model response.")) - payload = json.loads(sandbox.files[f"{directory}/input.json"]) + payload = staged(sandbox) payload["command"][:2] = [node, entrypoint] identities = directory / "node-identities.jsonl" preload = directory / "identity.cjs" @@ -69,12 +69,14 @@ async def hold_model_call(request: web.Request) -> web.Response: payload["env"]["NODE_OPTIONS"] = f"--require={preload}" payload["env"]["NODE_COMPILE_CACHE"] = str(directory / "compile-cache") payload["env"]["HOME"] = str(tmp_path / "task-home") - (directory / "home/.openclaw/openclaw.json").write_text(json.dumps(payload["config"])) - (directory / "input.json").write_text(json.dumps(payload)) + (directory / "home/.openclaw/openclaw.json").write_text( + sandbox.files[f"{directory}/home/.openclaw/openclaw.json"] + ) + (directory / "prompt.txt").write_text(payload.pop("prompt")) + (directory / "params.sh").write_text(shell_assignments(payload)) process = await asyncio.create_subprocess_exec( - sys.executable, - "-I", - process_supervisor.__file__, + "bash", + str(SUPERVISOR_SCRIPT), "--timeout", "90", "--cleanup-timeout", @@ -84,10 +86,9 @@ async def hold_model_call(request: web.Request) -> web.Response: "--stop-file", str(directory / "stop.request"), "--", - sys.executable, - "-I", - sandbox_runner.__file__, - str(directory / "input.json"), + "bash", + str(HARNESS_RUNNER_SCRIPT), + str(directory / "params.sh"), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index 9971820d60c..8deea828818 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -5,6 +5,8 @@ import json import math import shlex +import subprocess +import tempfile from copy import deepcopy from pathlib import Path from types import SimpleNamespace @@ -88,6 +90,30 @@ def events(*, stop_reason="stop") -> str: ) +def staged(sandbox) -> dict: + """The harness runner's PARAMS as bash reads them, with the staged prompt.""" + script = ( + 'source "$1"; printf "%s\\0" "$cwd" "$stdin" "$stdout" "$stderr" "${#command[@]}" "${command[@]}" "${env[@]}"' + ) + params = sandbox.files[f"{sandbox.directory}/params.sh"] + with tempfile.NamedTemporaryFile("w", suffix=".sh") as file: + file.write(params) + file.flush() + values = subprocess.run( + ["bash", "-c", script, "_", file.name], capture_output=True, text=True, check=True + ).stdout.split("\0")[:-1] + cwd, stdin, stdout, stderr, count, rest = *values[:4], int(values[4]), values[5:] + return { + "prompt": sandbox.files[f"{sandbox.directory}/prompt.txt"], + "cwd": cwd, + "stdin": stdin, + "stdout": stdout, + "stderr": stderr, + "command": rest[:count], + "env": dict(item.split("=", 1) for item in rest[count:]), + } + + class Sandbox: def __init__(self): self.files = {} @@ -116,7 +142,7 @@ def pty(self): raise AssertionError("This provider supports exec only; no PTY API") async def run_exec(self, command, **kwargs): - if "--receipt" in command and "process_supervisor.py" in command: + if "--receipt" in command and "process_supervisor.sh" in command: return await self.launch(command=command, **kwargs) if "stop.request" in command: await self.request_stop() @@ -129,12 +155,10 @@ async def download(self, source, destination): Path(destination).write_text(self.files[source]) async def create(self, **kwargs): - payload_path = next(path for path in self.files if path.endswith("/input.json")) - payload = json.loads(self.files[payload_path]) - assert payload["cwd"] == getattr(self, "expected_workdir", "/app") + self.directory = next(path for path in self.files if path.endswith("/params.sh")).rsplit("/", 1)[0] + assert staged(self)["cwd"] == getattr(self, "expected_workdir", "/app") assert kwargs["cwd"] == getattr(self, "expected_workdir", "/app") - assert "sandbox_runner.py" in kwargs["command"] - self.directory = payload["directory"] + assert f"{self.directory}/harness_runner.sh {self.directory}/params.sh" in kwargs["command"] self.started.set() if not self.blocked: self.exited.set() @@ -269,7 +293,7 @@ def test_http_sandbox_flow_runs_openclaw_in_borrowed_sandbox(setup, model_timeou assert body["usage"]["output_tokens_details"]["reasoning_tokens"] is None assert body["metadata"]["harness_execution"] == "sandbox" assert "_ng_agent_observations" not in body - payload = json.loads(sandbox.files[f"{sandbox.directory}/input.json"]) + payload = staged(sandbox) assert payload["prompt"] == "Fix the code" assert payload["command"][0].endswith("/node/bin/node") assert "--timeout 900" in sandbox.launch.await_args.kwargs["command"] @@ -693,7 +717,7 @@ def test_instructions_and_text_parts_reach_openclaw_without_other_provider_crede }, ) assert response.status_code == 200, response.text - payload = json.loads(sandbox.files[f"{sandbox.directory}/input.json"]) + payload = staged(sandbox) assert payload["prompt"] == "config instruction\n\nrequest instruction\n\ninput instruction\n\ntask" models = json.loads(sandbox.files[f"{sandbox.directory}/home/.openclaw/openclaw.json"]) assert list(models["models"]["providers"]) == ["nemo"] @@ -1325,9 +1349,6 @@ async def test_sandbox_envelope_fallback_preserves_known_and_unknown_cache_detai @pytest.mark.parametrize("owned_root", ["sessions", "runtime"]) @pytest.mark.parametrize("overlap", ["same", "ancestor", "descendant", "symlink", "dotdot"]) def test_workdir_cannot_overlap_adapter_storage(tmp_path, owned_root, overlap): - import subprocess - import sys - from responses_api_agents.openclaw_agent.sandbox import _SANDBOX_PATH_CHECK sessions, runtime = tmp_path / "sessions", tmp_path / "runtime" @@ -1347,7 +1368,7 @@ def test_workdir_cannot_overlap_adapter_storage(tmp_path, owned_root, overlap): else: workdir = owned / "repo" / ".." result = subprocess.run( - [sys.executable, "-c", _SANDBOX_PATH_CHECK, str(workdir), str(sessions), str(runtime)], + ["sh", "-c", _SANDBOX_PATH_CHECK, "_", str(workdir), str(sessions), str(runtime)], capture_output=True, text=True, errors="replace", @@ -1358,16 +1379,14 @@ def test_workdir_cannot_overlap_adapter_storage(tmp_path, owned_root, overlap): def test_workdir_validation_accepts_separate_repository_and_rejects_missing(tmp_path): - import subprocess - import sys - from responses_api_agents.openclaw_agent.sandbox import _SANDBOX_PATH_CHECK workdir = tmp_path / "task" command = [ - sys.executable, + "sh", "-c", _SANDBOX_PATH_CHECK, + "_", str(workdir), str(tmp_path / "sessions"), str(tmp_path / "runtime"), @@ -1380,6 +1399,17 @@ def test_workdir_validation_accepts_separate_repository_and_rejects_missing(tmp_ assert valid.returncode == 0, valid.stderr assert not (tmp_path / "sessions").exists() assert not (tmp_path / "runtime").exists() + # A root that does not exist yet still resolves through its existing ancestors, symlinks included. + (tmp_path / "alias").symlink_to(workdir) + nested = subprocess.run( + [*command[:5], str(tmp_path / "alias/new/sessions"), command[6]], + capture_output=True, + text=True, + errors="replace", + timeout=5, + ) + assert nested.returncode != 0 + assert f"overlaps adapter-owned storage: {workdir.resolve()}/new/sessions" in nested.stderr async def test_path_check_failure_never_uploads_or_installs(setup): @@ -1897,11 +1927,11 @@ async def test_close_retains_capture_after_provider_release(setup, tmp_path, own await agent.seed_agent_session(request, body) state = agent._session_records[body.agent_session_id].state directory = state.session.session_dir - assert f"{directory}/process_supervisor.py" not in sandbox.files + assert f"{directory}/process_supervisor.sh" not in sandbox.files sandbox.blocked = True task = asyncio.create_task(agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="task"))) await asyncio.wait_for(sandbox.started.wait(), 2) - assert f"{directory}/process_supervisor.py" in sandbox.files + assert f"{directory}/process_supervisor.sh" in sandbox.files async def release(): assert state.session.cleanup["cleanup_confirmed"] is True @@ -1971,7 +2001,7 @@ def test_task_timeout_metadata_reaches_sandbox_supervisor( assert float(command[command.index("--timeout") + 1]) == expected assert agent.config.timeout == 1800 assert response.json()["metadata"]["agent_timeout_sec"] == str(float(expected)) - payload = json.loads(sandbox.files[f"{sandbox.directory}/input.json"]) + payload = staged(sandbox) cli = payload["command"] assert cli[cli.index("--timeout") + 1] == str(math.ceil(expected)) closed = client.post("/v1/agent_sessions/close", json=close_body(session_id)) diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py b/responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py deleted file mode 100644 index 4f73e452f4c..00000000000 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox_runner.py +++ /dev/null @@ -1,147 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Test worker I/O separately from the shared Linux supervisor's cleanup contract.""" - -import json -import os -import shutil -import signal -import subprocess -import sys -import time -from pathlib import Path - -import pytest - -from nemo_gym.agent_utils import process_supervisor -from responses_api_agents.openclaw_agent import sandbox_runner - - -linux_only = pytest.mark.skipif(sys.platform != "linux", reason="Linux process supervision required") - - -def launch(tmp_path, code, *, timeout=3, python=sys.executable, env=None, supervised=True): - params = { - "directory": str(tmp_path), - "command": [python, "-c", code], - "cwd": str(tmp_path), - "env": env or {}, - "prompt": "task input", - } - path = tmp_path / "input.json" - path.write_text(json.dumps(params)) - command = [python, "-I", sandbox_runner.__file__, str(path)] - if supervised: - command = [ - python, - "-I", - process_supervisor.__file__, - "--timeout", - str(timeout), - "--cleanup-timeout", - "0.5", - "--receipt", - str(tmp_path / "cleanup.json"), - "--stop-file", - str(tmp_path / "stop.request"), - "--", - *command, - ] - return subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - -def result(tmp_path, process): - try: - stdout, stderr = process.communicate(timeout=10) - assert process.returncode == 0, (stdout, stderr) - return json.loads((tmp_path / "cleanup.json").read_text()) - finally: - if process.poll() is None: - process.kill() - process.wait() - - -@pytest.mark.parametrize( - "python", - [ - sys.executable, - pytest.param( - shutil.which("python3.8"), - marks=pytest.mark.skipif(shutil.which("python3.8") is None, reason="Python 3.8 unavailable"), - id="python38", - ), - ], -) -def test_worker_io_and_runtime_are_independent_of_cleanup(tmp_path, monkeypatch, python): - monkeypatch.setenv("HARNESS_INHERITED", "kept") - monkeypatch.setenv("HARNESS_OVERRIDE", "old") - process = launch( - tmp_path, - "import json,os,sys,pathlib; print(json.dumps([" - "pathlib.Path('prompt.txt').read_text(),os.environ['HARNESS_INHERITED'],os.environ['HARNESS_OVERRIDE']]))", - python=python, - env={"HARNESS_OVERRIDE": "new"}, - supervised=False, - ) - stdout, stderr = process.communicate(timeout=10) - assert process.returncode == 0, (stdout, stderr) - assert json.loads((tmp_path / "stdout.log").read_text()) == ["task input", "kept", "new"] - runtime = json.loads((tmp_path / "runtime.json").read_text()) - assert runtime["hostname"] == os.uname().nodename - assert runtime["pid"] == process.pid - assert not (tmp_path / "cleanup.json").exists() - assert not (tmp_path / "result.json").exists() - - -@linux_only -@pytest.mark.parametrize("ending", ["natural", "timeout", "cancel"]) -def test_task_descendants_survive_until_owner_teardown(tmp_path, ending): - try: - # A completed tool leaves its detached service adopted before close. - tool = ( - "import pathlib,subprocess,sys; " - "p=subprocess.Popen([sys.executable,'-c','import time; time.sleep(60)']," - "start_new_session=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL); " - "pathlib.Path('child.pid').write_text(str(p.pid))" - ) - code = ( - "import subprocess,sys,time,pathlib; " - f"subprocess.run([sys.executable,'-c',{tool!r}],check=True); " - "pathlib.Path('ready').touch(); " + ("time.sleep(60)" if ending != "natural" else "pass") - ) - process = launch(tmp_path, code) - if ending == "cancel": - for _ in range(200): - if (tmp_path / "ready").exists(): - break - time.sleep(0.01) - process.send_signal(signal.SIGTERM) - summary = result(tmp_path, process) - assert summary["cleanup_confirmed"] is True - assert summary["timed_out"] is (ending == "timeout") - assert set(summary) == {"return_code", "timed_out", "cleanup_confirmed", "error"} - pid = int((tmp_path / "child.pid").read_text()) - assert Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[0] != "Z" - finally: - if (tmp_path / "child.pid").exists(): - try: - os.kill(int((tmp_path / "child.pid").read_text()), signal.SIGKILL) - except ProcessLookupError: - pass - - -@linux_only -def test_spawn_error_does_not_prevent_cleanup(tmp_path): - summary = result(tmp_path, launch(tmp_path, "raise RuntimeError('failed')")) - assert summary["cleanup_confirmed"] is True - assert summary["return_code"] != 0 - - -@linux_only -def test_stop_marker_fences_launch_without_inventing_worker_identity(tmp_path): - (tmp_path / "stop.request").touch() - summary = result(tmp_path, launch(tmp_path, "open('unexpected', 'w').close()")) - assert summary["cleanup_confirmed"] is True - assert summary["return_code"] is None - assert not (tmp_path / "runtime.json").exists() - assert not (tmp_path / "unexpected").exists() From b1b6823776595d5e1c20fcb03a00bf91d47190db Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Mon, 26 Oct 2026 09:29:02 -0700 Subject: [PATCH 08/32] refactor(opencode): run the harness without Python in the sandbox Task images now need only bash. The shared harness_runner.sh replaces sandbox_runner.py: PARAMS carry the CLI argv, environment and stdio files, and a short setup records the task HOME/XDG directories for the shell.env hook, removes a stale initialization receipt and creates the private CLI directories. The SQLite snapshot moves to the agent server: after confirmed cleanup the adapter downloads opencode.db (and its -wal/-shm files when present) and runs the same export there, producing the same export.json and observations.db. The bootstrap no longer installs python3 and checks workdir overlap with POSIX shell, and the runtime installer serializes with flock, or a reclaimable lock directory without it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Hemil Desai Signed-off-by: yaoyu-33 --- benchmarks/swe_together/opencode.yaml | 1 - responses_api_agents/opencode_agent/README.md | 9 +- responses_api_agents/opencode_agent/app.py | 49 ++--- .../install_opencode_runtime.sh | 70 +++---- .../opencode_agent/sandbox.py | 169 ++++++++++------ .../opencode_agent/sandbox_runner.py | 152 --------------- .../opencode_agent/snapshot.py | 55 ++++++ .../opencode_agent/tests/test_bootstrap.py | 7 +- .../opencode_agent/tests/test_continuation.py | 30 ++- .../tests/test_exec_transport.py | 32 ++-- .../tests/test_install_runtime.py | 56 +++++- .../tests/test_lifecycle_contract.py | 12 +- .../tests/test_sandbox_runner.py | 181 +++++++++++------- .../tests/test_sandbox_sessions.py | 112 +++++++---- .../tests/test_session_review.py | 30 +-- .../tests/test_task_environment.py | 55 +----- 16 files changed, 514 insertions(+), 506 deletions(-) delete mode 100644 responses_api_agents/opencode_agent/sandbox_runner.py create mode 100644 responses_api_agents/opencode_agent/snapshot.py diff --git a/benchmarks/swe_together/opencode.yaml b/benchmarks/swe_together/opencode.yaml index 1f3a0bf8541..013b6cf9dad 100644 --- a/benchmarks/swe_together/opencode.yaml +++ b/benchmarks/swe_together/opencode.yaml @@ -61,7 +61,6 @@ swe_together_candidate: model: ${policy_model_name} opencode_version: 1.15.13 sandbox_model_id: ${policy_model_name} - sandbox_python_executable: /opt/gym-runtime/python/bin/python3 sandbox_path_prefix: [/opt/gym-runtime] sandbox_executable_path: /opt/gym-runtime/opencode sandbox_executable_sha256: 8e97f83175c84daeb51e96b34baf9ee9839ae54e5e848a3cce1e0e32dcf41a6c # pragma: allowlist secret (public runtime digest) diff --git a/responses_api_agents/opencode_agent/README.md b/responses_api_agents/opencode_agent/README.md index b9a895f7d10..711be94cf34 100644 --- a/responses_api_agents/opencode_agent/README.md +++ b/responses_api_agents/opencode_agent/README.md @@ -73,8 +73,8 @@ task files must use a Resources-owned sandbox. Missing access and provider is an error, never a fallback to local execution. Session setup installs the pinned OpenCode runtime inside the sandbox. The image -needs Linux, Python 3.8+ with SQLite and `fcntl`, and Bash. Missing Python/Bash can -be installed with apt-get or apk when running as root; otherwise preinstall them. +needs Linux and Bash; it does not need Python. Missing Bash can be installed with +apt-get or apk when running as root; otherwise preinstall it. Online installation needs GitHub access, curl, tar, gzip and a Debian or RHEL CA bundle. Missing required packages are installed with apt-get or apk as root; non-root images must preinstall them. System `rg` is a best-effort optimization: @@ -95,7 +95,7 @@ Natural CLI exit cannot distinguish unfinished tools in other sessions from serv those processes may still change task files. The stop-time cutoff also has adoption/PID races and cannot classify later service forks. It is not a security boundary. The CLI must exit before -SQLite observations are captured; services must use task-owned files rather than the +its SQLite database is copied to the agent server and exported there; services must use task-owned files rather than the agent session directory, which close removes. Capture requires confirmed termination. Missing evidence remains retryable; a final negative receipt skips capture and verification but permits connection release. Successful close receipts are retained for `session_close_retry_window_seconds` (default 300). @@ -208,8 +208,7 @@ Ordinary single-turn sessions keep their existing protocol and evidence shape. For a sandbox without runtime-download egress, set `sandbox_executable_path` to an existing sandbox-visible OpenCode binary and `sandbox_executable_sha256` to its SHA-256. Seed verifies bytes and version before copying it into the adapter cache. -`sandbox_python_executable` selects an existing Python interpreter (default -`python3`); `sandbox_path_prefix` adds executable directories for installation and +`sandbox_path_prefix` adds executable directories for installation and CLI/tool subprocesses while retaining the image PATH. These paths must already exist; the sandbox owner supplies mounts or image contents. diff --git a/responses_api_agents/opencode_agent/app.py b/responses_api_agents/opencode_agent/app.py index 3ca098c5651..68697318275 100644 --- a/responses_api_agents/opencode_agent/app.py +++ b/responses_api_agents/opencode_agent/app.py @@ -37,7 +37,7 @@ AgentContinuationCapabilities, ) from nemo_gym.agent_outcome import AgentOutcome -from nemo_gym.agent_utils.sandbox_session import SandboxSession +from nemo_gym.agent_utils.sandbox_session import HARNESS_RUNNER_FILE, HARNESS_RUNNER_SCRIPT, SandboxSession from nemo_gym.agent_utils.task_timeout import AGENT_TIMEOUT_METADATA_KEY, resolve_agent_timeout from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import ( @@ -136,11 +136,10 @@ def _extract_instruction(body_input) -> tuple[str, Optional[str]]: return user_message, system_message -def _sandbox_prepare_command(workdir: str, directory: str, runtime: str, *, python: str = "python3") -> str: +def _sandbox_prepare_command(workdir: str, directory: str, runtime: str) -> str: # Bootstrap with POSIX sh; the runtime installer itself requires Bash. bootstrap = """set -eu set -- -command -v python3 >/dev/null 2>&1 || set -- "$@" python3 command -v bash >/dev/null 2>&1 || set -- "$@" bash if [ "$#" -gt 0 ]; then [ "$(id -u)" = 0 ] || { echo "OpenCode sandbox execution requires $*: preinstall these tools or use a root image." >&2; exit 1; } @@ -155,17 +154,28 @@ def _sandbox_prepare_command(workdir: str, directory: str, runtime: str, *, pyth fi fi """ - validate_paths = ( - "from pathlib import Path; import sys; " - "workdir,*roots=[Path(p).resolve() for p in sys.argv[1:]]; " - "assert workdir.is_dir(), 'OpenCode task workdir is missing'; " - "assert all(workdir != root and workdir not in root.parents and root not in workdir.parents " - "for root in roots), 'OpenCode runtime/session storage overlaps the task workdir'" - ) + # Resolve symlinks like realpath -m: storage roots need not exist yet. + validate_paths = """resolve() { + if [ -e "$1" ]; then (cd -P "$1" && pwd -P); else + parent=${1%/*} + parent=$(resolve "${parent:-/}") && printf '%s/%s\\n' "${parent%/}" "${1##*/}" + fi +} +[ -d "$1" ] || { echo 'OpenCode task workdir is missing' >&2; exit 1; } +workdir=$(resolve "$1") +shift +for root in "$@"; do + root=$(resolve "$root") + case "$root/" in "${workdir%/}"/*) overlap=1 ;; esac + case "$workdir/" in "${root%/}"/*) overlap=1 ;; esac +done +[ -z "${overlap-}" ] || { echo 'OpenCode runtime/session storage overlaps the task workdir' >&2; exit 1; } +""" return ( - bootstrap.replace("command -v python3", "command -v " + quote(python)) - + f"{quote(python)} -I -c {quote(validate_paths)} {quote(workdir)} " - f"{quote(str(PurePosixPath(directory).parent))} {quote(runtime)} && mkdir -p {quote(directory)}" + bootstrap + + f"set -- {quote(workdir)} {quote(str(PurePosixPath(directory).parent))} {quote(runtime)}\n" + + validate_paths + + f"mkdir -p {quote(directory)}" ) @@ -195,7 +205,6 @@ class OpenCodeAgentConfig(BaseResponsesAPIAgentConfig): opencode_version: str = OPENCODE_VERSION # Optional verified, preinstalled runtime for sandboxes without download egress. - sandbox_python_executable: str = "python3" sandbox_path_prefix: list[str] = Field(default_factory=list) sandbox_executable_path: str | None = None sandbox_executable_sha256: str | None = Field(default=None, pattern=r"^[a-fA-F0-9]{64}$") @@ -730,7 +739,6 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Open sandbox=sandbox, session_dir=directory, workdir=workdir, harness="OpenCode", owns_sandbox=owns_sandbox ), runtime=runtime, - python_executable=self.config.sandbox_python_executable, model_ref=self.config.model_server, ) prepared_directory = False @@ -744,9 +752,7 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Open f"Cannot create OpenCode sandbox workdir {workdir}: {workspace.stderr or workspace.stdout}" ) state.sandbox_id = sandbox.sandbox_id - command = _sandbox_prepare_command( - workdir, directory, runtime, python=self.config.sandbox_python_executable - ) + command = _sandbox_prepare_command(workdir, directory, runtime) result = await sandbox.exec(command, timeout_s=self.config.sandbox_install_timeout_seconds) self._check_sandbox_setup(command, result) prepared_directory = True @@ -760,13 +766,12 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Open self.config.opencode_version, self.config.sandbox_executable_path or "", self.config.sandbox_executable_sha256 or "", - self.config.sandbox_python_executable, ":".join(self.config.sandbox_path_prefix), ) ) result = await sandbox.exec(command, cwd=workdir, timeout_s=self.config.sandbox_install_timeout_seconds) self._check_sandbox_setup(command, result) - await sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), f"{directory}/sandbox_runner.py") + await sandbox.upload(HARNESS_RUNNER_SCRIPT, f"{directory}/{HARNESS_RUNNER_FILE}") await sandbox.upload(Path(__file__).with_name("task_environment.mjs"), f"{directory}/task_environment.mjs") if self._model_call_capture_enabled(): await sandbox.upload(OBSERVABILITY_PATCH, f"{directory}/{OBSERVABILITY_PATCH.name}") @@ -1137,13 +1142,9 @@ async def _session_response( payload = { "instructions": system, "path_prefix": self.config.sandbox_path_prefix, - "directory": state.session.session_dir, "cwd": state.session.workdir, "command": command, "prompt": prompt, - "harness_session_id": state.harness_session_id, - "previous_message_ids": state.message_ids, - "activation_id": state.activation_id, "env": { "HOME": f"{state.persistent_directory}/home", "XDG_DATA_HOME": f"{state.persistent_directory}/data", diff --git a/responses_api_agents/opencode_agent/install_opencode_runtime.sh b/responses_api_agents/opencode_agent/install_opencode_runtime.sh index 5715ae96d0c..0367d189c08 100644 --- a/responses_api_agents/opencode_agent/install_opencode_runtime.sh +++ b/responses_api_agents/opencode_agent/install_opencode_runtime.sh @@ -7,19 +7,27 @@ runtime=$1 version=$2 source_binary=${3:-} source_sha256=${4:-} -python_executable=${5:-python3} -path_prefix=${6:-} +path_prefix=${5:-} if [ -n "$path_prefix" ]; then export PATH="$path_prefix:$PATH"; fi -require() { - command -v "$1" >/dev/null 2>&1 || { echo "OpenCode sandbox execution requires $1 in the task image" >&2; exit 1; } -} -require "$python_executable" -"$python_executable" -c 'import sys; assert sys.platform == "linux" and sys.version_info >= (3, 8), "OpenCode sandbox execution requires Linux and Python >=3.8 in the task image"' +[ "$(uname -s)" = Linux ] || { echo 'OpenCode sandbox execution requires a Linux task image' >&2; exit 1; } # Serialize preparation of the version-scoped cache in a shared task sandbox. -# The lock descriptor remains open across exec and releases on installer exit. -if [ "${NG_OPENCODE_INSTALL_LOCKED:-0}" != 1 ]; then - exec "$python_executable" -c 'import fcntl,os,sys; fd=os.open(sys.argv[1]+".lock",os.O_CREAT|os.O_RDWR,0o600); fcntl.flock(fd,fcntl.LOCK_EX); os.set_inheritable(fd,True); os.environ["NG_OPENCODE_INSTALL_LOCKED"]="1"; os.execvp("bash",["bash",*sys.argv[2:]])' "$runtime" "$0" "$@" +if command -v flock >/dev/null 2>&1; then + # The kernel releases the lock when the installer exits, however it exits. + exec 9>"$runtime.lock" + flock 9 +else + # The lock directory names its owner, so the lock of a killed installer can be reclaimed. + until mkdir "$runtime.lock.d" 2>/dev/null; do + owner= + { read -r owner <"$runtime.lock.d/pid"; } 2>/dev/null || true + if [ -n "$owner" ] && ! kill -0 "$owner" 2>/dev/null; then + rm -rf "$runtime.lock.d" + fi + sleep 0.1 + done + echo "$$" >"$runtime.lock.d/pid" + trap 'rm -rf "$runtime.lock.d"' EXIT fi case "$(uname -m)" in x86_64) arch=x64-baseline ;; @@ -39,31 +47,23 @@ export HOME="$runtime/home" XDG_CACHE_HOME="$runtime/cache" XDG_DATA_HOME="$runt if [ -n "$source_binary" ]; then [ -n "$source_sha256" ] || { echo 'A preinstalled OpenCode binary requires its SHA-256' >&2; exit 1; } # Publish a verified new inode so active sessions can keep executing the old one. - "$python_executable" -I - "$source_binary" "$source_sha256" "$runtime/opencode" "$version" <<'PY' -import hashlib -import os -import shutil -import subprocess -import sys -import tempfile -from pathlib import Path - -source, expected, target, version = sys.argv[1:] -if not Path(source).is_file(): - raise ValueError("OpenCode executable digest mismatch") -fd, staged = tempfile.mkstemp(prefix=".opencode-", dir=Path(target).parent) -os.close(fd) -try: - shutil.copyfile(source, staged) - if hashlib.sha256(Path(staged).read_bytes()).hexdigest() != expected: - raise ValueError("OpenCode executable digest mismatch") - os.chmod(staged, 0o755) - if subprocess.check_output([staged, "--version"], text=True).rstrip("\n") != version: - raise ValueError("OpenCode executable version mismatch") - os.replace(staged, target) -finally: - Path(staged).unlink(missing_ok=True) -PY + command -v sha256sum >/dev/null 2>&1 || { echo 'Verifying a preinstalled OpenCode binary requires sha256sum' >&2; exit 1; } + [ -f "$source_binary" ] || { echo 'OpenCode executable digest mismatch' >&2; exit 1; } + staged=$(mktemp "$runtime/.opencode-XXXXXX") + cp "$source_binary" "$staged" + digest=$(sha256sum "$staged") + if [ "${digest%% *}" != "$source_sha256" ]; then + rm -f "$staged" + echo 'OpenCode executable digest mismatch' >&2 + exit 1 + fi + chmod 755 "$staged" + if [ "$("$staged" --version)" != "$version" ]; then + rm -f "$staged" + echo 'OpenCode executable version mismatch' >&2 + exit 1 + fi + mv -f "$staged" "$runtime/opencode" fi cached=false if [ -x "$runtime/opencode" ] && [ "$("$runtime/opencode" --version)" = "$version" ]; then diff --git a/responses_api_agents/opencode_agent/sandbox.py b/responses_api_agents/opencode_agent/sandbox.py index 1772bbe4303..211adc962f9 100644 --- a/responses_api_agents/opencode_agent/sandbox.py +++ b/responses_api_agents/opencode_agent/sandbox.py @@ -12,7 +12,12 @@ from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError -from nemo_gym.agent_utils.sandbox_session import SandboxCommand, SandboxSession +from nemo_gym.agent_utils.sandbox_session import ( + HARNESS_RUNNER_FILE, + SandboxCommand, + SandboxSession, + shell_assignments, +) from nemo_gym.base_responses_api_agent import AgentSessionState from nemo_gym.config_types import ModelServerRef from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming @@ -20,6 +25,7 @@ from nemo_gym.sandbox.utils import read_text, upload_text from responses_api_agents.opencode_agent.artifacts import parse_opencode_observations from responses_api_agents.opencode_agent.observability import scope_opencode_activation +from responses_api_agents.opencode_agent.snapshot import snapshot LOG = logging.getLogger(__name__) @@ -45,7 +51,6 @@ class HarnessProcessInfo(BaseModel): model_config = ConfigDict(extra="forbid", strict=True) hostname: str pid: int - python: str | None = None def parse_runtime_info(payload: object) -> HarnessProcessInfo | None: @@ -57,6 +62,31 @@ def parse_runtime_info(payload: object) -> HarnessProcessInfo | None: return None +_TASK_DIRECTORY_KEYS = ("HOME", "XDG_DATA_HOME", "XDG_CONFIG_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME") +# Record the task's home and XDG directories, before OpenCode's overrides, for the +# task_environment.mjs shell.env hook. Defaults follow the XDG specification. +_TASK_ENV_SETUP = r"""json() { local value=${1//\\/\\\\}; printf '"%s"' "${value//\"/\\\"}"; } +home=${HOME:-~} +printf '{"HOME": %s, "XDG_DATA_HOME": %s, "XDG_CONFIG_HOME": %s, "XDG_CACHE_HOME": %s, "XDG_STATE_HOME": %s}' \ + "$(json "$home")" \ + "$(json "${XDG_DATA_HOME:-${home%/}/.local/share}")" \ + "$(json "${XDG_CONFIG_HOME:-${home%/}/.config}")" \ + "$(json "${XDG_CACHE_HOME:-${home%/}/.cache}")" \ + "$(json "${XDG_STATE_HOME:-${home%/}/.local/state}")" >"$task_env" || exit 1 +""" + + +def runner_setup(directory: str, private_directories: list[str]) -> str: + """Bash run in the sandbox before the CLI: fresh receipt, task directories, private directories.""" + return ( + # Never accept an initialization receipt left by an earlier invocation. + f"rm -f -- {quote(f'{directory}/task-environment.ready')} || exit 1\n" + f"task_env={quote(f'{directory}/task-env.json')}\n" + + _TASK_ENV_SETUP + + f"mkdir -p -- {' '.join(quote(path) for path in private_directories)}\n" + ) + + @dataclass class OpenCodeSandboxSession(AgentSessionState): """Keep OpenCode HOME/history under ``conversation_directory`` across launches. @@ -70,7 +100,6 @@ class OpenCodeSandboxSession(AgentSessionState): session: SandboxSession[str] runtime: str sandbox_id: str | None = None - python_executable: str = "python3" model_ref: ModelServerRef | None = None task: asyncio.Task[NeMoGymResponse] | None = None runtime_info: HarnessProcessInfo | None = None @@ -177,10 +206,6 @@ def retain_activation(self) -> None: }, ) - async def upload_json(self, name: str, payload: JsonValue) -> None: - """Stage adapter input using the shared text-transfer utility.""" - await upload_text(self.session.sandbox, path=f"{self.session.session_dir}/{name}", text=json.dumps(payload)) - async def read_text(self, name: str) -> str: """Read adapter output using file transfer.""" return await read_text(self.session.sandbox, path=f"{self.session.session_dir}/{name}") @@ -233,31 +258,45 @@ async def stage_activation(self, payload: dict[str, JsonValue]) -> SandboxComman await self.session.sandbox.upload( Path(__file__).with_name("task_environment.mjs"), f"{self.session.session_dir}/task_environment.mjs" ) - await self.upload_json("input.json", payload) + directory = self.session.session_dir + await upload_text(self.session.sandbox, path=f"{directory}/prompt.txt", text=payload["prompt"]) + env = payload["env"] + params = { + "cwd": payload["cwd"], + "command": payload["command"], + "env": env, + "stdin": f"{directory}/prompt.txt", + "stdout": f"{directory}/stdout.jsonl", + "stderr": f"{directory}/stderr.log", + "setup": runner_setup(directory, [env[key] for key in _TASK_DIRECTORY_KEYS]), + } + if payload.get("path_prefix"): + # Preinstalled runtime tools come first; the task's own PATH still follows. + params["setup"] += f"PATH={quote(':'.join(payload['path_prefix']))}:$PATH\n" + await upload_text(self.session.sandbox, path=f"{directory}/params.sh", text=shell_assignments(params)) return SandboxCommand( - argv=[ - self.python_executable, - "-I", - f"{self.persistent_directory}/sandbox_runner.py", - f"{self.session.session_dir}/input.json", - ], + argv=["bash", f"{self.persistent_directory}/{HARNESS_RUNNER_FILE}", f"{directory}/params.sh"] ) async def collect_artifacts(self, *, timeout: float) -> str: """Snapshot only after cleanup; keep observations before sandbox release.""" + with tempfile.TemporaryDirectory(prefix="opencode-snapshot-") as directory: + return await self._collect_artifacts(Path(directory), timeout=timeout) + + async def _collect_artifacts(self, directory: Path, *, timeout: float) -> str: previous_session_id = self.harness_session_id try: self.stderr = (await self.read_text("stderr.log"))[-16000:] except Exception: LOG.warning("OpenCode stderr log unavailable", exc_info=True) try: - await self.snapshot(timeout) + await self.snapshot(directory, timeout) except Exception as error: if self.stderr: raise RuntimeError(format_error_message(str(error), stderr=self.stderr)) from error raise + export_text = (directory / "export.json").read_text() if self.is_continuation: - export_text = await self._read_export() export = json.loads(export_text) harness_session_id = export.get("session_id") if not isinstance(harness_session_id, str) or not harness_session_id: @@ -271,33 +310,30 @@ async def collect_artifacts(self, *, timeout: float) -> str: except Exception: LOG.warning("OpenCode event log unavailable", exc_info=True) try: - with tempfile.TemporaryDirectory(prefix="opencode-observations-") as directory: - path = Path(directory) / "observations.db" - await self.session.sandbox.download(f"{self.session.session_dir}/observations.db", path) - trajectory = TrajectoryRecord( - task_id=self.request.task_id.task_id, rollout_id=self.request.episode_id.capture_key - ) - observations = await asyncio.to_thread( - parse_opencode_observations, - path, - self.request.episode_id.capture_key, + trajectory = TrajectoryRecord( + task_id=self.request.task_id.task_id, rollout_id=self.request.episode_id.capture_key + ) + observations = await asyncio.to_thread( + parse_opencode_observations, + directory / "observations.db", + self.request.episode_id.capture_key, + trajectory, + require_terminal_finish=True, + model_ref=self.model_ref, + message_ids=set(export.get("activation_message_ids", [])) if self.is_continuation else None, + ) + if self.is_continuation: + assert self.activation_id is not None + scope_opencode_activation( + observations, trajectory, - require_terminal_finish=True, - model_ref=self.model_ref, - message_ids=set(export.get("activation_message_ids", [])) if self.is_continuation else None, + invocation_prefix=self.invocation_prefix, + activation_id=self.activation_id, ) - if self.is_continuation: - assert self.activation_id is not None - scope_opencode_activation( - observations, - trajectory, - invocation_prefix=self.invocation_prefix, - activation_id=self.activation_id, - ) - self.activation_observations = observations - self.activation_trajectory = trajectory - else: - self.observations = observations + self.activation_observations = observations + self.activation_trajectory = trajectory + else: + self.observations = observations except Exception: LOG.exception("Failed to parse OpenCode observations") observations = AgentObservationBundle( @@ -321,7 +357,7 @@ async def collect_artifacts(self, *, timeout: float) -> str: # Preserve the single-turn evidence/error order. The response layer # parses this raw export and translates malformed content to 502. await self._check_task_environment() - return await self._read_export() + return export_text try: await self._check_task_environment() except Exception as error: @@ -343,17 +379,6 @@ async def collect_artifacts(self, *, timeout: float) -> str: ) from error return export_text - async def _read_export(self) -> str: - try: - return await self.read_text("export.json") - except Exception as error: - logs = await self.session.read_output_log() - raise RuntimeError( - format_error_message( - "OpenCode sandbox runner returned no valid result", output=logs, stderr=self.stderr - ) - ) from error - async def _check_task_environment(self) -> None: try: if await self.read_text("task-environment.ready") != "ready\n": @@ -363,15 +388,33 @@ async def _check_task_environment(self) -> None: format_error_message("OpenCode task environment initialization was not confirmed", stderr=self.stderr) ) from error - async def snapshot(self, timeout: float) -> None: - """Capture SQLite after CLI completion; borrowed task services may remain alive.""" + async def snapshot(self, directory: Path, timeout: float) -> None: + """Copy SQLite out after CLI completion and export it here; borrowed task services may remain alive.""" if self.session.cleanup is None or not self.session.cleanup["cleanup_confirmed"]: raise RuntimeError("OpenCode transcript capture requires confirmed cleanup") - captured = await self.session.sandbox.exec( - f"{quote(self.python_executable)} -I {quote(self.persistent_directory + '/sandbox_runner.py')} " - f"--snapshot {quote(self.session.session_dir)} {quote(self.persistent_directory)}", - cwd=self.session.workdir, - timeout_s=timeout, - ) - if captured.return_code != 0 or captured.error_type: - raise RuntimeError(f"OpenCode transcript capture failed: {captured.stderr}") + database = f"{self.persistent_directory}/data/opencode" + try: + async with asyncio.timeout(timeout): + # A stopped CLI may leave its write-ahead log behind; it holds the latest messages. + listed = await self.session.sandbox.exec( + f"cd {quote(database)} && for name in opencode.db opencode.db-wal opencode.db-shm; " + 'do [ ! -e "$name" ] || echo "$name"; done', + cwd=self.session.workdir, + timeout_s=timeout, + ) + if listed.return_code != 0 or listed.error_type: + raise RuntimeError(listed.stderr) + names = (listed.stdout or "").split() + if "opencode.db" not in names: + raise RuntimeError("OpenCode session database is missing") + for name in names: + await self.session.sandbox.download(f"{database}/{name}", directory / name) + await asyncio.to_thread( + snapshot, + directory / "opencode.db", + directory, + harness_session_id=self.harness_session_id, + previous_message_ids=self.message_ids, + ) + except Exception as error: + raise RuntimeError(f"OpenCode transcript capture failed: {error}") from error diff --git a/responses_api_agents/opencode_agent/sandbox_runner.py b/responses_api_agents/opencode_agent/sandbox_runner.py deleted file mode 100644 index e5847371ca5..00000000000 --- a/responses_api_agents/opencode_agent/sandbox_runner.py +++ /dev/null @@ -1,152 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Capture OpenCode output; the supervisor bounds execution; the sandbox owner stops task services.""" - -from __future__ import annotations - -import json -import os -import signal -import sqlite3 -import subprocess -import sys -from contextlib import ExitStack -from pathlib import Path -from typing import TypedDict - - -class RunnerInput(TypedDict): - """Harness input, independent of supervisor cleanup and runtime metadata.""" - - directory: str - prompt: str - command: list[str] - cwd: str - env: dict[str, str] - path_prefix: list[str] - harness_session_id: str | None - previous_message_ids: list[str] - - -def run(params: RunnerInput) -> int: - """Run the CLI with isolated files and wait for it before returning captured output.""" - directory = Path(params["directory"]) - (directory / "runtime.json").write_text( - json.dumps({"hostname": os.uname().nodename, "pid": os.getpid(), "python": sys.executable}) - ) - if params.get("path_prefix"): - params["env"]["PATH"] = os.pathsep.join([*params["path_prefix"], os.environ.get("PATH", "")]) - # Never accept an initialization receipt left by an earlier invocation. - (directory / "task-environment.ready").unlink(missing_ok=True) - # Save only task-directory variables for the shell.env hook, before CLI overrides. - task_home = os.environ.get("HOME") or str(Path.home()) - task_env = {"HOME": task_home} - for key, suffix in ( - ("XDG_DATA_HOME", ".local/share"), - ("XDG_CONFIG_HOME", ".config"), - ("XDG_CACHE_HOME", ".cache"), - ("XDG_STATE_HOME", ".local/state"), - ): - task_env[key] = os.environ.get(key) or str(Path(task_home) / suffix) - (directory / "task-env.json").write_text(json.dumps(task_env)) - for key in ("HOME", "XDG_DATA_HOME", "XDG_CONFIG_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME"): - if key in params["env"]: - Path(params["env"][key]).mkdir(parents=True, exist_ok=True) - (directory / "prompt.txt").write_text(params["prompt"]) - stopping = False - - def interrupt(*_: object) -> None: - nonlocal stopping - # Defer TERM until Popen returns so a signal cannot lose the child handle. - stopping = True - - signal.signal(signal.SIGTERM, interrupt) - with ExitStack() as stack: - stdout = stack.enter_context((directory / "stdout.jsonl").open("wb")) - stderr = stack.enter_context((directory / "stderr.log").open("wb")) - stdin = stack.enter_context((directory / "prompt.txt").open("rb")) - process = subprocess.Popen( - params["command"], - cwd=params["cwd"], - env={**os.environ, **params["env"]}, - stdin=stdin, - stdout=stdout, - stderr=stderr, - ) - try: - while True: - if stopping: - process.terminate() - stopping = False - try: - return process.wait(timeout=0.05) - except subprocess.TimeoutExpired: - pass - finally: - # An I/O/capture error must not leave the CLI changing the task. - # Only stop our direct CLI; task services belong to the sandbox owner. - if process.poll() is None: - process.kill() - process.wait() - - -def snapshot(directory: Path, *, session_directory: Path | None = None) -> None: - """Export root messages and retain the full session tree for observations.""" - session_directory = session_directory or directory - database = session_directory / "data/opencode/opencode.db" - input_path = directory / "input.json" - params = json.loads(input_path.read_text()) if input_path.exists() else {} - previous_ids = set(params.get("previous_message_ids", [])) - with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as source: - with sqlite3.connect(directory / "observations.db") as destination: - source.backup(destination) - source.row_factory = sqlite3.Row - sessions = source.execute( - "select id from session where parent_id is null order by time_created, id" - ).fetchall() - if len(sessions) != 1: - raise RuntimeError(f"Expected one OpenCode root session, found {len(sessions)}") - rows = source.execute( - "select id, data from message where session_id=? order by time_created, id", (sessions[0]["id"],) - ).fetchall() - session_id = sessions[0]["id"] - expected_id = params.get("harness_session_id") - if expected_id is not None and expected_id != session_id: - raise RuntimeError(f"OpenCode resumed unexpected session {session_id!r}") - messages = [] - for row in rows: - if row["id"] in previous_ids: - continue - info = json.loads(row["data"]) - parts = [ - json.loads(part[0]) - for part in source.execute( - "select data from part where message_id=? order by time_created, id", (row["id"],) - ) - ] - messages.append({"id": row["id"], "info": info, "parts": parts}) - all_messages = source.execute("select id, data from message order by time_created, id").fetchall() - usage_messages = [json.loads(row["data"]) for row in all_messages if row["id"] not in previous_ids] - (directory / "export.json").write_text( - json.dumps( - { - "session_id": session_id, - "message_ids": [row["id"] for row in all_messages], - "activation_message_ids": [row["id"] for row in all_messages if row["id"] not in previous_ids], - "messages": messages, - "usage_messages": usage_messages, - } - ) - ) - - -def main() -> None: - if sys.argv[1] == "--snapshot": - snapshot(Path(sys.argv[2]), session_directory=Path(sys.argv[3]) if len(sys.argv) > 3 else None) - return - params = json.loads(Path(sys.argv[1]).read_text()) - raise SystemExit(run(params)) - - -if __name__ == "__main__": - main() diff --git a/responses_api_agents/opencode_agent/snapshot.py b/responses_api_agents/opencode_agent/snapshot.py new file mode 100644 index 00000000000..9a22f4a7db2 --- /dev/null +++ b/responses_api_agents/opencode_agent/snapshot.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Export an OpenCode session database copied out of the sandbox after the CLI stopped.""" + +import json +import sqlite3 +from collections.abc import Collection +from pathlib import Path + + +def snapshot( + database: Path, directory: Path, *, harness_session_id: str | None, previous_message_ids: Collection[str] +) -> None: + """Export root messages and retain the full session tree for observations.""" + previous_ids = set(previous_message_ids) + with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as source: + with sqlite3.connect(directory / "observations.db") as destination: + source.backup(destination) + source.row_factory = sqlite3.Row + sessions = source.execute( + "select id from session where parent_id is null order by time_created, id" + ).fetchall() + if len(sessions) != 1: + raise RuntimeError(f"Expected one OpenCode root session, found {len(sessions)}") + rows = source.execute( + "select id, data from message where session_id=? order by time_created, id", (sessions[0]["id"],) + ).fetchall() + session_id = sessions[0]["id"] + if harness_session_id is not None and harness_session_id != session_id: + raise RuntimeError(f"OpenCode resumed unexpected session {session_id!r}") + messages = [] + for row in rows: + if row["id"] in previous_ids: + continue + info = json.loads(row["data"]) + parts = [ + json.loads(part[0]) + for part in source.execute( + "select data from part where message_id=? order by time_created, id", (row["id"],) + ) + ] + messages.append({"id": row["id"], "info": info, "parts": parts}) + all_messages = source.execute("select id, data from message order by time_created, id").fetchall() + usage_messages = [json.loads(row["data"]) for row in all_messages if row["id"] not in previous_ids] + (directory / "export.json").write_text( + json.dumps( + { + "session_id": session_id, + "message_ids": [row["id"] for row in all_messages], + "activation_message_ids": [row["id"] for row in all_messages if row["id"] not in previous_ids], + "messages": messages, + "usage_messages": usage_messages, + } + ) + ) diff --git a/responses_api_agents/opencode_agent/tests/test_bootstrap.py b/responses_api_agents/opencode_agent/tests/test_bootstrap.py index b12bdc4317e..20c893d3357 100644 --- a/responses_api_agents/opencode_agent/tests/test_bootstrap.py +++ b/responses_api_agents/opencode_agent/tests/test_bootstrap.py @@ -5,7 +5,6 @@ import shlex import shutil import subprocess -import sys import pytest @@ -13,13 +12,13 @@ @pytest.mark.parametrize("package_manager", ["apk", "apt-get"]) -@pytest.mark.parametrize("missing", [(), ("bash",), ("python3",), ("python3", "bash")]) +@pytest.mark.parametrize("missing", [(), ("bash",)]) @pytest.mark.parametrize("failure", [None, "nonroot", "install", "unavailable"]) def test_posix_bootstrap_installs_only_missing_prerequisites(tmp_path, package_manager, missing, failure): """Exercise the actual preparation through sh without invoking a host package manager.""" bindir = tmp_path / "bin" bindir.mkdir() - binaries = {"python3": sys.executable, "bash": shutil.which("bash"), "mkdir": shutil.which("mkdir")} + binaries = {"bash": shutil.which("bash"), "mkdir": shutil.which("mkdir")} assert all(binaries.values()) for name, executable in binaries.items(): if name not in missing: @@ -33,7 +32,7 @@ def test_posix_bootstrap_installs_only_missing_prerequisites(tmp_path, package_m '#!/bin/sh\nprintf "%s\\n" "$*" >> "$TEST_ROOT/packages.log"\n' + ("exit 100\n" if failure == "install" else "") + 'for name in "$@"; do\n' - + f'case "$name" in\n python3) ln -s {shlex.quote(sys.executable)} "$TEST_ROOT/bin/python3" ;;\n' + + 'case "$name" in\n' + f' bash) ln -s {shlex.quote(binaries["bash"])} "$TEST_ROOT/bin/bash" ;;\n' + "esac\ndone\n" ) diff --git a/responses_api_agents/opencode_agent/tests/test_continuation.py b/responses_api_agents/opencode_agent/tests/test_continuation.py index 958a28192f4..4887a953e75 100644 --- a/responses_api_agents/opencode_agent/tests/test_continuation.py +++ b/responses_api_agents/opencode_agent/tests/test_continuation.py @@ -4,6 +4,7 @@ import asyncio import json +import shlex import sqlite3 from pathlib import Path from types import SimpleNamespace @@ -17,7 +18,7 @@ from nemo_gym.base_responses_api_agent import AGENT_ACTIVATION_HEADER, AgentCloseSessionRequest from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming from responses_api_agents.opencode_agent.continuation import classify_opencode_run -from responses_api_agents.opencode_agent.sandbox_runner import snapshot +from responses_api_agents.opencode_agent.snapshot import snapshot from responses_api_agents.opencode_agent.tests.test_sandbox_sessions import seed, setup # noqa: F401 @@ -34,11 +35,11 @@ def install_artifact_runner(sandbox, tmp_path): payloads = [] async def launch(**kwargs): - path = [path for path in sandbox.files if path.endswith("/input.json")][-1] - payload = json.loads(sandbox.files[path]) + runner_params = next(arg for arg in shlex.split(kwargs["command"]) if arg.endswith("/params.sh")) + directory = runner_params.removesuffix("/params.sh") + payload = sandbox.params(directory) | {"directory": directory} payloads.append(payload) index = len(payloads) - 1 - directory = payload["directory"] sandbox.directory = directory sandbox.files[f"{directory}/task-environment.ready"] = "ready\n" sandbox.files[f"{directory}/stderr.log"] = "" @@ -114,7 +115,7 @@ async def launch(**kwargs): download = sandbox.download async def download_artifacts(source, destination): - if source.endswith("/observations.db"): + if source.endswith("/opencode.db"): Path(destination).write_bytes(database.read_bytes()) else: await download(source, destination) @@ -280,10 +281,9 @@ async def test_close_recovery_preserves_cancelled_activation(setup, tmp_path, mo @pytest.mark.parametrize("cancelled", [False, True]) -@pytest.mark.parametrize("artifact_failure", ["download", "parse"]) @pytest.mark.parametrize("prior_success", [False, True]) async def test_close_without_recovered_root_preserves_activation( - setup, tmp_path, monkeypatch, cancelled, artifact_failure, prior_success + setup, tmp_path, monkeypatch, cancelled, prior_success ): agent, sandbox = setup install_artifact_runner(sandbox, tmp_path) @@ -308,9 +308,7 @@ async def test_close_without_recovered_root_preserves_activation( download = sandbox.download async def broken_observations(source, destination): - if source.endswith("/observations.db"): - if artifact_failure == "download": - raise OSError("observations unavailable") + if source.endswith("/opencode.db"): Path(destination).write_bytes(b"not a SQLite database") else: await download(source, destination) @@ -374,7 +372,7 @@ def test_plain_http_delta_replay_and_cumulative_close(setup, tmp_path): assert payloads[1]["command"][-2:] == ["--session", "opencode-session"] assert payloads[0]["env"]["HOME"] == payloads[1]["env"]["HOME"] assert payloads[0]["env"]["XDG_DATA_HOME"] == payloads[1]["env"]["XDG_DATA_HOME"] - assert set(payloads[1]["previous_message_ids"]) == {"u0", "a0"} + assert set(sandbox.snapshots[1]["previous_message_ids"]) == {"u0", "a0"} receipt = close(client, request) invocations = roots(receipt["agent_observations"]) assert [r["activation_id"] for r in invocations] == [1, 2] @@ -413,7 +411,7 @@ async def slow_exec(command, **kwargs): return execute.return_value async def slow_upload(source, destination): - if stage == "upload" and destination.endswith("process_supervisor.py"): + if stage == "upload" and destination.endswith("process_supervisor.sh"): await asyncio.sleep(0.2) await upload(source, destination) @@ -903,7 +901,7 @@ async def interrupted(**kwargs): assert first.status_code == 200 and first.json()["status"] == "incomplete", first.text sandbox.result.update(timed_out=False, return_code=0) assert activate(client, 2).status_code == 200 - assert payloads[1]["harness_session_id"] == "opencode-session" + assert sandbox.snapshots[1]["harness_session_id"] == "opencode-session" assert [r["status"] for r in roots(close(client, request)["agent_observations"])] == [ "incomplete", "completed", @@ -936,8 +934,7 @@ def test_snapshot_scopes_delta_and_keeps_cumulative_store(tmp_path): ) activation = tmp_path / "activation" activation.mkdir() - (activation / "input.json").write_text(json.dumps({"harness_session_id": "root", "previous_message_ids": ["m0"]})) - snapshot(activation, session_directory=persistent) + snapshot(database, activation, harness_session_id="root", previous_message_ids=["m0"]) export = json.loads((activation / "export.json").read_text()) assert export["session_id"] == "root" assert export["activation_message_ids"] == ["m1"] @@ -945,6 +942,5 @@ def test_snapshot_scopes_delta_and_keeps_cumulative_store(tmp_path): assert [info["tokens"]["input"] for info in export["usage_messages"]] == [2] with sqlite3.connect(activation / "observations.db") as con: assert con.execute("select count(*) from message").fetchone()[0] == 2 - (activation / "input.json").write_text(json.dumps({"harness_session_id": "wrong"})) with pytest.raises(RuntimeError, match="unexpected session"): - snapshot(activation, session_directory=persistent) + snapshot(database, activation, harness_session_id="wrong", previous_message_ids=[]) diff --git a/responses_api_agents/opencode_agent/tests/test_exec_transport.py b/responses_api_agents/opencode_agent/tests/test_exec_transport.py index 289dbe2046d..41e9a2966fd 100644 --- a/responses_api_agents/opencode_agent/tests/test_exec_transport.py +++ b/responses_api_agents/opencode_agent/tests/test_exec_transport.py @@ -13,9 +13,8 @@ import pytest -from nemo_gym.agent_utils.sandbox_session import SandboxSession +from nemo_gym.agent_utils.sandbox_session import HARNESS_RUNNER_FILE, HARNESS_RUNNER_SCRIPT, SandboxSession from nemo_gym.sandbox.providers.base import SandboxExecResult -from responses_api_agents.opencode_agent import sandbox_runner from responses_api_agents.opencode_agent.sandbox import OpenCodeSandboxSession from responses_api_agents.opencode_agent.tests.test_sandbox_sessions import seed @@ -43,13 +42,13 @@ async def disconnect(self): self.disconnected = True async def exec(self, command, *, cwd=None, timeout_s=30): - if "--snapshot" in command: + if "opencode.db-wal" in command: self.snapshots += 1 # The CLI writer must finish before capture; task services can remain. if self.worker_pid is not None: with pytest.raises(ProcessLookupError): os.kill(int(self.worker_pid.read_text()), 0) - launch = "--receipt" in command and "process_supervisor.py" in command + launch = "--receipt" in command and "process_supervisor.sh" in command if launch: self.deadline = timeout_s self.delayed_command = command @@ -86,8 +85,8 @@ def make_session(tmp_path): ), runtime=str(tmp_path / "runtime"), ) - shutil.copyfile(sandbox_runner.__file__, directory / "sandbox_runner.py") - assert not (directory / "process_supervisor.py").exists() + shutil.copyfile(HARNESS_RUNNER_SCRIPT, directory / HARNESS_RUNNER_FILE) + assert not (directory / "process_supervisor.sh").exists() database = directory / "data/opencode/opencode.db" database.parent.mkdir(parents=True) with sqlite3.connect(database) as connection: @@ -100,15 +99,22 @@ def make_session(tmp_path): return state, provider, workdir -def payload(state, code, timeout=0.5): +def payload(state, code): + directory = state.session.session_dir return { - "directory": state.session.session_dir, "prompt": "task", "cwd": state.request.sandbox_access.workdir, "command": [sys.executable, "-c", code], - "env": {}, - "timeout": timeout, - "cleanup_timeout": 1, + "env": { + key: f"{directory}/{name}" + for key, name in ( + ("HOME", "home"), + ("XDG_DATA_HOME", "data"), + ("XDG_CONFIG_HOME", "config"), + ("XDG_CACHE_HOME", "cache"), + ("XDG_STATE_HOME", "state"), + ) + }, } @@ -132,9 +138,7 @@ async def test_exec_only_supervision_leaves_task_children_for_sandbox_owner(tmp_ + {"natural": "print('{}')", "crash": "raise SystemExit(7)"}.get(ending, "time.sleep(60)") ) deadline = 0.5 if ending == "timeout" else 60 - task = state.task = asyncio.create_task( - state.execute(payload(state, code, timeout=deadline), timeout=deadline, close_timeout=3) - ) + task = state.task = asyncio.create_task(state.execute(payload(state, code), timeout=deadline, close_timeout=3)) try: async with asyncio.timeout(5): while not (workdir / "ready").exists(): diff --git a/responses_api_agents/opencode_agent/tests/test_install_runtime.py b/responses_api_agents/opencode_agent/tests/test_install_runtime.py index 88dd97fd522..c26929ba695 100644 --- a/responses_api_agents/opencode_agent/tests/test_install_runtime.py +++ b/responses_api_agents/opencode_agent/tests/test_install_runtime.py @@ -6,6 +6,7 @@ import shutil import subprocess import sys +import time from pathlib import Path import pytest @@ -20,12 +21,11 @@ def sandbox(tmp_path: Path) -> tuple[Path, dict[str, str]]: """Run the real installer with an isolated PATH and no access to the host package manager.""" bindir = tmp_path / "bin" bindir.mkdir() - for name in ("bash", "uname", "getconf", "mkdir", "cp", "tar", "gzip", "chmod"): + for name in ("bash", "uname", "getconf", "mkdir", "cp", "tar", "gzip", "chmod", "rm", "sleep"): executable = shutil.which(name) if executable is None: pytest.skip(f"Installer test requires {name}") (bindir / name).symlink_to(executable) - (bindir / "python3").symlink_to(sys.executable) scripts = { "rg": '#!/bin/bash\necho "ripgrep test fixture"\n', "id": '#!/bin/bash\necho "${TEST_UID:-0}"\n', @@ -217,14 +217,54 @@ def test_optional_ripgrep_failure_does_not_block_runtime(sandbox, cached, unavai assert not (root / "packages.log").exists() -@pytest.mark.skipif(shutil.which("python3.8") is None, reason="Python 3.8 is not installed") -def test_python38_can_bootstrap_runtime(sandbox: tuple[Path, dict[str, str]]) -> None: +def cache_runtime(root: Path) -> None: + (root / "runtime").mkdir() + (root / "runtime/opencode").write_text("#!/bin/bash\necho 1.17.11\n") + (root / "runtime/opencode").chmod(0o755) + + +def test_lock_of_a_killed_installer_is_reclaimed(sandbox: tuple[Path, dict[str, str]]) -> None: root, env = sandbox - (root / "bin/python3").unlink() - (root / "bin/python3").symlink_to(shutil.which("python3.8")) + cache_runtime(root) + killed = subprocess.Popen(["true"]) + killed.wait() + (root / "runtime.lock.d").mkdir() + (root / "runtime.lock.d/pid").write_text(f"{killed.pid}\n") result = run_installer(root, env) - assert result.returncode == 19, result.stderr - assert (root / "download.log").exists() + assert result.returncode == 0, result.stderr + assert not (root / "runtime.lock.d").exists() + + +@pytest.mark.parametrize("lock", ["mkdir", "flock"]) +def test_concurrent_installers_wait_for_the_lock_holder(sandbox: tuple[Path, dict[str, str]], lock: str) -> None: + root, env = sandbox + cache_runtime(root) + if lock == "flock": + if shutil.which("flock") is None: + pytest.skip("flock is not installed") + (root / "bin/flock").symlink_to(shutil.which("flock")) + # One process holds the lock, so killing it releases the lock. + held = 'exec 9>"$1.lock"; flock 9; : >"$1.held"; exec sleep 30' + holder = subprocess.Popen(["bash", "-c", held, "_", str(root / "runtime")]) + while not (root / "runtime.held").exists(): + time.sleep(0.01) + else: + holder = subprocess.Popen(["sleep", "30"]) + (root / "runtime.lock.d").mkdir() + (root / "runtime.lock.d/pid").write_text(f"{holder.pid}\n") + try: + installer = subprocess.Popen( + [str(root / "bin/bash"), str(root / "install.sh"), str(root / "runtime"), "1.17.11"], env=env + ) + time.sleep(0.5) + assert installer.poll() is None + # The waiting installer proceeds once the holder exits, however it exits. + holder.kill() + holder.wait() + assert installer.wait(timeout=10) == 0 + finally: + holder.kill() + holder.wait() def test_musl_bootstraps_with_apk_and_downloads_matching_binary(sandbox: tuple[Path, dict[str, str]]) -> None: diff --git a/responses_api_agents/opencode_agent/tests/test_lifecycle_contract.py b/responses_api_agents/opencode_agent/tests/test_lifecycle_contract.py index 85325403a26..70494addd59 100644 --- a/responses_api_agents/opencode_agent/tests/test_lifecycle_contract.py +++ b/responses_api_agents/opencode_agent/tests/test_lifecycle_contract.py @@ -17,12 +17,12 @@ async def session(setup): body = seed() await agent.seed_agent_session(Request({"type": "http", "session": {}}), body) state = agent._session_records[body.agent_session_id].state + private = ("HOME", "XDG_DATA_HOME", "XDG_CONFIG_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME") payload = { - "directory": state.session.session_dir, "cwd": state.session.workdir, "prompt": "task", "command": [], - "env": {}, + "env": {key: f"{state.session.session_dir}/{key}" for key in private}, } return state, sandbox, payload @@ -84,7 +84,7 @@ async def test_output_failure_does_not_erase_cleanup_confirmation(setup): execute = sandbox.exec.side_effect async def fail_output(command, **kwargs): - if "--snapshot" in command: + if "opencode.db-wal" in command: assert state.session.cleanup["cleanup_confirmed"] is True return SimpleNamespace(return_code=1, error_type=None, stderr="database unreadable") return await execute(command, **kwargs) @@ -106,7 +106,7 @@ async def test_cancel_captures_partial_output_only_after_cleanup_and_keeps_origi snapshots = [] async def capture(command, **kwargs): - if "--snapshot" in command: + if "opencode.db-wal" in command: assert state.session.cleanup["cleanup_confirmed"] is True snapshots.append(command) if capture_fails: @@ -129,12 +129,12 @@ async def capture(command, **kwargs): @pytest.mark.parametrize( "receipt", [None, {"cleanup_confirmed": False, "return_code": 0, "error": None, "timed_out": False}] ) -async def test_snapshot_requires_positive_cleanup_before_any_io(setup, receipt): +async def test_snapshot_requires_positive_cleanup_before_any_io(setup, tmp_path, receipt): state, sandbox, _ = await session(setup) state.session.cleanup = receipt sandbox.exec.reset_mock() with pytest.raises(RuntimeError, match="requires confirmed cleanup"): - await state.snapshot(1) + await state.snapshot(tmp_path, 1) sandbox.exec.assert_not_awaited() await state.close(1) diff --git a/responses_api_agents/opencode_agent/tests/test_sandbox_runner.py b/responses_api_agents/opencode_agent/tests/test_sandbox_runner.py index 59d1406da4f..e607ca84d07 100644 --- a/responses_api_agents/opencode_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/opencode_agent/tests/test_sandbox_runner.py @@ -1,53 +1,63 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Test worker I/O separately from the shared Linux supervisor's cleanup contract.""" +"""Run the staged OpenCode activation through the bash runner and supervisor, and export its SQLite store.""" import json import os import shutil import signal +import sqlite3 import subprocess import sys import time from pathlib import Path +from types import SimpleNamespace import pytest -from nemo_gym.agent_utils import process_supervisor -from responses_api_agents.opencode_agent import sandbox_runner +from nemo_gym.agent_utils.sandbox_session import HARNESS_RUNNER_FILE, HARNESS_RUNNER_SCRIPT +from nemo_gym.agent_utils.supervisor_client import SUPERVISOR_SCRIPT +from responses_api_agents.opencode_agent.sandbox import OpenCodeSandboxSession +from responses_api_agents.opencode_agent.snapshot import snapshot linux_only = pytest.mark.skipif(sys.platform != "linux", reason="Linux process supervision required") +PRIVATE_KEYS = ("HOME", "XDG_DATA_HOME", "XDG_CONFIG_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME") -def launch(tmp_path, code, *, timeout=3, python=sys.executable, env=None, supervised=True): - params = { - "directory": str(tmp_path), - "command": [python, "-c", code], +class LocalSandbox: + """File transfer onto this machine, where the staged runner then executes.""" + + async def upload(self, source, destination): + Path(destination).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + + +async def stage(tmp_path: Path, command: list[str], *, env: dict[str, str] | None = None) -> list[str]: + """Stage an activation with the adapter, then return its argv with the runner it uploaded at seed time.""" + shutil.copyfile(HARNESS_RUNNER_SCRIPT, tmp_path / HARNESS_RUNNER_FILE) + state = SimpleNamespace( + session=SimpleNamespace(sandbox=LocalSandbox(), session_dir=str(tmp_path)), + persistent_directory=str(tmp_path), + is_continuation=False, + ) + payload = { + "instructions": "", "cwd": str(tmp_path), - "env": env or {}, + "command": command, "prompt": "task input", + "env": {key: str(tmp_path / "private" / key) for key in PRIVATE_KEYS} | (env or {}), } - path = tmp_path / "input.json" - path.write_text(json.dumps(params)) - command = [python, "-I", sandbox_runner.__file__, str(path)] - if supervised: - command = [ - python, - "-I", - process_supervisor.__file__, - "--timeout", - str(timeout), - "--cleanup-timeout", - "0.5", - "--receipt", - str(tmp_path / "cleanup.json"), - "--stop-file", - str(tmp_path / "stop.request"), - "--", - *command, - ] - return subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return (await OpenCodeSandboxSession.stage_activation(state, payload)).argv + + +def supervise(tmp_path: Path, argv: list[str], *, timeout: float = 3) -> subprocess.Popen: + return subprocess.Popen( + ["bash", str(SUPERVISOR_SCRIPT), "--timeout", str(timeout), "--cleanup-timeout", "0.5"] + + ["--receipt", str(tmp_path / "cleanup.json"), "--stop-file", str(tmp_path / "stop.request"), "--", *argv], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) def result(tmp_path, process): @@ -61,41 +71,51 @@ def result(tmp_path, process): process.wait() -@pytest.mark.parametrize( - "python", - [ - sys.executable, - pytest.param( - shutil.which("python3.8"), - marks=pytest.mark.skipif(shutil.which("python3.8") is None, reason="Python 3.8 unavailable"), - id="python38", - ), - ], -) -def test_worker_io_and_runtime_are_independent_of_cleanup(tmp_path, monkeypatch, python): - monkeypatch.setenv("HARNESS_INHERITED", "kept") - monkeypatch.setenv("HARNESS_OVERRIDE", "old") - process = launch( +async def test_runner_records_task_directories_before_private_overrides(tmp_path): + task_home = tmp_path / 'task "home" \\ dir' + (tmp_path / "task-environment.ready").write_text("ready\n") + argv = await stage( tmp_path, - "import json,os,sys,pathlib; print(json.dumps([" - "sys.stdin.read(),os.environ['HARNESS_INHERITED'],os.environ['HARNESS_OVERRIDE']]))", - python=python, + ["bash", "-c", 'cat; printf "|%s|%s|%s" "$HOME" "$HARNESS_INHERITED" "$HARNESS_OVERRIDE"'], env={"HARNESS_OVERRIDE": "new"}, - supervised=False, ) - stdout, stderr = process.communicate(timeout=10) - assert process.returncode == 0, (stdout, stderr) - assert json.loads((tmp_path / "stdout.jsonl").read_text()) == ["task input", "kept", "new"] - runtime = json.loads((tmp_path / "runtime.json").read_text()) - assert runtime["hostname"] == os.uname().nodename - assert runtime["pid"] == process.pid - assert not (tmp_path / "cleanup.json").exists() - assert not (tmp_path / "result.json").exists() + inherited = {key: value for key, value in os.environ.items() if not key.startswith("XDG_")} + env = inherited | { + "HOME": f"{task_home}/", + "XDG_CONFIG_HOME": "/task/config", + "XDG_CACHE_HOME": "", + "HARNESS_INHERITED": "kept", + "HARNESS_OVERRIDE": "old", + } + completed = subprocess.run(argv, env=env, capture_output=True, text=True, timeout=10) + assert completed.returncode == 0, completed.stderr + private = tmp_path / "private" + assert (tmp_path / "stdout.jsonl").read_text() == f"task input|{private / 'HOME'}|kept|new" + assert json.loads((tmp_path / "task-env.json").read_text()) == { + "HOME": f"{task_home}/", + "XDG_DATA_HOME": f"{task_home}/.local/share", + "XDG_CONFIG_HOME": "/task/config", + "XDG_CACHE_HOME": f"{task_home}/.cache", + "XDG_STATE_HOME": f"{task_home}/.local/state", + } + # Never accept a receipt left by an earlier invocation. + assert not (tmp_path / "task-environment.ready").exists() + assert all((private / key).is_dir() for key in PRIVATE_KEYS) + assert set(json.loads((tmp_path / "runtime.json").read_text())) == {"hostname", "pid"} + + +async def test_unset_home_falls_back_to_the_account_home(tmp_path): + argv = await stage(tmp_path, ["true"]) + env = {key: value for key, value in os.environ.items() if key != "HOME" and not key.startswith("XDG_")} + completed = subprocess.run(argv, env=env, capture_output=True, text=True, timeout=10) + assert completed.returncode == 0, completed.stderr + home = os.path.expanduser("~") + assert json.loads((tmp_path / "task-env.json").read_text())["XDG_STATE_HOME"] == f"{home}/.local/state" @linux_only @pytest.mark.parametrize("ending", ["natural", "timeout", "cancel"]) -def test_task_descendants_survive_until_owner_teardown(tmp_path, ending): +async def test_task_descendants_survive_until_owner_teardown(tmp_path, ending): try: # A completed tool leaves its detached service adopted before close. tool = ( @@ -109,13 +129,13 @@ def test_task_descendants_survive_until_owner_teardown(tmp_path, ending): f"subprocess.run([sys.executable,'-c',{tool!r}],check=True); " "pathlib.Path('ready').touch(); " + ("time.sleep(60)" if ending != "natural" else "pass") ) - process = launch(tmp_path, code) + process = supervise(tmp_path, await stage(tmp_path, [sys.executable, "-c", code])) if ending == "cancel": for _ in range(200): if (tmp_path / "ready").exists(): break time.sleep(0.01) - process.send_signal(signal.SIGTERM) + (tmp_path / "stop.request").touch() summary = result(tmp_path, process) assert summary["cleanup_confirmed"] is True assert summary["timed_out"] is (ending == "timeout") @@ -131,27 +151,27 @@ def test_task_descendants_survive_until_owner_teardown(tmp_path, ending): @linux_only -def test_spawn_error_does_not_prevent_cleanup(tmp_path): - summary = result(tmp_path, launch(tmp_path, "raise RuntimeError('failed')")) +async def test_cli_failure_is_reported_in_the_receipt(tmp_path): + summary = result( + tmp_path, supervise(tmp_path, await stage(tmp_path, [sys.executable, "-c", "raise SystemExit(3)"])) + ) assert summary["cleanup_confirmed"] is True - assert summary["return_code"] != 0 + assert summary["return_code"] == 3 @linux_only -def test_stop_marker_fences_launch_without_inventing_worker_identity(tmp_path): +async def test_stop_marker_fences_launch_without_inventing_worker_identity(tmp_path): (tmp_path / "stop.request").touch() - summary = result(tmp_path, launch(tmp_path, "open('unexpected', 'w').close()")) + argv = await stage(tmp_path, [sys.executable, "-c", "open('unexpected', 'w').close()"]) + summary = result(tmp_path, supervise(tmp_path, argv)) assert summary["cleanup_confirmed"] is True assert summary["return_code"] is None assert not (tmp_path / "runtime.json").exists() assert not (tmp_path / "unexpected").exists() -def test_snapshot_keeps_root_output_and_child_usage(tmp_path): - import sqlite3 - - database = tmp_path / "data/opencode/opencode.db" - database.parent.mkdir(parents=True) +def session_store(database: Path) -> sqlite3.Connection: + database.parent.mkdir(parents=True, exist_ok=True) con = sqlite3.connect(database) con.execute("pragma journal_mode=wal") con.executescript(""" @@ -170,11 +190,28 @@ def test_snapshot_keeps_root_output_and_child_usage(tmp_path): (str(index), str(index), session_id, json.dumps({"type": "text", "text": session_id}), index), ) con.commit() - sandbox_runner.snapshot(tmp_path) - export = json.loads((tmp_path / "export.json").read_text()) + return con + + +@pytest.mark.parametrize( + "copied", [("opencode.db", "opencode.db-wal"), ("opencode.db", "opencode.db-wal", "opencode.db-shm")] +) +def test_snapshot_of_a_copied_store_keeps_root_output_and_child_usage(tmp_path, copied): + sandbox = tmp_path / "sandbox" + con = session_store(sandbox / "opencode.db") + # The CLI was stopped without a checkpoint: its latest messages are only in the write-ahead log. + assert (sandbox / "opencode.db-wal").stat().st_size > 0 + local = tmp_path / "local" + local.mkdir() + for name in copied: + shutil.copyfile(sandbox / name, local / name) + con.close() + snapshot(local / "opencode.db", local, harness_session_id=None, previous_message_ids=[]) + export = json.loads((local / "export.json").read_text()) + assert export["session_id"] == "root" + assert export["message_ids"] == export["activation_message_ids"] == ["0", "1"] assert len(export["messages"]) == 1 assert export["messages"][0]["parts"][0]["text"] == "root" assert [info["tokens"]["input"] for info in export["usage_messages"]] == [1, 2] - with sqlite3.connect(tmp_path / "observations.db") as copy: + with sqlite3.connect(local / "observations.db") as copy: assert copy.execute("select count(*) from session").fetchone()[0] == 2 - con.close() diff --git a/responses_api_agents/opencode_agent/tests/test_sandbox_sessions.py b/responses_api_agents/opencode_agent/tests/test_sandbox_sessions.py index 927ec21f492..3ac6115905f 100644 --- a/responses_api_agents/opencode_agent/tests/test_sandbox_sessions.py +++ b/responses_api_agents/opencode_agent/tests/test_sandbox_sessions.py @@ -5,6 +5,7 @@ import json import shlex import sqlite3 +import subprocess from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch @@ -23,7 +24,7 @@ from nemo_gym.server_utils import ServerClient from nemo_gym.tasks.harbor.materialize import materialize_task from nemo_gym.tasks.harbor.task import load_task -from responses_api_agents.opencode_agent.app import OpenCodeAgent, OpenCodeAgentConfig +from responses_api_agents.opencode_agent.app import OpenCodeAgent, OpenCodeAgentConfig, _sandbox_prepare_command def seed() -> AgentSeedSessionRequest: @@ -90,12 +91,21 @@ def rejected_events(*, after_work: bool = False) -> str: else: export["messages"][-1] = rejected return json.dumps(export) +def read_params(text: str) -> dict: + """Source runner PARAMS with bash, as the sandbox runner does.""" + script = 'source /dev/stdin; printf "%s\\0" "$cwd" "$stdin" "$setup" "${#command[@]}" "${command[@]}" "${env[@]}"' + fields = subprocess.run(["bash", "-c", script], input=text, capture_output=True, text=True, check=True).stdout + cwd, stdin, setup, count, *rest = fields.split("\0")[:-1] + count = int(count) + env = dict(item.split("=", 1) for item in rest[count:]) + return {"cwd": cwd, "stdin": stdin, "setup": setup, "command": rest[:count], "env": env} class Sandbox: def __init__(self): self.sandbox_id = "provider-sandbox-id" self.files = {} + self.snapshots = [] self.result = { "return_code": 0, "timed_out": False, @@ -122,8 +132,10 @@ def pty(self): raise AssertionError("This provider supports exec only; no PTY API") async def run_exec(self, command, **kwargs): - if "--receipt" in command and "process_supervisor.py" in command: + if "--receipt" in command and "process_supervisor.sh" in command: return await self.launch(command=command, **kwargs) + if "opencode.db-wal" in command: + return SimpleNamespace(return_code=0, stdout="opencode.db\n", stderr="", error_type=None) if "stop.request" in command: await self.request_stop() return self.exec.return_value @@ -132,15 +144,29 @@ async def upload(self, source, destination): self.files[destination] = Path(source).read_text() async def download(self, source, destination): + if source.endswith("/opencode.db") and source not in self.files: + Path(destination).write_bytes(b"") + return Path(destination).write_text(self.files[source]) + def params(self, directory=None): + """The PARAMS and prompt staged for the runner in ``directory`` (default: the last launch).""" + params = read_params(self.files[f"{directory or self.directory}/params.sh"]) + params["prompt"] = self.files[params["stdin"]] + return params + + def snapshot(self, database, directory, *, harness_session_id, previous_message_ids): + """Stand in for the SQLite export (see test_snapshot.py): the staged export, the copied database.""" + self.snapshots.append({"harness_session_id": harness_session_id, "previous_message_ids": previous_message_ids}) + (directory / "export.json").write_text(self.files[f"{self.directory}/export.json"]) + (directory / "observations.db").write_bytes(database.read_bytes()) + async def create(self, **kwargs): - payload_path = next(path for path in self.files if path.endswith("/input.json")) - payload = json.loads(self.files[payload_path]) - assert payload["cwd"] == getattr(self, "expected_workdir", "/app") + runner_params = next(arg for arg in shlex.split(kwargs["command"]) if arg.endswith("/params.sh")) + self.directory = runner_params.removesuffix("/params.sh") + assert self.params()["cwd"] == getattr(self, "expected_workdir", "/app") assert kwargs["cwd"] == getattr(self, "expected_workdir", "/app") - assert "sandbox_runner.py" in kwargs["command"] - self.directory = payload["directory"] + assert "/harness_runner.sh " in kwargs["command"] self.started.set() if not self.blocked: self.exited.set() @@ -196,6 +222,7 @@ def setup(): patch(f"{module}.get_global_config_dict", return_value={}), patch(f"{module}.create_provider"), patch(f"{module}.AsyncSandbox.connect", AsyncMock(return_value=sandbox)), + patch("responses_api_agents.opencode_agent.sandbox.snapshot", sandbox.snapshot), ): agent = OpenCodeAgent(config=config, server_client=client) yield agent, sandbox @@ -265,7 +292,7 @@ def capture_observations(sandbox, tmp_path): original = sandbox.download async def download(source, destination): - if source.endswith("/observations.db"): + if source.endswith("/opencode.db"): Path(destination).write_bytes(database.read_bytes()) else: await original(source, destination) @@ -294,7 +321,7 @@ def test_http_session_flow_runs_opencode_in_borrowed_sandbox(setup, output_budge "message", ] assert body["usage"]["total_tokens"] == 16 - payload = json.loads(sandbox.files[f"{sandbox.directory}/input.json"]) + payload = sandbox.params() assert payload["prompt"] == "Fix the code" assert payload["command"][0].endswith("nemo-gym-opencode-runtime-1.17.11/opencode") assert payload["env"]["HOME"] == f"{sandbox.directory}/home" @@ -471,7 +498,7 @@ def test_instructions_and_text_parts_reach_opencode(setup): }, ) assert result.status_code == 200, result.text - payload = json.loads(sandbox.files[f"{sandbox.directory}/input.json"]) + payload = sandbox.params() config = json.loads(payload["env"]["OPENCODE_CONFIG_CONTENT"]) assert sandbox.files[config["instructions"][0]] == "outer\n\nsystem" assert payload["prompt"] == "task" @@ -1071,35 +1098,40 @@ async def test_adapter_workdir_overlap_is_rejected_before_connection(setup, work sandbox.exec.assert_not_awaited() -async def test_resolved_workdir_check_runs_before_session_files_are_created(setup, tmp_path): - import shlex - import subprocess - import sys - - agent, sandbox = setup - await agent.seed_agent_session(Request({"type": "http", "session": {}}), seed()) - command = shlex.split(sandbox.exec.await_args_list[0].args[0]) - script = command[command.index("-I") + 2] +@pytest.mark.parametrize( + ("workdir", "runtime", "overlaps"), + [ + ("task", "runtime", True), # a symlink to the session root + ("sessions/nested", "runtime", True), + ("repo", "link/runtime", True), # a missing root resolved through a symlinked parent + ("repo", "runtime", False), + ("repo/../repo", "runtime", False), + ], +) +def test_resolved_workdir_check_runs_before_session_files_are_created(tmp_path, workdir, runtime, overlaps): sessions = tmp_path / "sessions" - sessions.mkdir() - workdir = tmp_path / "task" - workdir.symlink_to(sessions, target_is_directory=True) - rejected = subprocess.run( - [sys.executable, "-I", "-c", script, str(workdir), str(sessions), str(tmp_path / "runtime")], - capture_output=True, - text=True, - ) - assert rejected.returncode != 0 - assert "overlaps the task workdir" in rejected.stderr - ordinary = tmp_path / "repo" - ordinary.mkdir() - accepted = subprocess.run( - [sys.executable, "-I", "-c", script, str(ordinary), str(sessions), str(tmp_path / "runtime")], - capture_output=True, - text=True, - ) - assert accepted.returncode == 0, accepted.stderr - assert list(sessions.iterdir()) == [] + (sessions / "nested").mkdir(parents=True) + (tmp_path / "task").symlink_to(sessions, target_is_directory=True) + (tmp_path / "repo").mkdir() + (tmp_path / "link").symlink_to(tmp_path / "repo", target_is_directory=True) + session = sessions / "session" + command = _sandbox_prepare_command(str(tmp_path / workdir), str(session), str(tmp_path / runtime)) + result = subprocess.run(["/bin/sh", "-c", command], capture_output=True, text=True) + if overlaps: + assert result.returncode != 0 + assert "overlaps the task workdir" in result.stderr + assert not session.exists() + else: + assert result.returncode == 0, result.stderr + assert session.is_dir() + + +def test_missing_workdir_is_rejected(tmp_path): + command = _sandbox_prepare_command(str(tmp_path / "missing"), str(tmp_path / "s/session"), str(tmp_path / "r")) + result = subprocess.run(["/bin/sh", "-c", command], capture_output=True, text=True) + assert result.returncode != 0 + assert "OpenCode task workdir is missing" in result.stderr + assert not (tmp_path / "s").exists() def test_close_response_cookie_blocks_legacy_run(setup): @@ -1429,11 +1461,11 @@ async def test_close_keeps_captured_events_after_sandbox_release(setup, tmp_path await agent.seed_agent_session(request, body) state = agent._session_records[body.agent_session_id].state directory = state.session.session_dir - assert f"{directory}/process_supervisor.py" not in sandbox.files + assert f"{directory}/process_supervisor.sh" not in sandbox.files sandbox.blocked = True task = asyncio.create_task(agent.responses(request, NeMoGymResponseCreateParamsNonStreaming(input="task"))) await asyncio.wait_for(sandbox.started.wait(), 2) - assert f"{directory}/process_supervisor.py" in sandbox.files + assert f"{directory}/process_supervisor.sh" in sandbox.files async def release(): assert state.session.cleanup["cleanup_confirmed"] is True diff --git a/responses_api_agents/opencode_agent/tests/test_session_review.py b/responses_api_agents/opencode_agent/tests/test_session_review.py index 9c6b45cb34e..fb3414cd313 100644 --- a/responses_api_agents/opencode_agent/tests/test_session_review.py +++ b/responses_api_agents/opencode_agent/tests/test_session_review.py @@ -46,7 +46,7 @@ def test_session_uploads_shared_plugin_only_with_model_capture(setup, enabled, t created.raise_for_status() result = client.post("/ng-rollout/opencode-smoke-a2/v1/responses", json={"input": "task"}) result.raise_for_status() - payload = json.loads(sandbox.files[f"{sandbox.directory}/input.json"]) + payload = sandbox.params() config = json.loads(payload["env"]["OPENCODE_CONFIG_CONTENT"]) plugin = Path(sandbox.directory) / OBSERVABILITY_PATCH.name task_env_plugin = Path(sandbox.directory) / "task_environment.mjs" @@ -117,7 +117,7 @@ def test_observation_parse_failure_preserves_response(setup, caplog): assert "Failed to parse OpenCode observations" in caplog.text -@pytest.mark.parametrize("failure", ["provider_timeout", "missing_export", "snapshot"]) +@pytest.mark.parametrize("failure", ["provider_timeout", "missing_database", "snapshot"]) @pytest.mark.parametrize("model_rejected", [False, True]) def test_execution_failure_keeps_cleanup_observations_and_diagnostics(setup, tmp_path, failure, model_rejected): agent, sandbox = setup @@ -131,10 +131,6 @@ def test_execution_failure_keeps_cleanup_observations_and_diagnostics(setup, tmp async def read(source, destination): if source.endswith("/stderr.log"): Path(destination).write_text("x" * 18000 + "ROOT CAUSE") - elif source.endswith("/output.log"): - Path(destination).write_text("output" * 6000 + "OUTPUT TAIL") - elif failure == "missing_export" and source.endswith("/export.json"): - raise FileNotFoundError("missing export") else: await download(source, destination) @@ -142,8 +138,10 @@ async def run(command, **kwargs): result = await execute(command, **kwargs) if failure == "provider_timeout" and "--receipt" in command: return SimpleNamespace(return_code=0, error_type="timeout", stdout="", stderr="deadline") - if failure == "snapshot" and "--snapshot" in command: - return SimpleNamespace(return_code=1, error_type=None, stdout="", stderr="no root session") + if failure == "missing_database" and "opencode.db-wal" in command: + return SimpleNamespace(return_code=0, error_type=None, stdout="", stderr="") + if failure == "snapshot" and "opencode.db-wal" in command: + return SimpleNamespace(return_code=1, error_type=None, stdout="", stderr="no session directory") return result sandbox.download = read @@ -163,9 +161,9 @@ async def run(command, **kwargs): else: assert "ROOT CAUSE" in str(error.value) assert len(str(error.value)) <= 16000 - if failure == "missing_export": - assert "returned no valid result" in str(error.value) - assert "OUTPUT TAIL" in str(error.value) + assert "OpenCode transcript capture failed" in str(error.value) + if failure == "missing_database": + assert "session database is missing" in str(error.value) assert len(state.stderr) == 16000 assert state.session.cleanup["cleanup_confirmed"] is True assert any(record.kind == "sandbox" for record in state.observations.records) @@ -260,7 +258,7 @@ async def read(source, destination): client.post("/v1/agent_sessions/close", json=close_body(created.json()["agent_session_id"])).raise_for_status() -@pytest.mark.parametrize("failure,status", [("receipt", 500), ("missing_export", 500), ("malformed_export", 502)]) +@pytest.mark.parametrize("failure,status", [("receipt", 500), ("malformed_export", 502)]) def test_single_turn_failure_preserves_observations_and_runtime(setup, tmp_path, failure, status): agent, sandbox = setup capture_observations(sandbox, tmp_path) @@ -268,14 +266,6 @@ def test_single_turn_failure_preserves_observations_and_runtime(setup, tmp_path, sandbox.task_environment_receipt = None elif failure == "malformed_export": sandbox.events = "{malformed" - download = sandbox.download - - async def read(source, destination): - if failure == "missing_export" and source.endswith("/export.json"): - raise FileNotFoundError("missing export") - await download(source, destination) - - sandbox.download = read with TestClient(agent.setup_webserver(), raise_server_exceptions=False) as client: created = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")) session_id = created.json()["agent_session_id"] diff --git a/responses_api_agents/opencode_agent/tests/test_task_environment.py b/responses_api_agents/opencode_agent/tests/test_task_environment.py index 9dc561ccb2a..7818127b57f 100644 --- a/responses_api_agents/opencode_agent/tests/test_task_environment.py +++ b/responses_api_agents/opencode_agent/tests/test_task_environment.py @@ -7,23 +7,25 @@ import os import shutil import subprocess -import sys from pathlib import Path import pytest -from responses_api_agents.opencode_agent import sandbox_runner +from responses_api_agents.opencode_agent.tests.test_sandbox_runner import stage + + +PLUGIN = Path(__file__).parents[1] / "task_environment.mjs" @pytest.mark.skipif(shutil.which("node") is None, reason="Node.js is required for the OpenCode plugin") @pytest.mark.parametrize("custom_xdg", [False, True]) -def test_shell_environment_keeps_task_files_after_session_close(tmp_path: Path, custom_xdg: bool) -> None: +async def test_shell_environment_keeps_task_files_after_session_close(tmp_path: Path, custom_xdg: bool) -> None: session = tmp_path / "session" session.mkdir() home = tmp_path / "task-home" home.mkdir() plugin = session / "task_environment.mjs" - shutil.copyfile(Path(sandbox_runner.__file__).with_name(plugin.name), plugin) + shutil.copyfile(PLUGIN, plugin) # Use a subprocess-local environment: no task directories come from the agent-server host. env = {key: value for key, value in os.environ.items() if not key.startswith("XDG_")} env["HOME"] = str(home) @@ -49,22 +51,8 @@ def test_shell_environment_keeps_task_files_after_session_close(tmp_path: Path, console.log(JSON.stringify({cli: process.env.XDG_CONFIG_HOME, cliHome: process.env.HOME, tool: output.env})); """ (session / "probe.mjs").write_text(script) - params = { - "directory": str(session), - "command": [shutil.which("node"), str(session / "probe.mjs")], - "cwd": str(session), - "env": cli_env, - "prompt": "", - } - input_path = session / "input.json" - input_path.write_text(json.dumps(params)) - result = subprocess.run( - [sys.executable, "-I", sandbox_runner.__file__, str(input_path)], - env=env, - capture_output=True, - text=True, - timeout=10, - ) + argv = await stage(session, [shutil.which("node"), str(session / "probe.mjs")], env=cli_env) + result = subprocess.run(argv, env=env, capture_output=True, text=True, timeout=10) assert result.returncode == 0, result.stderr assert (session / "task-environment.ready").read_text() == "ready\n" observed = json.loads((session / "stdout.jsonl").read_text()) @@ -84,7 +72,7 @@ def test_shell_environment_keeps_task_files_after_session_close(tmp_path: Path, @pytest.mark.skipif(shutil.which("node") is None, reason="Node.js is required for the OpenCode plugin") @pytest.mark.parametrize("failure", ["missing", "malformed", "missing_home", "relative_home", "invalid_xdg"]) def test_invalid_task_environment_cannot_acknowledge_hook_initialization(tmp_path: Path, failure: str) -> None: - shutil.copyfile(Path(sandbox_runner.__file__).with_name("task_environment.mjs"), tmp_path / "task_environment.mjs") + shutil.copyfile(PLUGIN, tmp_path / "task_environment.mjs") env = { "HOME": "/task/home", "XDG_CONFIG_HOME": "/task/config", @@ -116,29 +104,6 @@ def test_invalid_task_environment_cannot_acknowledge_hook_initialization(tmp_pat assert not (tmp_path / "task-environment.ready").exists() -def test_runner_removes_stale_task_environment_receipt_before_launch(tmp_path: Path) -> None: - receipt = tmp_path / "task-environment.ready" - receipt.write_text("ready\n") - params = { - "directory": str(tmp_path), - "command": [sys.executable, "-c", "print('CLI continued without loading the hook')"], - "cwd": str(tmp_path), - "env": {}, - "prompt": "", - } - input_path = tmp_path / "input.json" - input_path.write_text(json.dumps(params)) - result = subprocess.run( - [sys.executable, "-I", sandbox_runner.__file__, str(input_path)], - capture_output=True, - text=True, - timeout=10, - ) - assert result.returncode == 0, result.stderr - assert "CLI continued" in (tmp_path / "stdout.jsonl").read_text() - assert not receipt.exists() - - @pytest.mark.parametrize("task_environment", ["valid", "missing", "malformed", "plugin_error"]) def test_pinned_cli_reports_whether_required_plugin_initialized(tmp_path: Path, task_environment: str) -> None: configured_binary = os.environ.get("GYM_TEST_OPENCODE_BINARY") @@ -150,7 +115,7 @@ def test_pinned_cli_reports_whether_required_plugin_initialized(tmp_path: Path, pytest.skip("PATH OpenCode is not 1.17.11; set GYM_TEST_OPENCODE_BINARY to the pinned executable") assert version.returncode == 0 and version.stdout.strip() == "1.17.11" plugin = tmp_path / "task_environment.mjs" - shutil.copyfile(Path(sandbox_runner.__file__).with_name(plugin.name), plugin) + shutil.copyfile(PLUGIN, plugin) env = { **os.environ, "HOME": str(tmp_path / "home"), From 1dbb63991ac414fcffd6dcd2c5ef66819291101b Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Mon, 26 Oct 2026 09:32:49 -0700 Subject: [PATCH 09/32] refactor(pi_agent): run the harness without Python in the sandbox Pi now runs under the shared bash harness runner: the adapter stages params.sh and prompt.txt, and the runner records [seen_at, event] lines. Session setup no longer installs python3, the runtime installer checks for the musl C++ library by path instead of with ctypes, and the host parser skips recorded lines that are not [number, object] records. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Hemil Desai Signed-off-by: yaoyu-33 --- responses_api_agents/pi_agent/README.md | 6 +- responses_api_agents/pi_agent/app.py | 63 +++++---- .../pi_agent/install_pi_runtime.sh | 6 +- responses_api_agents/pi_agent/sandbox.py | 38 ++++-- .../pi_agent/sandbox_runner.py | 89 ------------ .../pi_agent/tests/test_bootstrap.py | 8 +- .../pi_agent/tests/test_install_runtime.py | 49 ++----- .../pi_agent/tests/test_sandbox.py | 96 ++++++------- .../pi_agent/tests/test_sandbox_runner.py | 127 ++++++------------ 9 files changed, 170 insertions(+), 312 deletions(-) delete mode 100644 responses_api_agents/pi_agent/sandbox_runner.py diff --git a/responses_api_agents/pi_agent/README.md b/responses_api_agents/pi_agent/README.md index 082f254980c..223709e700c 100644 --- a/responses_api_agents/pi_agent/README.md +++ b/responses_api_agents/pi_agent/README.md @@ -82,10 +82,10 @@ task working directory. Without access, configure agent-owned creation as noted Session setup rejects `pi_version: latest`. Pi's `resources_server` setting is required only for its compatibility `/run`, not agent sessions. -Supported task images are Linux x86_64/aarch64 glibc or x86_64 musl/Alpine with Python 3.8+, -bash, tar/gzip, and SHA-256 utilities. Session setup uses POSIX sh to install missing Python/Bash +Supported task images are Linux x86_64/aarch64 glibc or x86_64 musl/Alpine with +bash, tar/gzip, and SHA-256 utilities; no Python is needed. Session setup uses POSIX sh to install missing Bash with apt-get or apk when running as root, before launching the Bash runtime installer; -otherwise preinstall them in the image. Setup fails on either a nonzero exit or a provider error. +otherwise preinstall it in the image. Pi runs under Gym's shared Bash harness runner and process supervisor. Setup fails on either a nonzero exit or a provider error. Older Alpine images also need patchelf so Pi's Node can use a private, checksum-verified C++ library without replacing the task's system library. The pinned Node version has no arm64 musl build. diff --git a/responses_api_agents/pi_agent/app.py b/responses_api_agents/pi_agent/app.py index 0ca94d8ae69..d74f7e36511 100644 --- a/responses_api_agents/pi_agent/app.py +++ b/responses_api_agents/pi_agent/app.py @@ -32,7 +32,7 @@ from pydantic import BaseModel, ConfigDict, Field, PrivateAttr from nemo_gym.agent_outcome import AgentOutcome -from nemo_gym.agent_utils.sandbox_session import SandboxSession +from nemo_gym.agent_utils.sandbox_session import HARNESS_RUNNER_FILE, HARNESS_RUNNER_SCRIPT, SandboxSession from nemo_gym.agent_utils.task_timeout import AGENT_TIMEOUT_METADATA_KEY, resolve_agent_timeout from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import ( @@ -173,7 +173,6 @@ def _sandbox_prepare_command(directory: str) -> str: # Keep the bootstrap POSIX sh: Bash must exist before its installer can run. bootstrap = """set -eu set -- -command -v python3 >/dev/null 2>&1 || set -- "$@" python3 command -v bash >/dev/null 2>&1 || set -- "$@" bash if [ "$#" -gt 0 ]; then [ "$(id -u)" = 0 ] || { echo "Pi in the task sandbox requires $*: preinstall these tools or use a root image." >&2; exit 1; } @@ -270,6 +269,29 @@ def parse_pi_events(stdout: str | bytes) -> tuple[list[Any], dict[str, int]]: return output_items, {"input_tokens": input_tokens, "output_tokens": output_tokens} +def _parse_recorded_events(raw: str) -> list[tuple[float, dict[str, Any]]]: + """Read the runner's ``[seen_at, event]`` lines, skipping any it kept that are not valid records.""" + events = [] + # JSONL: split on "\n" only; str.splitlines() would also split inside strings on U+2028/U+0085. + for line in raw.split("\n"): + if not line.strip(): + continue + try: + record = json.loads(line) + except (ValueError, RecursionError): + record = None + if ( + isinstance(record, list) + and len(record) == 2 + and type(record[0]) in (int, float) + and isinstance(record[1], dict) + ): + events.append((float(record[0]), record[1])) + else: + LOG.warning("Skipping malformed Pi event record") + return events + + async def _read_pi_stdout(stream: asyncio.StreamReader) -> tuple[str, list[tuple[float, dict[str, Any]]]]: lines: list[str] = [] events: list[tuple[float, dict[str, Any]]] = [] @@ -733,7 +755,7 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> PiSa f"Pi sandbox installation failed (exit {installed.return_code}, " f"error_type={installed.error_type}): {details[-16000:]}" ) - await sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), f"{directory}/sandbox_runner.py") + await sandbox.upload(HARNESS_RUNNER_SCRIPT, f"{directory}/{HARNESS_RUNNER_FILE}") except BaseException as error: try: await state.close(self.config.session_close_timeout_seconds) @@ -890,34 +912,28 @@ async def _sandbox_response( command += ["--thinking", self.config.thinking] if system: command += ["--append-system-prompt", system] - payload = { - "directory": state.session.session_dir, - "command": command, - "prompt": prompt, - "cwd": state.session.workdir, - "env": { - "PI_CODING_AGENT_DIR": f"{state.session.session_dir}/home/.pi/agent", - "PI_SKIP_VERSION_CHECK": "1", - "PI_TELEMETRY": "0", - "NEMO_GYM_PI_BASH_TIMEOUT": str(self.config.sandbox_bash_timeout_seconds), - **mcp_env, - }, + env = { + "PI_CODING_AGENT_DIR": f"{state.session.session_dir}/home/.pi/agent", + "PI_SKIP_VERSION_CHECK": "1", + "PI_TELEMETRY": "0", + "NEMO_GYM_PI_BASH_TIMEOUT": str(self.config.sandbox_bash_timeout_seconds), + **mcp_env, } try: async with self.sem: raw = await state.execute( - payload, timeout=timeout, close_timeout=self.config.session_close_timeout_seconds + command=command, + env=env, + prompt=prompt, + timeout=timeout, + close_timeout=self.config.session_close_timeout_seconds, ) except BaseException as failure: # Common finalization captures before close removes the session directory. raw = state.session.artifacts if state.session.artifacts is not None else "" - events, output = [], [] - for line in (part for part in raw.split("\n") if part.strip()): + events, output = _parse_recorded_events(raw), [] + for _, event in events: try: - observed_at, event = json.loads(line) - if not isinstance(event, dict): - continue - events.append((float(observed_at), event)) parsed, _ = parse_pi_events(json.dumps(event)) output.extend(parsed) except (ValueError, TypeError, AttributeError): @@ -948,8 +964,7 @@ async def _sandbox_response( if isinstance(record, AgentInvocation): record.status = "incomplete" raise - # JSONL: split on "\n" only; str.splitlines() would also split inside strings on U+2028/U+0085. - events = [tuple(json.loads(line)) for line in raw.split("\n") if line.strip()] + events = _parse_recorded_events(raw) output = [] usage = {"input_tokens": 0, "output_tokens": 0} cached_tokens: int | None = 0 diff --git a/responses_api_agents/pi_agent/install_pi_runtime.sh b/responses_api_agents/pi_agent/install_pi_runtime.sh index eb5a59a136c..e56b8045ac2 100644 --- a/responses_api_agents/pi_agent/install_pi_runtime.sh +++ b/responses_api_agents/pi_agent/install_pi_runtime.sh @@ -27,10 +27,6 @@ install_packages() { } test "$(uname -s)" = Linux -if ! command -v python3 >/dev/null 2>&1; then - install_packages python3 -fi -python3 -c 'import sys; assert sys.version_info >= (3, 8), "Pi requires Python >=3.8"' case "$(uname -m)" in x86_64) arch=x64 ;; aarch64) arch=arm64 ;; @@ -58,7 +54,7 @@ if [ ! -f "$runtime/ready" ]; then if ! command -v curl >/dev/null 2>&1; then packages+=(curl ca-certificates) fi - if [ "$platform" = linux-x64-musl ] && ! python3 -c 'import ctypes; ctypes.CDLL("libstdc++.so.6")' 2>/dev/null; then + if [ "$platform" = linux-x64-musl ] && [ ! -e /usr/lib/libstdc++.so.6 ] && [ ! -e /lib/libstdc++.so.6 ]; then packages+=(libstdc++) fi install_packages "${packages[@]}" diff --git a/responses_api_agents/pi_agent/sandbox.py b/responses_api_agents/pi_agent/sandbox.py index 78113925726..4407140203b 100644 --- a/responses_api_agents/pi_agent/sandbox.py +++ b/responses_api_agents/pi_agent/sandbox.py @@ -9,7 +9,12 @@ from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError -from nemo_gym.agent_utils.sandbox_session import SandboxCommand, SandboxSession +from nemo_gym.agent_utils.sandbox_session import ( + HARNESS_RUNNER_FILE, + SandboxCommand, + SandboxSession, + shell_assignments, +) from nemo_gym.base_responses_api_agent import AgentSessionState from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming from nemo_gym.rollout_observability import AgentObservationBundle @@ -25,7 +30,6 @@ class HarnessProcessInfo(BaseModel): model_config = ConfigDict(extra="forbid", strict=True) hostname: str pid: int - python: str | None = None def parse_runtime_info(payload: object) -> HarnessProcessInfo | None: @@ -72,26 +76,32 @@ async def close(self, timeout: float) -> None: raise # The agent base replays the activation error independently of close. - async def execute(self, payload: dict[str, JsonValue], *, timeout: float, close_timeout: float) -> str: + async def execute( + self, *, command: list[str], env: dict[str, str], prompt: str, timeout: float, close_timeout: float + ) -> str: """Run through the common lifecycle; the adapter validates terminal events.""" return await self.session.execute( - stage_activation=lambda: self.stage_activation(payload), + stage_activation=lambda: self.stage_activation(command=command, env=env, prompt=prompt), collect=self.collect_artifacts, timeout=timeout, close_timeout=close_timeout, ) - async def stage_activation(self, payload: dict[str, JsonValue]) -> SandboxCommand: + async def stage_activation(self, *, command: list[str], env: dict[str, str], prompt: str) -> SandboxCommand: """Stage the invocation and return the Pi worker command.""" - await self.upload_json("input.json", payload) - return SandboxCommand( - argv=[ - "python3", - "-I", - f"{self.session.session_dir}/sandbox_runner.py", - f"{self.session.session_dir}/input.json", - ], - ) + directory = self.session.session_dir + await upload_text(self.session.sandbox, path=f"{directory}/prompt.txt", text=prompt) + params = { + "cwd": self.session.workdir, + "command": command, + "env": env, + "stdin": f"{directory}/prompt.txt", + "stdout": f"{directory}/stdout.log", + "stderr": f"{directory}/stderr.log", + "events": f"{directory}/events.jsonl", + } + await upload_text(self.session.sandbox, path=f"{directory}/params.sh", text=shell_assignments(params)) + return SandboxCommand(argv=["bash", f"{directory}/{HARNESS_RUNNER_FILE}", f"{directory}/params.sh"]) async def collect_artifacts(self) -> str: """Keep partial events even when worker metadata is unavailable.""" diff --git a/responses_api_agents/pi_agent/sandbox_runner.py b/responses_api_agents/pi_agent/sandbox_runner.py deleted file mode 100644 index dfbc52eb541..00000000000 --- a/responses_api_agents/pi_agent/sandbox_runner.py +++ /dev/null @@ -1,89 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Capture Pi JSON events; the supervisor bounds execution; the sandbox owner stops task services.""" - -import json -import os -import selectors -import signal -import subprocess -import sys -from contextlib import ExitStack -from pathlib import Path -from time import time - - -def run(params: dict) -> int: - """Run Pi with isolated input/environment and flush timestamped events as they arrive.""" - directory = Path(params["directory"]) - (directory / "runtime.json").write_text(json.dumps({"hostname": os.uname().nodename, "pid": os.getpid()})) - (directory / "prompt.txt").write_text(params["prompt"]) - stopping = False - - def interrupt(*_): - nonlocal stopping - stopping = True - - signal.signal(signal.SIGTERM, interrupt) - with ExitStack() as stack: - stderr = stack.enter_context((directory / "stderr.log").open("wb")) - stdin = stack.enter_context((directory / "prompt.txt").open("rb")) - events = stack.enter_context((directory / "events.jsonl").open("w")) - process = subprocess.Popen( - params["command"], - cwd=params["cwd"], - env={**os.environ, **params["env"]}, - stdin=stdin, - stdout=subprocess.PIPE, - stderr=stderr, - ) - try: - pending = bytearray() - - def capture(line: bytes) -> None: - try: - event = json.loads(line.decode(errors="replace")) - except (ValueError, RecursionError): - return - if isinstance(event, dict): - events.write(json.dumps([time(), event]) + "\n") - events.flush() - - # A detached tool can inherit stdout. Drain available data after Pi exits, - # without waiting for that tool to close its pipe. Resources may retain - # task services through verification. - with selectors.DefaultSelector() as selector: - selector.register(process.stdout, selectors.EVENT_READ) - while True: - if stopping and process.poll() is None: - process.terminate() - stopping = False - ready = selector.select(timeout=0.05) - if ready: - chunk = os.read(process.stdout.fileno(), 64 * 1024) - if not chunk: - break - pending.extend(chunk) - while b"\n" in pending: - line, _, pending = pending.partition(b"\n") - capture(line) - elif process.poll() is not None: - break - if pending: - capture(pending) - return process.wait() - finally: - # An I/O/capture error must not leave the CLI changing the task. - # Only stop our direct CLI; task services belong to the sandbox owner. - if process.poll() is None: - process.kill() - process.wait() - - -def main() -> None: - params = json.loads(Path(sys.argv[1]).read_text()) - raise SystemExit(run(params)) - - -if __name__ == "__main__": - main() diff --git a/responses_api_agents/pi_agent/tests/test_bootstrap.py b/responses_api_agents/pi_agent/tests/test_bootstrap.py index 46d2e7bfd7b..1d70b12151d 100644 --- a/responses_api_agents/pi_agent/tests/test_bootstrap.py +++ b/responses_api_agents/pi_agent/tests/test_bootstrap.py @@ -5,7 +5,6 @@ import shlex import shutil import subprocess -import sys import pytest @@ -13,13 +12,13 @@ @pytest.mark.parametrize("package_manager", ["apk", "apt-get"]) -@pytest.mark.parametrize("missing", [(), ("bash",), ("python3",), ("python3", "bash")]) +@pytest.mark.parametrize("missing", [(), ("bash",)]) @pytest.mark.parametrize("failure", [None, "nonroot", "install", "unavailable"]) def test_posix_bootstrap_installs_only_missing_prerequisites(tmp_path, package_manager, missing, failure): """Exercise the actual preparation through sh without invoking a host package manager.""" bindir = tmp_path / "bin" bindir.mkdir() - binaries = {"python3": sys.executable, "bash": shutil.which("bash"), "mkdir": shutil.which("mkdir")} + binaries = {"bash": shutil.which("bash"), "mkdir": shutil.which("mkdir")} assert all(binaries.values()) for name, executable in binaries.items(): if name not in missing: @@ -33,8 +32,7 @@ def test_posix_bootstrap_installs_only_missing_prerequisites(tmp_path, package_m '#!/bin/sh\nprintf "%s\\n" "$*" >> "$TEST_ROOT/packages.log"\n' + ("exit 100\n" if failure == "install" else "") + 'for name in "$@"; do\n' - + f'case "$name" in\n python3) ln -s {shlex.quote(sys.executable)} "$TEST_ROOT/bin/python3" ;;\n' - + f' bash) ln -s {shlex.quote(binaries["bash"])} "$TEST_ROOT/bin/bash" ;;\n' + + f'case "$name" in\n bash) ln -s {shlex.quote(binaries["bash"])} "$TEST_ROOT/bin/bash" ;;\n' + "esac\ndone\n" ) manager.chmod(0o755) diff --git a/responses_api_agents/pi_agent/tests/test_install_runtime.py b/responses_api_agents/pi_agent/tests/test_install_runtime.py index 365512d1203..c1c155535ce 100644 --- a/responses_api_agents/pi_agent/tests/test_install_runtime.py +++ b/responses_api_agents/pi_agent/tests/test_install_runtime.py @@ -12,6 +12,8 @@ INSTALLER = Path(__file__).parents[1] / "install_pi_runtime.sh" pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Pi requires Linux") +# On musl the installer looks for the task's C++ library where musl's loader does; this host may lack it there. +MUSL_LIBSTDCXX = [] if any(Path(d, "libstdc++.so.6").exists() for d in ("/lib", "/usr/lib")) else ["libstdc++"] @pytest.fixture @@ -24,14 +26,12 @@ def sandbox(tmp_path: Path) -> tuple[Path, dict[str, str]]: if executable is None: pytest.skip(f"Installer test requires {name}") (bindir / name).symlink_to(executable) - (bindir / "python3").symlink_to(sys.executable) scripts = { "id": '#!/bin/bash\necho "${TEST_UID:-0}"\n', "apt-get": """#!/bin/bash echo "$*" >> "$TEST_ROOT/packages.log" if [ "$1" = install ]; then [ "${TEST_INSTALL_FAIL:-0}" = 0 ] || exit 100 - case " $* " in *" python3 "*) ln -s "$TEST_PYTHON" "$TEST_ROOT/bin/python3" ;; esac case " $* " in *" curl "*) cp "$TEST_ROOT/curl" "$TEST_ROOT/bin/curl" ;; esac fi """, @@ -43,7 +43,7 @@ def sandbox(tmp_path: Path) -> tuple[Path, dict[str, str]]: curl = tmp_path / "curl" curl.write_text('#!/bin/bash\necho "$*" > "$TEST_ROOT/download.log"\nexit 19\n') curl.chmod(0o755) - return tmp_path, os.environ | {"PATH": str(bindir), "TEST_ROOT": str(tmp_path), "TEST_PYTHON": sys.executable} + return tmp_path, os.environ | {"PATH": str(bindir), "TEST_ROOT": str(tmp_path)} def run_installer(root: Path, env: dict[str, str]) -> subprocess.CompletedProcess[str]: @@ -85,29 +85,6 @@ def test_missing_curl_is_installed_before_downloading_node(sandbox: tuple[Path, assert not (root / "runtime/ready").exists() -def test_missing_python_is_bootstrapped_before_runtime_checks(sandbox: tuple[Path, dict[str, str]]) -> None: - root, env = sandbox - (root / "bin/python3").unlink() - result = run_installer(root, env) - assert result.returncode == 19, result.stderr - assert (root / "packages.log").read_text().splitlines() == [ - "update", - "install -y --no-install-recommends python3", - "update", - "install -y --no-install-recommends curl ca-certificates", - ] - assert (root / "download.log").exists() - - -def test_missing_python_without_root_explains_remedy(sandbox: tuple[Path, dict[str, str]]) -> None: - root, env = sandbox - (root / "bin/python3").unlink() - result = run_installer(root, env | {"TEST_UID": "1000"}) - assert result.returncode == 1 - assert "Pi requires python3: preinstall these packages" in result.stderr - assert not (root / "download.log").exists() - - def test_existing_curl_does_not_install_packages(sandbox: tuple[Path, dict[str, str]]) -> None: root, env = sandbox shutil.copy(root / "curl", root / "bin/curl") @@ -151,16 +128,6 @@ def test_cached_runtime_needs_no_package_installation(sandbox: tuple[Path, dict[ assert not (root / "download.log").exists() -@pytest.mark.skipif(shutil.which("python3.8") is None, reason="Python 3.8 is not installed") -def test_python38_can_bootstrap_runtime(sandbox: tuple[Path, dict[str, str]]) -> None: - root, env = sandbox - (root / "bin/python3").unlink() - (root / "bin/python3").symlink_to(shutil.which("python3.8")) - result = run_installer(root, env) - assert result.returncode == 19, result.stderr - assert (root / "download.log").exists() - - def test_unknown_libc_explains_failure_before_install(sandbox: tuple[Path, dict[str, str]]) -> None: root, env = sandbox getconf = root / "bin/getconf" @@ -178,7 +145,9 @@ def test_musl_installs_curl_with_apk_and_selects_musl_node(musl_sandbox: tuple[P root, env = musl_sandbox result = run_installer(root, env) assert result.returncode == 19, result.stderr - assert (root / "packages.log").read_text().splitlines() == ["add --no-cache curl ca-certificates"] + assert (root / "packages.log").read_text().splitlines() == [ + " ".join(["add --no-cache curl ca-certificates", *MUSL_LIBSTDCXX]) + ] assert ( "https://unofficial-builds.nodejs.org/download/release/v22.19.0/node-v22.19.0-linux-x64-musl.tar.gz" in (root / "download.log").read_text() @@ -190,9 +159,13 @@ def test_musl_existing_dependencies_need_no_root(musl_sandbox: tuple[Path, dict[ root, env = musl_sandbox shutil.copy(root / "curl", root / "bin/curl") result = run_installer(root, env | {"TEST_UID": "1000"}) + assert not (root / "packages.log").exists() + if MUSL_LIBSTDCXX: + assert result.returncode == 1 + assert "Pi requires libstdc++: preinstall these packages" in result.stderr + return assert result.returncode == 19, result.stderr assert "linux-x64-musl.tar.gz" in (root / "download.log").read_text() - assert not (root / "packages.log").exists() def test_musl_arm64_explains_unavailable_build(musl_sandbox: tuple[Path, dict[str, str]]) -> None: diff --git a/responses_api_agents/pi_agent/tests/test_sandbox.py b/responses_api_agents/pi_agent/tests/test_sandbox.py index bcf92ce4264..68ecd487c0d 100644 --- a/responses_api_agents/pi_agent/tests/test_sandbox.py +++ b/responses_api_agents/pi_agent/tests/test_sandbox.py @@ -10,6 +10,7 @@ import shutil import signal import socket +import subprocess import sys import threading from contextlib import asynccontextmanager, contextmanager @@ -24,6 +25,7 @@ from omegaconf import OmegaConf from pydantic import ValidationError +from nemo_gym.agent_utils.sandbox_session import HARNESS_RUNNER_FILE, HARNESS_RUNNER_SCRIPT from nemo_gym.agent_utils.supervisor_client import parse_cleanup_receipt from nemo_gym.base_responses_api_agent import AgentCloseSessionRequest, AgentSeedSessionRequest from nemo_gym.episode_types import EpisodeId, TaskId @@ -31,7 +33,7 @@ from nemo_gym.server_utils import ServerClient from nemo_gym.tasks.harbor.materialize import materialize_task from nemo_gym.tasks.harbor.task import load_task -from responses_api_agents.pi_agent.app import PiAgent, PiAgentConfig, PiAgentRunRequest +from responses_api_agents.pi_agent.app import PiAgent, PiAgentConfig, PiAgentRunRequest, _parse_recorded_events def seed(*, session_id: str = "pi-session") -> AgentSeedSessionRequest: @@ -94,6 +96,22 @@ def events(*, stop_reason="stop") -> str: ) +def staged(sandbox) -> dict: + """The activation as the runner sees it: params.sh sourced by bash, plus the staged prompt.""" + script = 'source /dev/stdin; printf "%s\\0" "$cwd" "${#command[@]}" "${command[@]}" "${env[@]}"' + params = sandbox.files[f"{sandbox.directory}/params.sh"] + values = subprocess.run( + ["bash", "-c", script], input=params, capture_output=True, text=True, check=True + ).stdout.split("\0")[:-1] + count = int(values[1]) + return { + "cwd": values[0], + "command": values[2 : 2 + count], + "env": dict(item.split("=", 1) for item in values[2 + count :]), + "prompt": sandbox.files[f"{sandbox.directory}/prompt.txt"], + } + + class Sandbox: def __init__(self): self.files = {} @@ -117,7 +135,7 @@ def __init__(self): self.cleanup_available = True async def execute(self, command, **kwargs): - if "--receipt" in command and "process_supervisor.py" in command: + if "--receipt" in command and "process_supervisor.sh" in command: return await self.launch(command, **kwargs) if "ln -s stop " in command: await self.signal() @@ -139,13 +157,11 @@ def save_result(self): self.files[f"{self.directory}/events.jsonl"] = self.events async def run(self, command, **kwargs): - payload_path = next(path for path in self.files if path.endswith("/input.json")) - payload = json.loads(self.files[payload_path]) - assert payload["cwd"] == getattr(self, "expected_workdir", "/app") + self.directory = next(path.rsplit("/", 1)[0] for path in self.files if path.endswith("/params.sh")) + assert staged(self)["cwd"] == getattr(self, "expected_workdir", "/app") assert kwargs["cwd"] == getattr(self, "expected_workdir", "/app") - assert "sandbox_runner.py" in command - assert "process_supervisor.py" in command - self.directory = payload["directory"] + assert f"bash {self.directory}/{HARNESS_RUNNER_FILE} {self.directory}/params.sh" in command + assert "process_supervisor.sh" in command self.started.set() if not self.blocked: self.exited.set() @@ -154,7 +170,7 @@ async def run(self, command, **kwargs): return SimpleNamespace(return_code=0, error_type=None, stdout="", stderr="") async def stop_runner(self): - self.directory = next(path.rsplit("/", 1)[0] for path in self.files if path.endswith("/input.json")) + self.directory = next(path.rsplit("/", 1)[0] for path in self.files if path.endswith("/params.sh")) self.result["timed_out"] = True self.save_result() self.exited.set() @@ -216,7 +232,7 @@ def test_http_session_flow_runs_pi_in_borrowed_sandbox(setup): assert created.status_code == 200, created.text session_id = created.json()["agent_session_id"] directory = agent._session_records[session_id].state.session.session_dir - assert f"{directory}/process_supervisor.py" not in sandbox.files + assert f"{directory}/process_supervisor.sh" not in sandbox.files installer = f"{directory}/install_pi_runtime.sh" assert installer in sandbox.files assert agent.config.resources_server is None @@ -227,7 +243,7 @@ def test_http_session_flow_runs_pi_in_borrowed_sandbox(setup): assert install_call.kwargs["timeout_s"] == agent.config.sandbox_install_timeout_seconds result = client.post("/ng-rollout/pi-smoke-a2/v1/responses", json={"input": "Fix the code"}) assert result.status_code == 200, result.text - assert f"{directory}/process_supervisor.py" in sandbox.files + assert f"{directory}/process_supervisor.sh" in sandbox.files body = result.json() assert body["status"] == "completed" assert [item["type"] for item in body["output"]] == [ @@ -240,7 +256,7 @@ def test_http_session_flow_runs_pi_in_borrowed_sandbox(setup): assert body["usage"]["input_tokens_details"]["cached_tokens"] is None assert body["metadata"]["harness_execution"] == "sandbox" assert "_ng_agent_observations" not in body - payload = json.loads(sandbox.files[f"{sandbox.directory}/input.json"]) + payload = staged(sandbox) assert payload["prompt"] == "Fix the code" assert payload["command"][0].endswith("/node/bin/node") assert "PATH" not in payload["env"] @@ -536,7 +552,7 @@ async def test_sandbox_runtime_guards_reach_the_pi_invocation(setup): request, session_id, task = await activate(agent, sandbox) response = await task assert response.status == "completed" - payload = json.loads(sandbox.files[f"{sandbox.directory}/input.json"]) + payload = staged(sandbox) assert payload["env"]["NEMO_GYM_PI_BASH_TIMEOUT"] == "123" assert f"{sandbox.directory}/runtime-guards.mjs" in payload["command"] assert "tool_call" in sandbox.files[f"{sandbox.directory}/runtime-guards.mjs"] @@ -561,7 +577,7 @@ async def test_mcp_grants_reach_pi_through_the_gym_extension(setup): request, session_id, task = await activate(agent, sandbox, body) response = await task assert response.status == "completed" - payload = json.loads(sandbox.files[f"{sandbox.directory}/input.json"]) + payload = staged(sandbox) extension = f"{sandbox.directory}/gym_mcp.mjs" assert sandbox.files[extension] == Path(__file__).parents[1].joinpath("gym_mcp.mjs").read_text() assert payload["command"][payload["command"].index(extension) - 1] == "--extension" @@ -603,7 +619,7 @@ async def test_no_mcp_grant_stages_no_tool_extension(setup): agent, sandbox = setup request, session_id, task = await activate(agent, sandbox) await task - payload = json.loads(sandbox.files[f"{sandbox.directory}/input.json"]) + payload = staged(sandbox) assert not any(path.endswith("gym_mcp.mjs") for path in [*payload["command"], *sandbox.files]) assert "NEMO_GYM_PI_MCP_CONFIG" not in payload["env"] await agent.close_agent_session(request, AgentCloseSessionRequest(**close_body(session_id))) @@ -891,7 +907,7 @@ def test_instructions_and_text_parts_reach_pi_without_other_provider_credentials }, ) assert response.status_code == 200, response.text - payload = json.loads(sandbox.files[f"{sandbox.directory}/input.json"]) + payload = staged(sandbox) assert payload["command"][-2:] == [ "--append-system-prompt", "config instruction\n\nrequest instruction\n\ninput instruction", @@ -1478,10 +1494,8 @@ async def _run_locally(command, *, limit=8, **kwargs): def _run_session_locally(sandbox, state, directory: Path, *, limit: float = 8) -> None: """Point a seeded session's files and commands at local paths and processes.""" - from responses_api_agents.pi_agent import sandbox_runner - state.session.session_dir = str(directory) - shutil.copyfile(sandbox_runner.__file__, directory / "sandbox_runner.py") + shutil.copyfile(HARNESS_RUNNER_SCRIPT, directory / HARNESS_RUNNER_FILE) sandbox.upload = AsyncMock(side_effect=shutil.copyfile) sandbox.download = AsyncMock(side_effect=shutil.copyfile) sandbox.exec.side_effect = functools.partial(_run_locally, limit=limit) @@ -1509,7 +1523,7 @@ async def test_close_fences_delayed_launch_before_and_after_removing_session(loc commands = [] async def queued_exec(command, **kwargs): - if "--receipt" in command and "process_supervisor.py" in command: + if "--receipt" in command and "process_supervisor.sh" in command: commands.append(command) waiting.set() if failure == "cancel": @@ -1518,7 +1532,7 @@ async def queued_exec(command, **kwargs): return await execute(command, **kwargs) state.session.sandbox.exec.side_effect = queued_exec - state.task = asyncio.create_task(state.execute({}, timeout=5, close_timeout=2)) + state.task = asyncio.create_task(state.execute(command=["true"], env={}, prompt="", timeout=5, close_timeout=2)) await asyncio.wait_for(waiting.wait(), 2) if failure == "cancel": state.task.cancel() @@ -1551,21 +1565,11 @@ async def test_launch_claim_without_receipt_blocks_close(local_session): @pytest.mark.skipif(sys.platform != "linux", reason="Linux sandbox contract") async def test_adapter_uses_shared_supervisor_and_captures_real_events(local_session): state, _ = local_session - payload = { - "directory": state.session.session_dir, - "cwd": state.request.sandbox_access.workdir, - # This stdlib-only event producer must not load the host's editable installs or site hooks. - "command": [ - sys.executable, - "-I", - "-S", - "-c", - "import json,sys; print(json.dumps({'prompt':sys.stdin.read()}))", - ], - "env": {}, - "prompt": "real invocation", - } - raw = await state.execute(payload, timeout=3, close_timeout=2) + # Diagnostics and malformed lines reach stdout too; only [seen_at, object] records reach the transcript. + script = 'echo starting; echo "{\\"prompt\\": \\"$(cat)\\", \\"cwd\\": \\"$PWD\\"}"; echo "{broken"' + raw = await state.execute( + command=["bash", "-c", script], env={}, prompt="real invocation", timeout=3, close_timeout=2 + ) diagnostics = { name: path.read_text(errors="replace") for name in ("cleanup.json", "runtime.json", "output.log", "stderr.log", "events.jsonl") @@ -1573,8 +1577,9 @@ async def test_adapter_uses_shared_supervisor_and_captures_real_events(local_ses } assert state.session.cleanup["cleanup_confirmed"] and state.session.cleanup["return_code"] == 0, diagnostics assert raw.strip(), diagnostics - _, event = json.loads(raw) - assert event == {"prompt": "real invocation"} + assert "{broken" in raw + events = _parse_recorded_events(raw) + assert [event for _, event in events] == [{"prompt": "real invocation", "cwd": state.session.workdir}] assert state.runtime_info.hostname await state.close(2) assert not Path(state.session.session_dir).exists() @@ -1590,14 +1595,9 @@ async def test_close_confirms_cleanup_before_provider_cancellation(local_session "child=subprocess.Popen([sys.executable,'-c','import time; time.sleep(60)'],start_new_session=True); " f"Path({str(processes_path)!r}).write_text(json.dumps([os.getpid(),child.pid])); time.sleep(60)" ) - payload = { - "directory": state.session.session_dir, - "cwd": state.request.sandbox_access.workdir, - "command": [sys.executable, "-c", code], - "env": {}, - "prompt": "", - } - state.task = asyncio.create_task(state.execute(payload, timeout=30, close_timeout=3)) + state.task = asyncio.create_task( + state.execute(command=[sys.executable, "-c", code], env={}, prompt="", timeout=30, close_timeout=3) + ) processes = [] try: async with asyncio.timeout(5): @@ -1610,8 +1610,8 @@ async def test_close_confirms_cleanup_before_provider_cancellation(local_session cli_pid, task_pid, wrapper_pid = processes assert not Path(f"/proc/{cli_pid}").exists() assert not Path(f"/proc/{wrapper_pid}").exists() - # The in-flight detached tool is drained before borrowed-session release. - assert not Path(f"/proc/{task_pid}").exists() + # A tool that left the harness process group belongs to the sandbox owner, not the supervisor. + assert Path(f"/proc/{task_pid}/stat").read_text().rsplit(")", 1)[1].split()[0] != "Z" state.session.sandbox.disconnect.assert_awaited_once() state.session.sandbox.stop.assert_not_awaited() finally: diff --git a/responses_api_agents/pi_agent/tests/test_sandbox_runner.py b/responses_api_agents/pi_agent/tests/test_sandbox_runner.py index e9104f16684..96786a4cf65 100644 --- a/responses_api_agents/pi_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/pi_agent/tests/test_sandbox_runner.py @@ -1,7 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Exercise the real Linux supervisor, not a mocked cleanup acknowledgement.""" +"""Run Pi's staged activation through the real bash runner and supervisor, not a mocked receipt.""" +import asyncio import json import os import shutil @@ -13,46 +14,36 @@ import pytest -from nemo_gym.agent_utils import process_supervisor -from responses_api_agents.pi_agent import sandbox_runner +from nemo_gym.agent_utils.sandbox_session import HARNESS_RUNNER_FILE, HARNESS_RUNNER_SCRIPT, SandboxSession +from nemo_gym.agent_utils.supervisor_client import SUPERVISOR_SCRIPT +from responses_api_agents.pi_agent.sandbox import PiSandboxSession +from responses_api_agents.pi_agent.tests.test_sandbox import seed pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Linux process supervision is required") -def launch(tmp_path, code, timeout=3, python=sys.executable, env=None): - request = { - "directory": str(tmp_path), - "command": [python, "-c", code], - "cwd": str(tmp_path), - "env": env or {}, - "timeout": timeout, - "cleanup_timeout": 2, - "prompt": "task input", - } - path = tmp_path / "input.json" - path.write_text(json.dumps(request)) - process = subprocess.Popen( - [ - python, - "-I", - process_supervisor.__file__, - "--timeout", - str(timeout), - "--cleanup-timeout", - "0.5", - "--receipt", - str(tmp_path / "cleanup.json"), - "--", - python, - "-I", - str(Path(sandbox_runner.__file__)), - str(path), - ], +class LocalFiles: + async def upload(self, source, destination): + shutil.copyfile(source, destination) + + +def launch(tmp_path, script, timeout=3, env=None): + """Stage ``script`` as Pi's CLI exactly as an activation does, then start it under the supervisor.""" + shutil.copyfile(HARNESS_RUNNER_SCRIPT, tmp_path / HARNESS_RUNNER_FILE) + state = PiSandboxSession( + request=seed(), + session=SandboxSession(sandbox=LocalFiles(), session_dir=str(tmp_path), workdir=str(tmp_path), harness="Pi"), + runtime="/unused", + ) + command = asyncio.run(state.stage_activation(command=["bash", "-c", script], env=env or {}, prompt="task input")) + return subprocess.Popen( + ["bash", str(SUPERVISOR_SCRIPT), "--timeout", str(timeout), "--cleanup-timeout", "0.5"] + + ["--stop-file", str(tmp_path / "stop.request"), "--receipt", str(tmp_path / "cleanup.json")] + + ["--", *command.argv], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) - return process def result(tmp_path, process): @@ -66,72 +57,37 @@ def result(tmp_path, process): process.wait() -def test_capture_and_stdin(tmp_path): - process = launch( - tmp_path, - "import json,os,sys; os.write(1,b'\\xff\\n'); print('[]'); " - "sys.stdout.write(json.dumps({'type':'test', 'prompt':sys.stdin.read()}))", +def test_capture_stdin_and_environment(tmp_path, monkeypatch): + monkeypatch.setenv("PI_TEST_INHERITED", "inherited") + monkeypatch.setenv("PI_TEST_OVERRIDE", "old") + script = ( + "printf '\\377\\n[]\\n'; " + 'printf \'{"prompt": "%s", "inherited": "%s", "override": "%s"}\' ' + '"$(cat)" "$PI_TEST_INHERITED" "$PI_TEST_OVERRIDE"' ) - summary = result(tmp_path, process) + summary = result(tmp_path, launch(tmp_path, script, env={"PI_TEST_OVERRIDE": "new"})) assert summary["cleanup_confirmed"] is True assert summary["return_code"] == 0 recorded_at, event = json.loads((tmp_path / "events.jsonl").read_text()) assert recorded_at > 0 - assert event["prompt"] == "task input" - - -@pytest.mark.skipif(shutil.which("python3.8") is None, reason="Python 3.8 is not installed") -def test_python38_preserves_environment_and_task_child(tmp_path, monkeypatch): - try: - monkeypatch.setenv("PI_TEST_INHERITED", "inherited") - monkeypatch.setenv("PI_TEST_OVERRIDE", "old") - code = ( - "import json,os,pathlib,subprocess,sys; " - "p=subprocess.Popen([sys.executable,'-c','import time; time.sleep(60)'],start_new_session=True); " - "pathlib.Path('child.pid').write_text(str(p.pid)); " - "print(json.dumps({'inherited':os.environ['PI_TEST_INHERITED']," - "'override':os.environ['PI_TEST_OVERRIDE'],'prompt':sys.stdin.read()}))" - ) - process = launch(tmp_path, code, python=shutil.which("python3.8"), env={"PI_TEST_OVERRIDE": "new"}) - summary = result(tmp_path, process) - assert summary["return_code"] == 0, summary - assert summary["cleanup_confirmed"] is True - _, event = json.loads((tmp_path / "events.jsonl").read_text()) - assert event == {"inherited": "inherited", "override": "new", "prompt": "task input"} - pid = int((tmp_path / "child.pid").read_text()) - assert Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[0] != "Z" - finally: - if (tmp_path / "child.pid").exists(): - try: - os.kill(int((tmp_path / "child.pid").read_text()), signal.SIGKILL) - except ProcessLookupError: - pass + assert event == {"prompt": "task input", "inherited": "inherited", "override": "new"} + assert set(json.loads((tmp_path / "runtime.json").read_text())) == {"hostname", "pid"} @pytest.mark.parametrize("ending", ["natural", "timeout", "cancel"]) def test_task_descendants_survive_until_owner_teardown(tmp_path, ending): try: # A completed tool leaves its detached service adopted before close. - tool = ( - "import pathlib,subprocess,sys; " - "p=subprocess.Popen([sys.executable,'-c','import time; time.sleep(60)']," - "start_new_session=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL); " - "pathlib.Path('child.pid').write_text(str(p.pid))" - ) - code = ( - "import subprocess,sys,time,pathlib; " - f"subprocess.run([sys.executable,'-c',{tool!r}],check=True); " - "pathlib.Path('ready').touch(); " + ("time.sleep(60)" if ending != "natural" else "pass") + script = "setsid sleep 60 /dev/null 2>&1 & echo $! > child.pid; touch ready; " + ( + "sleep 60" if ending != "natural" else "true" ) - # Allow the nested interpreters to start before testing harness termination, - # including when subprocess coverage adds interpreter startup overhead. - process = launch(tmp_path, code, timeout=3) + process = launch(tmp_path, script, timeout=3) if ending == "cancel": for _ in range(200): if (tmp_path / "ready").exists(): break time.sleep(0.01) - process.send_signal(signal.SIGTERM) + (tmp_path / "stop.request").touch() summary = result(tmp_path, process) assert summary["cleanup_confirmed"] is True assert summary["timed_out"] is (ending == "timeout") @@ -145,8 +101,7 @@ def test_task_descendants_survive_until_owner_teardown(tmp_path, ending): pass -def test_spawn_error_is_not_success(tmp_path): - process = launch(tmp_path, "raise RuntimeError('failed')") - summary = result(tmp_path, process) +def test_cli_failure_is_not_success(tmp_path): + summary = result(tmp_path, launch(tmp_path, "exit 7")) assert summary["cleanup_confirmed"] is True - assert summary["return_code"] != 0 + assert summary["return_code"] == 7 From 83197ddcb65c3f4976ecd8cd3cc89c1788d58d86 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Mon, 26 Oct 2026 09:34:46 -0700 Subject: [PATCH 10/32] fix(opencode): keep pre-port outcomes when the session database is unavailable A missing or undownloadable session database again leaves no export, so collection reports "OpenCode sandbox runner returned no valid result" with the output log, as when export.json was missing before. The export no longer depends on copying the observation store: a failed copy, like the failed observations.db download before, only adds observation gaps. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Hemil Desai Signed-off-by: yaoyu-33 --- .../opencode_agent/sandbox.py | 45 ++++++++++++++----- .../opencode_agent/snapshot.py | 12 ++++- .../opencode_agent/tests/test_continuation.py | 6 ++- .../tests/test_sandbox_runner.py | 8 ++++ .../tests/test_sandbox_sessions.py | 3 +- .../tests/test_session_review.py | 28 +++++++++--- 6 files changed, 81 insertions(+), 21 deletions(-) diff --git a/responses_api_agents/opencode_agent/sandbox.py b/responses_api_agents/opencode_agent/sandbox.py index 211adc962f9..f6a3c81a1be 100644 --- a/responses_api_agents/opencode_agent/sandbox.py +++ b/responses_api_agents/opencode_agent/sandbox.py @@ -295,8 +295,8 @@ async def _collect_artifacts(self, directory: Path, *, timeout: float) -> str: if self.stderr: raise RuntimeError(format_error_message(str(error), stderr=self.stderr)) from error raise - export_text = (directory / "export.json").read_text() if self.is_continuation: + export_text = await self._read_export(directory) export = json.loads(export_text) harness_session_id = export.get("session_id") if not isinstance(harness_session_id, str) or not harness_session_id: @@ -313,9 +313,12 @@ async def _collect_artifacts(self, directory: Path, *, timeout: float) -> str: trajectory = TrajectoryRecord( task_id=self.request.task_id.task_id, rollout_id=self.request.episode_id.capture_key ) + database = directory / "observations.db" + if not database.is_file(): + raise FileNotFoundError(f"OpenCode observations were not captured: {database}") observations = await asyncio.to_thread( parse_opencode_observations, - directory / "observations.db", + database, self.request.episode_id.capture_key, trajectory, require_terminal_finish=True, @@ -357,7 +360,7 @@ async def _collect_artifacts(self, directory: Path, *, timeout: float) -> str: # Preserve the single-turn evidence/error order. The response layer # parses this raw export and translates malformed content to 502. await self._check_task_environment() - return export_text + return await self._read_export(directory) try: await self._check_task_environment() except Exception as error: @@ -379,6 +382,17 @@ async def _collect_artifacts(self, directory: Path, *, timeout: float) -> str: ) from error return export_text + async def _read_export(self, directory: Path) -> str: + try: + return (directory / "export.json").read_text() + except OSError as error: + logs = await self.session.read_output_log() + raise RuntimeError( + format_sandbox_error( + "OpenCode sandbox runner returned no valid result", output=logs, stderr=self.stderr + ) + ) from error + async def _check_task_environment(self) -> None: try: if await self.read_text("task-environment.ready") != "ready\n": @@ -393,22 +407,31 @@ async def snapshot(self, directory: Path, timeout: float) -> None: if self.session.cleanup is None or not self.session.cleanup["cleanup_confirmed"]: raise RuntimeError("OpenCode transcript capture requires confirmed cleanup") database = f"{self.persistent_directory}/data/opencode" + deadline = asyncio.get_running_loop().time() + timeout try: - async with asyncio.timeout(timeout): + async with asyncio.timeout_at(deadline): # A stopped CLI may leave its write-ahead log behind; it holds the latest messages. listed = await self.session.sandbox.exec( - f"cd {quote(database)} && for name in opencode.db opencode.db-wal opencode.db-shm; " - 'do [ ! -e "$name" ] || echo "$name"; done', + "for name in opencode.db opencode.db-wal opencode.db-shm; " + f'do [ ! -e {quote(database)}/"$name" ] || echo "$name"; done', cwd=self.session.workdir, timeout_s=timeout, ) - if listed.return_code != 0 or listed.error_type: - raise RuntimeError(listed.stderr) - names = (listed.stdout or "").split() - if "opencode.db" not in names: - raise RuntimeError("OpenCode session database is missing") + if listed.return_code != 0 or listed.error_type: + raise RuntimeError(listed.stderr) + except Exception as error: + raise RuntimeError(f"OpenCode transcript capture failed: {error}") from error + names = (listed.stdout or "").split() + if "opencode.db" not in names: + return # No export: collection reports the missing result with the CLI logs. + try: + async with asyncio.timeout_at(deadline): for name in names: await self.session.sandbox.download(f"{database}/{name}", directory / name) + except Exception: + LOG.warning("OpenCode session database unavailable", exc_info=True) + return + try: await asyncio.to_thread( snapshot, directory / "opencode.db", diff --git a/responses_api_agents/opencode_agent/snapshot.py b/responses_api_agents/opencode_agent/snapshot.py index 9a22f4a7db2..dc2a6eb29d2 100644 --- a/responses_api_agents/opencode_agent/snapshot.py +++ b/responses_api_agents/opencode_agent/snapshot.py @@ -3,19 +3,21 @@ """Export an OpenCode session database copied out of the sandbox after the CLI stopped.""" import json +import logging import sqlite3 from collections.abc import Collection from pathlib import Path +LOG = logging.getLogger(__name__) + + def snapshot( database: Path, directory: Path, *, harness_session_id: str | None, previous_message_ids: Collection[str] ) -> None: """Export root messages and retain the full session tree for observations.""" previous_ids = set(previous_message_ids) with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as source: - with sqlite3.connect(directory / "observations.db") as destination: - source.backup(destination) source.row_factory = sqlite3.Row sessions = source.execute( "select id from session where parent_id is null order by time_created, id" @@ -53,3 +55,9 @@ def snapshot( } ) ) + # A missing observation store costs only observation evidence, not the export. + try: + with sqlite3.connect(directory / "observations.db") as destination: + source.backup(destination) + except sqlite3.Error: + LOG.exception("Failed to copy OpenCode observations") diff --git a/responses_api_agents/opencode_agent/tests/test_continuation.py b/responses_api_agents/opencode_agent/tests/test_continuation.py index 4887a953e75..d9ef300624c 100644 --- a/responses_api_agents/opencode_agent/tests/test_continuation.py +++ b/responses_api_agents/opencode_agent/tests/test_continuation.py @@ -281,9 +281,10 @@ async def test_close_recovery_preserves_cancelled_activation(setup, tmp_path, mo @pytest.mark.parametrize("cancelled", [False, True]) +@pytest.mark.parametrize("artifact_failure", ["copy", "parse"]) @pytest.mark.parametrize("prior_success", [False, True]) async def test_close_without_recovered_root_preserves_activation( - setup, tmp_path, monkeypatch, cancelled, prior_success + setup, tmp_path, monkeypatch, cancelled, artifact_failure, prior_success ): agent, sandbox = setup install_artifact_runner(sandbox, tmp_path) @@ -309,7 +310,8 @@ async def test_close_without_recovered_root_preserves_activation( async def broken_observations(source, destination): if source.endswith("/opencode.db"): - Path(destination).write_bytes(b"not a SQLite database") + # The export survives; only the observation store is unusable. + Path(destination).write_bytes(b"" if artifact_failure == "copy" else b"not a SQLite database") else: await download(source, destination) diff --git a/responses_api_agents/opencode_agent/tests/test_sandbox_runner.py b/responses_api_agents/opencode_agent/tests/test_sandbox_runner.py index e607ca84d07..54485e79db6 100644 --- a/responses_api_agents/opencode_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/opencode_agent/tests/test_sandbox_runner.py @@ -215,3 +215,11 @@ def test_snapshot_of_a_copied_store_keeps_root_output_and_child_usage(tmp_path, assert [info["tokens"]["input"] for info in export["usage_messages"]] == [1, 2] with sqlite3.connect(local / "observations.db") as copy: assert copy.execute("select count(*) from session").fetchone()[0] == 2 + + +def test_failed_observation_copy_keeps_the_export(tmp_path): + con = session_store(tmp_path / "opencode.db") + (tmp_path / "observations.db").mkdir() # SQLite cannot open the destination + snapshot(tmp_path / "opencode.db", tmp_path, harness_session_id=None, previous_message_ids=[]) + con.close() + assert json.loads((tmp_path / "export.json").read_text())["session_id"] == "root" diff --git a/responses_api_agents/opencode_agent/tests/test_sandbox_sessions.py b/responses_api_agents/opencode_agent/tests/test_sandbox_sessions.py index 3ac6115905f..00163428bbc 100644 --- a/responses_api_agents/opencode_agent/tests/test_sandbox_sessions.py +++ b/responses_api_agents/opencode_agent/tests/test_sandbox_sessions.py @@ -159,7 +159,8 @@ def snapshot(self, database, directory, *, harness_session_id, previous_message_ """Stand in for the SQLite export (see test_snapshot.py): the staged export, the copied database.""" self.snapshots.append({"harness_session_id": harness_session_id, "previous_message_ids": previous_message_ids}) (directory / "export.json").write_text(self.files[f"{self.directory}/export.json"]) - (directory / "observations.db").write_bytes(database.read_bytes()) + if database.stat().st_size: # an empty database stands for a failed observation copy + (directory / "observations.db").write_bytes(database.read_bytes()) async def create(self, **kwargs): runner_params = next(arg for arg in shlex.split(kwargs["command"]) if arg.endswith("/params.sh")) diff --git a/responses_api_agents/opencode_agent/tests/test_session_review.py b/responses_api_agents/opencode_agent/tests/test_session_review.py index fb3414cd313..a7c46792d17 100644 --- a/responses_api_agents/opencode_agent/tests/test_session_review.py +++ b/responses_api_agents/opencode_agent/tests/test_session_review.py @@ -131,6 +131,8 @@ def test_execution_failure_keeps_cleanup_observations_and_diagnostics(setup, tmp async def read(source, destination): if source.endswith("/stderr.log"): Path(destination).write_text("x" * 18000 + "ROOT CAUSE") + elif source.endswith("/output.log"): + Path(destination).write_text("output" * 6000 + "OUTPUT TAIL") else: await download(source, destination) @@ -161,9 +163,11 @@ async def run(command, **kwargs): else: assert "ROOT CAUSE" in str(error.value) assert len(str(error.value)) <= 16000 - assert "OpenCode transcript capture failed" in str(error.value) if failure == "missing_database": - assert "session database is missing" in str(error.value) + assert "returned no valid result" in str(error.value) + assert "OUTPUT TAIL" in str(error.value) + else: + assert "OpenCode transcript capture failed" in str(error.value) assert len(state.stderr) == 16000 assert state.session.cleanup["cleanup_confirmed"] is True assert any(record.kind == "sandbox" for record in state.observations.records) @@ -258,7 +262,7 @@ async def read(source, destination): client.post("/v1/agent_sessions/close", json=close_body(created.json()["agent_session_id"])).raise_for_status() -@pytest.mark.parametrize("failure,status", [("receipt", 500), ("malformed_export", 502)]) +@pytest.mark.parametrize("failure,status", [("receipt", 500), ("missing_database", 500), ("malformed_export", 502)]) def test_single_turn_failure_preserves_observations_and_runtime(setup, tmp_path, failure, status): agent, sandbox = setup capture_observations(sandbox, tmp_path) @@ -266,6 +270,15 @@ def test_single_turn_failure_preserves_observations_and_runtime(setup, tmp_path, sandbox.task_environment_receipt = None elif failure == "malformed_export": sandbox.events = "{malformed" + elif failure == "missing_database": + execute = sandbox.run_exec + + async def run(command, **kwargs): + if "opencode.db-wal" in command: + return SimpleNamespace(return_code=0, error_type=None, stdout="", stderr="") + return await execute(command, **kwargs) + + sandbox.exec.side_effect = run with TestClient(agent.setup_webserver(), raise_server_exceptions=False) as client: created = client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")) session_id = created.json()["agent_session_id"] @@ -275,5 +288,10 @@ def test_single_turn_failure_preserves_observations_and_runtime(setup, tmp_path, assert state.runtime_info.hostname == "task-container" closed = client.post("/v1/agent_sessions/close", json=close_body(session_id)) closed.raise_for_status() - invocation = next(r for r in closed.json()["agent_observations"]["records"] if r["kind"] == "agent_invocation") - assert invocation["conversation"][-1]["content"][0]["text"] == "Fixed" + observations = closed.json()["agent_observations"] + if failure == "missing_database": + # Without the session database only the observation gaps remain. + assert {gap["code"] for gap in observations["gaps"]} >= {"agent_artifact_unavailable"} + else: + invocation = next(r for r in observations["records"] if r["kind"] == "agent_invocation") + assert invocation["conversation"][-1]["content"][0]["text"] == "Fixed" From edeb1e64ba53f5fc2ed1ae31206f20e75b216077 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Mon, 26 Oct 2026 09:36:41 -0700 Subject: [PATCH 11/32] style: format bash supervisor tests Co-Authored-By: Claude Opus 5.5 Signed-off-by: Hemil Desai Signed-off-by: yaoyu-33 --- responses_api_agents/hermes_agent/tests/test_app.py | 3 ++- tests/unit_tests/test_harness_runner.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/responses_api_agents/hermes_agent/tests/test_app.py b/responses_api_agents/hermes_agent/tests/test_app.py index dc8c7d07f9a..314da88a651 100644 --- a/responses_api_agents/hermes_agent/tests/test_app.py +++ b/responses_api_agents/hermes_agent/tests/test_app.py @@ -455,7 +455,8 @@ async def download(remote_path, local_path): Path(local_path).write_text( json.dumps( { - "cleanup_confirmed": commands is None or any("touch" in cmd and "stop.request" in cmd for cmd in commands), + "cleanup_confirmed": commands is None + or any("touch" in cmd and "stop.request" in cmd for cmd in commands), "error": None, } ), diff --git a/tests/unit_tests/test_harness_runner.py b/tests/unit_tests/test_harness_runner.py index f9c69f8e554..1ae2e0ec639 100644 --- a/tests/unit_tests/test_harness_runner.py +++ b/tests/unit_tests/test_harness_runner.py @@ -88,7 +88,7 @@ def test_events_mode_forwards_term_and_ignores_a_detached_stdout_holder(tmp_path # A detached tool keeps stdout open; the runner must still finish when the CLI does. script = ( "setsid sleep 30 & echo $! > holder.pid; " - "trap 'echo \"{\\\"type\\\": \\\"stopped\\\"}\"; exit 0' TERM; echo '{\"type\": \"ready\"}'; " + 'trap \'echo "{\\"type\\": \\"stopped\\"}"; exit 0\' TERM; echo \'{"type": "ready"}\'; ' "while :; do sleep 0.05; done" ) params = _params(tmp_path, command=["bash", "-c", script], events=str(tmp_path / "events.jsonl")) From 55f59ef24b966c989476ecdc58b0194185d77665 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 26 Oct 2026 15:18:43 -0400 Subject: [PATCH 12/32] refactor(agent): adapt GitHub-only harness paths to the bash supervisor The bash supervisor no longer takes an interpreter, so NOOA drops the removed SandboxCommand.python field (its own pinned Python still runs its worker), and the Codex worker keeps its Python runner under the bash supervisor: its sandbox-local 429 retry proxy still needs Python. OpenCode drops sandbox_python_executable; sandbox_path_prefix now reaches the CLI through the runner setup. Tests and READMEs name process_supervisor.sh. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- responses_api_agents/codex_agent/README.md | 2 +- .../codex_agent/tests/test_exec_transport.py | 2 +- .../codex_agent/tests/test_sandbox_runner.py | 87 +------------------ .../tests/test_sandbox_sessions.py | 8 +- .../nooa_agent/sandbox_runner.py | 1 - .../tests/test_checkpoint_recovery.py | 2 +- .../nooa_agent/tests/test_sandbox_runner.py | 16 ++-- .../opencode_agent/sandbox.py | 2 +- .../tests/test_sandbox_sessions.py | 8 +- responses_api_agents/pi_agent/README.md | 2 +- 10 files changed, 24 insertions(+), 106 deletions(-) diff --git a/responses_api_agents/codex_agent/README.md b/responses_api_agents/codex_agent/README.md index eb56e7624d7..8a7b4311e00 100644 --- a/responses_api_agents/codex_agent/README.md +++ b/responses_api_agents/codex_agent/README.md @@ -134,7 +134,7 @@ uploads the supervisor at activation and owns its control files and log access; confirms process cleanup, captures artifacts, then releases the provider connection or owned sandbox. Concurrent closes share that work; missing evidence or failed release remains retryable, and interrupted activations keep captured observations after session files are removed. -`supervisor_client.py` handles controller-side launch/stop commands; `process_supervisor.py` +`supervisor_client.py` handles controller-side launch/stop commands; `process_supervisor.sh` runs inside the sandbox. Benchmark Resources session identity/verdict state stays separate. Missing worker identity or exit diagnostics become `runtime_info_unavailable` or `worker_exit_code_unavailable` observation gaps. When Codex fails before it starts a turn, the end diff --git a/responses_api_agents/codex_agent/tests/test_exec_transport.py b/responses_api_agents/codex_agent/tests/test_exec_transport.py index 5e91ba0273c..415a50e380f 100644 --- a/responses_api_agents/codex_agent/tests/test_exec_transport.py +++ b/responses_api_agents/codex_agent/tests/test_exec_transport.py @@ -40,7 +40,7 @@ async def disconnect(self): self.disconnected = True async def exec(self, command, *, cwd=None, timeout_s=30): - launch = "--receipt" in command and "process_supervisor.py" in command + launch = "--receipt" in command and "process_supervisor.sh" in command if launch: self.deadline = timeout_s self.delayed_command = command diff --git a/responses_api_agents/codex_agent/tests/test_sandbox_runner.py b/responses_api_agents/codex_agent/tests/test_sandbox_runner.py index 958268f7837..10307db5e48 100644 --- a/responses_api_agents/codex_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/codex_agent/tests/test_sandbox_runner.py @@ -13,7 +13,7 @@ import pytest -from nemo_gym.agent_utils import process_supervisor +from nemo_gym.agent_utils.supervisor_client import SUPERVISOR_SCRIPT from responses_api_agents.codex_agent import sandbox_runner @@ -34,9 +34,8 @@ def launch(tmp_path, code, timeout=3, python=sys.executable): path.write_text(json.dumps(request)) process = subprocess.Popen( [ - python, - "-I", - process_supervisor.__file__, + "bash", + str(SUPERVISOR_SCRIPT), "--timeout", str(timeout), "--cleanup-timeout", @@ -129,45 +128,6 @@ def test_stop_marker_prevents_process_launch(tmp_path): assert not (tmp_path / "should-not-exist").exists() -def test_sigterm_during_spawn_does_not_lose_child_handle(tmp_path): - # Deliver SIGTERM after the real child exists but before Popen returns to run(). - # Isolate signal handlers from pytest, and always reap the test child. - driver = """ -import json, os, runpy, signal, subprocess, sys -runner = runpy.run_path(sys.argv[1]) -spawn = subprocess.Popen -children = [] -def interrupted_spawn(*args, **kwargs): - child = spawn(*args, **kwargs) - children.append(child) - os.kill(os.getpid(), signal.SIGTERM) - return child -subprocess.Popen = interrupted_spawn -try: - summary = runner['_supervise']([sys.executable, '-c', 'import time; time.sleep(60)'], timeout=5, cleanup_timeout=2) - summary['child_alive'] = any(child.poll() is None for child in children) - print(json.dumps(summary)) -finally: - for child in children: - if child.poll() is None: - child.kill() - child.wait() -""" - completed = subprocess.run( - [sys.executable, "-c", driver, process_supervisor.__file__, str(tmp_path)], - capture_output=True, - text=True, - errors="replace", - timeout=10, - check=True, - ) - summary = json.loads(completed.stdout) - assert summary["timed_out"] is False - assert summary["cleanup_confirmed"] is True - assert summary["error"] is None - assert summary["child_alive"] is False - - def test_runtime_environment_does_not_inherit_provider_credentials(tmp_path, monkeypatch): monkeypatch.setenv("SANDBOX_PROVIDER_API_KEY", "must-not-inherit") task_environment = {"HOME": str(tmp_path / "task-home"), "XDG_DATA_HOME": str(tmp_path / "task-data")} @@ -183,47 +143,6 @@ def test_runtime_environment_does_not_inherit_provider_credentials(tmp_path, mon assert event == {"SANDBOX_PROVIDER_API_KEY": None, **task_environment} -def test_lost_spawn_handle_requires_sandbox_teardown(tmp_path): - # A lost handle cannot confirm termination; the owner must destroy the sandbox. - driver = """ -import json, os, runpy, subprocess, sys -runner = runpy.run_path(sys.argv[1]) -spawn = subprocess.Popen -children = [] -def lost_handle(*args, **kwargs): - child = spawn(*args, **kwargs) - children.append(child) - raise OSError('launch handle lost after process creation') -subprocess.Popen = lost_handle -try: - summary = runner['_supervise']([sys.executable, '-c', 'import time; time.sleep(60)'], timeout=5, cleanup_timeout=2) - try: - os.kill(children[0].pid, 0) - summary['child_alive'] = True - except ProcessLookupError: - summary['child_alive'] = False - print(json.dumps(summary)) -finally: - for child in children: - if child.poll() is None: - child.kill() - child.wait() -""" - completed = subprocess.run( - [sys.executable, "-c", driver, process_supervisor.__file__, str(tmp_path)], - capture_output=True, - text=True, - errors="replace", - timeout=10, - check=True, - ) - summary = json.loads(completed.stdout) - assert summary["return_code"] != 0 - assert summary["error"] == "Harness launch failed: launch handle lost after process creation" - assert summary["cleanup_confirmed"] is False - assert summary["child_alive"] is True - - @pytest.mark.skipif(shutil.which("python3.8") is None, reason="Python 3.8 is not installed") def test_python38_worker_captures_output_and_preserves_task_children(tmp_path): try: diff --git a/responses_api_agents/codex_agent/tests/test_sandbox_sessions.py b/responses_api_agents/codex_agent/tests/test_sandbox_sessions.py index 0e1fa0f61fd..fe61b52fc19 100644 --- a/responses_api_agents/codex_agent/tests/test_sandbox_sessions.py +++ b/responses_api_agents/codex_agent/tests/test_sandbox_sessions.py @@ -124,7 +124,7 @@ def publish(self): self.files[f"{self.directory}/events.jsonl"] = self.events async def execute(self, command, **kwargs): - if "exec python3 -I " in command and "process_supervisor.py" in command: + if "exec bash " in command and "process_supervisor.sh" in command: return await self.launch(command=command, **kwargs) if "touch " in command and "stop.request" in command: self.signals.append("SIGTERM") @@ -192,7 +192,7 @@ def test_http_agent_session_flow_runs_codex_in_borrowed_sandbox(setup): assert created.status_code == 200, created.text session_id = created.json()["agent_session_id"] directory = agent._session_records[session_id].state.session.session_dir - assert f"{directory}/process_supervisor.py" not in sandbox.files + assert f"{directory}/process_supervisor.sh" not in sandbox.files installer = f"{directory}/install_codex_runtime.sh" assert installer in sandbox.files for filename in ("sandbox_runner.py", "sandbox_transport.py"): @@ -205,7 +205,7 @@ def test_http_agent_session_flow_runs_codex_in_borrowed_sandbox(setup): assert install_call.kwargs["timeout_s"] == agent.config.sandbox_install_timeout_seconds result = client.post("/ng-rollout/codex-smoke-a2/v1/responses", json={"input": "Fix the code"}) assert result.status_code == 200, result.text - assert f"{directory}/process_supervisor.py" in sandbox.files + assert f"{directory}/process_supervisor.sh" in sandbox.files body = result.json() assert body["status"] == "completed" assert [item["type"] for item in body["output"]] == [ @@ -1774,7 +1774,7 @@ async def test_close_fences_delayed_launch_before_and_after_removing_session(loc commands = [] async def queued_exec(command, **kwargs): - if "--receipt" in command and "process_supervisor.py" in command: + if "--receipt" in command and "process_supervisor.sh" in command: commands.append(command) waiting.set() if failure == "cancel": diff --git a/responses_api_agents/nooa_agent/sandbox_runner.py b/responses_api_agents/nooa_agent/sandbox_runner.py index 5bc08a3165d..0ac75f96260 100644 --- a/responses_api_agents/nooa_agent/sandbox_runner.py +++ b/responses_api_agents/nooa_agent/sandbox_runner.py @@ -83,7 +83,6 @@ async def _stage_activation(self, request: NOOARunRequest) -> SandboxCommand: ) await upload_text(self.sandbox, path=f"{self.directory}/input.json", text=payload.model_dump_json()) return SandboxCommand( - python=self.python, argv=[ self.python, "-I", diff --git a/responses_api_agents/nooa_agent/tests/test_checkpoint_recovery.py b/responses_api_agents/nooa_agent/tests/test_checkpoint_recovery.py index d96a2eea58f..ffbd1eb1be1 100644 --- a/responses_api_agents/nooa_agent/tests/test_checkpoint_recovery.py +++ b/responses_api_agents/nooa_agent/tests/test_checkpoint_recovery.py @@ -53,7 +53,7 @@ async def download(self, source: str, destination: Path) -> None: async def exec( self, command: str, *, cwd: str, timeout_s: float | None, preserve_background_services: bool = False ) -> SandboxExecResult: - if "process_supervisor.py" in command: + if "process_supervisor.sh" in command: # Run the actual entrypoint from this checkout. The installed sandbox's # isolated interpreter and Linux descendant reaper are not under test. with (self.directory / "output.log").open("wb") as stdout: diff --git a/responses_api_agents/nooa_agent/tests/test_sandbox_runner.py b/responses_api_agents/nooa_agent/tests/test_sandbox_runner.py index 7b5d0d69802..4ed54cf2a13 100644 --- a/responses_api_agents/nooa_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/nooa_agent/tests/test_sandbox_runner.py @@ -8,7 +8,7 @@ import pytest -from nemo_gym.agent_utils.process_supervisor import CleanupReceipt +from nemo_gym.agent_utils.supervisor_client import CleanupReceipt from nemo_gym.agent_utils.supervisor_client import HarnessTerminationError from nemo_gym.sandbox.providers.base import SandboxExecResult from responses_api_agents.nooa_agent.result import finalize_run_result @@ -88,8 +88,8 @@ async def test_launch_quotes_paths_runs_in_task_and_preserves_cookies(limit: int assert "'/opt/nooa env/bin/python'" in command assert "-I -m responses_api_agents.nooa_agent.sandbox_entrypoint" in command assert "sandbox_supervisor" not in command - assert "/process_supervisor.py" in command - assert r.directory + "/process_supervisor.py" in s.files + assert "/process_supervisor.sh" in command + assert r.directory + "/process_supervisor.sh" in s.files launch_command = await r._stage_activation(request) assert launch_command.argv[-2:] == [r.directory + "/input.json", r.directory + "/result.json"] assert len(launch_command.argv) == 6 @@ -393,7 +393,7 @@ async def upload(source: Path, destination: str) -> None: with pytest.raises(asyncio.CancelledError): await execution assert not r.session.launch_started - assert all("process_supervisor.py" not in call.args[0] for call in s.exec.await_args_list) + assert all("process_supervisor.sh" not in call.args[0] for call in s.exec.await_args_list) s.disconnect.assert_awaited_once() @@ -551,7 +551,7 @@ async def download(source: str, destination: Path) -> None: await original_download(source, destination) async def execute(command: str, **kwargs) -> SandboxExecResult: - if "process_supervisor.py" in command: + if "process_supervisor.sh" in command: raise original_error return SandboxExecResult(stdout="", stderr="", return_code=0) @@ -697,7 +697,7 @@ async def test_cancelling_active_worker_captures_checkpoint_before_release() -> execution_stopped = asyncio.Event() async def execute(command: str, **kwargs) -> SandboxExecResult: - if "process_supervisor.py" in command: + if "process_supervisor.sh" in command: active.set() try: await asyncio.Event().wait() @@ -896,7 +896,7 @@ async def test_cancel_without_any_artifact_preserves_bare_cancellation() -> None active = asyncio.Event() async def execute(command: str, **kwargs) -> SandboxExecResult: - if "process_supervisor.py" in command: + if "process_supervisor.sh" in command: active.set() await asyncio.Event().wait() elif command.startswith("[ -f "): @@ -984,7 +984,7 @@ async def download_artifact(source: str, destination: Path) -> None: await download(source, destination) async def execute(command: str, **kwargs) -> SandboxExecResult: - if "process_supervisor.py" in command: + if "process_supervisor.sh" in command: raise provider_error return SandboxExecResult("", "", 0) diff --git a/responses_api_agents/opencode_agent/sandbox.py b/responses_api_agents/opencode_agent/sandbox.py index f6a3c81a1be..df41c452e12 100644 --- a/responses_api_agents/opencode_agent/sandbox.py +++ b/responses_api_agents/opencode_agent/sandbox.py @@ -388,7 +388,7 @@ async def _read_export(self, directory: Path) -> str: except OSError as error: logs = await self.session.read_output_log() raise RuntimeError( - format_sandbox_error( + format_error_message( "OpenCode sandbox runner returned no valid result", output=logs, stderr=self.stderr ) ) from error diff --git a/responses_api_agents/opencode_agent/tests/test_sandbox_sessions.py b/responses_api_agents/opencode_agent/tests/test_sandbox_sessions.py index 00163428bbc..dbf6003ca5e 100644 --- a/responses_api_agents/opencode_agent/tests/test_sandbox_sessions.py +++ b/responses_api_agents/opencode_agent/tests/test_sandbox_sessions.py @@ -1770,7 +1770,6 @@ async def test_reported_native_error_is_bounded_without_losing_classification(se def test_explicit_native_model_and_runtime_paths_are_passed_to_the_harness(setup): agent, sandbox = setup agent.config.sandbox_model_id = "gpt-5.5" - agent.config.sandbox_python_executable = "/runtime/python/bin/python3" agent.config.sandbox_path_prefix = ["/runtime/bin"] agent.config.sandbox_executable_path = "/runtime/bin/opencode" agent.config.sandbox_executable_sha256 = "1" * 64 @@ -1778,9 +1777,10 @@ def test_explicit_native_model_and_runtime_paths_are_passed_to_the_harness(setup assert client.post("/v1/agent_sessions", json=seed().model_dump(mode="json")).status_code == 200 result = client.post("/ng-rollout/opencode-smoke-a2/v1/responses", json={"input": "Fix code"}) assert result.status_code == 200, result.text - payload = json.loads(sandbox.files[f"{sandbox.directory}/input.json"]) - config = json.loads(payload["env"]["OPENCODE_CONFIG_CONTENT"]) + params = sandbox.params() + config = json.loads(params["env"]["OPENCODE_CONFIG_CONTENT"]) assert config["model"] == config["small_model"] == "nemo_gym/gpt-5.5" assert set(config["provider"]["nemo_gym"]["models"]) == {"gpt-5.5"} assert config["provider"]["nemo_gym"]["models"]["gpt-5.5"]["reasoning"] is True - assert payload["path_prefix"] == ["/runtime/bin"] + # The runner puts the preinstalled runtime first and keeps the image PATH after it. + assert params["setup"].endswith("PATH=/runtime/bin:$PATH\n") diff --git a/responses_api_agents/pi_agent/README.md b/responses_api_agents/pi_agent/README.md index 223709e700c..1324693120b 100644 --- a/responses_api_agents/pi_agent/README.md +++ b/responses_api_agents/pi_agent/README.md @@ -179,7 +179,7 @@ uploads the supervisor at activation and owns its control files and log access; confirms process cleanup, captures artifacts, then releases the provider connection or owned sandbox. Concurrent closes share that work; missing evidence or failed release remains retryable, and interrupted activations keep captured observations after session files are removed. -`supervisor_client.py` handles controller-side launch/stop commands; `process_supervisor.py` +`supervisor_client.py` handles controller-side launch/stop commands; `process_supervisor.sh` runs inside the sandbox. Benchmark Resources session identity/verdict state stays separate. Missing worker identity or exit diagnostics become `runtime_info_unavailable` or `worker_exit_code_unavailable` observation gaps. Positive cleanup confirmation remains From a298d67fdf8e87a62df485372673e1620cef5e8a Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 26 Oct 2026 15:21:11 -0400 Subject: [PATCH 13/32] fix(agent_utils): keep runner events on old bash and ignore inherited runner names On bash < 4.2 printf has no strftime, so every event was stamped with an empty time and the host parsers dropped them all; fall back to date. Strip a trailing CR before recording a JSON line. Inherited variables named like runner PARAMS (setup, events, stdout, ...) no longer configure the runner or leak into the CLI. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- nemo_gym/agent_utils/harness_runner.sh | 13 ++++++++++--- tests/unit_tests/test_harness_runner.py | 26 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/nemo_gym/agent_utils/harness_runner.sh b/nemo_gym/agent_utils/harness_runner.sh index 79cb7df3502..4c08e547595 100755 --- a/nemo_gym/agent_utils/harness_runner.sh +++ b/nemo_gym/agent_utils/harness_runner.sh @@ -15,6 +15,8 @@ # Writes runtime.json beside PARAMS. shopt -s extglob +# Inherited variables with these names must not configure the runner or leak into the CLI. +unset -v cwd command env stdin stdout stderr setup unset_env events source "$1" || exit 1 printf '{"hostname": "%s", "pid": %d}\n' "$HOSTNAME" "$$" >"$(dirname "$1")/runtime.json" eval "${setup-}" || exit 1 @@ -34,12 +36,17 @@ exec 3<"$stdout" 4>"$events" cli=$! trap 'kill -TERM "$cli" 2>/dev/null' TERM pending="" -emit() { [[ $1 == {* ]] && printf '[%s, %s]\n' "$seen" "$1" >&4; } -# Stamp new complete lines, keeping a partial one for the next pass. Bash < 5 has whole seconds. +emit() { + local line=${1%$'\r'} + [[ $line == {* ]] && printf '[%s, %s]\n' "$seen" "$line" >&4 +} +# Stamp new complete lines, keeping a partial one for the next pass. Bash < 5 has whole seconds; +# bash < 4.2 has no strftime in printf and forks date. stamp() { local line seen=${EPOCHREALTIME/,/.} - [[ -n $seen ]] || printf -v seen '%(%s)T' -1 + [[ -n $seen ]] || printf -v seen '%(%s)T' -1 2>/dev/null + [[ $seen =~ ^[0-9.]+$ ]] || seen=$(date +%s) while IFS= read -r line <&3; do emit "$pending$line" pending="" diff --git a/tests/unit_tests/test_harness_runner.py b/tests/unit_tests/test_harness_runner.py index 1ae2e0ec639..30fdae7b54c 100644 --- a/tests/unit_tests/test_harness_runner.py +++ b/tests/unit_tests/test_harness_runner.py @@ -83,6 +83,32 @@ def test_events_keep_json_objects_with_the_time_they_were_seen(tmp_path: Path) - assert all(before - 1 <= seen <= time.time() for seen, _ in events) + +def test_events_keep_numeric_times_and_crlf_lines_without_epochrealtime(tmp_path: Path) -> None: + # Bash < 5 has no EPOCHREALTIME and bash < 4.2 no printf strftime; both still stamp numbers. + script = 'printf \'{"type": "a"}\\r\\n{"type": "partial"}\'; sleep 0.2' + params = _params(tmp_path, command=["bash", "-c", script], events=str(tmp_path / "events.jsonl")) + result = subprocess.run( + ["bash", "-c", 'unset EPOCHREALTIME; . "$0" "$1"', str(HARNESS_RUNNER_SCRIPT), str(params)], + capture_output=True, + text=True, + timeout=20, + ) + assert result.returncode == 0, result.stderr + events = [json.loads(line) for line in (tmp_path / "events.jsonl").read_text().splitlines()] + assert [event for _, event in events] == [{"type": "a"}, {"type": "partial"}] + assert all(isinstance(seen, (int, float)) and seen > 0 for seen, _ in events) + + +def test_inherited_runner_names_neither_configure_the_runner_nor_reach_the_cli(tmp_path: Path) -> None: + params = _params(tmp_path, command=["bash", "-c", 'echo "${stdout-unset}|${setup-unset}"']) + inherited = {"setup": f"touch {tmp_path}/evaluated", "events": str(tmp_path / "events.jsonl"), "stdout": "leak"} + result = _run(params, env=os.environ | inherited) + assert result.returncode == 0, result.stderr + assert (tmp_path / "stdout.log").read_text() == "unset|unset\n" + assert not (tmp_path / "evaluated").exists() + assert not (tmp_path / "events.jsonl").exists() + @pytest.mark.skipif(sys.platform != "linux", reason="Linux process semantics") def test_events_mode_forwards_term_and_ignores_a_detached_stdout_holder(tmp_path: Path) -> None: # A detached tool keeps stdout open; the runner must still finish when the CLI does. From 626cab1a463c5eec1531af9dc768116990d9ea04 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 26 Oct 2026 15:28:22 -0400 Subject: [PATCH 14/32] fix(agent_utils): stop tools that left the harness group and harden the bash supervisor A stopped CLI's tools that run in their own session (OpenCode and Codex start every shell tool that way) kept writing to the task after a confirmed receipt. Before stopping the harness, the supervisor now records its descendants outside its process group from /proc and kills them, and the groups they lead, with the harness; confirmation waits for them too. Services orphaned before the stop are still left for the sandbox owner. The supervisor also: - reads each /proc/PID/stat whole and never evaluates its fields as arithmetic, so a process name holding a newline cannot run code or hide a group member; - refuses to launch without a readable /proc, and counts unreadable process state as running; - bounds the wait for a KILLed harness and writes a negative receipt instead; - exits on a missing option value or separator instead of looping or running part of the command, and ignores an inherited errexit. Launches render fixed-point seconds, and a deadline shorter than the supervisor's centisecond resolution expires before launch instead of leaving no receipt. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- nemo_gym/agent_utils/process_supervisor.sh | 129 +++++++++++++++--- nemo_gym/agent_utils/sandbox_session.py | 3 +- nemo_gym/agent_utils/supervisor_client.py | 7 +- tests/unit_tests/test_process_supervisor.py | 45 +++++- tests/unit_tests/test_sandbox_session.py | 18 +++ .../test_sandbox_supervisor_client.py | 16 +++ 6 files changed, 192 insertions(+), 26 deletions(-) diff --git a/nemo_gym/agent_utils/process_supervisor.sh b/nemo_gym/agent_utils/process_supervisor.sh index eba1e108a03..98a85232856 100755 --- a/nemo_gym/agent_utils/process_supervisor.sh +++ b/nemo_gym/agent_utils/process_supervisor.sh @@ -2,15 +2,25 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Process supervisor uploaded beside a sandboxed harness. Needs only bash, /proc, sleep and mv. +# Process supervisor uploaded beside a sandboxed harness. Needs only bash, a readable /proc, a sleep +# that accepts fractions of a second, and mv. # # Runs COMMAND in its own process group until it exits, the deadline passes or the stop file -# appears. A stopped harness gets TERM, then the whole group gets KILL. The receipt confirms that -# no group member can still run. Processes that left the group (detached services) are left for -# the sandbox owner. +# appears. A stopped harness gets TERM, then the whole group gets KILL, and so do the descendants +# that were still running in other process groups when it was stopped (tools a CLI started in their +# own session). The receipt confirms that none of them can still run. Processes that left the +# harness before the stop (detached services) are left for the sandbox owner. # # usage: process_supervisor.sh --timeout SECONDS --cleanup-timeout SECONDS --receipt PATH --stop-file PATH -- COMMAND... +# An inherited BASH_ENV or SHELLOPTS must not turn an expected failure into a missing receipt. +set +o errexit +o nounset +o pipefail + +usage() { + echo "usage: process_supervisor.sh --timeout S --cleanup-timeout S --receipt PATH --stop-file PATH -- COMMAND..." >&2 + exit 2 +} + # Seconds as integer centiseconds, rounded down; fails unless positive. centiseconds() { [[ $1 =~ ^([0-9]*)\.?([0-9]*)$ ]] || return 1 @@ -22,21 +32,85 @@ centiseconds() { # Linux uptime in centiseconds, without forking. now() { local uptime - read -r uptime _ /dev/null <"/proc/$1/stat" + [[ $stat == *') '* ]] || return 1 + fields=(${stat##*) }) + [[ ${fields[1]} =~ ^[0-9]+$ && ${fields[2]} =~ ^[0-9]+$ && ${fields[19]} =~ ^[0-9]+$ ]] +} + +# Succeeds while a member of one of the given process groups can still run. Zombies cannot, and +# only their parent can reap them. A process whose state cannot be read counts as running. group_alive() { - local stat fields - for stat in /proc/[0-9]*/stat; do - read -r stat 2>/dev/null <"$stat" || continue - fields=(${stat##*) }) - ((fields[2] == harness)) && [[ ${fields[0]} != [ZX] ]] && return 0 + local entry group + for entry in /proc/[0-9]*; do + if ! stat_fields "${entry#/proc/}"; then + [[ -e $entry ]] && return 0 + continue + fi + [[ ${fields[0]} == [ZX] ]] && continue + for group; do + [[ ${fields[2]} == "$group" ]] && return 0 + done done return 1 } +# Record the harness's descendants outside its process group as "PID START", and the groups they +# lead. They stop being descendants once the harness dies, so this runs before the harness is stopped. +track_descendants() { + local entry pid changed=1 + local -a parent=() group=() start=() inside=() + for entry in /proc/[0-9]*; do + pid=${entry#/proc/} + [[ $pid =~ ^[0-9]+$ ]] && stat_fields "$pid" || continue + parent[pid]=${fields[1]} group[pid]=${fields[2]} start[pid]=${fields[19]} + done + inside[harness]=1 + while ((changed)); do + changed=0 + for pid in "${!parent[@]}"; do + [[ -z ${inside[pid]} && -n ${inside[parent[pid]]} ]] || continue + inside[pid]=1 changed=1 + done + done + for pid in "${!inside[@]}"; do + [[ ${group[pid]} == "$harness" ]] && continue + tracked+=("$pid ${start[pid]}") + [[ ${group[pid]} == "$pid" ]] && tracked_groups+=("$pid") + done +} + +# Succeeds while a tracked descendant (same PID and start time) can still run. +tracked_alive() { + local entry + for entry in "${tracked[@]}"; do + stat_fields "${entry% *}" && [[ ${fields[19]} == "${entry#* }" && ${fields[0]} != [ZX] ]] && return 0 + done + return 1 +} + +kill_harness() { + local entry group + kill -KILL -- "-$harness" 2>/dev/null + for entry in "${tracked[@]}"; do + stat_fields "${entry% *}" && [[ ${fields[19]} == "${entry#* }" ]] && kill -KILL "${entry% *}" 2>/dev/null + done + for group in "${tracked_groups[@]}"; do + kill -KILL -- "-$group" 2>/dev/null + done + return 0 +} + stopped() { ((stopping)) || [[ -e $stop_file ]]; } write_receipt() { @@ -44,7 +118,8 @@ write_receipt() { "$1" "$return_code" "$2" "$timed_out" >"$receipt.tmp" && mv -f "$receipt.tmp" "$receipt" } -while [[ $1 == --* && $1 != -- ]]; do +while (($#)) && [[ $1 != -- ]]; do + (($# >= 2)) || usage case $1 in --timeout) timeout=$(centiseconds "$2") || { echo "--timeout must be finite and positive" >&2; exit 2; } ;; --cleanup-timeout) cleanup=$(centiseconds "$2") || { echo "--cleanup-timeout must be finite and positive" >&2; exit 2; } ;; @@ -54,18 +129,20 @@ while [[ $1 == --* && $1 != -- ]]; do esac shift 2 done +[[ $1 == -- ]] || usage shift -[[ -n ${timeout-} && -n ${cleanup-} && -n ${receipt-} && -n ${stop_file-} && $# -gt 0 ]] || { - echo "usage: process_supervisor.sh --timeout S --cleanup-timeout S --receipt PATH --stop-file PATH -- COMMAND..." >&2 - exit 2 -} +[[ -n ${timeout-} && -n ${cleanup-} && -n ${receipt-} && -n ${stop_file-} && $# -gt 0 ]] || usage -return_code=null timed_out=false stopping=0 +return_code=null timed_out=false stopping=0 tracked=() tracked_groups=() trap 'stopping=1' TERM if stopped; then write_receipt true null # Nothing was launched. exit 0 fi +if ! now; then + write_receipt true '"Sandbox process supervision requires a readable /proc"' # Nothing was launched. + exit 1 +fi # Job control gives the harness its own process group. set -m @@ -81,6 +158,7 @@ while kill -0 "$harness" 2>/dev/null && ! stopped && ((NOW < deadline)); do done if kill -0 "$harness" 2>/dev/null; then stopped || timed_out=true + track_descendants kill -TERM "$harness" 2>/dev/null grace=$((NOW + cleanup)) while kill -0 "$harness" 2>/dev/null && ((NOW < grace)); do @@ -90,13 +168,24 @@ if kill -0 "$harness" 2>/dev/null; then fi trap '' TERM -kill -KILL -- "-$harness" 2>/dev/null +kill_harness +# A process in uninterruptible sleep outlives KILL; report it rather than wait without bound. +now +deadline=$((NOW + cleanup)) +while kill -0 "$harness" 2>/dev/null; do + now + if ((NOW >= deadline)); then + write_receipt false '"Harness termination was not confirmed: the harness process did not exit"' + exit 1 + fi + sleep 0.01 +done wait "$harness" return_code=$? now deadline=$((NOW + cleanup)) -while group_alive; do - kill -KILL -- "-$harness" 2>/dev/null +while group_alive "$harness" "${tracked_groups[@]}" || tracked_alive; do + kill_harness now if ((NOW >= deadline)); then write_receipt false '"Harness termination was not confirmed: its process group still runs"' diff --git a/nemo_gym/agent_utils/sandbox_session.py b/nemo_gym/agent_utils/sandbox_session.py index bce08431686..a69e9429da7 100644 --- a/nemo_gym/agent_utils/sandbox_session.py +++ b/nemo_gym/agent_utils/sandbox_session.py @@ -184,7 +184,8 @@ async def execute( raise RuntimeError(f"{self.harness} sandbox session closed before launch") if deadline is not None: timeout = min(timeout, deadline - monotonic()) - if timeout <= 0: + # The supervisor counts whole centiseconds; less than one cannot run the harness. + if timeout < 0.01: raise TimeoutError(f"{self.harness} activation deadline expired before launch") cleanup_timeout, provider_timeout = supervision_timeouts(timeout=timeout, close_timeout=close_timeout) launch = supervised_launch_command( diff --git a/nemo_gym/agent_utils/supervisor_client.py b/nemo_gym/agent_utils/supervisor_client.py index 451ca4b1338..d6fe80004ac 100644 --- a/nemo_gym/agent_utils/supervisor_client.py +++ b/nemo_gym/agent_utils/supervisor_client.py @@ -120,6 +120,11 @@ def supervision_timeouts(*, timeout: float, close_timeout: float) -> tuple[float return cleanup_timeout, exec_timeout(timeout=timeout, cleanup_timeout=cleanup_timeout) +def _seconds(value: float) -> str: + """Seconds as the supervisor parses them: fixed-point, at least one centisecond.""" + return str(value) if 0.01 <= value < 1e16 else f"{max(value, 0.01):.2f}" + + def supervised_launch_command( *, session_dir: str, @@ -143,7 +148,7 @@ def supervised_launch_command( f"ln -s launch {claim_path} 2>/dev/null || exit 0; " f"echo $$ > {quote(f'{session_dir}/{SUPERVISOR_PID_FILE}')} && " f"exec bash {supervisor} " - f"--timeout {timeout} --cleanup-timeout {cleanup_timeout} " + f"--timeout {_seconds(timeout)} --cleanup-timeout {_seconds(cleanup_timeout)} " f"--stop-file {quote(f'{session_dir}/{STOP_REQUEST_FILE}')} " f"--receipt {quote(f'{session_dir}/{CLEANUP_RECEIPT_FILE}')} -- {join(command)} " f">{quote(f'{session_dir}/{OUTPUT_LOG_FILE}')} 2>&1" diff --git a/tests/unit_tests/test_process_supervisor.py b/tests/unit_tests/test_process_supervisor.py index 2524d76975d..281458e0d0c 100644 --- a/tests/unit_tests/test_process_supervisor.py +++ b/tests/unit_tests/test_process_supervisor.py @@ -4,6 +4,7 @@ """Exercise the uploaded bash supervisor as the sandbox runs it, without Gym imports.""" import json +import os import shutil import subprocess import sys @@ -59,6 +60,39 @@ def test_supervisor_requires_a_command_before_writing_a_receipt(tmp_path: Path) assert not (tmp_path / "cleanup.json").exists() +@pytest.mark.parametrize("missing", ["a value", "the separator"]) +def test_malformed_arguments_neither_hang_nor_run_part_of_the_command(tmp_path: Path, missing: str) -> None: + options = ["--timeout", "1", "--cleanup-timeout", "1", "--receipt", "cleanup.json", "--stop-file", "stop"] + arguments = [*options, "--receipt"] if missing == "a value" else [*options, "touch", "ran"] + result = subprocess.run(["bash", SUPERVISOR, *arguments], capture_output=True, text=True, timeout=10, cwd=tmp_path) + assert result.returncode == 2, result.stderr + assert not (tmp_path / "ran").exists() and not (tmp_path / "cleanup.json").exists() + + +@linux_only +def test_process_names_are_never_evaluated(tmp_path: Path) -> None: + # The kernel prints names unescaped. A bystander named like an arithmetic expression must not run + # code through the supervisor's /proc scan. + (tmp_path / "p").write_text(f"#!/bin/sh\ntouch {tmp_path}/evaluated\n") + (tmp_path / "p").chmod(0o755) + name = repr(b") a b c[$(p)]\n") + bystander = subprocess.Popen( + [sys.executable, "-c", f"import ctypes,time; ctypes.CDLL(None).prctl(15, {name}, 0, 0, 0); time.sleep(30)"] + ) + try: + deadline = time.monotonic() + 5 + while Path(f"/proc/{bystander.pid}/comm").read_text().strip() != ") a b c[$(p)]": + assert time.monotonic() < deadline + time.sleep(0.01) + result = _supervise(tmp_path, "true", env=os.environ | {"PATH": f"{tmp_path}:{os.environ['PATH']}"}) + assert result.returncode == 0, result.stderr + assert _receipt(tmp_path)["cleanup_confirmed"] is True + assert not (tmp_path / "evaluated").exists() + finally: + bystander.kill() + bystander.wait() + + @linux_only @pytest.mark.parametrize("command, return_code", [("exit 0", 0), ("exit 7", 7), ("kill -KILL $$", 137)]) def test_natural_exit_reports_the_harness_status(tmp_path: Path, command: str, return_code: int) -> None: @@ -252,24 +286,27 @@ def test_only_detached_services_survive_harness_group_termination(tmp_path: Path @linux_only +@pytest.mark.parametrize("detached", [False, True]) @pytest.mark.parametrize("cooperative", [True, False]) @pytest.mark.parametrize("ending", ["timeout", "cancel"]) def test_foreground_tool_cannot_mutate_task_after_confirmed_receipt( - tmp_path: Path, cooperative: bool, ending: str + tmp_path: Path, cooperative: bool, ending: str, detached: bool ) -> None: # A separate owner reaps any escaped process even when this regression fails. driver = r""" import ctypes, json, os, pathlib, signal, subprocess, sys, time assert ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) == 0 supervisor, root, cooperative, ending = sys.argv[1], pathlib.Path(sys.argv[2]), sys.argv[3] == 'True', sys.argv[4] +detached = sys.argv[5] == 'True' writer = "import pathlib,time\nwhile not pathlib.Path('release').exists(): time.sleep(.01)\npathlib.Path('mutated').touch()" -# The writer is orphaned when the tool dies, but stays in the harness group. +# The writer is orphaned when the tool dies. It stays in the harness group, or in the group of a +# tool the CLI started in its own session, as OpenCode and Codex do. tool = ("import pathlib,subprocess,sys,time\n" "child=subprocess.Popen([sys.executable,'-c'," + repr(writer) + "])\n" "pathlib.Path('writer.pid').write_text(str(child.pid))\nchild.wait()\n") cli = ("import pathlib,signal,subprocess,sys\n" + ("" if cooperative else "signal.signal(signal.SIGTERM, signal.SIG_IGN)\n") - + "child=subprocess.Popen([sys.executable,'-c'," + repr(tool) + "])\n" + + "child=subprocess.Popen([sys.executable,'-c'," + repr(tool) + "], start_new_session=" + str(detached) + ")\n" + "pathlib.Path('tool.pid').write_text(str(child.pid))\nchild.wait()\n") process = None try: @@ -303,7 +340,7 @@ def test_foreground_tool_cannot_mutate_task_after_confirmed_receipt( except ChildProcessError: pass """ result = subprocess.run( - [sys.executable, "-I", "-c", driver, SUPERVISOR, str(tmp_path), str(cooperative), ending], + [sys.executable, "-I", "-c", driver, SUPERVISOR, str(tmp_path), str(cooperative), ending, str(detached)], capture_output=True, text=True, errors="replace", diff --git a/tests/unit_tests/test_sandbox_session.py b/tests/unit_tests/test_sandbox_session.py index f4406543d5c..c1f58fcc658 100644 --- a/tests/unit_tests/test_sandbox_session.py +++ b/tests/unit_tests/test_sandbox_session.py @@ -233,6 +233,24 @@ async def stage(): await session.close(timeout=1) +async def test_deadline_shorter_than_supervisor_resolution_never_launches(session, receipt_reader): + from time import monotonic + + collect = AsyncMock() + with pytest.raises(TimeoutError, match="expired before launch"): + await session.execute( + stage_activation=AsyncMock(return_value=SandboxCommand(argv=["true"])), + collect=collect, + timeout=0.004, + close_timeout=1, + deadline=monotonic() + 10, + ) + assert not session.launch_started + session.sandbox.exec.assert_not_awaited() + collect.assert_not_awaited() + await session.close(timeout=1) + + async def test_close_during_input_preparation_prevents_launch(session, receipt_reader): preparing = asyncio.Event() diff --git a/tests/unit_tests/test_sandbox_supervisor_client.py b/tests/unit_tests/test_sandbox_supervisor_client.py index c8427aa4c5d..9c83d4ab3b3 100644 --- a/tests/unit_tests/test_sandbox_supervisor_client.py +++ b/tests/unit_tests/test_sandbox_supervisor_client.py @@ -4,6 +4,7 @@ import asyncio import json +import re import shutil import sys from pathlib import Path @@ -132,6 +133,21 @@ async def test_stop_wins_claim_and_fences_delayed_launch(session, caplog): assert receipt["return_code"] is None +@pytest.mark.parametrize("seconds", [5e-05, 0.004, 1e16]) +async def test_launch_renders_seconds_the_supervisor_accepts(session, seconds): + # Python renders tiny and huge floats with exponents, which the supervisor would reject after the claim. + sandbox, session_dir = session + shutil.copyfile(SUPERVISOR_SCRIPT, session_dir / SUPERVISOR_FILE) + (session_dir / STOP_REQUEST_FILE).touch() + command = supervised_launch_command( + session_dir=str(session_dir), command=["true"], timeout=seconds, cleanup_timeout=seconds + ) + rendered = re.search(r"--timeout (\S+) --cleanup-timeout (\S+) ", command).groups() + assert all(re.fullmatch(r"[0-9]+\.[0-9]+", value) for value in rendered), rendered + assert (await sandbox.exec(command)).return_code == 0 + assert json.loads((session_dir / CLEANUP_RECEIPT_FILE).read_text())["cleanup_confirmed"] is True + + async def test_missing_receipt_after_launch_is_not_cleanup_confirmation(session): sandbox, session_dir = session (session_dir / LAUNCH_CLAIM_FILE).symlink_to("launch") From 6ca344c5d67b9d6c6bae92e972625ea0360dc15d Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 26 Oct 2026 15:29:05 -0400 Subject: [PATCH 15/32] fix(agent_utils): give the events-mode CLI default interrupts and no runner descriptors Without job control bash starts an async command with INT and QUIT ignored, and the CLI's tools inherited that. The CLI also inherited the runner's read and events descriptors, so a tool could write raw lines into events.jsonl. Reset the two signals (bash 5 honours the reset) and close both descriptors for the CLI. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- nemo_gym/agent_utils/harness_runner.sh | 4 +++- tests/unit_tests/test_harness_runner.py | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/nemo_gym/agent_utils/harness_runner.sh b/nemo_gym/agent_utils/harness_runner.sh index 4c08e547595..bad3a64371b 100755 --- a/nemo_gym/agent_utils/harness_runner.sh +++ b/nemo_gym/agent_utils/harness_runner.sh @@ -32,7 +32,9 @@ cd "$cwd" || exit 1 # Stamp stdout lines as they appear. A file, unlike a pipe, cannot be held open by a detached tool. : >"$stdout" exec 3<"$stdout" 4>"$events" -"${command[@]}" <"$stdin" >>"$stdout" 2>"$stderr" & +# Without job control an async command ignores INT and QUIT, which its tools would inherit; the +# runner's own descriptors stay out of the CLI. +{ trap - INT QUIT; exec "${command[@]}" <"$stdin" >>"$stdout" 2>"$stderr" 3<&- 4>&-; } & cli=$! trap 'kill -TERM "$cli" 2>/dev/null' TERM pending="" diff --git a/tests/unit_tests/test_harness_runner.py b/tests/unit_tests/test_harness_runner.py index 30fdae7b54c..f6bb10947ea 100644 --- a/tests/unit_tests/test_harness_runner.py +++ b/tests/unit_tests/test_harness_runner.py @@ -109,6 +109,28 @@ def test_inherited_runner_names_neither_configure_the_runner_nor_reach_the_cli(t assert not (tmp_path / "evaluated").exists() assert not (tmp_path / "events.jsonl").exists() + +def test_events_mode_keeps_runner_descriptors_out_of_the_cli(tmp_path: Path) -> None: + script = 'echo \'{"type": "injected"}\' >&4; echo \'{"type": "ok"}\'' + params = _params(tmp_path, command=["bash", "-c", script], events=str(tmp_path / "events.jsonl")) + result = _run(params) + assert result.returncode == 0, result.stderr + events = [json.loads(line)[1] for line in (tmp_path / "events.jsonl").read_text().splitlines()] + assert events == [{"type": "ok"}] + assert "Bad file descriptor" in (tmp_path / "stderr.log").read_text() + + +@pytest.mark.skipif( + subprocess.run(["bash", "-c", "((BASH_VERSINFO[0] >= 5))"]).returncode != 0, reason="bash 5 restores INT" +) +def test_events_mode_cli_keeps_default_interrupt_handling(tmp_path: Path) -> None: + code = "import json, signal; print(json.dumps({'int': str(signal.getsignal(signal.SIGINT))}))" + params = _params(tmp_path, command=[sys.executable, "-c", code], events=str(tmp_path / "events.jsonl")) + result = _run(params) + assert result.returncode == 0, result.stderr + (event,) = [json.loads(line)[1] for line in (tmp_path / "events.jsonl").read_text().splitlines()] + assert "default_int_handler" in event["int"] + @pytest.mark.skipif(sys.platform != "linux", reason="Linux process semantics") def test_events_mode_forwards_term_and_ignores_a_detached_stdout_holder(tmp_path: Path) -> None: # A detached tool keeps stdout open; the runner must still finish when the CLI does. From fa254f1c032cbc9cc9a481797628902aaaeae9bf Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 26 Oct 2026 15:32:55 -0400 Subject: [PATCH 16/32] fix(agent): make installer locks without flock crash-safe and portable to older bash Without flock, the Claude installer never reclaimed the lock of a killed installer, and OpenCode's reclaim could delete a lock another installer had just taken (or skip an owner killed before it wrote its PID). Both now take a symlink that names the owner by PID and start time, and remove a dead owner's lock only under a second lock after re-reading it. The Claude installer lowercases the trusted digest without bash 4 expansions. The Pi installer no longer fails under set -u on bash < 4.4 when it needs no packages, and checks every default musl library directory for libstdc++. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- .../install_claude_runtime.sh | 34 ++++++++-- .../tests/test_install_claude_runtime.py | 64 ++++++++++++++++++- .../install_opencode_runtime.sh | 29 ++++++--- .../tests/test_install_runtime.py | 39 +++++++---- .../pi_agent/install_pi_runtime.sh | 7 +- .../pi_agent/tests/test_install_runtime.py | 4 +- 6 files changed, 148 insertions(+), 29 deletions(-) diff --git a/responses_api_agents/claude_code_agent/install_claude_runtime.sh b/responses_api_agents/claude_code_agent/install_claude_runtime.sh index 51e00d7e0e9..603ff79a3ee 100644 --- a/responses_api_agents/claude_code_agent/install_claude_runtime.sh +++ b/responses_api_agents/claude_code_agent/install_claude_runtime.sh @@ -8,12 +8,12 @@ # Prints the version of the verified runtime. --check-only exits 3 when the runtime is absent or unhealthy. set -euo pipefail -version="" runtime="" expected="" source="" check_only="" compressed="" temporary="" lock_dir="" +version="" runtime="" expected="" source="" check_only="" compressed="" temporary="" lock_link="" while (($#)); do case $1 in --version) version=$2 && shift 2 ;; --runtime) runtime=$2 && shift 2 ;; - --sha256) expected=${2,,} && shift 2 ;; + --sha256) expected=$(tr A-F a-f <<<"$2") && shift 2 ;; --source) source=$2 && shift 2 ;; --check-only) check_only=1 && shift ;; --gzip) compressed=1 && shift ;; @@ -34,15 +34,39 @@ verify() { echo "$actual" } -trap 'rm -f -- "$temporary"; [[ -z $lock_dir ]] || rmdir -- "$lock_dir"' EXIT +# A process as "PID START" from /proc, so a reused PID never matches its former owner; PID alone +# where /proc is missing. +process_identity() { + local stat= + IFS= read -r -d '' stat 2>/dev/null <"/proc/$1/stat" || true + if [[ $stat == *') '* ]]; then + set -- "$1" ${stat##*) } + echo "$1 ${21}" + elif kill -0 "$1" 2>/dev/null; then + echo "$1" + fi +} + +trap 'rm -f -- "$temporary"; [[ -z $lock_link ]] || rm -f -- "$lock_link"' EXIT mkdir -p -- "$(dirname -- "$runtime")" # Concurrent sessions in one sandbox share the runtime; only one may replace it. if command -v flock >/dev/null 2>&1; then exec 9>>"$runtime.lock" flock 9 else - until mkdir -- "$runtime.lock.d" 2>/dev/null; do sleep 0.1; done - lock_dir=$runtime.lock.d + # Without flock, a symlink names the owner. A lock whose owner is gone is removed under a second + # lock and only after re-reading it, so a lock taken in the meantime is never removed. + owner=$(process_identity $$) + until ln -s "$owner" "$runtime.lock.owner" 2>/dev/null; do + current=$(readlink "$runtime.lock.owner" 2>/dev/null) || current= + if [[ -n $current && $(process_identity "${current%% *}") != "$current" ]] && + mkdir "$runtime.lock.reclaim" 2>/dev/null; then + [[ $(readlink "$runtime.lock.owner" 2>/dev/null) != "$current" ]] || rm -f "$runtime.lock.owner" + rmdir "$runtime.lock.reclaim" + fi + sleep 0.1 + done + lock_link=$runtime.lock.owner fi verify "$runtime/claude" 2>/dev/null && exit 0 diff --git a/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py b/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py index b1ccb242ab3..04ac6e8be63 100644 --- a/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py +++ b/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py @@ -5,6 +5,7 @@ import hashlib import shutil import subprocess +import time from concurrent.futures import ThreadPoolExecutor from pathlib import Path @@ -15,8 +16,12 @@ INSTALLER = Path(host_runtime.__file__).with_name("install_claude_runtime.sh") pytestmark = pytest.mark.skipif( - shutil.which("bash") is None or shutil.which("sha256sum") is None, reason="bash and sha256sum are required" + any(shutil.which(tool) is None for tool in ("bash", "sha256sum", "timeout")), + reason="bash, sha256sum and timeout are required", ) +# The installer's tools without flock, for its fallback lock. +FALLBACK_TOOLS = ("bash", "sha256sum", "timeout", "mktemp", "gzip", "cat", "chmod", "mv", "rm", "mkdir", "dirname") +FALLBACK_TOOLS += ("ln", "readlink", "rmdir", "sleep", "tr", "kill") @pytest.fixture @@ -93,3 +98,60 @@ def test_installer_verifies_decompressed_native_bytes_and_check_only_reports_abs assert (runtime / "claude").read_bytes() == source.read_bytes() checked = install(runtime, "--sha256", checksum, "--check-only") assert (checked.returncode, checked.stdout) == (0, "2.1.108 (Claude Code)\n") + + +def _without_flock(tmp_path: Path) -> dict[str, str]: + bindir = tmp_path / "bin" + bindir.mkdir() + for tool in FALLBACK_TOOLS: + if shutil.which(tool) is None: + continue + (bindir / tool).symlink_to(shutil.which(tool)) + return {"PATH": str(bindir)} + + +def _identity(pid: int) -> str: + stat = Path(f"/proc/{pid}/stat") + return f"{pid} {stat.read_text().rsplit(') ', 1)[1].split()[19]}" if stat.exists() else str(pid) + + +def test_lock_left_by_a_killed_installer_is_reclaimed_without_flock(tmp_path, binary): + source, checksum = binary + exited = subprocess.Popen(["true"]) + exited.wait() + (tmp_path / "runtime.lock.owner").symlink_to(_identity(exited.pid)) + result = subprocess.run( + ["bash", str(INSTALLER), "--version", "2.1.108", "--runtime", str(tmp_path / "runtime")] + + ["--sha256", checksum, "--source", str(source)], + env=_without_flock(tmp_path), + capture_output=True, + text=True, + timeout=30, + ) + assert (result.returncode, result.stdout) == (0, "2.1.108 (Claude Code)\n"), result.stderr + assert not (tmp_path / "runtime.lock.owner").is_symlink() + + +def test_installer_without_flock_waits_for_a_live_lock_owner(tmp_path, binary): + source, checksum = binary + holder = subprocess.Popen(["sleep", "30"]) + try: + (tmp_path / "runtime.lock.owner").symlink_to(_identity(holder.pid)) + installer = subprocess.Popen( + ["bash", str(INSTALLER), "--version", "2.1.108", "--runtime", str(tmp_path / "runtime")] + + ["--sha256", checksum, "--source", str(source)], + env=_without_flock(tmp_path), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + time.sleep(0.5) + assert installer.poll() is None + holder.kill() + holder.wait() + # The owner is gone, so its lock is reclaimed and the install proceeds. + stdout, stderr = installer.communicate(timeout=20) + assert (installer.returncode, stdout) == (0, "2.1.108 (Claude Code)\n"), stderr + finally: + holder.kill() + holder.wait() diff --git a/responses_api_agents/opencode_agent/install_opencode_runtime.sh b/responses_api_agents/opencode_agent/install_opencode_runtime.sh index 0367d189c08..398c459fa8c 100644 --- a/responses_api_agents/opencode_agent/install_opencode_runtime.sh +++ b/responses_api_agents/opencode_agent/install_opencode_runtime.sh @@ -11,23 +11,36 @@ path_prefix=${5:-} if [ -n "$path_prefix" ]; then export PATH="$path_prefix:$PATH"; fi [ "$(uname -s)" = Linux ] || { echo 'OpenCode sandbox execution requires a Linux task image' >&2; exit 1; } +# A process as "PID START" from /proc, so a reused PID never matches its former owner; PID alone +# where /proc is missing. +process_identity() { + local stat= + IFS= read -r -d '' stat 2>/dev/null <"/proc/$1/stat" || true + if [[ $stat == *') '* ]]; then + set -- "$1" ${stat##*) } + echo "$1 ${21}" + elif kill -0 "$1" 2>/dev/null; then + echo "$1" + fi +} # Serialize preparation of the version-scoped cache in a shared task sandbox. if command -v flock >/dev/null 2>&1; then # The kernel releases the lock when the installer exits, however it exits. exec 9>"$runtime.lock" flock 9 else - # The lock directory names its owner, so the lock of a killed installer can be reclaimed. - until mkdir "$runtime.lock.d" 2>/dev/null; do - owner= - { read -r owner <"$runtime.lock.d/pid"; } 2>/dev/null || true - if [ -n "$owner" ] && ! kill -0 "$owner" 2>/dev/null; then - rm -rf "$runtime.lock.d" + # Without flock, a symlink names the owner. A lock whose owner is gone is removed under a second + # lock and only after re-reading it, so a lock taken in the meantime is never removed. + owner=$(process_identity $$) + until ln -s "$owner" "$runtime.lock.owner" 2>/dev/null; do + current=$(readlink "$runtime.lock.owner" 2>/dev/null) || current= + if [[ -n $current && $(process_identity "${current%% *}") != "$current" ]] && mkdir "$runtime.lock.reclaim" 2>/dev/null; then + [[ $(readlink "$runtime.lock.owner" 2>/dev/null) != "$current" ]] || rm -f "$runtime.lock.owner" + rmdir "$runtime.lock.reclaim" fi sleep 0.1 done - echo "$$" >"$runtime.lock.d/pid" - trap 'rm -rf "$runtime.lock.d"' EXIT + trap 'rm -f "$runtime.lock.owner"' EXIT fi case "$(uname -m)" in x86_64) arch=x64-baseline ;; diff --git a/responses_api_agents/opencode_agent/tests/test_install_runtime.py b/responses_api_agents/opencode_agent/tests/test_install_runtime.py index c26929ba695..fc70796c9e7 100644 --- a/responses_api_agents/opencode_agent/tests/test_install_runtime.py +++ b/responses_api_agents/opencode_agent/tests/test_install_runtime.py @@ -21,7 +21,8 @@ def sandbox(tmp_path: Path) -> tuple[Path, dict[str, str]]: """Run the real installer with an isolated PATH and no access to the host package manager.""" bindir = tmp_path / "bin" bindir.mkdir() - for name in ("bash", "uname", "getconf", "mkdir", "cp", "tar", "gzip", "chmod", "rm", "sleep"): + tools = ("bash", "uname", "getconf", "mkdir", "cp", "tar", "gzip", "chmod", "rm", "sleep", "ln", "readlink", "rmdir") + for name in (*tools, "mktemp", "sha256sum", "mv"): executable = shutil.which(name) if executable is None: pytest.skip(f"Installer test requires {name}") @@ -223,19 +224,34 @@ def cache_runtime(root: Path) -> None: (root / "runtime/opencode").chmod(0o755) -def test_lock_of_a_killed_installer_is_reclaimed(sandbox: tuple[Path, dict[str, str]]) -> None: +def identity(pid: int) -> str: + """The lock owner as the installer records it: PID and start time.""" + return f"{pid} {Path(f'/proc/{pid}/stat').read_text().rsplit(') ', 1)[1].split()[19]}" + + +@pytest.mark.parametrize("owner", ["exited", "reused"]) +def test_lock_of_a_killed_installer_is_reclaimed(sandbox: tuple[Path, dict[str, str]], owner: str) -> None: root, env = sandbox cache_runtime(root) - killed = subprocess.Popen(["true"]) - killed.wait() - (root / "runtime.lock.d").mkdir() - (root / "runtime.lock.d/pid").write_text(f"{killed.pid}\n") - result = run_installer(root, env) - assert result.returncode == 0, result.stderr - assert not (root / "runtime.lock.d").exists() + process = subprocess.Popen(["true"] if owner == "exited" else ["sleep", "30"]) + try: + if owner == "exited": + process.wait() + recorded = f"{process.pid} 1" + else: + # A live process that reuses the owner's PID started later, so it does not hold the lock. + recorded = f"{process.pid} {int(identity(process.pid).split()[1]) - 1}" + (root / "runtime.lock.owner").symlink_to(recorded) + result = run_installer(root, env) + assert result.returncode == 0, result.stderr + assert not (root / "runtime.lock.owner").is_symlink() + assert not (root / "runtime.lock.reclaim").exists() + finally: + process.kill() + process.wait() -@pytest.mark.parametrize("lock", ["mkdir", "flock"]) +@pytest.mark.parametrize("lock", ["owner", "flock"]) def test_concurrent_installers_wait_for_the_lock_holder(sandbox: tuple[Path, dict[str, str]], lock: str) -> None: root, env = sandbox cache_runtime(root) @@ -250,8 +266,7 @@ def test_concurrent_installers_wait_for_the_lock_holder(sandbox: tuple[Path, dic time.sleep(0.01) else: holder = subprocess.Popen(["sleep", "30"]) - (root / "runtime.lock.d").mkdir() - (root / "runtime.lock.d/pid").write_text(f"{holder.pid}\n") + (root / "runtime.lock.owner").symlink_to(identity(holder.pid)) try: installer = subprocess.Popen( [str(root / "bin/bash"), str(root / "install.sh"), str(root / "runtime"), "1.17.11"], env=env diff --git a/responses_api_agents/pi_agent/install_pi_runtime.sh b/responses_api_agents/pi_agent/install_pi_runtime.sh index e56b8045ac2..3f081cd6435 100644 --- a/responses_api_agents/pi_agent/install_pi_runtime.sh +++ b/responses_api_agents/pi_agent/install_pi_runtime.sh @@ -54,10 +54,13 @@ if [ ! -f "$runtime/ready" ]; then if ! command -v curl >/dev/null 2>&1; then packages+=(curl ca-certificates) fi - if [ "$platform" = linux-x64-musl ] && [ ! -e /usr/lib/libstdc++.so.6 ] && [ ! -e /lib/libstdc++.so.6 ]; then + # musl searches /lib, /usr/local/lib and /usr/lib by default. + if [ "$platform" = linux-x64-musl ] && [ ! -e /lib/libstdc++.so.6 ] && [ ! -e /usr/local/lib/libstdc++.so.6 ] \ + && [ ! -e /usr/lib/libstdc++.so.6 ]; then packages+=(libstdc++) fi - install_packages "${packages[@]}" + # Bash < 4.4 treats an empty array as unbound under set -u. + install_packages ${packages[@]+"${packages[@]}"} mkdir -p "$runtime" cd "$runtime" # gzip and sha256sum -c work with Alpine's BusyBox; xz and --strict may not. diff --git a/responses_api_agents/pi_agent/tests/test_install_runtime.py b/responses_api_agents/pi_agent/tests/test_install_runtime.py index c1c155535ce..3064bec963a 100644 --- a/responses_api_agents/pi_agent/tests/test_install_runtime.py +++ b/responses_api_agents/pi_agent/tests/test_install_runtime.py @@ -13,7 +13,9 @@ INSTALLER = Path(__file__).parents[1] / "install_pi_runtime.sh" pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Pi requires Linux") # On musl the installer looks for the task's C++ library where musl's loader does; this host may lack it there. -MUSL_LIBSTDCXX = [] if any(Path(d, "libstdc++.so.6").exists() for d in ("/lib", "/usr/lib")) else ["libstdc++"] +MUSL_LIBSTDCXX = ( + [] if any(Path(d, "libstdc++.so.6").exists() for d in ("/lib", "/usr/local/lib", "/usr/lib")) else ["libstdc++"] +) @pytest.fixture From 28448d10a5bdb415887aa0793eda4e098c0f2ad9 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 26 Oct 2026 15:32:55 -0400 Subject: [PATCH 17/32] fix(opencode,kilocode): close host-side SQLite connections sqlite3's context manager commits but does not close, so a failed export kept the copied databases open while the snapshot directory was removed. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- responses_api_agents/kilocode_agent/observability.py | 5 +++-- responses_api_agents/opencode_agent/snapshot.py | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/responses_api_agents/kilocode_agent/observability.py b/responses_api_agents/kilocode_agent/observability.py index 85baca792a4..a20b99dd677 100644 --- a/responses_api_agents/kilocode_agent/observability.py +++ b/responses_api_agents/kilocode_agent/observability.py @@ -4,6 +4,7 @@ import json import sqlite3 +from contextlib import closing from pathlib import Path from nemo_gym.openai_utils import ( @@ -21,8 +22,8 @@ def read_kilo_snapshot( directory: Path, *, fallback_invocation_id: str ) -> tuple[AgentObservationBundle, NeMoGymResponseUsage | None]: """Read ``kilo.db`` copied with its WAL files, through one consistent copy that includes WAL contents.""" - with sqlite3.connect(directory / "kilo.db") as source: - with sqlite3.connect(directory / "observations.db") as destination: + with closing(sqlite3.connect(directory / "kilo.db")) as source: + with closing(sqlite3.connect(directory / "observations.db")) as destination: source.backup(destination) return read_kilo_observations(directory / "observations.db", fallback_invocation_id=fallback_invocation_id) diff --git a/responses_api_agents/opencode_agent/snapshot.py b/responses_api_agents/opencode_agent/snapshot.py index dc2a6eb29d2..d9fa8856d53 100644 --- a/responses_api_agents/opencode_agent/snapshot.py +++ b/responses_api_agents/opencode_agent/snapshot.py @@ -6,6 +6,7 @@ import logging import sqlite3 from collections.abc import Collection +from contextlib import closing from pathlib import Path @@ -17,7 +18,8 @@ def snapshot( ) -> None: """Export root messages and retain the full session tree for observations.""" previous_ids = set(previous_message_ids) - with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as source: + # Close both connections even on errors; the caller removes the directory afterwards. + with closing(sqlite3.connect(f"file:{database}?mode=ro", uri=True)) as source: source.row_factory = sqlite3.Row sessions = source.execute( "select id from session where parent_id is null order by time_created, id" @@ -57,7 +59,7 @@ def snapshot( ) # A missing observation store costs only observation evidence, not the export. try: - with sqlite3.connect(directory / "observations.db") as destination: + with closing(sqlite3.connect(directory / "observations.db")) as destination: source.backup(destination) except sqlite3.Error: LOG.exception("Failed to copy OpenCode observations") From 844e9c6221f348ed95718ef1afb7fb6d1c963c2b Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 26 Oct 2026 15:34:54 -0400 Subject: [PATCH 18/32] docs(agent_utils): describe the stop-time cleanup of tools in their own sessions The receipt docstring, the harness guide and the Codex, Pi, OpenCode, OpenClaw, Hermes and Terminal-Bench READMEs now describe what the bash supervisor does: it kills the harness group and, after a forced stop, the descendants that ran in other process groups when the stop began; processes that left the harness earlier stay with the sandbox owner. The receipt documents shell exit statuses. The Kilo and local-provider tests cover a tool in its own session again. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- benchmarks/terminal_bench_2_1/README.md | 27 +++++++++---------- .../agent-server/add-or-update-a-harness.mdx | 14 +++++----- nemo_gym/agent_utils/supervisor_client.py | 9 ++++--- responses_api_agents/codex_agent/README.md | 13 ++++----- .../hermes_sandboxed_agent/README.md | 14 +++++----- .../tests/test_exec_transport.py | 12 +++++---- responses_api_agents/openclaw_agent/README.md | 13 ++++----- responses_api_agents/opencode_agent/README.md | 13 ++++----- responses_api_agents/pi_agent/README.md | 13 ++++----- tests/unit_tests/test_local_provider.py | 10 ++++--- 10 files changed, 75 insertions(+), 63 deletions(-) diff --git a/benchmarks/terminal_bench_2_1/README.md b/benchmarks/terminal_bench_2_1/README.md index 563cdf30f90..262bb8ce775 100644 --- a/benchmarks/terminal_bench_2_1/README.md +++ b/benchmarks/terminal_bench_2_1/README.md @@ -156,24 +156,23 @@ a successful smoke, and one task does not establish a benchmark baseline. ## Tasks that start services -The supervisor stops live processes in the harness group and reaps its children before -capture or verification. On timeout or close, it first records its already adopted -children. It then stops the harness and repeatedly drains newly adopted children, -including in-flight tools in separate sessions, within the cleanup deadline. Previously -adopted children outside the harness group remain for Resources to stop after verification -or episode failure. This includes unfinished background jobs such as `make &` left by an +The supervisor stops live processes in the harness group before capture or verification. +On timeout or close, it first records the harness's descendants in other process groups, +such as in-flight tools in separate sessions. It then stops the harness and kills the group, +those descendants and the groups they lead within the cleanup deadline. Processes that left +the harness before the stop remain for Resources to stop after verification or episode +failure. This includes unfinished background jobs such as `make &` left by an earlier tool call, as well as services; either can continue writing during verification. -To survive this forced-stop cleanup, a service must be outside the harness group and, -before the stop, adopted by the supervisor or descended from a process that was. In the -usual tool-launch path, let the process that launched it exit before the harness deadline -or stop so that adoption can occur. A service still parented by an in-flight tool is -adopted when that tool exits or is killed during the stop, and is drained with the -in-flight tools. +To survive this forced-stop cleanup, a service must be outside the harness group and no +longer descend from the harness when the stop begins. In the usual tool-launch path, let the +process that launched it exit before the harness deadline or stop. A service still parented +by an in-flight tool descends from the harness at the stop and is killed with the in-flight +tools. Natural CLI exit has no such cutoff: unfinished tools in separate sessions cannot be -distinguished from services and may still change task files. Adoption races, PID reuse, -and a preserved service orphaning more children during cleanup also limit this rule. +distinguished from services and may still change task files. PID reuse and a tool +that starts a new session after the stop is recorded also limit this rule. A confirmed receipt is not a guarantee that the sandbox is idle or a security boundary. Unconfirmed termination blocks verification; a final failure receipt still lets agent close release its connection for owner teardown. diff --git a/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx b/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx index e0ea053eb2a..babcf021228 100644 --- a/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx +++ b/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx @@ -334,7 +334,7 @@ The shared modules have separate responsibilities. All of them run on the Agent | --- | --- | | `agent_utils/sandbox_session.py` | `SandboxSession` owns the sandbox connection, the ownership flag, `workdir`, `session_dir`, the execution and finalization tasks, the termination receipt, artifacts, and retryable close state. It uploads the supervisor before launch. `SandboxCommand` carries the harness argv. `shell_assignments()` renders runner input as bash source. | | `agent_utils/supervisor_client.py` | Builds the launch and stop shell commands, defines control-file names and timeout budgets, validates termination receipts, and retires session directories safely. | -| `agent_utils/process_supervisor.sh` | Runs inside the Linux sandbox and needs only bash, `/proc`, `sleep`, and `mv`. It enforces the harness deadline, stops the harness process group, confirms that no member still runs, and writes the termination receipt. The sandbox needs neither Gym nor Python. | +| `agent_utils/process_supervisor.sh` | Runs inside the Linux sandbox and needs only bash, a readable `/proc`, a `sleep` that accepts fractions of a second, and `mv`. It enforces the harness deadline, stops the harness process group and the descendants it started in other groups, confirms that none still runs, and writes the termination receipt. The sandbox needs neither Gym nor Python. | | `agent_utils/harness_runner.sh` | Runs inside the sandbox with only bash. It sources the input that `shell_assignments()` wrote, applies the harness environment, runs the CLI with file-backed stdio, and optionally records each JSON line of stdout with the time it was seen. | | `agent_utils/uv_bootstrap.py` | `uv_for_sandbox()` chooses the `uv` binary to upload for sandboxed Python runtime installation. It reuses a compatible host binary or downloads the static musl build for the detected platform. If the platform probe fails, it warns and falls back to the host binary without confirming compatibility. Hermes uses it. | | `agent_utils/task_timeout.py` | `resolve_agent_timeout(metadata, ceiling=...)` returns the harness time budget: a task's positive `metadata.agent_timeout_sec`, which Harbor task rows carry, capped by the configured timeout. A missing or invalid value keeps the configured timeout. Hermes, Pi, Codex, OpenCode, and OpenClaw use it. | @@ -493,7 +493,7 @@ Adapters normally call `SandboxSession` instead of repeating its launch, stop, c | Helper | Purpose | | --- | --- | | `upload_text(sandbox, *, path, text)` and `read_text(sandbox, *, path)` | Transfer UTF-8 text through upload and download APIs without putting file contents into shell commands. `read_text()` replaces invalid UTF-8; the adapter validates the decoded payload. These are module functions in `nemo_gym/sandbox/utils.py`, not `AsyncSandbox` methods. | -| `supervised_launch_command()` | Requires an explicit supervisor interpreter and checks that it can load the supervisor before claiming the launch. Builds a quoted command with an atomic launch claim that fences delayed execution. | +| `supervised_launch_command()` | Checks that bash can parse the supervisor before claiming the launch. Builds a quoted command with an atomic launch claim that fences delayed execution. | | `stop_and_confirm_cleanup()` | Fences a pending launch or asks the supervisor to stop through `stop.request`, then requires an explicit termination receipt. It sends no signal, so a reused PID cannot be hit. Raises `HarnessTerminationError` for a well-formed negative receipt, which is final; missing or malformed evidence raises a retryable error. | | `remove_session_directory()` | Renames the session directory before deleting it, so its launch fence stays effective. Removes only adapter-owned session files, after the termination outcome is final and artifacts are collected. | | `parse_cleanup_receipt()` | Requires a literal `cleanup_confirmed: true`, and normalizes malformed optional diagnostics with a warning. A fenced launch that never ran has no harness exit code, so missing diagnostics never become a successful exit. | @@ -668,11 +668,11 @@ Serialize close requests. Keep successful close responses for a documented, boun ### Harness termination -The supervisor starts the harness in a new process group. At the deadline or on a stop request it sends `SIGTERM` to the harness and waits for the grace period. In every case it then kills the harness process group and waits until no member of the group can still run. Processes that left the group, such as tool commands started in their own sessions, are left for the sandbox owner. +The supervisor starts the harness in a new process group. At the deadline or on a stop request it first records the harness's descendants in other process groups, such as tool commands started in their own sessions, then sends `SIGTERM` to the harness and waits for the grace period. In every case it then kills the harness process group, and after a forced stop also the recorded descendants and the groups they lead, and waits until none of them can still run. Processes that left the harness before the stop are left for the sandbox owner. -Shell-tool commands can run in their own sessions, so a harness-group kill alone may not reach them. A background process left by a completed tool, even a plain `server &`, can then survive as a task service. Detachment depends on the harness and launch path: OpenClaw's [pinned child adapter](https://github.com/openclaw/openclaw/blob/e085fa1a3ffd32d0ea6917e1e6fb4ecbffbb77d2/src/process/supervisor/adapters/child.ts#L50-L100) disables detachment when `OPENCLAW_SERVICE_MARKER` is nonblank and can fall back to spawning with `detached: false`. Commands on those paths can remain in the harness group and stop at close. +Shell-tool commands can run in their own sessions, so a harness-group kill alone may not reach them; the forced-stop cleanup reaches them only while they still descend from the harness. A background process left by a completed tool, even a plain `server &`, can then survive as a task service. Detachment depends on the harness and launch path: OpenClaw's [pinned child adapter](https://github.com/openclaw/openclaw/blob/e085fa1a3ffd32d0ea6917e1e6fb4ecbffbb77d2/src/process/supervisor/adapters/child.ts#L50-L100) disables detachment when `OPENCLAW_SERVICE_MARKER` is nonblank and can fall back to spawning with `detached: false`. Commands on those paths can remain in the harness group and stop at close. -`cleanup_confirmed: true` means that no live harness-group processes remain. A zombie does not block confirmation, because it cannot run. The receipt does not mean the sandbox is idle, and it is not a security boundary: a tool command still running in its own session can change task files after the receipt. +`cleanup_confirmed: true` means that no live harness-group processes remain. A zombie does not block confirmation, because it cannot run. The receipt does not mean the sandbox is idle, and it is not a security boundary: after a natural exit, a tool command still running in its own session can change task files after the receipt. Services survive until the sandbox owner tears down the sandbox: the Resources Server after verification or episode failure, or the agent when it closes an owned sandbox. @@ -711,7 +711,7 @@ For both borrowed and owned sandboxes, finalize in this order: 2. Collect artifacts once. 3. Release the provider exec. -Cancelling provider exec too early can kill the provider's entire process group, including the supervisor when it is already the group leader, as with the local provider. `setsid()` alone does not guarantee cleanup; the close ordering does. Concurrent close calls join the same finalization. A retry after missing termination evidence or a failed release keeps the handle and does not capture artifacts again. Each finalization step, from staging cancellation through session-file removal and disconnect, has its own timeout; the shared timeout is a per-step budget, not a deadline for the whole close. +Cancelling provider exec too early can kill the provider's entire process group, including the supervisor when it is already the group leader, as with the local provider. Leaving the provider's process group alone would not guarantee cleanup; the close ordering does. Concurrent close calls join the same finalization. A retry after missing termination evidence or a failed release keeps the handle and does not capture artifacts again. Each finalization step, from staging cancellation through session-file removal and disconnect, has its own timeout; the shared timeout is a per-step budget, not a deadline for the whole close. For a borrowed sandbox, close retires the session paths and disconnects. For an owned sandbox, close tries the same graceful finalization before stopping the sandbox. If graceful termination fails, a confirmed provider teardown stands in for it, and unavailable artifacts are recorded as unavailable. If teardown fails, the state is kept for retry. In both modes, keep launch fences in place and reject late activations. @@ -748,7 +748,7 @@ Record evidence conformance and sandbox lifecycle validation separately, because - **Execution modes.** Test host calls with no sandbox access or provider, borrowed agent sessions, agent-owned sessions with a configured provider, and rejection of a seed with neither. Assert that host calls never create a sandbox and that sandbox failures never fall back to the host. Test the compatibility `/run` endpoint with and without a Resources Server binding. - **Installation.** Test cold installation, runtime reuse, failed and partial installation, and unsupported platforms. Include an image without Python: a CLI harness must run there. Claim offline or read-only mounted-runtime support only after you test those deployments. - **Failures and races.** Test setup failures, request mapping, unsupported controls, isolation, retries, cancellation, close races, and termination failures. Include supervisor upload failures, close during upload, interpreter or supervisor bootstrap failures before launch, termination before transport cancellation, artifact capture before teardown, capture failures, and malformed optional diagnostics. Check that a final negative receipt releases a borrowed connection without making the activation gradable. -- **Process termination.** Use real Linux processes. Cover tool commands in the harness group, with grandchildren, under a harness that exits on `SIGTERM` and one that must be killed, for both a deadline and a close. A tool in the harness group must not change the task after a confirmed receipt, and a detached service must survive. Check that every harness process shares the supervised process group, and that a zombie does not block confirmation. Read the process state from `/proc//stat`, because `os.kill(pid, 0)` also succeeds for a zombie. Run the `sandbox`-marked tests with the local provider, which exposes a surviving process that holds the exec output pipe open. +- **Process termination.** Use real Linux processes. Cover tool commands in the harness group, with grandchildren, under a harness that exits on `SIGTERM` and one that must be killed, for both a deadline and a close. A tool in the harness group, or in its own session while the harness still runs, must not change the task after a confirmed receipt, and a detached service must survive. Check that every harness process shares the supervised process group, and that a zombie does not block confirmation. Read the process state from `/proc//stat`, because `os.kill(pid, 0)` also succeeds for a zombie. Run the `sandbox`-marked tests with the local provider, which exposes a surviving process that holds the exec output pipe open. - **Test hygiene.** Bound every wait in lifecycle tests, for example with `asyncio.wait_for()`, so that a regression fails instead of hanging. Check that each new regression test fails when you revert the fix it covers. - **Task environment.** Confirm that tool shells keep the task's `HOME` and `XDG_*` directories, and that files a tool writes under `~` are visible to verification. - **Real smoke test.** Run a real model on a real task with tools and verification. Confirm that the harness runs in the sandbox, that the agent closes before verification, and that the sandbox owner cleans it up. Inspect failures, traces, artifacts, and grading coverage. diff --git a/nemo_gym/agent_utils/supervisor_client.py b/nemo_gym/agent_utils/supervisor_client.py index d6fe80004ac..f7d5985d0c3 100644 --- a/nemo_gym/agent_utils/supervisor_client.py +++ b/nemo_gym/agent_utils/supervisor_client.py @@ -47,9 +47,12 @@ class CleanupReceipt(TypedDict): """Outcome after bounded harness termination, as written by process_supervisor.sh. - ``cleanup_confirmed`` means no live harness-group members remain. Forced - stops also drain the harness's descendants. Services orphaned before the stop - remain until the owner closes the sandbox; this is not a sandbox-empty check. + ``cleanup_confirmed`` means no live harness-group members remain. Forced stops + also kill the harness's descendants that ran in other process groups when the + stop began, and the groups they lead. Processes that left the harness before the + stop remain until the owner closes the sandbox; this is not a sandbox-empty check. + ``return_code`` is the shell status: 128+N for a harness killed by signal N, and + 126 or 127 when the command could not be run. """ cleanup_confirmed: bool diff --git a/responses_api_agents/codex_agent/README.md b/responses_api_agents/codex_agent/README.md index 8a7b4311e00..d5cfe3dc7a1 100644 --- a/responses_api_agents/codex_agent/README.md +++ b/responses_api_agents/codex_agent/README.md @@ -143,13 +143,14 @@ gap. Positive cleanup confirmation remains required; the adapter validates the captured terminal events without inventing an exit code. One shared Linux supervisor per activation fences delayed launches. -The supervisor stops live harness-group processes and reaps its children. On timeout -or close it preserves children already adopted before the stop, typically services from -completed tools, and drains newly adopted children within the cleanup deadline, including -tools in separate sessions. The sandbox owner stops the preserved services later. +The supervisor stops live harness-group processes. On timeout or close it first records +the harness's descendants in other process groups, typically tools in their own sessions, +then kills the harness group, those descendants and the groups they lead within the cleanup +deadline. Processes that left the harness before the stop, typically services from completed +tools, remain for the sandbox owner to stop later. Natural CLI exit cannot distinguish unfinished tools in other sessions from services; -those processes may still change task files. The stop-time cutoff also has adoption/PID -races and cannot classify later service forks. It is not a security boundary. +those processes may still change task files. The stop-time cutoff also has PID races and +cannot classify later service forks. It is not a security boundary. Services must use task-owned files rather than the temporary agent session directory. Close confirms the receipt before cancelling provider execution, then removes adapter-owned files and disconnects. Missing/negative termination evidence diff --git a/responses_api_agents/hermes_sandboxed_agent/README.md b/responses_api_agents/hermes_sandboxed_agent/README.md index 120915fb03a..4b8c64b54ec 100644 --- a/responses_api_agents/hermes_sandboxed_agent/README.md +++ b/responses_api_agents/hermes_sandboxed_agent/README.md @@ -110,14 +110,14 @@ Completed parser reports follow Pro's required-pass rule, including empty report execution failures or unusable parser output remain inconclusive. The agent inherits Gym's standard aggregation; Slurm reporting includes coverage. -The supervisor stops live harness-group processes and reaps its children. At the harness -timeout it preserves children already adopted before the stop and outside the harness -group, and drains newly adopted children within the cleanup deadline, including tools in -separate sessions. This runner never signals the supervisor, so it does not request this -cleanup on close. The sandbox owner stops the preserved processes later. +The supervisor stops live harness-group processes. At the harness timeout it first records +the harness's descendants in other process groups, including tools in separate sessions, then +kills the harness group, those descendants and the groups they lead within the cleanup +deadline. This runner never requests an early stop, so it does not request this cleanup on +close. Processes that left the harness before the stop remain for the sandbox owner. Natural CLI exit cannot distinguish unfinished tools in other sessions from services; -those processes may still change task files. The stop-time cutoff also has adoption/PID -races and cannot classify later service forks. It is not a security boundary. +those processes may still change task files. The stop-time cutoff also has PID races and +cannot classify later service forks. It is not a security boundary. Supervisor diagnostics retain at most the last 1 MiB, including a truncation marker. Missing termination confirmation skips verification and stops the task sandbox. Resources cleanup retains failed handles for retry; final agent diff --git a/responses_api_agents/kilocode_agent/tests/test_exec_transport.py b/responses_api_agents/kilocode_agent/tests/test_exec_transport.py index e14050779d0..8a1a95d882f 100644 --- a/responses_api_agents/kilocode_agent/tests/test_exec_transport.py +++ b/responses_api_agents/kilocode_agent/tests/test_exec_transport.py @@ -21,7 +21,7 @@ from responses_api_agents.kilocode_agent.tests.test_sandbox import seed -pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Linux child-subreaper and /proc required") +pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Linux /proc required") class ExecOnlySandbox: @@ -90,13 +90,14 @@ def activate(state, code, timeout=0.5): ) +@pytest.mark.parametrize("detached", [False, True]) @pytest.mark.parametrize("ending", ["natural", "timeout", "cancel", "close"]) -async def test_exec_only_supervision_reaps_harness_group(tmp_path, ending): +async def test_exec_only_supervision_reaps_harness_group(tmp_path, ending, detached): state, provider, workdir = make_session(tmp_path) - # A tool that stays in Kilo's process group; detached services are left to the sandbox owner. + # A tool in Kilo's process group, or in its own session as Kilo starts shell tools. code = ( "import subprocess,sys,time,pathlib; " - "p=subprocess.Popen([sys.executable,'-c','import time; time.sleep(60)']); " + f"p=subprocess.Popen([sys.executable,'-c','import time; time.sleep(60)'],start_new_session={detached}); " "pathlib.Path('child.pid').write_text(str(p.pid)); " + ("print('{}')" if ending == "natural" else "time.sleep(60)") ) @@ -125,7 +126,8 @@ async def test_exec_only_supervision_reaps_harness_group(tmp_path, ending): stat = Path(f"/proc/{(workdir / 'child.pid').read_text()}/stat").read_text() except FileNotFoundError: stat = ") X" - assert stat.rpartition(")")[2].split()[0] in ("Z", "X") + # Natural CLI exit has no stop cutoff; the sandbox owner tears down a detached tool. + assert (stat.rpartition(")")[2].split()[0] in ("Z", "X")) is not (detached and ending == "natural") assert provider.cancelled_launch is False assert provider.deadline > deadline + 3 * 1 await state.close(timeout=3) diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index 82ea57a36fc..bda2b75207a 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -236,13 +236,14 @@ An explicit wall or model-output limit returns partial output as `incomplete`; u raise HTTP 502 even when a useful patch exists. Do not infer rollout success from reward alone. Close returns the captured observations, including salvaged transcript evidence after cancellation. -The supervisor stops live harness-group processes and reaps its children. On timeout -or close it preserves children already adopted before the stop, typically services from -completed tools, and drains newly adopted children within the cleanup deadline, including -tools in separate sessions. The sandbox owner stops the preserved services later. +The supervisor stops live harness-group processes. On timeout or close it first records +the harness's descendants in other process groups, typically tools in their own sessions, +then kills the harness group, those descendants and the groups they lead within the cleanup +deadline. Processes that left the harness before the stop, typically services from completed +tools, remain for the sandbox owner to stop later. Natural CLI exit cannot distinguish unfinished tools in other sessions from services; -those processes may still change task files. The stop-time cutoff also has adoption/PID -races and cannot classify later service forks. It is not a security boundary. +those processes may still change task files. The stop-time cutoff also has PID races and +cannot classify later service forks. It is not a security boundary. The sandbox launcher sets `OPENCLAW_NO_RESPAWN=1` to keep the CLI in the supervised group. Services must use task-owned files rather than the temporary agent session directory. Close is serialized and checks the receipt, process handle, adapter-owned file removal, and disconnect before succeeding. Unknown launch outcomes and missing diff --git a/responses_api_agents/opencode_agent/README.md b/responses_api_agents/opencode_agent/README.md index 711be94cf34..ad27370c47c 100644 --- a/responses_api_agents/opencode_agent/README.md +++ b/responses_api_agents/opencode_agent/README.md @@ -87,13 +87,14 @@ correlation plugin. The plugin is uploaded into each sandbox when model-call observability is enabled. Sessions disable automatic updates and models.dev fetches. The runner uses ordinary sandbox `exec`, with an internal deadline. -The supervisor stops live harness-group processes and reaps its children. On timeout -or close it preserves children already adopted before the stop, typically services from -completed tools, and drains newly adopted children within the cleanup deadline, including -tools in separate sessions. The sandbox owner stops the preserved services later. +The supervisor stops live harness-group processes. On timeout or close it first records +the harness's descendants in other process groups, typically tools in their own sessions, +then kills the harness group, those descendants and the groups they lead within the cleanup +deadline. Processes that left the harness before the stop, typically services from completed +tools, remain for the sandbox owner to stop later. Natural CLI exit cannot distinguish unfinished tools in other sessions from services; -those processes may still change task files. The stop-time cutoff also has adoption/PID -races and cannot classify later service forks. It is not a security boundary. +those processes may still change task files. The stop-time cutoff also has PID races and +cannot classify later service forks. It is not a security boundary. The CLI must exit before its SQLite database is copied to the agent server and exported there; services must use task-owned files rather than the agent session directory, which close removes. Capture requires confirmed termination. diff --git a/responses_api_agents/pi_agent/README.md b/responses_api_agents/pi_agent/README.md index 1324693120b..9c495154eb9 100644 --- a/responses_api_agents/pi_agent/README.md +++ b/responses_api_agents/pi_agent/README.md @@ -164,13 +164,14 @@ configuration must not replace the agent's cap with a larger value. Enforcement 7. Environment Server asks Resources to verify and close the task session. The benchmark owns its verification procedure and sandbox teardown. -The supervisor stops live harness-group processes and reaps its children. On timeout -or close it preserves children already adopted before the stop, typically services from -completed tools, and drains newly adopted children within the cleanup deadline, including -tools in separate sessions. The sandbox owner stops the preserved services later. +The supervisor stops live harness-group processes. On timeout or close it first records +the harness's descendants in other process groups, typically tools in their own sessions, +then kills the harness group, those descendants and the groups they lead within the cleanup +deadline. Processes that left the harness before the stop, typically services from completed +tools, remain for the sandbox owner to stop later. Natural CLI exit cannot distinguish unfinished tools in other sessions from services; -those processes may still change task files. The stop-time cutoff also has adoption/PID -races and cannot classify later service forks. It is not a security boundary. +those processes may still change task files. The stop-time cutoff also has PID races and +cannot classify later service forks. It is not a security boundary. Services must use task-owned files rather than the temporary agent session directory. Harness execution uses `nemo_gym.agent_utils.sandbox_session.SandboxSession`, shared with Hermes. diff --git a/tests/unit_tests/test_local_provider.py b/tests/unit_tests/test_local_provider.py index c9f9b78fa97..2568d17b293 100644 --- a/tests/unit_tests/test_local_provider.py +++ b/tests/unit_tests/test_local_provider.py @@ -34,14 +34,16 @@ @pytest.mark.skipif(sys.platform != "linux", reason="Linux /proc is required") -async def test_supervisor_deadline_finishes_before_provider_hard_timeout(tmp_path: Path) -> None: +@pytest.mark.parametrize("detached", [False, True]) +async def test_supervisor_deadline_finishes_before_provider_hard_timeout(tmp_path: Path, detached: bool) -> None: provider = LocalProvider() handle = await provider.create(SandboxSpec(workdir=str(tmp_path))) worker = tmp_path / "worker.py" worker.write_text( "import pathlib, signal, subprocess, sys, time\n" "signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" - "child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])\n" + "child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'], " + f"start_new_session={detached})\n" "pathlib.Path('child.pid').write_text(str(child.pid))\n" "time.sleep(60)\n" ) @@ -77,7 +79,9 @@ async def test_supervisor_deadline_finishes_before_provider_hard_timeout(tmp_pat assert receipt["return_code"] == 128 + signal.SIGKILL assert receipt["error"] is None child_pid = int((tmp_path / "child.pid").read_text()) - assert not Path(f"/proc/{child_pid}").exists(), "in-flight tool survived" + # A killed tool may linger as a zombie until init reaps it; it can no longer run. + stat = Path(f"/proc/{child_pid}/stat") + assert not stat.exists() or stat.read_text().rsplit(")", 1)[1].split()[0] in "ZX", "in-flight tool survived" finally: if (tmp_path / "child.pid").exists(): try: From 7552c46f20bf3ebd76ab9cdda9135fd15a38b181 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 26 Oct 2026 15:35:44 -0400 Subject: [PATCH 19/32] fix(agent_utils): reject shell words bash cannot hold; tighten Hermes stop test shell_assignments() now raises for NUL and non-string words, which bash would silently drop or which raised an opaque TypeError. The Hermes close test stops the supervisor through the stop file alone, as production does, and the sandboxed Hermes README states that the supervisor needs the task image's bash. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- nemo_gym/agent_utils/sandbox_session.py | 13 ++++++++++--- .../hermes_agent/tests/test_sandbox_runner.py | 2 +- .../hermes_sandboxed_agent/README.md | 1 + tests/unit_tests/test_sandbox_session.py | 5 ++++- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/nemo_gym/agent_utils/sandbox_session.py b/nemo_gym/agent_utils/sandbox_session.py index a69e9429da7..5e84dd2b0f5 100644 --- a/nemo_gym/agent_utils/sandbox_session.py +++ b/nemo_gym/agent_utils/sandbox_session.py @@ -56,20 +56,27 @@ def shell_assignments(values: Mapping[str, str | Sequence[str] | Mapping[str, st A string becomes a scalar and a sequence an array. A mapping becomes an array of ``KEY=VALUE`` words, ready for ``export`` or ``env``. Every word is shell-quoted. + Words must be strings without NUL, which bash cannot hold. """ + + def word(item: object) -> str: + if not isinstance(item, str) or "\0" in item: + raise ValueError(f"Invalid shell word, which must be a string without NUL: {item!r}") + return quote(item) + lines = [] for name, value in values.items(): if not _SHELL_NAME.fullmatch(name): raise ValueError(f"Invalid shell variable name: {name!r}") if isinstance(value, str): - lines.append(f"{name}={quote(value)}") + lines.append(f"{name}={word(value)}") continue if isinstance(value, Mapping): invalid = [key for key in value if not _SHELL_NAME.fullmatch(key)] if invalid: raise ValueError(f"Invalid environment variable names: {invalid!r}") - value = [f"{key}={item}" for key, item in value.items()] - lines.append(f"{name}=({' '.join(quote(item) for item in value)})") + value = [f"{key}={item}" if isinstance(item, str) else item for key, item in value.items()] + lines.append(f"{name}=({' '.join(word(item) for item in value)})") return "".join(f"{line}\n" for line in lines) diff --git a/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py b/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py index c6f56a52b91..71d139e189b 100644 --- a/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/hermes_agent/tests/test_sandbox_runner.py @@ -559,8 +559,8 @@ def run_conversation(self, *args, **kwargs): while not (tmp_path / "model.patch").exists() and process.poll() is None and time.monotonic() < deadline: time.sleep(0.01) assert (tmp_path / "model.patch").exists() + # Production sends no signal; the stop file alone must stop the supervisor. stop_request_path.touch() - process.terminate() _, stderr = process.communicate(timeout=10) assert process.returncode == 0, stderr finally: diff --git a/responses_api_agents/hermes_sandboxed_agent/README.md b/responses_api_agents/hermes_sandboxed_agent/README.md index 4b8c64b54ec..7a2b4f3d790 100644 --- a/responses_api_agents/hermes_sandboxed_agent/README.md +++ b/responses_api_agents/hermes_sandboxed_agent/README.md @@ -118,6 +118,7 @@ close. Processes that left the harness before the stop remain for the sandbox ow Natural CLI exit cannot distinguish unfinished tools in other sessions from services; those processes may still change task files. The stop-time cutoff also has PID races and cannot classify later service forks. It is not a security boundary. +The supervisor runs with the task image's bash; this adapter does not install it. Supervisor diagnostics retain at most the last 1 MiB, including a truncation marker. Missing termination confirmation skips verification and stops the task sandbox. Resources cleanup retains failed handles for retry; final agent diff --git a/tests/unit_tests/test_sandbox_session.py b/tests/unit_tests/test_sandbox_session.py index c1f58fcc658..230fd4c42f4 100644 --- a/tests/unit_tests/test_sandbox_session.py +++ b/tests/unit_tests/test_sandbox_session.py @@ -618,7 +618,10 @@ def test_shell_assignments_round_trip_through_bash(tmp_path): assert not (tmp_path / "unintended").exists() -@pytest.mark.parametrize("values", [{"bad-name": "x"}, {"env": {"BAD KEY": "x"}}]) +@pytest.mark.parametrize( + "values", + [{"bad-name": "x"}, {"env": {"BAD KEY": "x"}}, {"cwd": "a\0b"}, {"command": ["cli", 1]}, {"env": {"K": None}}], +) def test_shell_assignments_reject_names_bash_cannot_assign(values): with pytest.raises(ValueError, match="Invalid"): shell_assignments(values) From 01ac0f1b2f74ee95a17ab3b77ce94cee961ba8b9 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 26 Oct 2026 15:36:34 -0400 Subject: [PATCH 20/32] style: apply ruff formatting to the ported tests Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- .../nooa_agent/tests/test_sandbox_runner.py | 3 +-- .../opencode_agent/tests/test_install_runtime.py | 16 +++++++++++++++- .../tests/test_sandbox_sessions.py | 2 ++ tests/unit_tests/test_harness_runner.py | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/responses_api_agents/nooa_agent/tests/test_sandbox_runner.py b/responses_api_agents/nooa_agent/tests/test_sandbox_runner.py index 4ed54cf2a13..4d292298b80 100644 --- a/responses_api_agents/nooa_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/nooa_agent/tests/test_sandbox_runner.py @@ -8,8 +8,7 @@ import pytest -from nemo_gym.agent_utils.supervisor_client import CleanupReceipt -from nemo_gym.agent_utils.supervisor_client import HarnessTerminationError +from nemo_gym.agent_utils.supervisor_client import CleanupReceipt, HarnessTerminationError from nemo_gym.sandbox.providers.base import SandboxExecResult from responses_api_agents.nooa_agent.result import finalize_run_result from responses_api_agents.nooa_agent.runner import NOOARunFailure diff --git a/responses_api_agents/opencode_agent/tests/test_install_runtime.py b/responses_api_agents/opencode_agent/tests/test_install_runtime.py index fc70796c9e7..9166fbb7f35 100644 --- a/responses_api_agents/opencode_agent/tests/test_install_runtime.py +++ b/responses_api_agents/opencode_agent/tests/test_install_runtime.py @@ -21,7 +21,21 @@ def sandbox(tmp_path: Path) -> tuple[Path, dict[str, str]]: """Run the real installer with an isolated PATH and no access to the host package manager.""" bindir = tmp_path / "bin" bindir.mkdir() - tools = ("bash", "uname", "getconf", "mkdir", "cp", "tar", "gzip", "chmod", "rm", "sleep", "ln", "readlink", "rmdir") + tools = ( + "bash", + "uname", + "getconf", + "mkdir", + "cp", + "tar", + "gzip", + "chmod", + "rm", + "sleep", + "ln", + "readlink", + "rmdir", + ) for name in (*tools, "mktemp", "sha256sum", "mv"): executable = shutil.which(name) if executable is None: diff --git a/responses_api_agents/opencode_agent/tests/test_sandbox_sessions.py b/responses_api_agents/opencode_agent/tests/test_sandbox_sessions.py index dbf6003ca5e..eaadeb713e2 100644 --- a/responses_api_agents/opencode_agent/tests/test_sandbox_sessions.py +++ b/responses_api_agents/opencode_agent/tests/test_sandbox_sessions.py @@ -91,6 +91,8 @@ def rejected_events(*, after_work: bool = False) -> str: else: export["messages"][-1] = rejected return json.dumps(export) + + def read_params(text: str) -> dict: """Source runner PARAMS with bash, as the sandbox runner does.""" script = 'source /dev/stdin; printf "%s\\0" "$cwd" "$stdin" "$setup" "${#command[@]}" "${command[@]}" "${env[@]}"' diff --git a/tests/unit_tests/test_harness_runner.py b/tests/unit_tests/test_harness_runner.py index f6bb10947ea..28d1e47ea11 100644 --- a/tests/unit_tests/test_harness_runner.py +++ b/tests/unit_tests/test_harness_runner.py @@ -83,7 +83,6 @@ def test_events_keep_json_objects_with_the_time_they_were_seen(tmp_path: Path) - assert all(before - 1 <= seen <= time.time() for seen, _ in events) - def test_events_keep_numeric_times_and_crlf_lines_without_epochrealtime(tmp_path: Path) -> None: # Bash < 5 has no EPOCHREALTIME and bash < 4.2 no printf strftime; both still stamp numbers. script = 'printf \'{"type": "a"}\\r\\n{"type": "partial"}\'; sleep 0.2' @@ -131,6 +130,7 @@ def test_events_mode_cli_keeps_default_interrupt_handling(tmp_path: Path) -> Non (event,) = [json.loads(line)[1] for line in (tmp_path / "events.jsonl").read_text().splitlines()] assert "default_int_handler" in event["int"] + @pytest.mark.skipif(sys.platform != "linux", reason="Linux process semantics") def test_events_mode_forwards_term_and_ignores_a_detached_stdout_holder(tmp_path: Path) -> None: # A detached tool keeps stdout open; the runner must still finish when the CLI does. From ec0eadf965886f6e483c681093f2184a6d0ac1ed Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 26 Oct 2026 16:05:30 -0400 Subject: [PATCH 21/32] test(pi_agent): stop in-flight detached tools and keep services from completed tools The bash port's tests asserted that a tool still running in its own session survives a close, and modelled a completed tool's service as a direct child of the CLI. With the supervisor's stop-time cleanup, an in-flight detached tool is stopped again, as before the port; a service left by a tool that already exited still survives. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- responses_api_agents/pi_agent/tests/test_sandbox.py | 5 +++-- responses_api_agents/pi_agent/tests/test_sandbox_runner.py | 7 +++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/responses_api_agents/pi_agent/tests/test_sandbox.py b/responses_api_agents/pi_agent/tests/test_sandbox.py index 68ecd487c0d..b57fec980d5 100644 --- a/responses_api_agents/pi_agent/tests/test_sandbox.py +++ b/responses_api_agents/pi_agent/tests/test_sandbox.py @@ -1610,8 +1610,9 @@ async def test_close_confirms_cleanup_before_provider_cancellation(local_session cli_pid, task_pid, wrapper_pid = processes assert not Path(f"/proc/{cli_pid}").exists() assert not Path(f"/proc/{wrapper_pid}").exists() - # A tool that left the harness process group belongs to the sandbox owner, not the supervisor. - assert Path(f"/proc/{task_pid}/stat").read_text().rsplit(")", 1)[1].split()[0] != "Z" + # The in-flight detached tool is stopped before borrowed-session release; init may still hold it as a zombie. + stat = Path(f"/proc/{task_pid}/stat") + assert not stat.exists() or stat.read_text().rsplit(")", 1)[1].split()[0] in "ZX" state.session.sandbox.disconnect.assert_awaited_once() state.session.sandbox.stop.assert_not_awaited() finally: diff --git a/responses_api_agents/pi_agent/tests/test_sandbox_runner.py b/responses_api_agents/pi_agent/tests/test_sandbox_runner.py index 96786a4cf65..45d81bf198c 100644 --- a/responses_api_agents/pi_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/pi_agent/tests/test_sandbox_runner.py @@ -77,10 +77,9 @@ def test_capture_stdin_and_environment(tmp_path, monkeypatch): @pytest.mark.parametrize("ending", ["natural", "timeout", "cancel"]) def test_task_descendants_survive_until_owner_teardown(tmp_path, ending): try: - # A completed tool leaves its detached service adopted before close. - script = "setsid sleep 60 /dev/null 2>&1 & echo $! > child.pid; touch ready; " + ( - "sleep 60" if ending != "natural" else "true" - ) + # A completed tool leaves its detached service, which no longer descends from the CLI at the stop. + tool = "setsid sleep 60 /dev/null 2>&1 & echo $! > child.pid" + script = f"bash -c '{tool}'; touch ready; " + ("sleep 60" if ending != "natural" else "true") process = launch(tmp_path, script, timeout=3) if ending == "cancel": for _ in range(200): From a1f9e5ca892e7ec59209a7ad7b158449dcd46dc2 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 26 Oct 2026 16:46:05 -0400 Subject: [PATCH 22/32] docs(agent_utils): say which harnesses keep a Python shim in the sandbox The harness guide said Codex starts its CLI through harness_runner.sh. On main Codex keeps its Python sandbox_runner.py for its sandbox-local retry proxy, so it still needs the task image's Python; Hermes, Stirrup and NOOA run their Python shims on interpreters they install privately. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- .../latest/pages/agent-server/add-or-update-a-harness.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx b/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx index babcf021228..0ae61ea45ce 100644 --- a/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx +++ b/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx @@ -48,7 +48,7 @@ An adapter can stop at any level below. Each level adds work and enables a speci An adapter supplies the knowledge that is specific to the harness or framework: 1. **Install or load the agent.** For example, a framework adapter lists the framework in its `requirements.txt`, while Pi and Codex install a pinned runtime into the task sandbox at seed. -2. **Configure and invoke it.** Build the harness configuration from the adapter configuration and the request, then call the SDK or start the CLI. For example, Hermes writes a `config.yaml`, and Pi and Codex start their CLI through the shared `harness_runner.sh`. +2. **Configure and invoke it.** Build the harness configuration from the adapter configuration and the request, then call the SDK or start the CLI. For example, Hermes writes a `config.yaml`, Pi starts its CLI through the shared `harness_runner.sh`, and Codex starts its CLI from its own `sandbox_runner.py`, which also runs a sandbox-local retry proxy for model requests. 3. **Map the harness's inputs, results, and traces to Gym.** Convert the request into the harness's input, the harness's output into a `NeMoGymResponse`, and its model and tool activity into an `AgentObservationBundle`. 4. **Map Gym connections into the harness's own configuration.** Set the rollout-prefixed Model Server URL as the harness's model base URL, add tool grants to its tool configuration, and run it in `SandboxAccess.workdir`. For example, Hermes adds each MCP grant to the `mcp_servers` section of its configuration. @@ -317,13 +317,13 @@ The [`nemo_gym/agent_utils`](https://github.com/NVIDIA-NeMo/Gym/tree/main/nemo_g ### Code layout -Organize a new adapter under `responses_api_agents//`. The adapter's `sandbox.py` runs on the Agent Server. Inside the sandbox, the shared `harness_runner.sh` starts the harness CLI, or an adapter-specific `sandbox_runner.py` shim wraps a harness that is itself a Python program. +Organize a new adapter under `responses_api_agents//`. The adapter's `sandbox.py` runs on the Agent Server. Inside the sandbox, the shared `harness_runner.sh` starts the harness CLI, or an adapter-specific `sandbox_runner.py` shim wraps a harness that is itself a Python program or needs Python next to its CLI. Run such a shim with an interpreter the adapter installs privately, as Hermes, Stirrup and NOOA do, so the task image does not need Python; Codex is the exception and still uses the task image's Python 3.8 or newer. | Adapter file | Responsibility | | --- | --- | | `app.py` | Gym endpoints and configuration; request validation and mapping; session hooks; response and observation conversion; dispatch to host or sandbox execution. | | `sandbox.py` | A harness-specific session state class that composes `SandboxSession`. It installs the runtime at seed, stages input in `stage_activation()`, collects artifacts, and holds harness-specific state and diagnostics. | -| `sandbox_runner.py` (Python harnesses only) | Reads the staged input, invokes the harness SDK, writes the harness's output and optional process diagnostics, and keeps partial output where the harness supports it. Handles `SIGTERM` by stopping the harness and writing that partial output before it exits, because the supervisor sends `SIGTERM` at the deadline and on close. CLI harnesses use the shared `harness_runner.sh` instead, so the task image needs only bash. | +| `sandbox_runner.py` (only when the shim needs Python) | Reads the staged input, invokes the harness SDK, writes the harness's output and optional process diagnostics, and keeps partial output where the harness supports it. Handles `SIGTERM` by stopping the harness and writing that partial output before it exits, because the supervisor sends `SIGTERM` at the deadline and on close. CLI harnesses use the shared `harness_runner.sh` instead, so the task image needs only bash. | | `install__runtime.sh` or an existing harness-specific installer | Installs the pinned harness and its private dependencies, then checks that they run. Keep installation out of the activation path. | | `requirements.txt` | The adapter's isolated environment: `-e nemo-gym[dev,sandbox] @ ../../` plus packages that run on the Agent Server, such as a framework or SDK. Without the `sandbox` extra, an SDK-backed provider such as OpenSandbox cannot import its SDK, and the first sandbox session fails. | | `tests/` | Host dispatch, session setup, request semantics, output parsing, installation, and lifecycle regressions. | From a61af13a715cc5f7be5e9ee730321346a5b9c83d Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 26 Oct 2026 20:02:01 -0400 Subject: [PATCH 23/32] fix(agent): crash-safe installer queue, checked version probe, JSON-safe task dirs Address review findings on the bash runtime: - Without flock, the OpenCode and Claude installers now queue on numbered owner links that are never removed or reused, and skip a number whose owner is done or gone. There is no second reclaim lock left to strand, and an installer killed at any point cannot block or release another. - A preinstalled OpenCode binary must exit successfully from --version, not only print the version, before it replaces the cached runtime. - OpenCode's and Kilo's task-env.json writers escape every JSON control character, so task HOME/XDG paths with tabs or newlines stay valid JSON. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- .../install_claude_runtime.sh | 38 +++++++++++++------ .../tests/test_install_claude_runtime.py | 12 +++--- .../kilocode_agent/sandbox.py | 9 ++++- .../tests/test_task_environment.py | 14 +++++++ .../install_opencode_runtime.sh | 37 +++++++++++++----- .../opencode_agent/sandbox.py | 13 ++++++- .../tests/test_install_runtime.py | 25 +++++++----- .../tests/test_task_environment.py | 12 ++++++ 8 files changed, 122 insertions(+), 38 deletions(-) diff --git a/responses_api_agents/claude_code_agent/install_claude_runtime.sh b/responses_api_agents/claude_code_agent/install_claude_runtime.sh index 603ff79a3ee..b12f7154d86 100644 --- a/responses_api_agents/claude_code_agent/install_claude_runtime.sh +++ b/responses_api_agents/claude_code_agent/install_claude_runtime.sh @@ -8,7 +8,7 @@ # Prints the version of the verified runtime. --check-only exits 3 when the runtime is absent or unhealthy. set -euo pipefail -version="" runtime="" expected="" source="" check_only="" compressed="" temporary="" lock_link="" +version="" runtime="" expected="" source="" check_only="" compressed="" temporary="" queue="" ticket="" while (($#)); do case $1 in --version) version=$2 && shift 2 ;; @@ -47,26 +47,40 @@ process_identity() { fi } -trap 'rm -f -- "$temporary"; [[ -z $lock_link ]] || rm -f -- "$lock_link"' EXIT +trap 'rm -f -- "$temporary"; [[ -z $ticket ]] || : >"$queue/$ticket.done"' EXIT mkdir -p -- "$(dirname -- "$runtime")" # Concurrent sessions in one sandbox share the runtime; only one may replace it. if command -v flock >/dev/null 2>&1; then exec 9>>"$runtime.lock" flock 9 else - # Without flock, a symlink names the owner. A lock whose owner is gone is removed under a second - # lock and only after re-reading it, so a lock taken in the meantime is never removed. + # Without flock, installers queue: each takes the next number in the queue directory as a + # symlink naming its owner, then waits until every lower number is done or its owner is gone. + # Numbers are never removed or reused, so no installer, however it exits, can release another. + queue=$runtime.lock.d + mkdir -p "$queue" owner=$(process_identity $$) - until ln -s "$owner" "$runtime.lock.owner" 2>/dev/null; do - current=$(readlink "$runtime.lock.owner" 2>/dev/null) || current= - if [[ -n $current && $(process_identity "${current%% *}") != "$current" ]] && - mkdir "$runtime.lock.reclaim" 2>/dev/null; then - [[ $(readlink "$runtime.lock.owner" 2>/dev/null) != "$current" ]] || rm -f "$runtime.lock.owner" - rmdir "$runtime.lock.reclaim" - fi + while :; do + last=0 + for entry in "$queue"/*; do + number=${entry##*/} + number=${number%.done} + if [[ $number =~ ^[0-9]+$ ]] && ((number > last)); then last=$number; fi + done + ticket=$((last + 1)) + ln -s "$owner" "$queue/$ticket" 2>/dev/null && break + done + until + ahead= + for ((number = 1; number < ticket; number++)); do + [[ -e $queue/$number.done ]] && continue + holder=$(readlink "$queue/$number" 2>/dev/null) || continue + [[ $(process_identity "${holder%% *}") != "$holder" ]] || { ahead=1; break; } + done + [[ -z $ahead ]] + do sleep 0.1 done - lock_link=$runtime.lock.owner fi verify "$runtime/claude" 2>/dev/null && exit 0 diff --git a/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py b/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py index 04ac6e8be63..a48acaa8062 100644 --- a/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py +++ b/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py @@ -21,7 +21,7 @@ ) # The installer's tools without flock, for its fallback lock. FALLBACK_TOOLS = ("bash", "sha256sum", "timeout", "mktemp", "gzip", "cat", "chmod", "mv", "rm", "mkdir", "dirname") -FALLBACK_TOOLS += ("ln", "readlink", "rmdir", "sleep", "tr", "kill") +FALLBACK_TOOLS += ("ln", "readlink", "sleep", "tr", "kill") @pytest.fixture @@ -115,11 +115,12 @@ def _identity(pid: int) -> str: return f"{pid} {stat.read_text().rsplit(') ', 1)[1].split()[19]}" if stat.exists() else str(pid) -def test_lock_left_by_a_killed_installer_is_reclaimed_without_flock(tmp_path, binary): +def test_number_left_by_a_killed_installer_does_not_block_the_queue_without_flock(tmp_path, binary): source, checksum = binary exited = subprocess.Popen(["true"]) exited.wait() - (tmp_path / "runtime.lock.owner").symlink_to(_identity(exited.pid)) + (tmp_path / "runtime.lock.d").mkdir() + (tmp_path / "runtime.lock.d/1").symlink_to(_identity(exited.pid)) result = subprocess.run( ["bash", str(INSTALLER), "--version", "2.1.108", "--runtime", str(tmp_path / "runtime")] + ["--sha256", checksum, "--source", str(source)], @@ -129,14 +130,15 @@ def test_lock_left_by_a_killed_installer_is_reclaimed_without_flock(tmp_path, bi timeout=30, ) assert (result.returncode, result.stdout) == (0, "2.1.108 (Claude Code)\n"), result.stderr - assert not (tmp_path / "runtime.lock.owner").is_symlink() + assert (tmp_path / "runtime.lock.d/2.done").exists() def test_installer_without_flock_waits_for_a_live_lock_owner(tmp_path, binary): source, checksum = binary holder = subprocess.Popen(["sleep", "30"]) try: - (tmp_path / "runtime.lock.owner").symlink_to(_identity(holder.pid)) + (tmp_path / "runtime.lock.d").mkdir() + (tmp_path / "runtime.lock.d/1").symlink_to(_identity(holder.pid)) installer = subprocess.Popen( ["bash", str(INSTALLER), "--version", "2.1.108", "--runtime", str(tmp_path / "runtime")] + ["--sha256", checksum, "--source", str(source)], diff --git a/responses_api_agents/kilocode_agent/sandbox.py b/responses_api_agents/kilocode_agent/sandbox.py index 808c5999377..8dd720c880d 100644 --- a/responses_api_agents/kilocode_agent/sandbox.py +++ b/responses_api_agents/kilocode_agent/sandbox.py @@ -35,8 +35,15 @@ _ACTIVATION_SETUP = r"""rm -f -- "$session/task-environment.ready" || exit 1 mkdir -p -- "$session/home" "$session/data" "$session/config" "$session/cache" || exit 1 json_string() { - local value=${1//\\/\\\\} + local value=${1//\\/\\\\} code octal character escaped value=${value//\"/\\\"} + # JSON forbids raw control characters; a bash string cannot hold NUL. + for ((code = 1; code < 32; code++)); do + printf -v octal '%03o' "$code" + printf -v character "\\$octal" + printf -v escaped '\\u%04x' "$code" + value=${value//"$character"/"$escaped"} + done printf '"%s"' "$value" } home=${HOME:-$(unset HOME; echo ~)} diff --git a/responses_api_agents/kilocode_agent/tests/test_task_environment.py b/responses_api_agents/kilocode_agent/tests/test_task_environment.py index b568c924478..cd2b3737828 100644 --- a/responses_api_agents/kilocode_agent/tests/test_task_environment.py +++ b/responses_api_agents/kilocode_agent/tests/test_task_environment.py @@ -162,3 +162,17 @@ def test_pinned_cli_reports_whether_required_plugin_initialized(tmp_path: Path, assert receipt.read_text() == "ready\n" else: assert not receipt.exists() + + +@pytest.mark.parametrize("home", ["/task/a\tb", "/task/new\nline", '/task/"quoted"\\back']) +def test_recorded_task_directories_are_valid_json(tmp_path: Path, home: str) -> None: + from shlex import quote + + from responses_api_agents.kilocode_agent.sandbox import _ACTIVATION_SETUP + + env = {"PATH": os.environ["PATH"], "HOME": home, "XDG_CONFIG_HOME": f"{home}\x01\x1fconfig"} + setup = f"session={quote(str(tmp_path))}\n{_ACTIVATION_SETUP}" + result = subprocess.run(["bash", "-c", setup], env=env, capture_output=True, text=True, timeout=10) + assert result.returncode == 0, result.stderr + recorded = json.loads((tmp_path / "task-env.json").read_text()) + assert recorded["HOME"] == home and recorded["XDG_CONFIG_HOME"] == f"{home}\x01\x1fconfig" diff --git a/responses_api_agents/opencode_agent/install_opencode_runtime.sh b/responses_api_agents/opencode_agent/install_opencode_runtime.sh index 398c459fa8c..34487a24b2b 100644 --- a/responses_api_agents/opencode_agent/install_opencode_runtime.sh +++ b/responses_api_agents/opencode_agent/install_opencode_runtime.sh @@ -29,18 +29,34 @@ if command -v flock >/dev/null 2>&1; then exec 9>"$runtime.lock" flock 9 else - # Without flock, a symlink names the owner. A lock whose owner is gone is removed under a second - # lock and only after re-reading it, so a lock taken in the meantime is never removed. + # Without flock, installers queue: each takes the next number in the queue directory as a + # symlink naming its owner, then waits until every lower number is done or its owner is gone. + # Numbers are never removed or reused, so no installer, however it exits, can release another. + queue=$runtime.lock.d + mkdir -p "$queue" owner=$(process_identity $$) - until ln -s "$owner" "$runtime.lock.owner" 2>/dev/null; do - current=$(readlink "$runtime.lock.owner" 2>/dev/null) || current= - if [[ -n $current && $(process_identity "${current%% *}") != "$current" ]] && mkdir "$runtime.lock.reclaim" 2>/dev/null; then - [[ $(readlink "$runtime.lock.owner" 2>/dev/null) != "$current" ]] || rm -f "$runtime.lock.owner" - rmdir "$runtime.lock.reclaim" - fi + while :; do + last=0 + for entry in "$queue"/*; do + number=${entry##*/} + number=${number%.done} + if [[ $number =~ ^[0-9]+$ ]] && ((number > last)); then last=$number; fi + done + ticket=$((last + 1)) + ln -s "$owner" "$queue/$ticket" 2>/dev/null && break + done + trap ': >"$queue/$ticket.done"' EXIT + until + ahead= + for ((number = 1; number < ticket; number++)); do + [[ -e $queue/$number.done ]] && continue + holder=$(readlink "$queue/$number" 2>/dev/null) || continue + [[ $(process_identity "${holder%% *}") != "$holder" ]] || { ahead=1; break; } + done + [[ -z $ahead ]] + do sleep 0.1 done - trap 'rm -f "$runtime.lock.owner"' EXIT fi case "$(uname -m)" in x86_64) arch=x64-baseline ;; @@ -71,7 +87,8 @@ if [ -n "$source_binary" ]; then exit 1 fi chmod 755 "$staged" - if [ "$("$staged" --version)" != "$version" ]; then + # The probe must exit successfully as well as print the exact version. + if ! actual=$("$staged" --version) || [ "$actual" != "$version" ]; then rm -f "$staged" echo 'OpenCode executable version mismatch' >&2 exit 1 diff --git a/responses_api_agents/opencode_agent/sandbox.py b/responses_api_agents/opencode_agent/sandbox.py index df41c452e12..bcbdd58101e 100644 --- a/responses_api_agents/opencode_agent/sandbox.py +++ b/responses_api_agents/opencode_agent/sandbox.py @@ -65,7 +65,18 @@ def parse_runtime_info(payload: object) -> HarnessProcessInfo | None: _TASK_DIRECTORY_KEYS = ("HOME", "XDG_DATA_HOME", "XDG_CONFIG_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME") # Record the task's home and XDG directories, before OpenCode's overrides, for the # task_environment.mjs shell.env hook. Defaults follow the XDG specification. -_TASK_ENV_SETUP = r"""json() { local value=${1//\\/\\\\}; printf '"%s"' "${value//\"/\\\"}"; } +_TASK_ENV_SETUP = r"""json() { + local value=${1//\\/\\\\} code octal character escaped + value=${value//\"/\\\"} + # JSON forbids raw control characters; a bash string cannot hold NUL. + for ((code = 1; code < 32; code++)); do + printf -v octal '%03o' "$code" + printf -v character "\\$octal" + printf -v escaped '\\u%04x' "$code" + value=${value//"$character"/"$escaped"} + done + printf '"%s"' "$value" +} home=${HOME:-~} printf '{"HOME": %s, "XDG_DATA_HOME": %s, "XDG_CONFIG_HOME": %s, "XDG_CACHE_HOME": %s, "XDG_STATE_HOME": %s}' \ "$(json "$home")" \ diff --git a/responses_api_agents/opencode_agent/tests/test_install_runtime.py b/responses_api_agents/opencode_agent/tests/test_install_runtime.py index 9166fbb7f35..170bde2ff36 100644 --- a/responses_api_agents/opencode_agent/tests/test_install_runtime.py +++ b/responses_api_agents/opencode_agent/tests/test_install_runtime.py @@ -34,7 +34,6 @@ def sandbox(tmp_path: Path) -> tuple[Path, dict[str, str]]: "sleep", "ln", "readlink", - "rmdir", ) for name in (*tools, "mktemp", "sha256sum", "mv"): executable = shutil.which(name) @@ -244,7 +243,9 @@ def identity(pid: int) -> str: @pytest.mark.parametrize("owner", ["exited", "reused"]) -def test_lock_of_a_killed_installer_is_reclaimed(sandbox: tuple[Path, dict[str, str]], owner: str) -> None: +def test_number_of_a_killed_installer_does_not_block_the_queue( + sandbox: tuple[Path, dict[str, str]], owner: str +) -> None: root, env = sandbox cache_runtime(root) process = subprocess.Popen(["true"] if owner == "exited" else ["sleep", "30"]) @@ -255,11 +256,12 @@ def test_lock_of_a_killed_installer_is_reclaimed(sandbox: tuple[Path, dict[str, else: # A live process that reuses the owner's PID started later, so it does not hold the lock. recorded = f"{process.pid} {int(identity(process.pid).split()[1]) - 1}" - (root / "runtime.lock.owner").symlink_to(recorded) + (root / "runtime.lock.d").mkdir() + (root / "runtime.lock.d/1").symlink_to(recorded) result = run_installer(root, env) assert result.returncode == 0, result.stderr - assert not (root / "runtime.lock.owner").is_symlink() - assert not (root / "runtime.lock.reclaim").exists() + # The installer queued behind the dead owner's number and marked its own done. + assert (root / "runtime.lock.d/2").is_symlink() and (root / "runtime.lock.d/2.done").exists() finally: process.kill() process.wait() @@ -280,7 +282,8 @@ def test_concurrent_installers_wait_for_the_lock_holder(sandbox: tuple[Path, dic time.sleep(0.01) else: holder = subprocess.Popen(["sleep", "30"]) - (root / "runtime.lock.owner").symlink_to(identity(holder.pid)) + (root / "runtime.lock.d").mkdir() + (root / "runtime.lock.d/1").symlink_to(identity(holder.pid)) try: installer = subprocess.Popen( [str(root / "bin/bash"), str(root / "install.sh"), str(root / "runtime"), "1.17.11"], env=env @@ -356,7 +359,7 @@ def seed_binary( ) -@pytest.mark.parametrize("failure", ["digest", "version"]) +@pytest.mark.parametrize("failure", ["digest", "version", "unhealthy"]) def test_invalid_seed_preserves_existing_executable(sandbox, failure): root, env = sandbox binary = root / "preinstalled-opencode" @@ -364,9 +367,13 @@ def test_invalid_seed_preserves_existing_executable(sandbox, failure): assert seed_binary(root, env, binary).returncode == 0 target = root / "runtime/opencode" original, inode = target.read_bytes(), target.stat().st_ino - binary.write_text("#!/bin/bash\necho 1.17.10\n") + # An unhealthy probe prints the right version but fails. + binary.write_text( + "#!/bin/bash\necho 1.17.11\nexit 7\n" if failure == "unhealthy" else "#!/bin/bash\necho 1.17.10\n" + ) result = seed_binary(root, env, binary, digest="0" * 64 if failure == "digest" else None) - assert result.returncode != 0 and f"{failure} mismatch" in result.stderr + message = "digest mismatch" if failure == "digest" else "version mismatch" + assert result.returncode != 0 and message in result.stderr assert target.read_bytes() == original and target.stat().st_ino == inode assert not list(target.parent.glob(".opencode-*")) assert not (root / "download.log").exists() and not (root / "packages.log").exists() diff --git a/responses_api_agents/opencode_agent/tests/test_task_environment.py b/responses_api_agents/opencode_agent/tests/test_task_environment.py index 7818127b57f..0620b318b76 100644 --- a/responses_api_agents/opencode_agent/tests/test_task_environment.py +++ b/responses_api_agents/opencode_agent/tests/test_task_environment.py @@ -156,3 +156,15 @@ def test_pinned_cli_reports_whether_required_plugin_initialized(tmp_path: Path, else: assert "failed to load plugin" in result.stderr assert not receipt.exists() + + +@pytest.mark.parametrize("home", ["/task/a\tb", "/task/new\nline", '/task/"quoted"\\back']) +def test_recorded_task_directories_are_valid_json(tmp_path: Path, home: str) -> None: + from responses_api_agents.opencode_agent.sandbox import runner_setup + + env = {"PATH": os.environ["PATH"], "HOME": home, "XDG_CONFIG_HOME": f"{home}\x01\x1fconfig"} + setup = runner_setup(str(tmp_path), [str(tmp_path / "private")]) + result = subprocess.run(["bash", "-c", setup], env=env, capture_output=True, text=True, timeout=10) + assert result.returncode == 0, result.stderr + recorded = json.loads((tmp_path / "task-env.json").read_text()) + assert recorded["HOME"] == home and recorded["XDG_CONFIG_HOME"] == f"{home}\x01\x1fconfig" From c5fd0702b33f5aa1f7f6c9088eb2f476fdc5a67a Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 26 Oct 2026 20:15:55 -0400 Subject: [PATCH 24/32] test(opencode,openclaw): pass the bash runner's payload to the neutral-name tests The pkill tests merged from main called payload() with a timeout, which only the Python runner's input carried; the bash runner takes its timeout from execute(). Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- .../openclaw_agent/tests/test_exec_transport.py | 4 +--- .../opencode_agent/tests/test_exec_transport.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/responses_api_agents/openclaw_agent/tests/test_exec_transport.py b/responses_api_agents/openclaw_agent/tests/test_exec_transport.py index 89789d8fc5c..a9b2ea67e64 100644 --- a/responses_api_agents/openclaw_agent/tests/test_exec_transport.py +++ b/responses_api_agents/openclaw_agent/tests/test_exec_transport.py @@ -157,9 +157,7 @@ async def test_supervisor_and_runner_run_under_neutral_names(tmp_path): """A task agent's `pkill -f python3` or `killall python3` must not match the processes above OpenClaw.""" state, provider, workdir = make_session(tmp_path) code = "import pathlib,time; pathlib.Path('ready').touch(); time.sleep(60)" - task = state.task = asyncio.create_task( - state.execute(payload(state, code, timeout=60), timeout=60, close_timeout=3) - ) + task = state.task = asyncio.create_task(state.execute(payload(state, code), timeout=60, close_timeout=3)) try: async with asyncio.timeout(5): while not (workdir / "ready").exists(): diff --git a/responses_api_agents/opencode_agent/tests/test_exec_transport.py b/responses_api_agents/opencode_agent/tests/test_exec_transport.py index 8b22bff7845..bb16b3ab391 100644 --- a/responses_api_agents/opencode_agent/tests/test_exec_transport.py +++ b/responses_api_agents/opencode_agent/tests/test_exec_transport.py @@ -189,9 +189,7 @@ async def test_supervisor_and_runner_run_under_neutral_names(tmp_path): f"pathlib.Path({state.session.session_dir!r},'task-environment.ready').write_text('ready\\n'); " "pathlib.Path('ready').touch(); time.sleep(60)" ) - task = state.task = asyncio.create_task( - state.execute(payload(state, code, timeout=60), timeout=60, close_timeout=3) - ) + task = state.task = asyncio.create_task(state.execute(payload(state, code), timeout=60, close_timeout=3)) try: async with asyncio.timeout(5): while not (workdir / "ready").exists(): From 6198d4da4962454a9dc0508948b445a0f8c6a872 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Mon, 26 Oct 2026 20:45:07 -0400 Subject: [PATCH 25/32] test(agent_utils): find the stdlib runner as the tool's parent The pkill test from main assumed the supervisor's only child is the runner. The bash supervisor also forks its polling sleeps, so take the runner as the parent of the tool it started and check that it is a supervisor child. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- tests/unit_tests/test_sandbox_supervisor_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit_tests/test_sandbox_supervisor_client.py b/tests/unit_tests/test_sandbox_supervisor_client.py index d6e1ccf1851..53cceebf2ae 100644 --- a/tests/unit_tests/test_sandbox_supervisor_client.py +++ b/tests/unit_tests/test_sandbox_supervisor_client.py @@ -245,8 +245,10 @@ async def test_kills_by_interpreter_name_miss_the_supervisor_and_stdlib_runner(s while not tool_pid_file.exists() or not tool_pid_file.read_text(): await asyncio.sleep(0.01) supervisor_pid = int((session_dir / SUPERVISOR_PID_FILE).read_text()) - (runner_pid,) = map(int, Path(f"/proc/{supervisor_pid}/task/{supervisor_pid}/children").read_text().split()) tool_pid = int(tool_pid_file.read_text()) + # The bash supervisor's polling sleeps are children too; the runner is the tool's parent. + runner_pid = int(Path(f"/proc/{tool_pid}/stat").read_text().rsplit(")", 1)[1].split()[1]) + assert str(runner_pid) in Path(f"/proc/{supervisor_pid}/task/{supervisor_pid}/children").read_text().split() # The supervisor is bash; the runner runs through a link to the base interpreter. for pid, name in ((supervisor_pid, "bash"), (runner_pid, "gym-runner")): assert Path(f"/proc/{pid}/comm").read_text().strip() == name From bb1661b85ed219c8088cef69e87b87e342545f53 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 27 Oct 2026 12:20:49 -0400 Subject: [PATCH 26/32] fix(agent_utils): reach processes that leave the harness during a stop Review found that one descendant snapshot before TERM let a confirmed receipt leave processes running: a TERM handler that runs `setsid ... &` and exits, or a tool that keeps forking new sessions during the grace period. - The supervisor gives the harness a NEMO_GYM_SUPERVISOR environment marker unique to this run. On a forced stop it scans /proc when the stop begins, on every grace and drain round, and before the final KILL. It reaches the harness group, descendants of the harness, of its group and of reached processes, and processes carrying the marker. Marked processes that had already left the harness when the stop began are preserved as services, as the subreaper on main did. The Codex runner passes the marker through its env allowlist. - A recorded group is killed or awaited only while its ID is free or still held by the recorded leader (same start time), so a reused group ID is not hit. - Without a process group of its own, the supervisor restarts itself as a job in a new group, so a provider's group KILL cannot end it before the receipt. The shell left behind passes on TERM and the exit status. - Docstrings and READMEs state what a forced stop reaches and what it does not. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- benchmarks/terminal_bench_2_1/README.md | 8 +- nemo_gym/agent_utils/__init__.py | 4 +- nemo_gym/agent_utils/process_supervisor.sh | 185 +++++++++++++----- nemo_gym/agent_utils/supervisor_client.py | 29 ++- responses_api_agents/codex_agent/README.md | 11 +- .../codex_agent/sandbox_runner.py | 2 + .../hermes_sandboxed_agent/README.md | 8 +- responses_api_agents/openclaw_agent/README.md | 11 +- responses_api_agents/opencode_agent/README.md | 11 +- responses_api_agents/pi_agent/README.md | 11 +- tests/unit_tests/test_process_supervisor.py | 108 ++++++++++ 11 files changed, 298 insertions(+), 90 deletions(-) diff --git a/benchmarks/terminal_bench_2_1/README.md b/benchmarks/terminal_bench_2_1/README.md index 262bb8ce775..12959f44d59 100644 --- a/benchmarks/terminal_bench_2_1/README.md +++ b/benchmarks/terminal_bench_2_1/README.md @@ -157,10 +157,10 @@ a successful smoke, and one task does not establish a benchmark baseline. ## Tasks that start services The supervisor stops live processes in the harness group before capture or verification. -On timeout or close, it first records the harness's descendants in other process groups, -such as in-flight tools in separate sessions. It then stops the harness and kills the group, -those descendants and the groups they lead within the cleanup deadline. Processes that left -the harness before the stop remain for Resources to stop after verification or episode +On timeout or close, it stops the harness, then kills within the cleanup deadline the group, +the harness's descendants in other process groups (such as in-flight tools in separate +sessions) and processes that inherited the harness's `NEMO_GYM_SUPERVISOR` environment +marker. Processes that left the harness before the stop remain for Resources to stop after verification or episode failure. This includes unfinished background jobs such as `make &` left by an earlier tool call, as well as services; either can continue writing during verification. diff --git a/nemo_gym/agent_utils/__init__.py b/nemo_gym/agent_utils/__init__.py index deeb45a9987..0c4ec4d1065 100644 --- a/nemo_gym/agent_utils/__init__.py +++ b/nemo_gym/agent_utils/__init__.py @@ -10,7 +10,9 @@ Only process_supervisor.sh and harness_runner.sh are copied from this package into the task sandbox; they need bash, not Python. The supervisor enforces deadlines, stops -the harness process group and writes its termination receipt. Detached task services +the harness process group and writes its termination receipt. A forced stop also kills the +harness's descendants in other groups and processes that inherited its marker; processes +that had left the harness before the stop, and detached processes after a natural exit, remain for the sandbox owner to stop. Core sandbox APIs and providers live in :mod:`nemo_gym.sandbox`. """ diff --git a/nemo_gym/agent_utils/process_supervisor.sh b/nemo_gym/agent_utils/process_supervisor.sh index 98a85232856..f2ff0919ef2 100755 --- a/nemo_gym/agent_utils/process_supervisor.sh +++ b/nemo_gym/agent_utils/process_supervisor.sh @@ -6,10 +6,17 @@ # that accepts fractions of a second, and mv. # # Runs COMMAND in its own process group until it exits, the deadline passes or the stop file -# appears. A stopped harness gets TERM, then the whole group gets KILL, and so do the descendants -# that were still running in other process groups when it was stopped (tools a CLI started in their -# own session). The receipt confirms that none of them can still run. Processes that left the -# harness before the stop (detached services) are left for the sandbox owner. +# appears. When the harness exits by itself, its process group gets KILL. A stopped harness gets +# TERM, then KILL together with its process group and every process the stop reaches: each process +# that descends from the harness, from a member of its group or from another reached process, and +# each process that inherited the harness's NEMO_GYM_SUPERVISOR environment marker, such as a tool +# a CLI started in its own session or a service a tool daemonized during the stop. /proc is scanned +# when the stop begins and on every polling round until nothing reached still runs. +# +# Not reached: processes that had already left the harness when the stop began (detached +# services), which are left for the sandbox owner, and a process that leaves the harness between two +# scans after dropping the marker from its environment. The receipt confirms that no reached +# process can still run; it is not a sandbox-empty check. # # usage: process_supervisor.sh --timeout SECONDS --cleanup-timeout SECONDS --receipt PATH --stop-file PATH -- COMMAND... @@ -48,67 +55,114 @@ stat_fields() { [[ ${fields[1]} =~ ^[0-9]+$ && ${fields[2]} =~ ^[0-9]+$ && ${fields[19]} =~ ^[0-9]+$ ]] } -# Succeeds while a member of one of the given process groups can still run. Zombies cannot, and -# only their parent can reap them. A process whose state cannot be read counts as running. -group_alive() { - local entry group - for entry in /proc/[0-9]*; do - if ! stat_fields "${entry#/proc/}"; then - [[ -e $entry ]] && return 0 - continue - fi - [[ ${fields[0]} == [ZX] ]] && continue - for group; do - [[ ${fields[2]} == "$group" ]] && return 0 - done - done +# Succeeds when process $1, started at $2, inherited this supervisor's marker. A process that started +# before the supervisor cannot have, so its environment is not read. +marked() { + local variable + ((10#$2 >= 10#$started)) || return 1 + while IFS= read -r -d '' variable; do + [[ $variable == "$marker" ]] && return 0 + done 2>/dev/null <"/proc/$1/environ" return 1 } -# Record the harness's descendants outside its process group as "PID START", and the groups they -# lead. They stop being descendants once the harness dies, so this runs before the harness is stopped. -track_descendants() { - local entry pid changed=1 - local -a parent=() group=() start=() inside=() +# Succeeds while process group $1, whose leader started at $2, can still be the recorded one. A +# group keeps its ID while it has members, so a different process holding that PID means the group +# emptied and the ID was reused. +owned_group() { + if stat_fields "$1"; then + [[ ${fields[19]} == "$2" ]] + else + [[ ! -e /proc/$1 ]] + fi +} + +# Record in tracked[PID]=START and groups[PGID]=LEADER_START what a forced stop reaches: the +# harness, members of the groups already recorded, processes already tracked, processes that carry +# the marker without descending from a preserved one, and the descendants of all of these. The first +# scan of a stop preserves marked processes that it does not reach. +scan() { + local entry pid leader changed=1 + local -a parent=() group=() start=() inside=() kept=() for entry in /proc/[0-9]*; do pid=${entry#/proc/} [[ $pid =~ ^[0-9]+$ ]] && stat_fields "$pid" || continue - parent[pid]=${fields[1]} group[pid]=${fields[2]} start[pid]=${fields[19]} + start[pid]=${fields[19]} + [[ ${fields[0]} == [ZX] ]] || parent[pid]=${fields[1]} group[pid]=${fields[2]} + done + for pid in "${!preserved[@]}"; do + [[ ${start[pid]} == "${preserved[pid]}" ]] && kept[pid]=1 + done + while ((changed)); do + changed=0 + for pid in "${!parent[@]}"; do + [[ -z ${kept[pid]} && -n ${kept[parent[pid]]} ]] || continue + kept[pid]=1 changed=1 + done + done + for pid in "${!parent[@]}"; do + leader=${group[pid]} + if [[ -n ${kept[pid]} || $pid == "$$" ]]; then + continue + elif [[ -n ${tracked[pid]} && ${tracked[pid]} == "${start[pid]}" ]]; then + inside[pid]=1 + elif [[ -n ${groups[leader]+set} && ${start[leader]-${groups[leader]}} == "${groups[leader]}" ]]; then + inside[pid]=1 + elif ((preserving)) && marked "$pid" "${start[pid]}"; then + inside[pid]=1 + fi done - inside[harness]=1 + changed=1 while ((changed)); do changed=0 for pid in "${!parent[@]}"; do - [[ -z ${inside[pid]} && -n ${inside[parent[pid]]} ]] || continue + [[ -z ${inside[pid]} && -z ${kept[pid]} && -n ${inside[parent[pid]]} ]] || continue inside[pid]=1 changed=1 done done + if ((!preserving)); then + for pid in "${!parent[@]}"; do + [[ -z ${inside[pid]} && $pid != "$$" ]] && marked "$pid" "${start[pid]}" && preserved[pid]=${start[pid]} + done + preserving=1 + fi for pid in "${!inside[@]}"; do - [[ ${group[pid]} == "$harness" ]] && continue - tracked+=("$pid ${start[pid]}") - [[ ${group[pid]} == "$pid" ]] && tracked_groups+=("$pid") + tracked[pid]=${start[pid]} + [[ ${group[pid]} == "$pid" ]] && groups[pid]=${start[pid]} done } -# Succeeds while a tracked descendant (same PID and start time) can still run. -tracked_alive() { - local entry - for entry in "${tracked[@]}"; do - stat_fields "${entry% *}" && [[ ${fields[19]} == "${entry#* }" && ${fields[0]} != [ZX] ]] && return 0 +kill_reached() { + local pid leader + for leader in "${!groups[@]}"; do + owned_group "$leader" "${groups[leader]}" && kill -KILL -- "-$leader" 2>/dev/null done - return 1 + for pid in "${!tracked[@]}"; do + stat_fields "$pid" && [[ ${fields[19]} == "${tracked[pid]}" ]] && kill -KILL "$pid" 2>/dev/null + done + return 0 } -kill_harness() { - local entry group - kill -KILL -- "-$harness" 2>/dev/null - for entry in "${tracked[@]}"; do - stat_fields "${entry% *}" && [[ ${fields[19]} == "${entry#* }" ]] && kill -KILL "${entry% *}" 2>/dev/null +# Succeeds while a reached process can still run: a member of a recorded group that was not reused, +# or a tracked process with its start time. Zombies cannot run, and only their parent can reap them. +# A process whose state cannot be read counts as running. +reached_alive() { + local entry pid leader + local -a owned=() + for leader in "${!groups[@]}"; do + owned_group "$leader" "${groups[leader]}" && owned[leader]=1 done - for group in "${tracked_groups[@]}"; do - kill -KILL -- "-$group" 2>/dev/null + for entry in /proc/[0-9]*; do + pid=${entry#/proc/} + if ! [[ $pid =~ ^[0-9]+$ ]] || ! stat_fields "$pid"; then + [[ -e $entry ]] && return 0 + continue + fi + [[ ${fields[0]} == [ZX] ]] && continue + [[ -n ${owned[fields[2]]} ]] && return 0 + [[ -n ${tracked[pid]} && ${tracked[pid]} == "${fields[19]}" ]] && return 0 done - return 0 + return 1 } stopped() { ((stopping)) || [[ -e $stop_file ]]; } @@ -118,6 +172,10 @@ write_receipt() { "$1" "$return_code" "$2" "$timed_out" >"$receipt.tmp" && mv -f "$receipt.tmp" "$receipt" } +# Tests source the functions above without running the supervisor. +[[ ${BASH_SOURCE[0]} == "$0" ]] || return 0 + +arguments=("$@") while (($#)) && [[ $1 != -- ]]; do (($# >= 2)) || usage case $1 in @@ -133,22 +191,42 @@ done shift [[ -n ${timeout-} && -n ${cleanup-} && -n ${receipt-} && -n ${stop_file-} && $# -gt 0 ]] || usage -return_code=null timed_out=false stopping=0 tracked=() tracked_groups=() +# Leave the provider's process group, so that a KILL sent to it cannot end the supervisor before +# the receipt. This shell stays behind to pass on TERM and the exit status. +if [[ ${NEMO_GYM_SUPERVISOR-} != detached ]] && stat_fields $$ && [[ ${fields[2]} != "$$" ]]; then + set -m + NEMO_GYM_SUPERVISOR=detached "$BASH" "$0" "${arguments[@]}" & + supervisor=$! + set +m + trap 'kill -TERM "$supervisor" 2>/dev/null' TERM + while kill -0 "$supervisor" 2>/dev/null; do + wait "$supervisor" + done + wait "$supervisor" + exit +fi + +return_code=null timed_out=false stopping=0 preserving=0 forced=0 +tracked=() groups=() preserved=() trap 'stopping=1' TERM if stopped; then write_receipt true null # Nothing was launched. exit 0 fi -if ! now; then +if ! now || ! stat_fields $$; then write_receipt true '"Sandbox process supervision requires a readable /proc"' # Nothing was launched. exit 1 fi +started=${fields[19]} +identity=$$.$started +marker=NEMO_GYM_SUPERVISOR=$identity # Job control gives the harness its own process group. set -m -"$@" & +NEMO_GYM_SUPERVISOR=$identity "$@" & harness=$! set +m +stat_fields "$harness" && groups[harness]=${fields[19]} tracked[harness]=${fields[19]} || groups[harness]= now deadline=$((NOW + timeout)) @@ -158,17 +236,20 @@ while kill -0 "$harness" 2>/dev/null && ! stopped && ((NOW < deadline)); do done if kill -0 "$harness" 2>/dev/null; then stopped || timed_out=true - track_descendants + forced=1 + scan kill -TERM "$harness" 2>/dev/null grace=$((NOW + cleanup)) while kill -0 "$harness" 2>/dev/null && ((NOW < grace)); do sleep 0.05 + scan now done + scan fi trap '' TERM -kill_harness +kill_reached # A process in uninterruptible sleep outlives KILL; report it rather than wait without bound. now deadline=$((NOW + cleanup)) @@ -184,11 +265,13 @@ wait "$harness" return_code=$? now deadline=$((NOW + cleanup)) -while group_alive "$harness" "${tracked_groups[@]}" || tracked_alive; do - kill_harness +while :; do + ((forced)) && scan + reached_alive || break + kill_reached now if ((NOW >= deadline)); then - write_receipt false '"Harness termination was not confirmed: its process group still runs"' + write_receipt false '"Harness termination was not confirmed: a process it started still runs"' exit 1 fi sleep 0.01 diff --git a/nemo_gym/agent_utils/supervisor_client.py b/nemo_gym/agent_utils/supervisor_client.py index 98b7068b834..73e81c3a0f6 100644 --- a/nemo_gym/agent_utils/supervisor_client.py +++ b/nemo_gym/agent_utils/supervisor_client.py @@ -13,7 +13,7 @@ process_supervisor.sh Session Uploaded before launch bin/gym-runner Launch shell Interpreter link for a standard-library runner launch.claim Launch or stop shell Atomically claims launch or fences it -supervisor.pid Launch shell After claiming launch, before exec +supervisor.pid Launch shell After claiming launch, before exec (see below) stop.request Stop shell Polled by the supervisor to stop the harness cleanup.json Supervisor or stop After cleanup, or when stop fences launch output.log Launch shell Captures supervisor and harness diagnostics @@ -27,6 +27,10 @@ process name nor the supervisor's command line mentions python; a harness argv that keeps its own python (Hermes, Stirrup) still appears in the supervisor's command line. Gym stops the supervisor through the stop file, which does not depend on these names. + +``supervisor.pid`` holds the PID that executes the supervisor. When the provider did not +make the launch shell a process group leader, that process starts the supervisor in a new +group, passes TERM on to it and exits with its status. """ import json @@ -58,12 +62,15 @@ class CleanupReceipt(TypedDict): """Outcome after bounded harness termination, as written by process_supervisor.sh. - ``cleanup_confirmed`` means no live harness-group members remain. Forced stops - also kill the harness's descendants that ran in other process groups when the - stop began, and the groups they lead. Processes that left the harness before the - stop remain until the owner closes the sandbox; this is not a sandbox-empty check. - ``return_code`` is the shell status: 128+N for a harness killed by signal N, and - 126 or 127 when the command could not be run. + ``cleanup_confirmed`` means no live harness-group members remain. After a forced + stop it also means that no process the stop reached still runs: the harness's + descendants, including tools in their own sessions, and processes that inherited + the harness's ``NEMO_GYM_SUPERVISOR`` marker without having left the harness + before the stop. Processes that left the harness before the stop, and a process + that left between two scans without the marker, remain until the owner closes the + sandbox; this is not a sandbox-empty check. ``return_code`` is the shell status: + 128+N for a harness killed by signal N, and 126 or 127 when the command could not + be run. """ cleanup_confirmed: bool @@ -159,9 +166,11 @@ def supervised_launch_command( The supervisor needs only bash. Check that bash can parse it before claiming a launch: a failed bootstrap cannot write a cleanup receipt. Supervisor and harness - diagnostics share output.log. The supervisor stops the harness group; the sandbox - owner stops escaped services. Stops reach the supervisor through the stop file, - not a signal, so the harness keeps the default TERM disposition. + diagnostics share output.log. The supervisor leaves the provider's process group and + stops the harness group, and after a forced stop the processes it reached; the + sandbox owner stops services that left the harness earlier. Stops reach the + supervisor through the stop file, not a signal, so the harness keeps the default + TERM disposition. With ``stdlib_runner``, the harness argv starts with a Python interpreter, and the runner runs through ``bin/gym-runner``, a link to the binary behind that interpreter, diff --git a/responses_api_agents/codex_agent/README.md b/responses_api_agents/codex_agent/README.md index 64be29c3333..d84bbf500ee 100644 --- a/responses_api_agents/codex_agent/README.md +++ b/responses_api_agents/codex_agent/README.md @@ -143,11 +143,12 @@ gap. Positive cleanup confirmation remains required; the adapter validates the captured terminal events without inventing an exit code. One shared Linux supervisor per activation fences delayed launches. -The supervisor stops live harness-group processes. On timeout or close it first records -the harness's descendants in other process groups, typically tools in their own sessions, -then kills the harness group, those descendants and the groups they lead within the cleanup -deadline. Processes that left the harness before the stop, typically services from completed -tools, remain for the sandbox owner to stop later. +The supervisor stops live harness-group processes. On timeout or close it also kills, within +the cleanup deadline, the harness's descendants in other process groups (typically tools in +their own sessions) and processes that inherited the harness's `NEMO_GYM_SUPERVISOR` +environment marker, such as services a tool daemonizes during the stop. Processes that left +the harness before the stop, typically services from completed tools, remain for the sandbox +owner to stop later. Natural CLI exit cannot distinguish unfinished tools in other sessions from services; those processes may still change task files. The stop-time cutoff also has PID races and cannot classify later service forks. It is not a security boundary. diff --git a/responses_api_agents/codex_agent/sandbox_runner.py b/responses_api_agents/codex_agent/sandbox_runner.py index 66338b42c60..956088d7808 100644 --- a/responses_api_agents/codex_agent/sandbox_runner.py +++ b/responses_api_agents/codex_agent/sandbox_runner.py @@ -60,6 +60,8 @@ def interrupt(*_): "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME", + # Lets the supervisor find Codex's tools after they leave its process tree. + "NEMO_GYM_SUPERVISOR", ) if key in os.environ }, diff --git a/responses_api_agents/hermes_sandboxed_agent/README.md b/responses_api_agents/hermes_sandboxed_agent/README.md index 7a2b4f3d790..0f5096d89f6 100644 --- a/responses_api_agents/hermes_sandboxed_agent/README.md +++ b/responses_api_agents/hermes_sandboxed_agent/README.md @@ -110,10 +110,10 @@ Completed parser reports follow Pro's required-pass rule, including empty report execution failures or unusable parser output remain inconclusive. The agent inherits Gym's standard aggregation; Slurm reporting includes coverage. -The supervisor stops live harness-group processes. At the harness timeout it first records -the harness's descendants in other process groups, including tools in separate sessions, then -kills the harness group, those descendants and the groups they lead within the cleanup -deadline. This runner never requests an early stop, so it does not request this cleanup on +The supervisor stops live harness-group processes. At the harness timeout it also kills, +within the cleanup deadline, the harness's descendants in other process groups, including +tools in separate sessions, and processes that inherited the harness's `NEMO_GYM_SUPERVISOR` +environment marker. This runner never requests an early stop, so it does not request this cleanup on close. Processes that left the harness before the stop remain for the sandbox owner. Natural CLI exit cannot distinguish unfinished tools in other sessions from services; those processes may still change task files. The stop-time cutoff also has PID races and diff --git a/responses_api_agents/openclaw_agent/README.md b/responses_api_agents/openclaw_agent/README.md index 72169f4eb3b..bcdb96f2acc 100644 --- a/responses_api_agents/openclaw_agent/README.md +++ b/responses_api_agents/openclaw_agent/README.md @@ -236,11 +236,12 @@ An explicit wall or model-output limit returns partial output as `incomplete`; u raise HTTP 502 even when a useful patch exists. Do not infer rollout success from reward alone. Close returns the captured observations, including salvaged transcript evidence after cancellation. -The supervisor stops live harness-group processes. On timeout or close it first records -the harness's descendants in other process groups, typically tools in their own sessions, -then kills the harness group, those descendants and the groups they lead within the cleanup -deadline. Processes that left the harness before the stop, typically services from completed -tools, remain for the sandbox owner to stop later. +The supervisor stops live harness-group processes. On timeout or close it also kills, within +the cleanup deadline, the harness's descendants in other process groups (typically tools in +their own sessions) and processes that inherited the harness's `NEMO_GYM_SUPERVISOR` +environment marker, such as services a tool daemonizes during the stop. Processes that left +the harness before the stop, typically services from completed tools, remain for the sandbox +owner to stop later. Natural CLI exit cannot distinguish unfinished tools in other sessions from services; those processes may still change task files. The stop-time cutoff also has PID races and cannot classify later service forks. It is not a security boundary. diff --git a/responses_api_agents/opencode_agent/README.md b/responses_api_agents/opencode_agent/README.md index 35d532707b8..11c4a28e916 100644 --- a/responses_api_agents/opencode_agent/README.md +++ b/responses_api_agents/opencode_agent/README.md @@ -87,11 +87,12 @@ correlation plugin. The plugin is uploaded into each sandbox when model-call observability is enabled. Sessions disable automatic updates and models.dev fetches. The runner uses ordinary sandbox `exec`, with an internal deadline. -The supervisor stops live harness-group processes. On timeout or close it first records -the harness's descendants in other process groups, typically tools in their own sessions, -then kills the harness group, those descendants and the groups they lead within the cleanup -deadline. Processes that left the harness before the stop, typically services from completed -tools, remain for the sandbox owner to stop later. +The supervisor stops live harness-group processes. On timeout or close it also kills, within +the cleanup deadline, the harness's descendants in other process groups (typically tools in +their own sessions) and processes that inherited the harness's `NEMO_GYM_SUPERVISOR` +environment marker, such as services a tool daemonizes during the stop. Processes that left +the harness before the stop, typically services from completed tools, remain for the sandbox +owner to stop later. Natural CLI exit cannot distinguish unfinished tools in other sessions from services; those processes may still change task files. The stop-time cutoff also has PID races and cannot classify later service forks. It is not a security boundary. diff --git a/responses_api_agents/pi_agent/README.md b/responses_api_agents/pi_agent/README.md index 36644cd8512..6388b92d6b5 100644 --- a/responses_api_agents/pi_agent/README.md +++ b/responses_api_agents/pi_agent/README.md @@ -164,11 +164,12 @@ configuration must not replace the agent's cap with a larger value. Enforcement 7. Environment Server asks Resources to verify and close the task session. The benchmark owns its verification procedure and sandbox teardown. -The supervisor stops live harness-group processes. On timeout or close it first records -the harness's descendants in other process groups, typically tools in their own sessions, -then kills the harness group, those descendants and the groups they lead within the cleanup -deadline. Processes that left the harness before the stop, typically services from completed -tools, remain for the sandbox owner to stop later. +The supervisor stops live harness-group processes. On timeout or close it also kills, within +the cleanup deadline, the harness's descendants in other process groups (typically tools in +their own sessions) and processes that inherited the harness's `NEMO_GYM_SUPERVISOR` +environment marker, such as services a tool daemonizes during the stop. Processes that left +the harness before the stop, typically services from completed tools, remain for the sandbox +owner to stop later. Natural CLI exit cannot distinguish unfinished tools in other sessions from services; those processes may still change task files. The stop-time cutoff also has PID races and cannot classify later service forks. It is not a security boundary. diff --git a/tests/unit_tests/test_process_supervisor.py b/tests/unit_tests/test_process_supervisor.py index 281458e0d0c..a2172482867 100644 --- a/tests/unit_tests/test_process_supervisor.py +++ b/tests/unit_tests/test_process_supervisor.py @@ -6,6 +6,7 @@ import json import os import shutil +import signal import subprocess import sys import time @@ -128,6 +129,113 @@ def test_runs_with_only_bash_and_core_utilities_on_path(tmp_path: Path) -> None: assert receipt["cleanup_confirmed"] is True and receipt["timed_out"] is True, receipt +def _running(pid: int) -> bool: + try: + return Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[0] not in "ZX" + except FileNotFoundError: + return False + + +@linux_only +@pytest.mark.parametrize("escape", ["term_handler", "forking_tool"]) +def test_forced_stop_reaches_processes_that_leave_the_harness_during_the_stop(tmp_path: Path, escape: str) -> None: + # Each daemon starts its own session while its parent exits or is killed, so neither the harness + # group nor a scan of the harness's descendants finds it; the inherited marker does. + (tmp_path / "daemon.py").write_text("import os, time\nos.setsid()\ntime.sleep(30)\n") + spawn = f'"{sys.executable}" daemon.py & echo $! >> daemons' + worker = { + "term_handler": f"trap '{spawn}; exit 0' TERM; touch ready; while :; do sleep 0.05; done", + "forking_tool": f"trap '' TERM; touch ready; while :; do {spawn}; sleep 0.02; done", + }[escape] + (tmp_path / "worker.sh").write_text(worker) + try: + result = _supervise(tmp_path, "bash", "worker.sh", timeout="1", cwd=tmp_path) + assert result.returncode == 0, result.stderr + receipt = _receipt(tmp_path) + assert receipt["cleanup_confirmed"] is True and receipt["timed_out"] is True, receipt + daemons = [int(pid) for pid in (tmp_path / "daemons").read_text().split()] + assert daemons + assert not [pid for pid in daemons if _running(pid)] + finally: + for pid in (tmp_path / "daemons").read_text().split() if (tmp_path / "daemons").exists() else []: + try: + os.kill(int(pid), signal.SIGKILL) + except ProcessLookupError: + pass + + +@linux_only +@pytest.mark.parametrize("kind", ["process", "group"]) +@pytest.mark.parametrize("reused", [False, True]) +def test_reused_ids_are_neither_awaited_nor_killed(tmp_path: Path, kind: str, reused: bool) -> None: + # A recorded PID or process group ID now held by a process with another start time was reused. + bystander = subprocess.Popen(["sleep", "30"], start_new_session=True) + try: + start = int(Path(f"/proc/{bystander.pid}/stat").read_text().rsplit(")", 1)[1].split()[19]) + record = {"process": "tracked", "group": "groups"}[kind] + script = f'source "$1"; {record}[$2]=$3; reached_alive && echo alive || echo gone; kill_reached' + result = subprocess.run( + ["bash", "-c", script, "_", SUPERVISOR, str(bystander.pid), str(start + 1 if reused else start)], + capture_output=True, + text=True, + timeout=10, + ) + assert result.stdout == ("gone\n" if reused else "alive\n"), result.stderr + if reused: + assert bystander.poll() is None + else: + assert bystander.wait(timeout=5) == -signal.SIGKILL + finally: + bystander.kill() + bystander.wait() + + +@linux_only +def test_provider_group_kill_leaves_supervisor_alive_to_stop_the_harness(tmp_path: Path) -> None: + # The shell stays the provider group leader; the supervisor must leave that group. + worker = ( + "import json, os, pathlib, time; pathlib.Path('pids.tmp').write_text(json.dumps([os.getppid(), os.getpid()]));" + " os.replace('pids.tmp', 'pids.json'); time.sleep(60)" + ) + with (tmp_path / "output.log").open("wb") as output: + process = subprocess.Popen( + ["sh", "-c", '"$@" & wait', "provider", "bash", SUPERVISOR, "--timeout", "60", "--cleanup-timeout", "0.2"] + + ["--receipt", str(tmp_path / "cleanup.json"), "--stop-file", str(tmp_path / "stop.request")] + + ["--", sys.executable, "-c", worker], + cwd=tmp_path, + start_new_session=True, + stdout=output, + stderr=output, + ) + pids = [] + try: + deadline = time.monotonic() + 30 + while not (tmp_path / "pids.json").exists(): + assert time.monotonic() < deadline, (tmp_path / "output.log").read_text() + time.sleep(0.01) + pids = json.loads((tmp_path / "pids.json").read_text()) + supervisor_pid, worker_pid = pids + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=10) + assert _running(supervisor_pid) + (tmp_path / "stop.request").touch() + deadline = time.monotonic() + 10 + while not (tmp_path / "cleanup.json").exists(): + assert time.monotonic() < deadline, (tmp_path / "output.log").read_text() + time.sleep(0.01) + assert _receipt(tmp_path) == {"cleanup_confirmed": True, "return_code": 143, "error": None, "timed_out": False} + assert not _running(worker_pid) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + for pid in reversed(pids): + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + + @linux_only def test_existing_stop_file_fences_the_launch(tmp_path: Path) -> None: stop = tmp_path / "stop.request" From d303a762ca02b0cea6ffe73455cc4a4db3e67806 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 27 Oct 2026 12:20:49 -0400 Subject: [PATCH 27/32] fix(agent_utils): linear CR strip, errexit guard and distinct runner statuses - emit() strips a trailing CR without `${1%$'\r'}`, which tried the suffix at every position and took seconds per MB, long enough for Pi's agent_end event to outlast the deadline. - The runner resets errexit, nounset, pipefail and IFS, so a BASH_ENV with `set -e` no longer stops it at the first non-JSON line. - A failed exec redirection no longer falls through into events mode or launches the CLI. The runner exits 125 for its own failures (bad cwd, an exit in setup) and 126 or 127 when the CLI could not be started. - Drop `trap - INT QUIT`: it did not change what the CLI inherits, and its test passed without it. - Kilo tells runner failures apart from Kilo's own exit status, reports no exit code for a Kilo the runner never started, and none for a cancelled run. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- nemo_gym/agent_utils/harness_runner.sh | 33 ++++++---- nemo_gym/agent_utils/sandbox_session.py | 12 +++- responses_api_agents/kilocode_agent/app.py | 12 +++- .../kilocode_agent/sandbox.py | 7 +- .../kilocode_agent/tests/test_sandbox.py | 44 +++++++++++-- tests/unit_tests/test_harness_runner.py | 64 +++++++++++++++---- 6 files changed, 136 insertions(+), 36 deletions(-) diff --git a/nemo_gym/agent_utils/harness_runner.sh b/nemo_gym/agent_utils/harness_runner.sh index bad3a64371b..1e9d1f738fb 100755 --- a/nemo_gym/agent_utils/harness_runner.sh +++ b/nemo_gym/agent_utils/harness_runner.sh @@ -12,34 +12,45 @@ # setup bash code run first, before env is applied # unset_env array of extglob patterns naming inherited variables to drop # events file receiving each JSON object line of stdout as [seen_at, object] -# Writes runtime.json beside PARAMS. +# Writes runtime.json beside PARAMS. Exits with the command's status, 125 when the runner fails before +# starting it (including an exit from setup), or 126 or 127 when the command could not be started. +# An inherited BASH_ENV or SHELLOPTS must not stop the runner at an expected failure. +set +o errexit +o nounset +o pipefail +IFS=$' \t\n' shopt -s extglob # Inherited variables with these names must not configure the runner or leak into the CLI. unset -v cwd command env stdin stdout stderr setup unset_env events -source "$1" || exit 1 +trap 'exit 125' EXIT +source "$1" || exit printf '{"hostname": "%s", "pid": %d}\n' "$HOSTNAME" "$$" >"$(dirname "$1")/runtime.json" -eval "${setup-}" || exit 1 +eval "${setup-}" || exit for pattern in "${unset_env[@]}"; do for name in $(compgen -e); do [[ $name == $pattern ]] && unset "$name" done done ((${#env[@]})) && export "${env[@]}" -cd "$cwd" || exit 1 -[[ -n ${events-} ]] || exec "${command[@]}" <"$stdin" >"$stdout" 2>"$stderr" +cd "$cwd" || exit +# A failed redirection does not end bash at exec; the CLI did not start. +if [[ -z ${events-} ]]; then + trap - EXIT + exec "${command[@]}" <"$stdin" >"$stdout" 2>"$stderr" + exit 126 +fi # Stamp stdout lines as they appear. A file, unlike a pipe, cannot be held open by a detached tool. -: >"$stdout" -exec 3<"$stdout" 4>"$events" -# Without job control an async command ignores INT and QUIT, which its tools would inherit; the -# runner's own descriptors stay out of the CLI. -{ trap - INT QUIT; exec "${command[@]}" <"$stdin" >>"$stdout" 2>"$stderr" 3<&- 4>&-; } & +: >"$stdout" && exec 3<"$stdout" 4>"$events" || exit +trap - EXIT +# The runner's own descriptors stay out of the CLI. +{ exec "${command[@]}" <"$stdin" >>"$stdout" 2>"$stderr" 3<&- 4>&-; exit 126; } & cli=$! trap 'kill -TERM "$cli" 2>/dev/null' TERM pending="" emit() { - local line=${1%$'\r'} + # ${1%$'\r'} would try the suffix at every position of a long line. + local line=$1 + [[ ${line: -1} == $'\r' ]] && line=${line%?} [[ $line == {* ]] && printf '[%s, %s]\n' "$seen" "$line" >&4 } # Stamp new complete lines, keeping a partial one for the next pass. Bash < 5 has whole seconds; diff --git a/nemo_gym/agent_utils/sandbox_session.py b/nemo_gym/agent_utils/sandbox_session.py index 733f23c35ff..dd9125a3a6e 100644 --- a/nemo_gym/agent_utils/sandbox_session.py +++ b/nemo_gym/agent_utils/sandbox_session.py @@ -34,6 +34,10 @@ # Shared runner uploaded by adapters; reads the bash source written by shell_assignments(). HARNESS_RUNNER_FILE = "harness_runner.sh" HARNESS_RUNNER_SCRIPT = Path(__file__).with_name(HARNESS_RUNNER_FILE) +# The runner's exit statuses when the CLI never ran: its own failure (bad cwd, setup), and a CLI that +# could not be started. Any other status is the CLI's own. +HARNESS_RUNNER_FAILED = 125 +HARNESS_COMMAND_NOT_STARTED = (126, 127) # Where adapters upload a standard-library Python runner, relative to session_dir. Without a # .py suffix, a `pkill -f '\.py'` in the task cannot match the runner's command line. RUNNER_FILE = "sandbox_runner" @@ -111,8 +115,12 @@ class SandboxSession[Artifacts]: receipt skips capture, releases a borrowed connection and leaves teardown to Resources; an owned sandbox falls back to provider stop. Closed means release succeeded. - The supervisor stops the harness process group. Processes that left the group - (detached services or tools) remain until the sandbox owner destroys the sandbox. + The supervisor stops the harness process group; a forced stop also reaches the + harness's descendants in other groups and processes that inherited its + ``NEMO_GYM_SUPERVISOR`` marker. Processes that had left the harness when the stop + began, processes in other groups after a natural exit, and a process that left + between two scans without the marker remain until the sandbox owner destroys the + sandbox. Borrowed sandboxes stay available for Resources verification; owned sandboxes are stopped on close. Unconfirmed harness termination prevents capture and verification. diff --git a/responses_api_agents/kilocode_agent/app.py b/responses_api_agents/kilocode_agent/app.py index 23f5bafa9f7..482417c0bfc 100644 --- a/responses_api_agents/kilocode_agent/app.py +++ b/responses_api_agents/kilocode_agent/app.py @@ -31,7 +31,7 @@ from openai.types.responses import ResponseError from pydantic import ConfigDict, Field, JsonValue, PrivateAttr -from nemo_gym.agent_utils.sandbox_session import SandboxSession +from nemo_gym.agent_utils.sandbox_session import HARNESS_COMMAND_NOT_STARTED, HARNESS_RUNNER_FAILED, SandboxSession from nemo_gym.agent_utils.supervisor_client import CleanupReceipt from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import ( @@ -227,8 +227,12 @@ def _runtime_failure(artifacts: KiloArtifacts, cleanup: CleanupReceipt | None) - return "Kilo did not confirm its task-environment plugin" if cleanup["timed_out"]: return None + if cleanup["return_code"] == HARNESS_RUNNER_FAILED: + return f"Kilo runner failed before starting Kilo (exit {HARNESS_RUNNER_FAILED})" + if cleanup["return_code"] in HARNESS_COMMAND_NOT_STARTED: + return f"Kilo could not be started (exit {cleanup['return_code']})" if cleanup["return_code"] not in (0, None): - return f"Kilo runner exited {cleanup['return_code']}" + return f"Kilo exited {cleanup['return_code']}" if artifacts.exit_code not in (0, None): return f"Kilo exited {artifacts.exit_code}" if artifacts.exit_code is None: @@ -469,7 +473,9 @@ def _sandbox_observations(self, state: KiloSandboxSession) -> AgentObservationBu artifacts, state.session.cleanup, cancelled=state.task is not None and state.task.cancelled() ) sandbox.outcome, sandbox.error_type = outcome, error_type - sandbox.exit_code, sandbox.wall_time_s = artifacts.exit_code, artifacts.wall_time_s + # A cancelled Kilo exits because the stop told it to; that status is not one Kilo chose. + sandbox.exit_code = None if outcome == "cancelled" else artifacts.exit_code + sandbox.wall_time_s = artifacts.wall_time_s _, length_stop = _scan_run_events(artifacts.stdout) if artifacts.observations is None: # The CLI stdout carries only the root session; without the database there is no session tree. diff --git a/responses_api_agents/kilocode_agent/sandbox.py b/responses_api_agents/kilocode_agent/sandbox.py index 8dd720c880d..2dfbe37bc74 100644 --- a/responses_api_agents/kilocode_agent/sandbox.py +++ b/responses_api_agents/kilocode_agent/sandbox.py @@ -12,6 +12,8 @@ from time import perf_counter from nemo_gym.agent_utils.sandbox_session import ( + HARNESS_COMMAND_NOT_STARTED, + HARNESS_RUNNER_FAILED, HARNESS_RUNNER_FILE, HARNESS_RUNNER_SCRIPT, SandboxCommand, @@ -203,9 +205,12 @@ async def collect_artifacts(self) -> KiloArtifacts: stderr = await read_text(self.session.sandbox, path=f"{directory}/stderr.log") except Exception: stderr = await self.session.read_output_log() - # The runner execs Kilo, so the receipt holds Kilo's exit status; a timed-out Kilo has none of its own. + # The runner execs Kilo, so the receipt holds Kilo's exit status; a timed-out Kilo has none of its + # own, nor does a Kilo that the runner never started. cleanup = self.session.cleanup exit_code = cleanup["return_code"] if cleanup is not None and not cleanup["timed_out"] else None + if exit_code == HARNESS_RUNNER_FAILED or exit_code in HARNESS_COMMAND_NOT_STARTED: + exit_code = None try: runtime = json.loads(await read_text(self.session.sandbox, path=f"{directory}/runtime.json")) except Exception: diff --git a/responses_api_agents/kilocode_agent/tests/test_sandbox.py b/responses_api_agents/kilocode_agent/tests/test_sandbox.py index f3451ba1cb9..9ac96257dcd 100644 --- a/responses_api_agents/kilocode_agent/tests/test_sandbox.py +++ b/responses_api_agents/kilocode_agent/tests/test_sandbox.py @@ -10,7 +10,7 @@ import sys import threading from pathlib import Path -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException @@ -31,7 +31,7 @@ from nemo_gym.rollout_observability import AgentInvocation, AgentObservationBundle, join_model_call_observations from nemo_gym.sandbox.access import DirectSandboxConnection, SandboxAccess from nemo_gym.sandbox.providers.base import SandboxExecResult -from responses_api_agents.kilocode_agent.app import KiloCodeAgent, KiloCodeAgentRunRequest +from responses_api_agents.kilocode_agent.app import KiloCodeAgent, KiloCodeAgentRunRequest, _runtime_failure from responses_api_agents.kilocode_agent.sandbox import KiloArtifacts, KiloSandboxSession, _sandbox_prepare_command from responses_api_agents.kilocode_agent.tests.test_app import _make_model_server_agent @@ -294,15 +294,45 @@ async def test_connection_failure_never_falls_back_to_host() -> None: install.assert_not_called() -@pytest.mark.parametrize("timed_out", [False, True]) -async def test_artifacts_take_the_exit_code_from_cleanup_only_when_kilo_finished(timed_out: bool) -> None: +@pytest.mark.parametrize( + "return_code, timed_out, exit_code", + [(3, False, 3), (143, True, None), (125, False, None), (126, False, None), (127, False, None)], +) +async def test_artifacts_take_the_exit_code_from_cleanup_only_when_kilo_finished( + return_code: int, timed_out: bool, exit_code: int | None +) -> None: session = state() - session.session.cleanup = receipt(return_code=143 if timed_out else 3, timed_out=timed_out) + session.session.cleanup = receipt(return_code=return_code, timed_out=timed_out) with patch(f"{TRANSPORT}.read_text", side_effect=["partial", "diagnostic", RuntimeError("not written")]): artifacts = await session.collect_artifacts() assert (artifacts.stdout, artifacts.stderr) == ("partial", "diagnostic") - # The supervisor's TERM status after a timeout is not an exit Kilo chose. - assert artifacts.exit_code == (None if timed_out else 3) + # The supervisor's TERM status after a timeout, and the runner's statuses for a Kilo that never + # started, are not exits Kilo chose. + assert artifacts.exit_code == exit_code + + +@pytest.mark.parametrize( + "return_code, failure", + [ + (125, "Kilo runner failed before starting Kilo (exit 125)"), + (126, "Kilo could not be started (exit 126)"), + (127, "Kilo could not be started (exit 127)"), + (3, "Kilo exited 3"), + ], +) +def test_runtime_failure_tells_the_runner_from_kilo(return_code: int, failure: str) -> None: + artifacts = KiloArtifacts(task_environment_ready=True, stdout="", stderr="", exit_code=None) + assert _runtime_failure(artifacts, receipt(return_code=return_code)) == failure + + +def test_cancelled_run_reports_no_exit_code_of_its_own() -> None: + session = state() + session.session.cleanup = receipt(return_code=143) + session.session.artifacts = KiloArtifacts(task_environment_ready=True, stdout="", stderr="", exit_code=143) + session.task = MagicMock(cancelled=MagicMock(return_value=True)) + observations = _make_model_server_agent()._sandbox_observations(session) + sandbox = observations.records[-1] + assert (sandbox.outcome, sandbox.exit_code) == ("cancelled", None) async def test_identical_activations_join_and_replay_independent_copies() -> None: diff --git a/tests/unit_tests/test_harness_runner.py b/tests/unit_tests/test_harness_runner.py index 28d1e47ea11..d74b66b4b3a 100644 --- a/tests/unit_tests/test_harness_runner.py +++ b/tests/unit_tests/test_harness_runner.py @@ -99,6 +99,58 @@ def test_events_keep_numeric_times_and_crlf_lines_without_epochrealtime(tmp_path assert all(isinstance(seen, (int, float)) and seen > 0 for seen, _ in events) +def test_events_mode_stamps_long_lines_in_linear_time(tmp_path: Path) -> None: + # Pi's agent_end event carries the whole conversation; a quadratic strip would outlast the deadline. + code = "import json; print(json.dumps({'text': 'a' * 2_000_000})); print('b' * 1_000_000)" + params = _params(tmp_path, command=[sys.executable, "-c", code], events=str(tmp_path / "events.jsonl")) + started = time.monotonic() + result = _run(params, env=os.environ | {"LC_ALL": "C.UTF-8"}) + assert result.returncode == 0, result.stderr + assert time.monotonic() - started < 10 + (event,) = [json.loads(line)[1] for line in (tmp_path / "events.jsonl").read_text().splitlines()] + assert event == {"text": "a" * 2_000_000} + + +def test_inherited_errexit_does_not_stop_the_runner(tmp_path: Path) -> None: + (tmp_path / "bash_env").write_text("set -euo pipefail\n") + script = 'echo "not json"; echo \'{"type": "ok"}\'; exit 3' + params = _params(tmp_path, command=["bash", "-c", script], events=str(tmp_path / "events.jsonl")) + result = _run(params, env=os.environ | {"BASH_ENV": str(tmp_path / "bash_env")}) + assert result.returncode == 3, result.stderr + events = [json.loads(line)[1] for line in (tmp_path / "events.jsonl").read_text().splitlines()] + assert events == [{"type": "ok"}] + + +@pytest.mark.parametrize("events", [False, True]) +@pytest.mark.parametrize( + "failure, return_codes", + [("setup", {125}), ("cwd", {125}), ("stdin", {126}), ("command", {126, 127})], +) +def test_runner_failures_are_told_apart_from_the_command_status( + tmp_path: Path, failure: str, return_codes: set[int], events: bool +) -> None: + # A command that cannot start must not fall through into events mode or run anyway. + values = {"command": ["touch", str(tmp_path / "ran")]} + if failure == "setup": + values["setup"] = "exit 1" + elif failure == "cwd": + values["cwd"] = str(tmp_path / "missing") + elif failure == "stdin": + values["stdin"] = str(tmp_path / "missing") + else: + values["command"] = [str(tmp_path / "missing")] + if events: + values["events"] = str(tmp_path / "events.jsonl") + params = _params(tmp_path, **values) + (tmp_path / "stdout.log").write_text("earlier output\n") + result = _run(params) + assert result.returncode in return_codes, result.stderr + assert not (tmp_path / "ran").exists() + # Earlier output survives unless the runner got as far as opening stdout. + if failure in ("setup", "cwd") or (failure == "stdin" and not events): + assert (tmp_path / "stdout.log").read_text() == "earlier output\n" + + def test_inherited_runner_names_neither_configure_the_runner_nor_reach_the_cli(tmp_path: Path) -> None: params = _params(tmp_path, command=["bash", "-c", 'echo "${stdout-unset}|${setup-unset}"']) inherited = {"setup": f"touch {tmp_path}/evaluated", "events": str(tmp_path / "events.jsonl"), "stdout": "leak"} @@ -119,18 +171,6 @@ def test_events_mode_keeps_runner_descriptors_out_of_the_cli(tmp_path: Path) -> assert "Bad file descriptor" in (tmp_path / "stderr.log").read_text() -@pytest.mark.skipif( - subprocess.run(["bash", "-c", "((BASH_VERSINFO[0] >= 5))"]).returncode != 0, reason="bash 5 restores INT" -) -def test_events_mode_cli_keeps_default_interrupt_handling(tmp_path: Path) -> None: - code = "import json, signal; print(json.dumps({'int': str(signal.getsignal(signal.SIGINT))}))" - params = _params(tmp_path, command=[sys.executable, "-c", code], events=str(tmp_path / "events.jsonl")) - result = _run(params) - assert result.returncode == 0, result.stderr - (event,) = [json.loads(line)[1] for line in (tmp_path / "events.jsonl").read_text().splitlines()] - assert "default_int_handler" in event["int"] - - @pytest.mark.skipif(sys.platform != "linux", reason="Linux process semantics") def test_events_mode_forwards_term_and_ignores_a_detached_stdout_holder(tmp_path: Path) -> None: # A detached tool keeps stdout open; the runner must still finish when the CLI does. From c69adb86d174306a029023dade9830766aa35f0a Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 27 Oct 2026 12:20:49 -0400 Subject: [PATCH 28/32] fix(agent): clean up staged OpenCode copies; bootstrap bash for Claude Code - The OpenCode installer removes a partly staged binary from its single EXIT trap, however it exits, and a cached binary must exit successfully from --version to be reused. - Claude Code setup installs a missing bash on root images with apk or apt-get, as the other bash-runner adapters do, and otherwise fails with a message that names bash instead of a bare 127. - Reword "native" in the Claude installer docs, docstring and a test name. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- .../claude_code_agent/README.md | 5 +++-- responses_api_agents/claude_code_agent/app.py | 17 +++++++++++++++-- .../claude_code_agent/runtime.py | 2 +- .../tests/test_install_claude_runtime.py | 2 +- .../claude_code_agent/tests/test_sandbox.py | 14 ++++++++++++++ .../opencode_agent/install_opencode_runtime.sh | 14 ++++++++------ .../tests/test_install_runtime.py | 15 +++++++++++---- 7 files changed, 53 insertions(+), 16 deletions(-) diff --git a/responses_api_agents/claude_code_agent/README.md b/responses_api_agents/claude_code_agent/README.md index b7929db0459..8f5d01e7c55 100644 --- a/responses_api_agents/claude_code_agent/README.md +++ b/responses_api_agents/claude_code_agent/README.md @@ -231,8 +231,9 @@ sandbox_install_timeout_seconds: 600 session_close_timeout_seconds: 60 ``` -The image must include `bash`, coreutils, `gzip`, `tar`, and libraries required by the -native Claude binary; it needs no Python. Seed probes the sandbox's architecture and libc, downloads +The image must include coreutils, `gzip`, `tar`, and the libraries the Claude binary +requires; it needs no Python. Seed installs a missing `bash` on a root image with `apk` or +`apt-get`, and otherwise fails with a message that names it. Seed probes the sandbox's architecture and libc, downloads and caches the matching Linux x64/arm64 release on the **Gym host**, then compresses and uploads it in bounded pieces through the sandbox provider's file transfer API. The sandbox needs no internet access for runtime setup. The Gym host needs HTTPS diff --git a/responses_api_agents/claude_code_agent/app.py b/responses_api_agents/claude_code_agent/app.py index 268fb86253c..075389226a1 100644 --- a/responses_api_agents/claude_code_agent/app.py +++ b/responses_api_agents/claude_code_agent/app.py @@ -78,6 +78,19 @@ LOG = logging.getLogger(__name__) +# Bootstrap with POSIX sh; the runtime installer, the harness runner and the supervisor require Bash. +_BASH_BOOTSTRAP = """if ! command -v bash >/dev/null 2>&1; then + [ "$(id -u)" = 0 ] || { echo "Claude Code sandbox sessions require bash: preinstall it or use a root image." >&2; exit 1; } + if command -v apk >/dev/null 2>&1; then + apk add --no-cache bash || exit 1 + elif command -v apt-get >/dev/null 2>&1; then + apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends bash || exit 1 + else + echo "Claude Code sandbox sessions require bash: preinstall it (automatic installation requires apt-get or apk)." >&2 + exit 1 + fi +fi +""" class ClaudeCodeAgentConfig(BaseResponsesAPIAgentConfig): @@ -601,9 +614,9 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Clau result = await sandbox.exec(f"mkdir -p -- {quote(workdir)}", cwd="/", timeout_s=30) self._check_sandbox_setup(result) result = await sandbox.exec( - f"test -d {quote(workdir)} && umask 077 && mkdir -p {quote(directory)}/config", + _BASH_BOOTSTRAP + f"test -d {quote(workdir)} && umask 077 && mkdir -p {quote(directory)}/config", cwd="/", - timeout_s=30, + timeout_s=self.config.sandbox_install_timeout_seconds, ) self._check_sandbox_setup(result) await sandbox.upload(HARNESS_RUNNER_SCRIPT, f"{directory}/{HARNESS_RUNNER_FILE}") diff --git a/responses_api_agents/claude_code_agent/runtime.py b/responses_api_agents/claude_code_agent/runtime.py index b9ed3255e1f..964bf207070 100644 --- a/responses_api_agents/claude_code_agent/runtime.py +++ b/responses_api_agents/claude_code_agent/runtime.py @@ -29,7 +29,7 @@ def sha256_file(path: Path) -> str: - """Hash a native binary in bounded chunks.""" + """Hash a Claude binary in bounded chunks.""" digest = hashlib.sha256() with path.open("rb") as binary: for chunk in iter(lambda: binary.read(1024 * 1024), b""): diff --git a/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py b/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py index a48acaa8062..cdccbc6491a 100644 --- a/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py +++ b/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py @@ -88,7 +88,7 @@ def test_preinstalled_binary_needs_trusted_checksum_and_matching_version(tmp_pat assert not (tmp_path / "runtime" / "claude").exists() -def test_installer_verifies_decompressed_native_bytes_and_check_only_reports_absence(tmp_path, binary): +def test_installer_verifies_uncompressed_bytes_and_check_only_reports_absence(tmp_path, binary): source, checksum = binary runtime = tmp_path / "runtime" assert install(runtime, "--sha256", checksum, "--check-only").returncode == 3 diff --git a/responses_api_agents/claude_code_agent/tests/test_sandbox.py b/responses_api_agents/claude_code_agent/tests/test_sandbox.py index 12f544e7a0c..20ffe0760c3 100644 --- a/responses_api_agents/claude_code_agent/tests/test_sandbox.py +++ b/responses_api_agents/claude_code_agent/tests/test_sandbox.py @@ -4,6 +4,7 @@ import asyncio import io import json +import subprocess import tarfile import threading from types import SimpleNamespace @@ -73,6 +74,19 @@ async def test_seed_installs_inside_sandbox_and_preserves_ownership(monkeypatch, assert not any(command.startswith("mkdir -p -- /app") for command in commands) +def test_setup_without_bash_names_it(tmp_path): + # A non-root image without bash must fail at setup with a message, not with a bare 127 later. + bindir = tmp_path / "bin" + bindir.mkdir() + (bindir / "id").write_text("#!/bin/sh\necho 1000\n") + (bindir / "id").chmod(0o755) + result = subprocess.run( + ["/bin/sh", "-c", app._BASH_BOOTSTRAP], env={"PATH": str(bindir)}, capture_output=True, text=True, timeout=10 + ) + assert result.returncode == 1 + assert "require bash" in result.stderr + + @pytest.mark.asyncio @pytest.mark.parametrize("workdir", ["/", "/tmp", "relative", "/app/../tmp", "/tmp/nemo-gym-claude-runtime-2.1.108"]) async def test_bad_workdir_rejected_before_connect(monkeypatch, workdir): diff --git a/responses_api_agents/opencode_agent/install_opencode_runtime.sh b/responses_api_agents/opencode_agent/install_opencode_runtime.sh index 34487a24b2b..d56047fe280 100644 --- a/responses_api_agents/opencode_agent/install_opencode_runtime.sh +++ b/responses_api_agents/opencode_agent/install_opencode_runtime.sh @@ -23,6 +23,10 @@ process_identity() { echo "$1" fi } +staged= +ticket= +# However the installer exits, remove a partly staged binary and leave the installer queue. +trap '[ -z "$staged" ] || rm -f "$staged"; [ -z "$ticket" ] || : >"$queue/$ticket.done"' EXIT # Serialize preparation of the version-scoped cache in a shared task sandbox. if command -v flock >/dev/null 2>&1; then # The kernel releases the lock when the installer exits, however it exits. @@ -42,10 +46,8 @@ else number=${number%.done} if [[ $number =~ ^[0-9]+$ ]] && ((number > last)); then last=$number; fi done - ticket=$((last + 1)) - ln -s "$owner" "$queue/$ticket" 2>/dev/null && break + ln -s "$owner" "$queue/$((last + 1))" 2>/dev/null && { ticket=$((last + 1)); break; } done - trap ': >"$queue/$ticket.done"' EXIT until ahead= for ((number = 1; number < ticket; number++)); do @@ -82,21 +84,21 @@ if [ -n "$source_binary" ]; then cp "$source_binary" "$staged" digest=$(sha256sum "$staged") if [ "${digest%% *}" != "$source_sha256" ]; then - rm -f "$staged" echo 'OpenCode executable digest mismatch' >&2 exit 1 fi chmod 755 "$staged" # The probe must exit successfully as well as print the exact version. if ! actual=$("$staged" --version) || [ "$actual" != "$version" ]; then - rm -f "$staged" echo 'OpenCode executable version mismatch' >&2 exit 1 fi mv -f "$staged" "$runtime/opencode" + staged= fi cached=false -if [ -x "$runtime/opencode" ] && [ "$("$runtime/opencode" --version)" = "$version" ]; then +# A cached binary must also exit successfully, not only print the version. +if [ -x "$runtime/opencode" ] && actual=$("$runtime/opencode" --version) && [ "$actual" = "$version" ]; then cached=true fi missing=() diff --git a/responses_api_agents/opencode_agent/tests/test_install_runtime.py b/responses_api_agents/opencode_agent/tests/test_install_runtime.py index 170bde2ff36..e6c128ef9d2 100644 --- a/responses_api_agents/opencode_agent/tests/test_install_runtime.py +++ b/responses_api_agents/opencode_agent/tests/test_install_runtime.py @@ -145,11 +145,13 @@ def test_cached_runtime_needs_no_package_installation(sandbox: tuple[Path, dict[ assert not (root / "download.log").exists() -def test_wrong_cached_version_is_not_reused(sandbox): +# A cached binary that prints the right version and then fails is unhealthy. +@pytest.mark.parametrize("probe", ["echo 1.17.10", "echo 1.17.11; exit 7"]) +def test_wrong_or_unhealthy_cached_version_is_not_reused(sandbox, probe): root, env = sandbox (root / "runtime").mkdir() binary = root / "runtime/opencode" - binary.write_text("#!/bin/bash\necho 1.17.10\n") + binary.write_text(f"#!/bin/bash\n{probe}\n") binary.chmod(0o755) result = run_installer(root, env) assert result.returncode == 19, result.stderr @@ -359,7 +361,7 @@ def seed_binary( ) -@pytest.mark.parametrize("failure", ["digest", "version", "unhealthy"]) +@pytest.mark.parametrize("failure", ["digest", "version", "unhealthy", "staging"]) def test_invalid_seed_preserves_existing_executable(sandbox, failure): root, env = sandbox binary = root / "preinstalled-opencode" @@ -371,8 +373,13 @@ def test_invalid_seed_preserves_existing_executable(sandbox, failure): binary.write_text( "#!/bin/bash\necho 1.17.11\nexit 7\n" if failure == "unhealthy" else "#!/bin/bash\necho 1.17.10\n" ) + if failure == "staging": + # A command that fails while the copy is staged must not leave the copy behind. + (root / "bin/chmod").unlink() + (root / "bin/chmod").write_text("#!/bin/bash\nexit 1\n") + (root / "bin/chmod").chmod(0o755) result = seed_binary(root, env, binary, digest="0" * 64 if failure == "digest" else None) - message = "digest mismatch" if failure == "digest" else "version mismatch" + message = {"digest": "digest mismatch", "staging": "OpenCode setup failed"}.get(failure, "version mismatch") assert result.returncode != 0 and message in result.stderr assert target.read_bytes() == original and target.stat().st_ino == inode assert not list(target.parent.glob(".opencode-*")) From 7908559f87e8670722014440674512073b954cad Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 27 Oct 2026 12:20:49 -0400 Subject: [PATCH 29/32] docs(agent-server): describe sandbox Python and uv paths as exceptions - Codex's task-image Python, stdlib_runner, and Hermes and Stirrup building a uv environment in the sandbox are exceptions being removed; new Python harnesses bring a runtime prepared on the host, as NOOA and hermes_sandboxed_agent do. - The runner row names the utilities it needs and its exit statuses. - The migration note lists the removed python= parameters, the bash requirement, and the receipt and runner status changes. - Harness termination describes what a forced stop reaches, the marker, and supervisor.pid when the supervisor leaves the provider group. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- .../agent-server/add-or-update-a-harness.mdx | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx b/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx index 0593cfa432c..8612c3ad6f5 100644 --- a/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx +++ b/fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx @@ -317,7 +317,7 @@ The [`nemo_gym/agent_utils`](https://github.com/NVIDIA-NeMo/Gym/tree/main/nemo_g ### Code layout -Organize a new adapter under `responses_api_agents//`. The adapter's `sandbox.py` runs on the Agent Server. Inside the sandbox, the shared `harness_runner.sh` starts the harness CLI, or an adapter-specific `sandbox_runner.py` shim wraps a harness that is itself a Python program or needs Python next to its CLI. Run such a shim with an interpreter the adapter installs privately, as Hermes, Stirrup and NOOA do, so the task image does not need Python; Codex is the exception and still uses the task image's Python 3.8 or newer. +Organize a new adapter under `responses_api_agents//`. The adapter's `sandbox.py` runs on the Agent Server. Inside the sandbox, the shared `harness_runner.sh` starts the harness CLI. A harness that is itself a Python program runs an adapter-specific `sandbox_runner.py` shim with a Python runtime that does not come from the task image: a pinned standalone Python that the Agent Server verifies and uploads, as NOOA does, or a runtime prepared outside the task container and mounted read-only, as `hermes_sandboxed_agent` does. Three adapters still depend on Python or the network inside the sandbox, and these paths are being removed rather than copied: Codex runs its shim on the task image's Python 3.8 or newer, and Hermes and Stirrup build a `uv` virtual environment in the sandbox from the network. | Adapter file | Responsibility | | --- | --- | @@ -334,9 +334,9 @@ The shared modules have separate responsibilities. All of them run on the Agent | --- | --- | | `agent_utils/sandbox_session.py` | `SandboxSession` owns the sandbox connection, the ownership flag, `workdir`, `session_dir`, the execution and finalization tasks, the termination receipt, artifacts, and retryable close state. It uploads the supervisor before launch. `SandboxCommand` carries the harness argv. `shell_assignments()` renders runner input as bash source. | | `agent_utils/supervisor_client.py` | Builds the launch and stop shell commands, defines control-file names and timeout budgets, validates termination receipts, and retires session directories safely. | -| `agent_utils/process_supervisor.sh` | Runs inside the Linux sandbox and needs only bash, a readable `/proc`, a `sleep` that accepts fractions of a second, and `mv`. It enforces the harness deadline, stops the harness process group and the descendants it started in other groups, confirms that none still runs, and writes the termination receipt. The sandbox needs neither Gym nor Python. | -| `agent_utils/harness_runner.sh` | Runs inside the sandbox with only bash. It sources the input that `shell_assignments()` wrote, applies the harness environment, runs the CLI with file-backed stdio, and optionally records each JSON line of stdout with the time it was seen. | -| `agent_utils/uv_bootstrap.py` | `uv_for_sandbox()` chooses the `uv` binary to upload for sandboxed Python runtime installation. It reuses a compatible host binary or downloads the static musl build for the detected platform. If the platform probe fails, it warns and falls back to the host binary without confirming compatibility. Hermes uses it. | +| `agent_utils/process_supervisor.sh` | Runs inside the Linux sandbox and needs only bash, a readable `/proc`, a `sleep` that accepts fractions of a second, and `mv`. It leaves the provider's process group, enforces the harness deadline, stops the harness process group and, after a forced stop, the processes described under [Harness termination](#harness-termination), confirms that none still runs, and writes the termination receipt. The sandbox needs neither Gym nor Python. | +| `agent_utils/harness_runner.sh` | Runs inside the sandbox with bash (built with programmable completion, for `compgen`), `dirname`, `sleep`, and, on bash older than 4.2, `date`. It sources the input that `shell_assignments()` wrote, applies the harness environment, runs the CLI with file-backed stdio, and optionally records each JSON line of stdout with the time it was seen. It exits with the CLI's status, with 125 when it fails before starting the CLI (including an `exit` in its setup code), or with 126 or 127 when the CLI could not be started. | +| `agent_utils/uv_bootstrap.py` | `uv_for_sandbox()` chooses the `uv` binary to upload for sandboxed Python runtime installation. It reuses a compatible host binary or downloads the static musl build for the detected platform. If the platform probe fails, it warns and falls back to the host binary without confirming compatibility. Hermes and Stirrup use it to build their runtime inside the sandbox, a path that is being removed; do not use it in a new adapter. | | `agent_utils/task_timeout.py` | `resolve_agent_timeout(metadata, ceiling=...)` returns the harness time budget: a task's positive `metadata.agent_timeout_sec`, which Harbor task rows carry, capped by the configured timeout. A missing or invalid value keeps the configured timeout. Hermes, Pi, Codex, OpenCode, and OpenClaw use it. | | `sandbox/utils.py` | `upload_text()` and `read_text()` transfer UTF-8 text through `AsyncSandbox.upload()` and `download()`. Use those underlying APIs directly for binary files. | @@ -405,7 +405,7 @@ Keep harness-specific response parsing and model and tool behavior in the adapte **1. Define the session state.** Subclass `AgentSessionState` and compose a `SandboxSession`. Implement two hooks: -- `stage_activation` uploads the activation input and returns `SandboxCommand(argv=...)`. Install the runtime earlier, during seed. A CLI harness runs through the shared bash runner, whose name never mentions python. Set `stdlib_runner=True` only for a Python runner that needs nothing beyond the standard library and whose `argv` starts with its interpreter: the runner then runs as `bin/gym-runner`, so a `pkill -f python3` in the task cannot stop it. Upload such a runner without a `.py` suffix, as Codex does. +- `stage_activation` uploads the activation input and returns `SandboxCommand(argv=...)`. Install the runtime earlier, during seed. A CLI harness runs through the shared bash runner, whose name never mentions python. `stdlib_runner=True` exists only for Codex, whose standard-library runner still runs on the task image's Python: the runner then runs as `bin/gym-runner`, so a `pkill -f python3` in the task cannot stop it. Do not use it in a new adapter; it goes away once Codex no longer needs the task image's Python. - `collect` copies artifacts to the Agent Server after harness termination is confirmed, including partial output from interrupted runs. It may run a bounded snapshot command, but it must not restart the harness. Bound the size of what it downloads: trim large logs inside the sandbox first and keep the tail with a truncation marker. For example, the older `hermes_sandboxed_agent`, which calls the process supervisor directly, keeps the last 1 MiB of its runner log; apply the same bound in `collect`. ```python @@ -508,13 +508,19 @@ All control files live under `session_dir`, and their names are defined in `supe | `process_supervisor.sh` | `SandboxSession` uploads it after staging input and before launch. | | `bin/gym-runner` | The launch shell links it to the binary behind the interpreter of a runner that sets `stdlib_runner`, and runs the runner through it, so the runner's process name doesn't mention python. If the link can't be made, the runner runs unchanged. | | `launch.claim` | The launch or stop shell creates it atomically, either to claim execution or to fence a delayed launch. | -| `supervisor.pid` | The launch shell records the PID that becomes the supervisor when the shell executes it. | +| `supervisor.pid` | The launch shell records the PID that becomes the supervisor when the shell executes it. If the provider did not make that shell a process group leader, this process instead starts the supervisor in a new process group, passes `SIGTERM` on to it and exits with its status. | | `stop.request` | The stop shell writes this cancellation marker; the supervisor polls for it. A signal is not used, so the harness keeps the default `SIGTERM` disposition. | | `cleanup.json` | The supervisor records here whether the harness process group terminated. If a stop wins the launch claim, the stop shell writes the same receipt shape with `return_code: null` and `timed_out: false`. | | `output.log` | The launch shell redirects supervisor and harness diagnostics here. Task output is a separate, adapter-owned artifact. | To migrate an adapter that launches the process supervisor itself, such as `hermes_sandboxed_agent`, move launch, termination, collection, and release into `SandboxSession`. Receipt types and timeout helpers that used to live in `process_supervisor.py` are now in `nemo_gym.agent_utils.supervisor_client`. Update the imports, hook arguments, control-file assumptions, test doubles, and tests together. The adapter imports these modules at startup, so a stale import also breaks host calls. +Out-of-tree adapters written for the Python supervisor also need these changes: + +- `SandboxCommand(python=...)` and `supervised_launch_command(python=...)` are removed. The supervisor runs with bash, so the task image needs bash; an adapter that installed `python3` during setup only to run the supervisor or a runner no longer needs to. +- The receipt's `return_code` is now the shell status of the harness command: 128+N when signal N killed it, where the Python supervisor reported -N. A harness that could not be started now gives a confirmed receipt with 126 or 127 instead of a negative receipt with a launch error. +- A harness that runs through `harness_runner.sh` reports the runner's own failures as 125, before the CLI starts, so 125, 126 and 127 do not come from the CLI itself. + ## Prepare the runtime during session setup Implement the [agent-session API](https://github.com/NVIDIA-NeMo/Gym/blob/main/nemo_gym/base_responses_api_agent.py) on the supplied `SandboxAccess`: @@ -539,11 +545,11 @@ Run the installation check at every seed, and reuse a compatible runtime if one The supervisor and the shared runner need only bash in the task image. Do not require Python in the sandbox for a CLI harness: check paths, install runtimes, and post-process output with shell commands or on the Agent Server. -Hermes gets Python from its private runtime because Hermes itself is a Python program. The launch command checks that bash can parse the supervisor before it claims the launch. +A harness that is itself a Python program needs a Python runtime that the adapter brings, never the task image's. The launch command checks that bash can parse the supervisor before it claims the launch. The proposed shared installer must not depend on system package managers or root access. Not all existing adapter installers meet this goal yet. Until they migrate, document the supported images and any remaining prerequisites. -For example, [Hermes seed](https://github.com/NVIDIA-NeMo/Gym/blob/main/responses_api_agents/hermes_agent/sandbox.py) reuses a private virtual environment keyed to the pinned version, after checking that `run_agent` and `mcp` import. A cold setup uploads the `uv` binary that `uv_for_sandbox()` selects for the sandbox's platform, creates a private Python 3.13 environment, and installs the pinned Hermes package with its MCP extra. Installation still needs network access for Python and the Hermes dependencies. Seed also uploads harness support files into the runtime directory, so Hermes does not yet support offline installation or read-only runtime mounts. +[Hermes seed](https://github.com/NVIDIA-NeMo/Gym/blob/main/responses_api_agents/hermes_agent/sandbox.py) and Stirrup seed are exceptions that are being removed, not examples to follow. A cold setup uploads the `uv` binary that `uv_for_sandbox()` selects for the sandbox's platform, creates a private Python 3.13 environment in the sandbox, and installs the pinned package from the network. For a new Python harness, verify a pinned standalone Python on the Agent Server and upload it, as [NOOA](https://github.com/NVIDIA-NeMo/Gym/blob/main/responses_api_agents/nooa_agent/sandbox_runtime.py) does, or prepare the runtime outside the task container and mount it read-only, as [`hermes_sandboxed_agent/prepare_runtime.sh`](https://github.com/NVIDIA-NeMo/Gym/blob/main/responses_api_agents/hermes_sandboxed_agent/prepare_runtime.sh) does. The [shared installation proposal](https://github.com/NVIDIA-NeMo/Gym/pull/3961#issuecomment-6003983236) would add an Agent Server helper, `harness_install.py`, with `ensure_harness_installed(...)`. It would detect the platform, reuse or stage verified artifacts, run the adapter's installer at seed, and return the runtime location. Neither the helper nor its package location exists yet. @@ -669,11 +675,17 @@ Serialize close requests. Keep successful close responses for a documented, boun ### Harness termination -The supervisor starts the harness in a new process group. At the deadline or on a stop request it first records the harness's descendants in other process groups, such as tool commands started in their own sessions, then sends `SIGTERM` to the harness and waits for the grace period. In every case it then kills the harness process group, and after a forced stop also the recorded descendants and the groups they lead, and waits until none of them can still run. Processes that left the harness before the stop are left for the sandbox owner. +The supervisor first leaves the provider's process group, so a provider that kills the group of its exec cannot end the supervisor before the receipt. It starts the harness in a new process group and sets `NEMO_GYM_SUPERVISOR` in the harness's environment to a value unique to this supervisor. When the harness exits by itself, the supervisor kills the harness process group. At the deadline or on a stop request it records what the stop reaches, sends `SIGTERM` to the harness and waits for the grace period, then kills everything reached and waits until none of it can still run. A forced stop reaches: + +- the harness process group; +- every process that descends from the harness, from a member of its group, or from another reached process, such as tool commands started in their own sessions; +- every process that inherited the marker, such as a service that a tool daemonizes during the stop, unless it had already left the harness when the stop began, or descends from such a process. + +The supervisor scans `/proc` when the stop begins and on every polling round until nothing it reached still runs. Two kinds of process are not reached: processes that had left the harness when the stop began, which are left for the sandbox owner, and a process that leaves the harness between two scans after dropping the marker from its environment. Keep `NEMO_GYM_SUPERVISOR` in the environment of tool commands; the Codex runner passes it through its environment allowlist. -Shell-tool commands can run in their own sessions, so a harness-group kill alone may not reach them; the forced-stop cleanup reaches them only while they still descend from the harness. A background process left by a completed tool, even a plain `server &`, can then survive as a task service. Detachment depends on the harness and launch path: OpenClaw's [pinned child adapter](https://github.com/openclaw/openclaw/blob/e085fa1a3ffd32d0ea6917e1e6fb4ecbffbb77d2/src/process/supervisor/adapters/child.ts#L50-L100) disables detachment when `OPENCLAW_SERVICE_MARKER` is nonblank and can fall back to spawning with `detached: false`. Commands on those paths can remain in the harness group and stop at close. +Shell-tool commands can run in their own sessions, so a harness-group kill alone may not reach them; a forced stop does. A background process left by a completed tool, even a plain `server &`, can then survive as a task service. Detachment depends on the harness and launch path: OpenClaw's [pinned child adapter](https://github.com/openclaw/openclaw/blob/e085fa1a3ffd32d0ea6917e1e6fb4ecbffbb77d2/src/process/supervisor/adapters/child.ts#L50-L100) disables detachment when `OPENCLAW_SERVICE_MARKER` is nonblank and can fall back to spawning with `detached: false`. Commands on those paths can remain in the harness group and stop at close. -`cleanup_confirmed: true` means that no live harness-group processes remain. A zombie does not block confirmation, because it cannot run. The receipt does not mean the sandbox is idle, and it is not a security boundary: after a natural exit, a tool command still running in its own session can change task files after the receipt. +`cleanup_confirmed: true` means that no live harness-group processes remain and, after a forced stop, that nothing else the stop reached still runs. A zombie does not block confirmation, because it cannot run. The receipt does not mean the sandbox is idle, and it is not a security boundary: after a natural exit, a tool command still running in its own session can change task files after the receipt. Services survive until the sandbox owner tears down the sandbox: the Resources Server after verification or episode failure, or the agent when it closes an owned sandbox. From fb642f9d2d048ecbd84b406023ad126ea35fb4f6 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 27 Oct 2026 12:51:33 -0400 Subject: [PATCH 30/32] test(agent_utils): find the supervisor through the runner after it leaves the group The pkill test launches without a new session, so the supervisor now restarts itself in its own process group and supervisor.pid names the process that started it. Take the supervisor as the runner's parent, accept it as the recorded PID or that PID's child, and check that neither bash process matches the python kills. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- .../test_sandbox_supervisor_client.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/unit_tests/test_sandbox_supervisor_client.py b/tests/unit_tests/test_sandbox_supervisor_client.py index 53cceebf2ae..4a47a5af24f 100644 --- a/tests/unit_tests/test_sandbox_supervisor_client.py +++ b/tests/unit_tests/test_sandbox_supervisor_client.py @@ -217,6 +217,10 @@ async def test_stop_reaches_running_supervisor_through_the_stop_file(session, tm """ +def _parent(pid: int) -> int: + return int(Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[1]) + + def _alive(pid: int) -> bool: try: return Path(f"/proc/{pid}/stat").read_text().split(") ")[-1][0] != "Z" @@ -247,10 +251,15 @@ async def test_kills_by_interpreter_name_miss_the_supervisor_and_stdlib_runner(s supervisor_pid = int((session_dir / SUPERVISOR_PID_FILE).read_text()) tool_pid = int(tool_pid_file.read_text()) # The bash supervisor's polling sleeps are children too; the runner is the tool's parent. - runner_pid = int(Path(f"/proc/{tool_pid}/stat").read_text().rsplit(")", 1)[1].split()[1]) - assert str(runner_pid) in Path(f"/proc/{supervisor_pid}/task/{supervisor_pid}/children").read_text().split() + runner_pid = _parent(tool_pid) + # This launch shell is no process group leader, so the recorded PID started the supervisor + # in a new group; either way the runner's parent is the supervisor. + harness_supervisor = _parent(runner_pid) + assert harness_supervisor == supervisor_pid or str(harness_supervisor) in ( + Path(f"/proc/{supervisor_pid}/task/{supervisor_pid}/children").read_text().split() + ) # The supervisor is bash; the runner runs through a link to the base interpreter. - for pid, name in ((supervisor_pid, "bash"), (runner_pid, "gym-runner")): + for pid, name in ((supervisor_pid, "bash"), (harness_supervisor, "bash"), (runner_pid, "gym-runner")): assert Path(f"/proc/{pid}/comm").read_text().strip() == name cmdline = Path(f"/proc/{pid}/cmdline").read_bytes().replace(b"\0", b" ").decode() assert "python" not in cmdline and ".py" not in cmdline, cmdline @@ -265,8 +274,8 @@ async def test_kills_by_interpreter_name_miss_the_supervisor_and_stdlib_runner(s ["-x", "python3"], ): matched = {int(pid) for pid in subprocess.run(["pgrep", *args], capture_output=True).stdout.split()} - assert not matched & {supervisor_pid, runner_pid}, args - assert all(map(_alive, (supervisor_pid, runner_pid, tool_pid))) + assert not matched & {supervisor_pid, harness_supervisor, runner_pid}, args + assert all(map(_alive, (supervisor_pid, harness_supervisor, runner_pid, tool_pid))) finally: # Gym's own stop goes through the stop file, whatever the processes are called. receipt = await stop_and_confirm_cleanup( @@ -274,7 +283,7 @@ async def test_kills_by_interpreter_name_miss_the_supervisor_and_stdlib_runner(s ) await asyncio.wait_for(launch, timeout=10) assert receipt["cleanup_confirmed"] is True and receipt["error"] is None - assert not any(map(_alive, (supervisor_pid, runner_pid, tool_pid))) + assert not any(map(_alive, (supervisor_pid, harness_supervisor, runner_pid, tool_pid))) @pytest.mark.parametrize("confirmed", [False, "true", 1]) From 05bac0530f1247808a27c7882693e3af83e8cc5b Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 27 Oct 2026 15:48:24 -0400 Subject: [PATCH 31/32] perf(agent_utils): make the supervisor's forced stop cheaper on busy hosts A forced stop scans every process in /proc several times, and each /proc/PID/stat parse used a longest-prefix match (quadratic in bash) and three regex compilations. On a host with about 1,100 processes a stop took 1.65 s, so Hermes's close test, which allows 2 s and polls for the receipt once a second, failed there while CI passed. Parse with a shortest-prefix loop and glob digit checks, let the forced drain take liveness from its scan instead of a second /proc pass, and skip the extra scan once the harness has exited (the drain scans next). A stop on the same host now takes about 0.55 s. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- nemo_gym/agent_utils/process_supervisor.sh | 35 ++++++++++++++------- tests/unit_tests/test_process_supervisor.py | 25 +++++++++++++++ 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/nemo_gym/agent_utils/process_supervisor.sh b/nemo_gym/agent_utils/process_supervisor.sh index f2ff0919ef2..93c150a1b0e 100755 --- a/nemo_gym/agent_utils/process_supervisor.sh +++ b/nemo_gym/agent_utils/process_supervisor.sh @@ -46,13 +46,18 @@ now() { # Fields of /proc/PID/stat after the command name: [0] state, [1] ppid, [2] pgrp, [19] start time. # The kernel prints the name unescaped, so it can hold spaces, parentheses and newlines: read the -# whole file, split after the last ") ", and never evaluate a field as arithmetic. +# whole file, split after the last ") ", and never evaluate a field as arithmetic. Scans parse every +# process, so this strips the shortest prefix until no ") " is left and tests digits with globs: +# ${stat##*) } takes quadratic time and each =~ compiles a regex. stat_fields() { local stat= IFS= read -r -d '' stat 2>/dev/null <"/proc/$1/stat" [[ $stat == *') '* ]] || return 1 - fields=(${stat##*) }) - [[ ${fields[1]} =~ ^[0-9]+$ && ${fields[2]} =~ ^[0-9]+$ && ${fields[19]} =~ ^[0-9]+$ ]] + while [[ $stat == *') '* ]]; do + stat=${stat#*) } + done + fields=($stat) + [[ ${fields[1]} && ${fields[2]} && ${fields[19]} && ${fields[1]}${fields[2]}${fields[19]} != *[!0-9]* ]] } # Succeeds when process $1, started at $2, inherited this supervisor's marker. A process that started @@ -80,13 +85,17 @@ owned_group() { # Record in tracked[PID]=START and groups[PGID]=LEADER_START what a forced stop reaches: the # harness, members of the groups already recorded, processes already tracked, processes that carry # the marker without descending from a preserved one, and the descendants of all of these. The first -# scan of a stop preserves marked processes that it does not reach. +# scan of a stop preserves marked processes that it does not reach. Succeeds, like reached_alive, +# while a reached process can still run. scan() { - local entry pid leader changed=1 + local entry pid leader changed=1 unreadable=0 local -a parent=() group=() start=() inside=() kept=() for entry in /proc/[0-9]*; do pid=${entry#/proc/} - [[ $pid =~ ^[0-9]+$ ]] && stat_fields "$pid" || continue + if ! [[ $pid =~ ^[0-9]+$ ]] || ! stat_fields "$pid"; then + [[ -e $entry ]] && unreadable=1 + continue + fi start[pid]=${fields[19]} [[ ${fields[0]} == [ZX] ]] || parent[pid]=${fields[1]} group[pid]=${fields[2]} done @@ -130,6 +139,7 @@ scan() { tracked[pid]=${start[pid]} [[ ${group[pid]} == "$pid" ]] && groups[pid]=${start[pid]} done + ((${#inside[@]} || unreadable)) } kill_reached() { @@ -240,12 +250,12 @@ if kill -0 "$harness" 2>/dev/null; then scan kill -TERM "$harness" 2>/dev/null grace=$((NOW + cleanup)) - while kill -0 "$harness" 2>/dev/null && ((NOW < grace)); do - sleep 0.05 + # Once the harness has exited, the drain below scans for what it left. + while sleep 0.05; kill -0 "$harness" 2>/dev/null; do scan now + ((NOW < grace)) || break done - scan fi trap '' TERM @@ -266,8 +276,11 @@ return_code=$? now deadline=$((NOW + cleanup)) while :; do - ((forced)) && scan - reached_alive || break + if ((forced)); then + scan || break + else + reached_alive || break + fi kill_reached now if ((NOW >= deadline)); then diff --git a/tests/unit_tests/test_process_supervisor.py b/tests/unit_tests/test_process_supervisor.py index a2172482867..069f90c46c3 100644 --- a/tests/unit_tests/test_process_supervisor.py +++ b/tests/unit_tests/test_process_supervisor.py @@ -94,6 +94,31 @@ def test_process_names_are_never_evaluated(tmp_path: Path) -> None: bystander.wait() +@linux_only +def test_process_names_that_look_like_fields_are_parsed_after_the_last_parenthesis() -> None: + bystander = subprocess.Popen( + [ + sys.executable, + "-c", + "import ctypes,time; ctypes.CDLL(None).prctl(15, b'x) R 1 1 1 1', 0, 0, 0); time.sleep(30)", + ] + ) + try: + deadline = time.monotonic() + 5 + while Path(f"/proc/{bystander.pid}/comm").read_text().strip() != "x) R 1 1 1 1": + assert time.monotonic() < deadline + time.sleep(0.01) + fields = Path(f"/proc/{bystander.pid}/stat").read_text().rsplit(") ", 1)[1].split() + script = 'source "$1"; stat_fields "$2" && echo "${fields[1]} ${fields[2]} ${fields[19]}"' + result = subprocess.run( + ["bash", "-c", script, "_", SUPERVISOR, str(bystander.pid)], capture_output=True, text=True, timeout=10 + ) + assert result.stdout == f"{fields[1]} {fields[2]} {fields[19]}\n", result.stderr + finally: + bystander.kill() + bystander.wait() + + @linux_only @pytest.mark.parametrize("command, return_code", [("exit 0", 0), ("exit 7", 7), ("kill -KILL $$", 137)]) def test_natural_exit_reports_the_harness_status(tmp_path: Path, command: str, return_code: int) -> None: From 656e1739ebbd0c0c269547c8d41a3b87ccf332f7 Mon Sep 17 00:00:00 2001 From: yaoyu-33 Date: Tue, 27 Oct 2026 20:48:22 -0400 Subject: [PATCH 32/32] fix(agent): close gaps in the flock-less installer queue and root-workdir check Review on #5151 found three gaps in the bash ports: - The Claude installer set its queue ticket before linking it, so an installer stopped after losing a number marked the winner's ticket done and let the next installer run beside it. It now takes the ticket only once its link exists, as OpenCode's installer does. - Both flock-less queues treated a killed but unreaped owner as live, so later installers waited until their timeout. A zombie or dead process now has no identity and no longer holds its number. - OpenClaw's overlap check missed a workdir that resolves to /, such as "/.", because "/" plus a separator is "//". Both sides now drop a trailing slash first; the error names / when it is the owned path. Co-Authored-By: Claude Opus 5.5 Signed-off-by: yaoyu-33 --- .../install_claude_runtime.sh | 7 +- .../tests/test_install_claude_runtime.py | 87 +++++++++++++++---- .../openclaw_agent/sandbox.py | 7 +- .../openclaw_agent/tests/test_sandbox.py | 23 ++++- .../install_opencode_runtime.sh | 3 +- .../tests/test_install_runtime.py | 10 ++- 6 files changed, 113 insertions(+), 24 deletions(-) diff --git a/responses_api_agents/claude_code_agent/install_claude_runtime.sh b/responses_api_agents/claude_code_agent/install_claude_runtime.sh index b12f7154d86..edd3ff2395f 100644 --- a/responses_api_agents/claude_code_agent/install_claude_runtime.sh +++ b/responses_api_agents/claude_code_agent/install_claude_runtime.sh @@ -35,12 +35,13 @@ verify() { } # A process as "PID START" from /proc, so a reused PID never matches its former owner; PID alone -# where /proc is missing. +# where /proc is missing. A zombie or dead process has none: it can never finish its turn. process_identity() { local stat= IFS= read -r -d '' stat 2>/dev/null <"/proc/$1/stat" || true if [[ $stat == *') '* ]]; then set -- "$1" ${stat##*) } + case $2 in Z | X) return ;; esac echo "$1 ${21}" elif kill -0 "$1" 2>/dev/null; then echo "$1" @@ -67,8 +68,8 @@ else number=${number%.done} if [[ $number =~ ^[0-9]+$ ]] && ((number > last)); then last=$number; fi done - ticket=$((last + 1)) - ln -s "$owner" "$queue/$ticket" 2>/dev/null && break + # Hold the number only once its link is ours: on exit the trap marks this ticket done. + ln -s "$owner" "$queue/$((last + 1))" 2>/dev/null && { ticket=$((last + 1)); break; } done until ahead= diff --git a/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py b/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py index cdccbc6491a..15fb9ada664 100644 --- a/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py +++ b/responses_api_agents/claude_code_agent/tests/test_install_claude_runtime.py @@ -115,22 +115,79 @@ def _identity(pid: int) -> str: return f"{pid} {stat.read_text().rsplit(') ', 1)[1].split()[19]}" if stat.exists() else str(pid) -def test_number_left_by_a_killed_installer_does_not_block_the_queue_without_flock(tmp_path, binary): +def _zombie() -> subprocess.Popen: + """A killed child that is not reaped yet, so /proc still lists it with its start time.""" + process = subprocess.Popen(["sleep", "30"]) + process.kill() + stat = Path(f"/proc/{process.pid}/stat") + deadline = time.monotonic() + 5 + while stat.read_text().rsplit(") ", 1)[1].split()[0] != "Z": + assert time.monotonic() < deadline, "the killed child never became a zombie" + time.sleep(0.01) + return process + + +@pytest.mark.parametrize("owner", ["exited", "zombie"]) +def test_number_left_by_a_killed_installer_does_not_block_the_queue_without_flock(tmp_path, binary, owner): source, checksum = binary - exited = subprocess.Popen(["true"]) - exited.wait() - (tmp_path / "runtime.lock.d").mkdir() - (tmp_path / "runtime.lock.d/1").symlink_to(_identity(exited.pid)) - result = subprocess.run( - ["bash", str(INSTALLER), "--version", "2.1.108", "--runtime", str(tmp_path / "runtime")] - + ["--sha256", checksum, "--source", str(source)], - env=_without_flock(tmp_path), - capture_output=True, - text=True, - timeout=30, - ) - assert (result.returncode, result.stdout) == (0, "2.1.108 (Claude Code)\n"), result.stderr - assert (tmp_path / "runtime.lock.d/2.done").exists() + if owner == "zombie" and not Path("/proc").is_dir(): + pytest.skip("A zombie owner is only visible through /proc") + process = subprocess.Popen(["true"]) if owner == "exited" else _zombie() + try: + if owner == "exited": + process.wait() + (tmp_path / "runtime.lock.d").mkdir() + (tmp_path / "runtime.lock.d/1").symlink_to(_identity(process.pid)) + result = subprocess.run( + ["bash", str(INSTALLER), "--version", "2.1.108", "--runtime", str(tmp_path / "runtime")] + + ["--sha256", checksum, "--source", str(source)], + env=_without_flock(tmp_path), + capture_output=True, + text=True, + timeout=30, + ) + assert (result.returncode, result.stdout) == (0, "2.1.108 (Claude Code)\n"), result.stderr + assert (tmp_path / "runtime.lock.d/2.done").exists() + finally: + process.kill() + process.wait() + + +def test_installer_stopped_after_losing_a_number_leaves_the_winners_ticket_open(tmp_path, binary): + source, checksum = binary + winner = subprocess.Popen(["sleep", "30"]) + try: + env = _without_flock(tmp_path) + ln = tmp_path / "bin/ln" + real_ln = ln.resolve() + ln.unlink() + # Another installer links number 1 first; this installer is then stopped before it retries. + ln.write_text( + "#!/bin/bash\n" + "for target; do :; done\n" + f"if [[ $target == */runtime.lock.d/* ]]; then\n" + f' {real_ln} -s "{_identity(winner.pid)}" "$target"\n' + ' kill -TERM "$PPID"\n' + " exit 1\n" + "fi\n" + f'exec {real_ln} "$@"\n' + ) + ln.chmod(0o755) + result = subprocess.run( + ["bash", str(INSTALLER), "--version", "2.1.108", "--runtime", str(tmp_path / "runtime")] + + ["--sha256", checksum, "--source", str(source)], + env=env, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode != 0 + assert (tmp_path / "runtime.lock.d/1").readlink() == Path(_identity(winner.pid)) + # Marking number 1 done would let the next installer run beside the live winner. + assert not (tmp_path / "runtime.lock.d/1.done").exists() + finally: + winner.kill() + winner.wait() def test_installer_without_flock_waits_for_a_live_lock_owner(tmp_path, binary): diff --git a/responses_api_agents/openclaw_agent/sandbox.py b/responses_api_agents/openclaw_agent/sandbox.py index 028a7a1c536..afbc232df90 100644 --- a/responses_api_agents/openclaw_agent/sandbox.py +++ b/responses_api_agents/openclaw_agent/sandbox.py @@ -36,9 +36,10 @@ until [ -d "${path:-/}" ]; do rest=/${path##*/}$rest path=${path%/*}; done owned=$(cd "${path:-/}" && pwd -P) || exit 1 owned=${owned%/}$rest - case "$workdir/" in "$owned/"*) overlap=1 ;; *) overlap= ;; esac - case "$owned/" in "$workdir/"*) overlap=1 ;; esac - [ -z "$overlap" ] || { echo "OpenClaw task workdir overlaps adapter-owned storage: $owned" >&2; exit 1; } + # A path of / ends in its separator already; strip it so / contains every other path. + case "${workdir%/}/" in "${owned%/}/"*) overlap=1 ;; *) overlap= ;; esac + case "${owned%/}/" in "${workdir%/}/"*) overlap=1 ;; esac + [ -z "$overlap" ] || { echo "OpenClaw task workdir overlaps adapter-owned storage: ${owned:-/}" >&2; exit 1; } done """ diff --git a/responses_api_agents/openclaw_agent/tests/test_sandbox.py b/responses_api_agents/openclaw_agent/tests/test_sandbox.py index 8deea828818..962b8577719 100644 --- a/responses_api_agents/openclaw_agent/tests/test_sandbox.py +++ b/responses_api_agents/openclaw_agent/tests/test_sandbox.py @@ -1347,7 +1347,7 @@ async def test_sandbox_envelope_fallback_preserves_known_and_unknown_cache_detai @pytest.mark.parametrize("owned_root", ["sessions", "runtime"]) -@pytest.mark.parametrize("overlap", ["same", "ancestor", "descendant", "symlink", "dotdot"]) +@pytest.mark.parametrize("overlap", ["same", "ancestor", "descendant", "symlink", "dotdot", "root", "root_symlink"]) def test_workdir_cannot_overlap_adapter_storage(tmp_path, owned_root, overlap): from responses_api_agents.openclaw_agent.sandbox import _SANDBOX_PATH_CHECK @@ -1365,6 +1365,12 @@ def test_workdir_cannot_overlap_adapter_storage(tmp_path, owned_root, overlap): elif overlap == "symlink": workdir = tmp_path / "task-alias" workdir.symlink_to(owned) + elif overlap == "root": + # Passes the seed's lexical rejection of "/", and resolves to it. + workdir = "/." + elif overlap == "root_symlink": + workdir = tmp_path / "root-alias" + workdir.symlink_to("/") else: workdir = owned / "repo" / ".." result = subprocess.run( @@ -1378,6 +1384,21 @@ def test_workdir_cannot_overlap_adapter_storage(tmp_path, owned_root, overlap): assert "overlaps adapter-owned storage" in result.stderr +def test_adapter_storage_at_the_root_overlaps_any_workdir(tmp_path): + from responses_api_agents.openclaw_agent.sandbox import _SANDBOX_PATH_CHECK + + (tmp_path / "task").mkdir() + result = subprocess.run( + ["sh", "-c", _SANDBOX_PATH_CHECK, "_", str(tmp_path / "task"), "/", str(tmp_path / "runtime")], + capture_output=True, + text=True, + errors="replace", + timeout=5, + ) + assert result.returncode != 0 + assert "overlaps adapter-owned storage: /" in result.stderr + + def test_workdir_validation_accepts_separate_repository_and_rejects_missing(tmp_path): from responses_api_agents.openclaw_agent.sandbox import _SANDBOX_PATH_CHECK diff --git a/responses_api_agents/opencode_agent/install_opencode_runtime.sh b/responses_api_agents/opencode_agent/install_opencode_runtime.sh index d56047fe280..a03696af98a 100644 --- a/responses_api_agents/opencode_agent/install_opencode_runtime.sh +++ b/responses_api_agents/opencode_agent/install_opencode_runtime.sh @@ -12,12 +12,13 @@ if [ -n "$path_prefix" ]; then export PATH="$path_prefix:$PATH"; fi [ "$(uname -s)" = Linux ] || { echo 'OpenCode sandbox execution requires a Linux task image' >&2; exit 1; } # A process as "PID START" from /proc, so a reused PID never matches its former owner; PID alone -# where /proc is missing. +# where /proc is missing. A zombie or dead process has none: it can never finish its turn. process_identity() { local stat= IFS= read -r -d '' stat 2>/dev/null <"/proc/$1/stat" || true if [[ $stat == *') '* ]]; then set -- "$1" ${stat##*) } + case $2 in Z | X) return ;; esac echo "$1 ${21}" elif kill -0 "$1" 2>/dev/null; then echo "$1" diff --git a/responses_api_agents/opencode_agent/tests/test_install_runtime.py b/responses_api_agents/opencode_agent/tests/test_install_runtime.py index e6c128ef9d2..b3392fb4816 100644 --- a/responses_api_agents/opencode_agent/tests/test_install_runtime.py +++ b/responses_api_agents/opencode_agent/tests/test_install_runtime.py @@ -244,7 +244,7 @@ def identity(pid: int) -> str: return f"{pid} {Path(f'/proc/{pid}/stat').read_text().rsplit(') ', 1)[1].split()[19]}" -@pytest.mark.parametrize("owner", ["exited", "reused"]) +@pytest.mark.parametrize("owner", ["exited", "reused", "zombie"]) def test_number_of_a_killed_installer_does_not_block_the_queue( sandbox: tuple[Path, dict[str, str]], owner: str ) -> None: @@ -255,6 +255,14 @@ def test_number_of_a_killed_installer_does_not_block_the_queue( if owner == "exited": process.wait() recorded = f"{process.pid} 1" + elif owner == "zombie": + # Killed but not yet reaped: /proc still lists it with the recorded start time. + recorded = identity(process.pid) + process.kill() + deadline = time.monotonic() + 5 + while Path(f"/proc/{process.pid}/stat").read_text().rsplit(") ", 1)[1].split()[0] != "Z": + assert time.monotonic() < deadline, "the killed owner never became a zombie" + time.sleep(0.01) else: # A live process that reuses the owner's PID started later, so it does not hold the lock. recorded = f"{process.pid} {int(identity(process.pid).split()[1]) - 1}"