fix(agents): hide the sandbox supervisor and runners from pkill python

fern/versions/latest/pages/agent-server/add-or-update-a-harness.mdx

@@ -404,7 +404,7 @@ Keep harness-specific response parsing and model and tool behavior in the adapte
 
 **1. Define the session state.** Subclass `AgentSessionState` and compose a `SandboxSession`. Implement two hooks:
 
-- `stage_activation` uploads the activation input and returns `SandboxCommand(argv=..., python=...)`. `python` is the sandbox interpreter that runs the supervisor. Install the runtime earlier, during seed.
+- `stage_activation` uploads the activation input and returns `SandboxCommand(argv=..., python=...)`. `python` is the sandbox interpreter that runs the supervisor. Install the runtime earlier, during seed. Set `stdlib_runner=True` when the runner needs only the standard library and `argv` starts with `python`: the runner then runs as `bin/gym-runner`, like the supervisor, so a `pkill -f python3` in the task cannot stop it. Upload such a runner without a `.py` suffix, as Codex and Pi do.
 - `collect` copies artifacts to the Agent Server after harness termination is confirmed, including partial output from interrupted runs. It may run a bounded snapshot command, but it must not restart the harness. Bound the size of what it downloads: trim large logs inside the sandbox first and keep the tail with a truncation marker. For example, the older `hermes_sandboxed_agent`, which calls the process supervisor directly, keeps the last 1 MiB of its runner log; apply the same bound in `collect`.
 
 ```python
@@ -463,7 +463,7 @@ async with self.sem:  # asyncio.Semaphore(self.config.concurrency)
     )
 ```
 
-`execute()` uploads `process_supervisor.py` into `session_dir` after `stage_activation` returns, so adapters do not upload it themselves. It then launches the harness under the supervisor, confirms harness termination, collects artifacts, and releases the provider exec. Captured artifacts remain in `session.artifacts` even when execution raises. If the supervisor writes a final negative receipt, `execute()` raises `HarnessTerminationError` and does not capture artifacts. A nonzero harness exit code does not raise by itself, because the output can still be useful. Convert artifacts into a response and observations in the adapter.
+`execute()` uploads `process_supervisor.py` into `session_dir`, as `process_supervisor`, after `stage_activation` returns, so adapters do not upload it themselves. It then launches the harness under the supervisor, confirms harness termination, collects artifacts, and releases the provider exec. Captured artifacts remain in `session.artifacts` even when execution raises. If the supervisor writes a final negative receipt, `execute()` raises `HarnessTerminationError` and does not capture artifacts. A nonzero harness exit code does not raise by itself, because the output can still be useful. Convert artifacts into a response and observations in the adapter.
 
 **4. Close the session.** In `_close_agent_session_state()`, call `session.close()` first, then cancel the activation task if it is still running. Close joins any finalization in progress, never captures artifacts twice, and is safe to retry while termination evidence is missing. After a final negative receipt, close skips capture, releases a borrowed connection, and leaves teardown to the Resources Server; an owned sandbox is stopped through the provider. Build observations from `session.artifacts` if activation was interrupted before parsing them.
 
@@ -496,7 +496,9 @@ All control files live under `session_dir`, and their names are defined in `supe
 
 | File | Writer and purpose |
 | --- | --- |
-| `process_supervisor.py` | `SandboxSession` uploads it after staging input and before launch. |
+| `process_supervisor` | `SandboxSession` uploads `process_supervisor.py` under this name after staging input and before launch. |
+| `bin/gym-supervisor` | The launch shell links it to the binary behind `SandboxCommand.python` and runs the supervisor through it, so the supervisor's process name doesn't mention python. If the link can't be made, the supervisor runs under `python`. |
+| `bin/gym-runner` | The same link for a runner that sets `stdlib_runner`. |
 | `launch.claim` | The launch or stop shell creates it atomically, either to claim execution or to fence a delayed launch. |
 | `supervisor.pid` | The launch shell records the PID that becomes the supervisor when the shell executes it. |
 | `stop.request` | The stop shell writes this cancellation marker before signaling the supervisor. |

nemo_gym/agent_utils/sandbox_session.py

@@ -17,6 +17,7 @@
     STOP_REQUEST_FILE,
     SUPERVISOR_FILE,
     HarnessTerminationError,
+    check_stdlib_runner,
     remove_session_directory,
     stop_and_confirm_cleanup,
     supervised_launch_command,
@@ -28,6 +29,9 @@
 
 
 LOG = logging.getLogger(__name__)
+# Where adapters upload a standard-library runner, relative to session_dir. Without a
+# .py suffix, a `pkill -f '\.py'` in the task cannot match the runner's command line.
+RUNNER_FILE = "sandbox_runner"
 
 
 def _reuse_or_start[T](
@@ -40,10 +44,20 @@ def _reuse_or_start[T](
 
 @dataclass(frozen=True, kw_only=True)
 class SandboxCommand:
-    """Harness argv and the installed interpreter used to launch its supervisor."""
+    """Harness argv and the installed interpreter used to launch its supervisor.
+
+    ``stdlib_runner`` declares that the runner needs nothing beyond the standard library.
+    Its argv, which must start with ``python``, then runs under a neutral process name,
+    as the supervisor always does, so that ``pkill python3`` in the task cannot stop it.
+    The renamed interpreter is the base interpreter, outside any virtual environment.
+    """
 
     argv: list[str]
     python: str
+    stdlib_runner: bool = False
+
+    def __post_init__(self) -> None:
+        check_stdlib_runner(self.argv, python=self.python, stdlib_runner=self.stdlib_runner)
 
 
 @dataclass(kw_only=True)
@@ -169,6 +183,7 @@ async def execute(
                 timeout=timeout,
                 cleanup_timeout=cleanup_timeout,
                 python=command.python,
+                stdlib_runner=command.stdlib_runner,
             )
             self.launch_started = True
             self._exec_task = asyncio.create_task(

nemo_gym/agent_utils/supervisor_client.py

@@ -10,13 +10,25 @@
 ===================== ==================== ===========================================
 Filename              Writer               When
 ===================== ==================== ===========================================
-process_supervisor.py Session              Uploaded before launch
+process_supervisor    Session              Uploaded before launch
+bin/gym-supervisor    Launch shell         Interpreter link for the supervisor
+bin/gym-runner        Launch shell         Interpreter link for a standard-library runner
 launch.claim          Launch or stop shell Atomically claims launch or fences it
 supervisor.pid        Launch shell         After claiming launch, before exec
 stop.request          Stop shell           Before signalling the supervisor
 cleanup.json          Supervisor or stop   After cleanup, or when stop fences launch
 output.log            Launch shell         Captures supervisor and harness diagnostics
 ===================== ==================== ===========================================
+
+Agents in the task sandbox often clean up with ``pkill -f python3``, ``pkill python3`` or
+``killall python3``. In a reference harness whose agent CLI is a native binary these
+commands are harmless; here they would stop the supervisor, which then ends the
+harness. So the supervisor, and a standard-library runner that opts in, run under
+interpreter links with neutral names, from files without a ``.py`` suffix. Their process
+names never mention python, and neither do their command lines when the runner opts in; a
+harness argv that keeps its own python (Hermes, Stirrup) still appears in the supervisor's
+command line. Gym's own stop path signals the recorded supervisor PID and does not depend
+on these names.
 """
 
 import json
@@ -31,14 +43,18 @@
 from nemo_gym.sandbox.utils import read_text
 
 
-SUPERVISOR_FILE = "process_supervisor.py"
+SUPERVISOR_FILE = "process_supervisor"
+SUPERVISOR_INTERPRETER = "bin/gym-supervisor"
+RUNNER_INTERPRETER = "bin/gym-runner"
 LAUNCH_CLAIM_FILE = "launch.claim"
 SUPERVISOR_PID_FILE = "supervisor.pid"
 STOP_REQUEST_FILE = "stop.request"
 CLEANUP_RECEIPT_FILE = "cleanup.json"
 OUTPUT_LOG_FILE = "output.log"
 
 _CLEANUP_RECEIPT = TypeAdapter(CleanupReceipt)
+# The interpreter binary itself, past virtual-environment links and launcher shims.
+_RESOLVE_INTERPRETER = 'import os; print(os.readlink("/proc/self/exe"))'
 LOG = logging.getLogger(__name__)
 
 
@@ -91,32 +107,60 @@ def supervision_timeouts(*, timeout: float, close_timeout: float) -> tuple[float
     return cleanup_timeout, exec_timeout(timeout=timeout, cleanup_timeout=cleanup_timeout)
 
 
+def check_stdlib_runner(command: list[str], *, python: str, stdlib_runner: bool) -> None:
+    """Reject ``stdlib_runner`` for a harness argv that the runner link cannot replace."""
+    if stdlib_runner and command[:1] != [python]:
+        raise ValueError(
+            f"stdlib_runner requires a harness argv that starts with the interpreter {python!r}; got {command[:1]!r}"
+        )
+
+
 def supervised_launch_command(
     *,
     session_dir: str,
     command: list[str],
     timeout: float,
     cleanup_timeout: float,
     python: str,
+    stdlib_runner: bool = False,
 ) -> str:
     """Fence delayed launches and run a harness command under the shared supervisor.
 
     The adapter must install or select the interpreter explicitly. Check that it
     can load the supervisor before claiming a launch: a failed bootstrap cannot
     write a cleanup receipt. Supervisor and harness diagnostics share output.log.
     The supervisor stops the harness group; the sandbox owner stops escaped services.
+
+    The supervisor runs through ``bin/gym-supervisor``, a link to the binary behind
+    ``python``, so that python-matching kills in the task skip it. With ``stdlib_runner``,
+    the harness command, which must start with ``python``, runs through ``bin/gym-runner``
+    the same way. The links resolve a virtual environment to its base interpreter, so only
+    standard-library runners may opt in. When the link cannot be made or cannot load
+    the supervisor, both processes keep ``python``.
     """
+    check_stdlib_runner(command, python=python, stdlib_runner=stdlib_runner)
     supervisor = quote(f"{session_dir}/{SUPERVISOR_FILE}")
     claim_path = quote(f"{session_dir}/{LAUNCH_CLAIM_FILE}")
+    link = quote(f"{session_dir}/{SUPERVISOR_INTERPRETER}")
+    harness = join(command)
+    runner = ""
+    if stdlib_runner:
+        runner_link = quote(f"{session_dir}/{RUNNER_INTERPRETER}")
+        runner = f'run={quote(python)}; [ "$py" = {link} ] && ln -sfn "$real" {runner_link} && run={runner_link}; '
+        harness = f'"$run" {join(command[1:])}'
     return (
         f"[ -d {quote(session_dir)} ] && [ ! -L {claim_path} ] || exit 0; "
-        f"{quote(python)} -I {supervisor} --help >/dev/null || exit $?; "
-        f"trap '' TERM; ln -s launch {claim_path} 2>/dev/null || exit 0; "
+        f"py={quote(python)}; "
+        f"real=$({quote(python)} -I -c {quote(_RESOLVE_INTERPRETER)} 2>/dev/null) && "
+        f'{{ [ -d {quote(f"{session_dir}/bin")} ] || mkdir {quote(f"{session_dir}/bin")}; }} && ln -sfn "$real" {link} && '
+        f"{link} -I {supervisor} --help >/dev/null 2>&1 && py={link}; "
+        f'[ "$py" = {link} ] || {quote(python)} -I {supervisor} --help >/dev/null || exit $?; '
+        f"{runner}trap '' TERM; ln -s launch {claim_path} 2>/dev/null || exit 0; "
         f"echo $$ > {quote(f'{session_dir}/{SUPERVISOR_PID_FILE}')} && "
-        f"exec {quote(python)} -I {supervisor} "
+        f'exec "$py" -I {supervisor} '
         f"--timeout {timeout} --cleanup-timeout {cleanup_timeout} "
         f"--stop-file {quote(f'{session_dir}/{STOP_REQUEST_FILE}')} "
-        f"--receipt {quote(f'{session_dir}/{CLEANUP_RECEIPT_FILE}')} -- {join(command)} "
+        f"--receipt {quote(f'{session_dir}/{CLEANUP_RECEIPT_FILE}')} -- {harness} "
         f">{quote(f'{session_dir}/{OUTPUT_LOG_FILE}')} 2>&1"
     )
 

responses_api_agents/codex_agent/README.md

@@ -157,6 +157,9 @@ blocks verification. Missing evidence stays retryable; a final negative receipt
 the borrowed connection without making the failed activation gradable. The borrowing
 agent never destroys the sandbox.
 Cleanup coordinates lifecycle; it is not a security boundary against hostile task code.
+The supervisor and the runner above Codex run as `gym-supervisor` and `gym-runner` (links to the
+sandbox's `python3`) from files without a `.py` suffix, so a task agent's `pkill -f python3`,
+`pkill python3` or `killall python3` does not end Codex, as it would not under Harbor.
 
 Responses and close observations preserve text, reasoning, tool events, partial output,
 CLI aggregate usage, and sandbox hostname/PID/version. Wall-time and the pinned CLI's explicit

responses_api_agents/codex_agent/app.py

@@ -35,7 +35,7 @@
 from pydantic import ConfigDict, Field, PrivateAttr, model_validator
 
 from nemo_gym.agent_outcome import AgentOutcome
-from nemo_gym.agent_utils.sandbox_session import SandboxSession
+from nemo_gym.agent_utils.sandbox_session import RUNNER_FILE, SandboxSession
 from nemo_gym.agent_utils.task_timeout import AGENT_TIMEOUT_METADATA_KEY, resolve_agent_timeout
 from nemo_gym.base_resources_server import NEMO_GYM_MCP_METADATA_KEY, BaseRunRequest, BaseVerifyResponse
 from nemo_gym.base_responses_api_agent import (
@@ -729,8 +729,9 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> Code
                     f"Codex installer exited {installed.return_code}, command={install_command}, "
                     f"error_type={installed.error_type}; stdout={installed.stdout}; stderr={installed.stderr}"
                 )
-            for filename in ("sandbox_runner.py", "sandbox_transport.py"):
-                await sandbox.upload(Path(__file__).with_name(filename), f"{directory}/{filename}")
+            await sandbox.upload(Path(__file__).with_name("sandbox_runner.py"), f"{directory}/{RUNNER_FILE}")
+            # The runner loads the transport from beside itself under this name.
+            await sandbox.upload(Path(__file__).with_name("sandbox_transport.py"), f"{directory}/sandbox_transport.py")
         except BaseException as error:
             try:
                 await state.close(self.config.session_close_timeout_seconds)

responses_api_agents/codex_agent/sandbox.py

@@ -10,7 +10,7 @@
 
 from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError
 
-from nemo_gym.agent_utils.sandbox_session import SandboxCommand, SandboxSession
+from nemo_gym.agent_utils.sandbox_session import RUNNER_FILE, SandboxCommand, SandboxSession
 from nemo_gym.base_responses_api_agent import AgentSessionState
 from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming
 from nemo_gym.rollout_observability import AgentObservationBundle
@@ -96,9 +96,10 @@ async def stage_activation(self, payload: dict[str, JsonValue]) -> SandboxComman
             argv=[
                 "python3",
                 "-I",
-                f"{self.session.session_dir}/sandbox_runner.py",
+                f"{self.session.session_dir}/{RUNNER_FILE}",
                 f"{self.session.session_dir}/input.json",
             ],
+            stdlib_runner=True,
         )
 
     async def collect_artifacts(self) -> str:
@@ -119,7 +120,7 @@ async def collect_artifacts(self) -> str:
         self.runtime_info = parse_runtime_info(runtime_payload)
         try:
             snapshot = await self.session.sandbox.exec(
-                f"python3 -I {shlex.quote(self.session.session_dir + '/sandbox_runner.py')} "
+                f"python3 -I {shlex.quote(f'{self.session.session_dir}/{RUNNER_FILE}')} "
                 f"--snapshot {shlex.quote(self.session.session_dir)}",
                 timeout_s=10,
             )

responses_api_agents/codex_agent/tests/test_exec_transport.py

@@ -40,7 +40,7 @@ async def disconnect(self):
         self.disconnected = True
 
     async def exec(self, command, *, cwd=None, timeout_s=30):
-        launch = "--receipt" in command and "process_supervisor.py" in command
+        launch = "--receipt" in command and "process_supervisor" in command
         if launch:
             self.deadline = timeout_s
             self.delayed_command = command
@@ -77,7 +77,7 @@ def make_session(tmp_path):
         ),
         runtime=str(tmp_path / "runtime"),
     )
-    shutil.copyfile(sandbox_runner.__file__, directory / "sandbox_runner.py")
+    shutil.copyfile(sandbox_runner.__file__, directory / "sandbox_runner")
     return state, provider, workdir
 
 

responses_api_agents/codex_agent/tests/test_sandbox_sessions.py

@@ -124,7 +124,7 @@ def publish(self):
         self.files[f"{self.directory}/events.jsonl"] = self.events
 
     async def execute(self, command, **kwargs):
-        if "exec python3 -I " in command and "process_supervisor.py" in command:
+        if "--receipt" in command and "process_supervisor" in command:
             return await self.launch(command=command, **kwargs)
         if "touch " in command and "stop.request" in command:
             self.signals.append("SIGTERM")
@@ -139,7 +139,7 @@ async def create(self, **kwargs):
         payload = json.loads(self.files[payload_path])
         assert payload["cwd"] == getattr(self, "expected_workdir", "/app")
         assert kwargs["cwd"] == getattr(self, "expected_workdir", "/app")
-        assert "sandbox_runner.py" in kwargs["command"]
+        assert "/sandbox_runner " in kwargs["command"] and "/bin/gym-runner" in kwargs["command"]
         self.directory = payload["directory"]
         self.started.set()
         if not self.blocked:
@@ -192,11 +192,12 @@ def test_http_agent_session_flow_runs_codex_in_borrowed_sandbox(setup):
             assert created.status_code == 200, created.text
             session_id = created.json()["agent_session_id"]
             directory = agent._session_records[session_id].state.session.session_dir
-            assert f"{directory}/process_supervisor.py" not in sandbox.files
+            assert f"{directory}/process_supervisor" not in sandbox.files
             installer = f"{directory}/install_codex_runtime.sh"
             assert installer in sandbox.files
-            for filename in ("sandbox_runner.py", "sandbox_transport.py"):
-                assert sandbox.files[f"{directory}/{filename}"] == (Path(__file__).parents[1] / filename).read_text()
+            staged = {"sandbox_runner": "sandbox_runner.py", "sandbox_transport.py": "sandbox_transport.py"}
+            for name, source in staged.items():
+                assert sandbox.files[f"{directory}/{name}"] == (Path(__file__).parents[1] / source).read_text()
             assert agent.config.resources_server is None
             assert not sandbox.launch.called
             assert not any(path.startswith("/app/") for path in sandbox.files)
@@ -205,7 +206,7 @@ def test_http_agent_session_flow_runs_codex_in_borrowed_sandbox(setup):
             assert install_call.kwargs["timeout_s"] == agent.config.sandbox_install_timeout_seconds
             result = client.post("/ng-rollout/codex-smoke-a2/v1/responses", json={"input": "Fix the code"})
             assert result.status_code == 200, result.text
-            assert f"{directory}/process_supervisor.py" in sandbox.files
+            assert f"{directory}/process_supervisor" in sandbox.files
             body = result.json()
             assert body["status"] == "completed"
             assert [item["type"] for item in body["output"]] == [
@@ -1735,7 +1736,7 @@ async def local_session(setup, tmp_path):
     state.session.session_dir = str(directory)
     from responses_api_agents.codex_agent import sandbox_runner
 
-    shutil.copyfile(sandbox_runner.__file__, directory / "sandbox_runner.py")
+    shutil.copyfile(sandbox_runner.__file__, directory / "sandbox_runner")
     sandbox.upload = AsyncMock(side_effect=shutil.copyfile)
     sandbox.download = AsyncMock(side_effect=shutil.copyfile)
 
@@ -1774,7 +1775,7 @@ async def test_close_fences_delayed_launch_before_and_after_removing_session(loc
     commands = []
 
     async def queued_exec(command, **kwargs):
-        if "--receipt" in command and "process_supervisor.py" in command:
+        if "--receipt" in command and "process_supervisor" in command:
             commands.append(command)
             waiting.set()
             if failure == "cancel":