From d70ea7e9d3926556cf14997ef5a3446dfe67a2b2 Mon Sep 17 00:00:00 2001 From: Nikhil Deshmudre Date: Mon, 2 Nov 2026 12:20:46 -0800 Subject: [PATCH] feat(nooa): restrict SWE task networking and prepare offline runtime Signed-off-by: Nikhil Deshmudre --- benchmarks/swebench/pro/nooa.yaml | 13 +- .../agent-server/nooa-swe-networking.mdx | 121 ++++++++ .../nooa-swe-bench-pro.mdx | 36 ++- resources_servers/swebench_pro/app.py | 29 +- .../swebench_pro/tests/test_app.py | 9 +- .../tests/test_task_network_policy.py | 192 ++++++++++++ responses_api_agents/nooa_agent/README.md | 4 + responses_api_agents/nooa_agent/app.py | 7 +- .../nooa_agent/bench_agent_adapter.py | 6 +- responses_api_agents/nooa_agent/config.py | 156 +++------- responses_api_agents/nooa_agent/gym_llm.py | 30 +- .../nooa_agent/install_nooa_runtime.sh | 29 +- responses_api_agents/nooa_agent/invocation.py | 128 ++++++++ .../nooa_agent/offline_runtime.py | 246 +++++++++++++++ .../nooa_agent/requirements.txt | 2 +- responses_api_agents/nooa_agent/runner.py | 5 +- .../nooa_agent/sandbox_entrypoint.py | 4 +- .../nooa_agent/sandbox_runner.py | 3 + .../nooa_agent/sandbox_runtime.py | 185 ++++++++++-- .../tests/test_bench_agent_adapter.py | 195 ++++++++++++ .../nooa_agent/tests/test_config.py | 22 ++ .../nooa_agent/tests/test_gym_llm.py | 73 ++++- .../nooa_agent/tests/test_observability.py | 44 +++ .../nooa_agent/tests/test_offline_runtime.py | 196 ++++++++++++ .../nooa_agent/tests/test_recipe.py | 87 +++++- .../nooa_agent/tests/test_runner.py | 27 +- .../tests/test_runtime_installer.py | 30 ++ .../tests/test_sandbox_entrypoint.py | 13 + .../nooa_agent/tests/test_sandbox_runner.py | 2 + .../nooa_agent/tests/test_sandbox_runtime.py | 283 +++++++++++++++++- .../nooa_agent/tests/test_sandbox_sessions.py | 32 +- 31 files changed, 2018 insertions(+), 191 deletions(-) create mode 100644 fern/versions/latest/pages/agent-server/nooa-swe-networking.mdx create mode 100644 resources_servers/swebench_pro/tests/test_task_network_policy.py create mode 100644 responses_api_agents/nooa_agent/invocation.py create mode 100644 responses_api_agents/nooa_agent/offline_runtime.py create mode 100644 responses_api_agents/nooa_agent/tests/test_bench_agent_adapter.py create mode 100644 responses_api_agents/nooa_agent/tests/test_offline_runtime.py diff --git a/benchmarks/swebench/pro/nooa.yaml b/benchmarks/swebench/pro/nooa.yaml index 3f3c2a8f98c..9697e9ecfa8 100644 --- a/benchmarks/swebench/pro/nooa.yaml +++ b/benchmarks/swebench/pro/nooa.yaml @@ -5,18 +5,24 @@ config_paths: - responses_api_agents/nooa_agent/configs/nooa_agent.yaml - environment_servers/single_agent_turn_legacy/configs/single_agent_turn_legacy.yaml +use_absolute_ip: true + # This flat-row recipe binds its own environment instead of the generic NOOA environment. -nooa_environment_server: - _delete_key: environment_servers +nooa_environment_server: null swebench_pro_nooa_resources_server: _inherit_from: swebench_pro_resources_server resources_servers: swebench_pro: allowed_agents: [nooa_agent] + # Only the task sandbox is restricted; fresh grading sandboxes retain their settings. + task_network_policy: + model_server: ${swebench_pro_nooa_agent.responses_api_agents.nooa_agent.model_server} sandbox_config: # Outlive the episode, patch extraction, and final cleanup budgets. ttl_s: 25200 + provider_options: + platform: {os: linux, arch: amd64} datasets: - name: swebench_pro type: benchmark @@ -34,6 +40,9 @@ swebench_pro_nooa_agent: nooa: execution_mode: sandboxed max_policy_calls: null + # Built on the controller before seeding; task installation performs no downloads. + runtime_wheelhouse: ${oc.env:NOOA_SWE_RUNTIME_WHEELHOUSE,null} + runtime_wheelhouse_required: true swebench_pro_nooa: _inherit_from: single_agent_turn_legacy diff --git a/fern/versions/latest/pages/agent-server/nooa-swe-networking.mdx b/fern/versions/latest/pages/agent-server/nooa-swe-networking.mdx new file mode 100644 index 00000000000..df5c1455d77 --- /dev/null +++ b/fern/versions/latest/pages/agent-server/nooa-swe-networking.mdx @@ -0,0 +1,121 @@ +--- +title: "Run NOOA SWE-bench Pro with restricted networking" +description: "Prepare NOOA offline and restrict only the SWE-bench Pro task sandbox on OpenSandbox." +position: 5 +--- + +# Run NOOA SWE-bench Pro with restricted networking + +The NOOA SWE-bench Pro recipe enables a task-only OpenSandbox network policy. +The SWE Resources server applies default-deny egress when it creates the task +sandbox, before handing the sandbox to NOOA. NOOA borrows that Resources-owned +sandbox. Fresh grading sandboxes continue using the benchmark's original +network settings, images, test artifacts, and dependency preparation. +Other SWE harnesses keep their existing network behavior. + +## Deployment requirements + +Use an OpenSandbox deployment with an egress sidecar and **`dns+nft` enforcement**. +DNS-only filtering can be bypassed with a literal IP address. Provider API +acceptance of a policy is not proof of enforcement; validate both blocked domain +requests and blocked literal-IP requests against the actual deployment. See +[OpenSandbox egress controls](/main/infrastructure/sandbox/opensandbox-best-practices#egress-controls-are-off-by-default). + +The allowlist contains the advertised **Gym model-server host**, which proxies +inference to the configured upstream model. NOOA uses the Gym Responses API; the Gym model adapter +may call a different upstream protocol, such as Chat Completions. It does not add package indexes, +GitHub, or the upstream model service to the task allowlist. Advertise a concrete +host reachable from OpenSandbox; loopback and wildcard bind addresses are rejected. +The policy model reference is derived from the NOOA agent's `model_server`, so overriding that +reference also changes the permitted endpoint. +The recipe uses `use_absolute_ip: true`, but a controller behind NAT still needs a +reachable advertised host and firewall rules for its selected server ports. + + +This policy allows **all ports and HTTP routes on the allowed host**. It does not +limit access to a model API path, and it cannot isolate other services sharing +that host or IP. Place the model endpoint accordingly and assess cohosted services. +The current helper is a host allowlist, not a model-request authorization boundary. + + +Only the OpenSandbox network-policy provider is supported for restricted tasks. +Unsupported providers, unreachable endpoint configuration, and provider policy +creation errors fail explicitly; they never trigger an unrestricted retry. +Choose the provider explicitly with a provider config overlay. The NOOA recipe does not load +OpenSandbox credentials when another provider is selected. Existing unrestricted local/Docker configurations +remain available only when the user explicitly sets `task_network_policy: null`; restricted configurations +never disable their policy automatically. An intentionally online runtime also needs +`runtime_wheelhouse_required: false` and `runtime_wheelhouse: null`. These choices permit internet access +and are unsuitable for a restricted evaluation. + +Configure the OpenSandbox deployment outside the task: no task environment variable +can turn a DNS-only deployment into `dns+nft` enforcement. + +## Prepare and run + +Use a controller with Python 3.13.14 or newer, package/GitHub access, and Docker +capable of running a Linux amd64 build container. Install Gym and the NOOA agent +requirements in your controller environment. Docker builds the runtime; the +benchmark task and grading sandboxes run on OpenSandbox. + +Prebuilding is intentional: Docker, package indexes, and GitHub are available on the build controller, +not necessarily on the agent host or inside restricted task sandboxes. Startup reports a setup error +when the required bundle path is unset; it does not attempt an online task installation. + +From the repository root, prepare a fresh bundle and the shared SWE dataset: + +```bash +export NOOA_SWE_RUNTIME_WHEELHOUSE=/absolute/path/to/new-owned-runtime-bundle +python -m responses_api_agents.nooa_agent.offline_runtime \ + --output "$NOOA_SWE_RUNTIME_WHEELHOUSE" --target x86_64/gnu +python benchmarks/swebench/pro/prepare.py +``` + +The bundle contains portable Python, the pinned NOOA packages, the Gym runtime, +and all dependency wheels. Its manifest binds the source, Python and wheel checksums, +architecture, and builder glibc version. Task setup uses local wheels with +`--no-index --no-deps` in a private prefix. It preserves the task image and its Python. +Missing, incompatible, or corrupted bundles fail setup without an online fallback. +Source or dependency changes require a fresh bundle directory. An existing empty output directory is +accepted; a nonempty incompatible directory is preserved and rejected. Use an absolute bundle path or +`~/...` (expanded by both preparation and agent configuration). Relative agent paths are rejected because +the server working directory differs from the preparation shell. An existing bundle +can be checked with the same command plus `--validate-only`. + +The recipe selects Linux amd64. An arm64 platform overlay requires a matching +`aarch64/gnu` bundle and compatible task images. Task glibc must be at least the +builder version; the default `manylinux2014` builder uses glibc 2.17, permitting older glibc task images +such as Ubuntu 18.04. A custom newer builder still requires its newer glibc; musl targets are unsupported. Prepared bundles can be transferred +between controllers with matching source and target. + +Supply `OPENSANDBOX_DOMAIN` and `OPENSANDBOX_API_KEY` privately, and configure +`policy_base_url`, `policy_api_key`, and `policy_model_name` through Gym's normal +`env.yaml` or CLI overrides. The Gym model-server host must be reachable from the +sandbox. Set the NOOA agent's optional `context_window` to the model's known context +size when needed. The planner and outgoing request use the same output cap: explicit Gym +`max_output_tokens` wins; otherwise NOOA's `max_tokens`, `max_completion_tokens`, or `max_output_tokens`, +including an `extra_body` cap, is sent as `max_output_tokens`. Conflicting NOOA caps are rejected. +When no cap is supplied, no request cap is invented; a planning fallback remains planning-only. + +```bash +gym env start --config benchmarks/swebench/pro/nooa.yaml \ + --config nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml \ + --config responses_api_models/vllm_model/configs/vllm_model.yaml + +gym eval run --no-serve --agent swebench_pro_nooa_agent \ + --input benchmarks/swebench/data/swebench_pro_benchmark.jsonl \ + --output /tmp/nooa-swebench-pro.jsonl --limit 1 --concurrency 1 +``` + +## Validate the deployment + +Before increasing concurrency, check a fresh task sandbox installs the offline +runtime and reaches the model server. Confirm external hostname and literal-IP +requests are denied, grading completes in a separate sandbox with its original +network settings, and both sandboxes are cleaned up. Unit tests and successful +policy creation do not establish live network enforcement. + +The recipe uses the existing SWE verifier and retains its reward and failure +semantics. NOOA execution, transport, worker, resource, and cleanup failures remain +episode failures. Inspect the rollout and verifier diagnostics before interpreting +benchmark results. diff --git a/fern/versions/latest/pages/evaluation-tutorials/nooa-swe-bench-pro.mdx b/fern/versions/latest/pages/evaluation-tutorials/nooa-swe-bench-pro.mdx index face4aa632b..7c14068bfcb 100644 --- a/fern/versions/latest/pages/evaluation-tutorials/nooa-swe-bench-pro.mdx +++ b/fern/versions/latest/pages/evaluation-tutorials/nooa-swe-bench-pro.mdx @@ -33,10 +33,11 @@ NOOA-started service to remain live are not supported by this lifecycle. Cold runtime setup requires Linux x86_64 or aarch64 with glibc, `bash`, `tar`, `gzip`, `sha256sum`, `flock`, `tee`, `mv` with `-T` support, and writable `/opt/nemo-gym-nooa` and `/tmp`. The controller supplies Python; -the sandbox needs outbound package and build-dependency access. Alpine/musl images are not supported. -Model and Resources URLs must be reachable from the sandbox. A host-loopback URL is not automatically -rewritten; configure `use_absolute_ip` / `default_host` for the deployment. No host virtual environment -or provider credentials are copied into the runner. +Alpine/musl images are not supported. Python packages normally require outbound access during cold installation. +The NOOA SWE-bench Pro recipe instead requires a controller-prepared offline wheelhouse because its task sandbox +allows only the Gym model-server host. Model and granted Resources URLs must be reachable from the sandbox; +configure `use_absolute_ip` / `default_host` for the deployment. No host virtual environment or provider +credentials are copied into the runner. ## Resource tools and observations @@ -65,26 +66,35 @@ counter reward, model usage, resource calls, and trajectory. This example does n ## SWE-bench Pro -The recipe reuses the existing Resources-owned task sandbox, patch extraction, and separate grading sandbox: +The recipe reuses the existing Resources-owned task sandbox, patch extraction, and separate grading sandbox. +It restricts only the task sandbox on OpenSandbox; grading retains its original networking. Prepare the +runtime outside the task sandbox before startup: ```bash +export NOOA_SWE_RUNTIME_WHEELHOUSE=/absolute/path/to/new-owned-runtime-bundle +python -m responses_api_agents.nooa_agent.offline_runtime \ + --output "$NOOA_SWE_RUNTIME_WHEELHOUSE" --target x86_64/gnu python benchmarks/swebench/pro/prepare.py -gym env start --config benchmarks/swebench/pro/nooa.yaml --config model-provider.yaml +gym env start --config benchmarks/swebench/pro/nooa.yaml --config model-provider.yaml \ + --config nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml gym eval run --no-serve \ --config benchmarks/swebench/pro/nooa.yaml --config model-provider.yaml \ + --config nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml \ --input benchmarks/swebench/data/swebench_pro_benchmark.jsonl \ --output /tmp/nooa-swebench-pro.jsonl --limit 1 --concurrency 1 \ +agent_name=swebench_pro_nooa_agent ``` -Use `model-provider.yaml` to supply the policy model and sandbox provider settings and credentials. Pass the -same recipe and overlay to both commands: `--no-serve` does not inherit routing from the running head server. -For Docker, configure Gym's published model -and Resources addresses to be reachable from the container before running. Preparation preserves upstream grading -metadata in the shared flat dataset. The existing `single_agent_turn_legacy` adapter separates that metadata -from the Responses request before invoking NOOA. Swap the provider configuration for -remote infrastructure without changing the agent recipe. +Use `model-provider.yaml` to supply the policy model and sandbox provider settings, and provide +`OPENSANDBOX_DOMAIN` and `OPENSANDBOX_API_KEY` privately. Pass the same recipe and overlay to both commands: +`--no-serve` does not inherit routing from the running head server. The model-server host must be reachable +from the sandbox; its reference is derived from the NOOA agent configuration. Restricted SWE tasks require +an OpenSandbox deployment with `dns+nft` enforcement; unsupported providers fail explicitly. The policy +allows all ports and routes on the model-server host, including any cohosted services. Preparation preserves upstream grading metadata in the shared flat dataset. +The existing `single_agent_turn_legacy` adapter separates that metadata from the Responses request before +invoking NOOA. See [NOOA SWE-bench Pro restricted networking](/main/agent-server/nooa-swe-networking) for offline +build prerequisites and live isolation checks. Other SWE harnesses keep their existing network behavior. The overall deadline belongs to the Environment Server (`default_episode_timeout_seconds`). The standard task contract has no separate NOOA agent deadline. This recipe sets the task sandbox TTL to **25,200 seconds diff --git a/resources_servers/swebench_pro/app.py b/resources_servers/swebench_pro/app.py index 91949e2b22f..aa3a574a5f1 100644 --- a/resources_servers/swebench_pro/app.py +++ b/resources_servers/swebench_pro/app.py @@ -32,9 +32,9 @@ from fastapi.exceptions import RequestValidationError from pydantic import BaseModel, ConfigDict, Field, ValidationError -from nemo_gym.agents import lifecycle_observation +from nemo_gym.agents import lifecycle_observation, restricted_network_policy, sandbox_server_url from nemo_gym.client import SESSION_ID_KEY -from nemo_gym.config import get_global_config_dict +from nemo_gym.config import ModelServerRef, get_global_config_dict from nemo_gym.core import ( VERIFIER_ERROR, EpisodeId, @@ -133,6 +133,18 @@ class PatchCaptureLostError(RuntimeError): """The agent sandbox did not report how ``git diff`` ended or did not return its output, so the patch is unknown.""" +class SWEBenchProTaskNetworkPolicy(BaseModel): + """Allow only the model server host for task egress, on every port and route. + + Requires an OpenSandbox deployment enforcing network policies with dns+nft. + Grading sandboxes retain the original sandbox_config.provider_options. + """ + + model_config = ConfigDict(extra="forbid") + + model_server: ModelServerRef + + class SWEBenchProResourcesServerConfig(BaseResourcesServerConfig): is_verifying_golden_patch: bool = False apply_anti_cheating: bool = True @@ -162,6 +174,7 @@ class SWEBenchProResourcesServerConfig(BaseResourcesServerConfig): image_template: str | None = None # Local SIF path with a checked provenance manifest. sandbox_provider: str sandbox_config: dict[str, Any] + task_network_policy: SWEBenchProTaskNetworkPolicy | None = None class SWEBenchProInstanceRequest(BaseModel): @@ -333,10 +346,18 @@ async def _create_sandbox( self, body: SWEBenchProInstanceRequest, files: dict[str, str] | None = None, + *, + task: bool = False, ) -> AsyncSandbox: global_config_dict = get_global_config_dict() provider_config = resolve_provider_config(self.config.sandbox_provider, global_config_dict) provider_metadata = resolve_provider_metadata(self.config.sandbox_provider, global_config_dict) + provider_options = self.config.sandbox_config.get("provider_options", {}) + if task and self.config.task_network_policy is not None: + model_url = sandbox_server_url(self.config.task_network_policy.model_server.name, require_reachable=True) + # Replace rather than extend any baseline allowlist. Grading keeps the original options. + provider_options = dict(provider_options or {}) + provider_options["network_policy"] = restricted_network_policy(next(iter(provider_config)), [model_url]) image_info = await asyncio.to_thread(self._image_info, body) spec = SandboxSpec( image=image_info["image"], @@ -353,7 +374,7 @@ async def _create_sandbox( }, resources=SandboxResources.from_mapping(self.config.sandbox_config.get("resources", {})), entrypoint=None, - provider_options=self.config.sandbox_config.get("provider_options", {}), + provider_options=provider_options, ) sandbox = AsyncSandbox(provider_config) await sandbox.start(spec) @@ -430,7 +451,7 @@ async def _typed_seed_response(self, session_id: str) -> ResourcesSeedSessionRes async def _seed_session(self, session_id: str, body: SWEBenchProInstanceRequest) -> SWEBenchProSeedSessionResponse: await self._stop_session_sandbox(session_id) - sandbox = await self._create_sandbox(body) + sandbox = await self._create_sandbox(body, task=True) # Keep ownership before initialization, including when cleanup itself fails. self._session_id_to_sandbox[session_id] = sandbox try: diff --git a/resources_servers/swebench_pro/tests/test_app.py b/resources_servers/swebench_pro/tests/test_app.py index 766d7479e7e..d8939132629 100644 --- a/resources_servers/swebench_pro/tests/test_app.py +++ b/resources_servers/swebench_pro/tests/test_app.py @@ -133,7 +133,7 @@ def typed_seed_json(session_id: str, rollout_id: str = "rollout", task_data: dic def stub_task_sandboxes(monkeypatch: MonkeyPatch, server: SWEBenchProResourcesServer) -> AsyncMock: """Make typed seeds open fake task sandboxes; return the ``_create_sandbox`` mock.""" create = AsyncMock( - side_effect=lambda body, files=None: SimpleNamespace( + side_effect=lambda body, files=None, *, task=False: SimpleNamespace( _handle=SimpleNamespace(sandbox_id="sandbox-id"), exec=AsyncMock(), serialize=AsyncMock(return_value={"sandbox_id": "sandbox-id"}), @@ -674,8 +674,11 @@ async def stop_verifier() -> None: ) verifier_sandbox = SimpleNamespace(stop=AsyncMock(side_effect=stop_verifier)) - async def create_sandbox(body: SWEBenchProInstanceRequest, files: dict | None = None) -> SimpleNamespace: - if files is None: + async def create_sandbox( + body: SWEBenchProInstanceRequest, files: dict | None = None, *, task: bool = False + ) -> SimpleNamespace: + assert task is (files is None) + if task: events.append("create-task") return task_sandbox events.append("create-verifier") diff --git a/resources_servers/swebench_pro/tests/test_task_network_policy.py b/resources_servers/swebench_pro/tests/test_task_network_policy.py new file mode 100644 index 00000000000..bc574376474 --- /dev/null +++ b/resources_servers/swebench_pro/tests/test_task_network_policy.py @@ -0,0 +1,192 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Check policy selection at Resources-owned creation, not live network isolation.""" + +from copy import deepcopy +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from pydantic import ValidationError +from pytest import MonkeyPatch + +from nemo_gym.client import SESSION_ID_KEY +from nemo_gym.core import EpisodeId, TaskId +from nemo_gym.server_runtime import ResourcesSeedSessionRequest +from resources_servers.swebench_pro.app import ( + SWEBenchProSeedSessionRequest, + SWEBenchProTaskNetworkPolicy, + SWEBenchProVerifyRequest, +) +from resources_servers.swebench_pro.tests.test_app import make_server, request_body +from resources_servers.swebench_pro.verification import VerificationResult + + +MODEL_POLICY = {"model_server": {"type": "responses_api_models", "name": "policy_model"}} + + +def sandbox_double() -> SimpleNamespace: + return SimpleNamespace( + _handle=SimpleNamespace(sandbox_id="sandbox-id"), + start=AsyncMock(), + exec=AsyncMock(return_value=SimpleNamespace(return_code=0, stdout="", stderr="")), + serialize=AsyncMock(return_value={"sandbox_id": "sandbox-id"}), + stop=AsyncMock(), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("typed", [False, True]) +async def test_task_only_policy_is_present_before_handoff_and_does_not_change_grading( + monkeypatch: MonkeyPatch, typed: bool +) -> None: + server = make_server(golden=False, apply_anti_cheating=False, task_network_policy=MODEL_POLICY) + original_options = { + "platform": {"os": "linux", "architecture": "amd64"}, + "network_policy": {"defaultAction": "allow", "egress": []}, + } + server.config.sandbox_config["provider_options"] = deepcopy(original_options) + monkeypatch.setattr( + "resources_servers.swebench_pro.app.get_global_config_dict", lambda: {"test": {"opensandbox": {}}} + ) + endpoint = MagicMock(return_value="http://model.example:8000/rollouts/one/v1") + monkeypatch.setattr("nemo_gym.agents.agent_tools.get_server_url", endpoint) + task_sandbox, grading_sandbox = sandbox_double(), sandbox_double() + create = MagicMock(side_effect=[task_sandbox, grading_sandbox]) + monkeypatch.setattr("resources_servers.swebench_pro.app.AsyncSandbox", create) + monkeypatch.setattr(server, "_extract_model_patch", AsyncMock(return_value="agent patch")) + verify = AsyncMock( + return_value=VerificationResult(completed=True, resolved=True, patch_applied=True, test_results={"tests": []}) + ) + monkeypatch.setattr("resources_servers.swebench_pro.app.run_verification", verify) + request = SimpleNamespace(session={SESSION_ID_KEY: "session"}) + if typed: + body = ResourcesSeedSessionRequest( + resources_session_id="session", + episode_id=EpisodeId(rollout_id="rollout"), + task_id=TaskId(taskset="swebench_pro", task_id="instance_example"), + task_data=request_body(), + ) + else: + body = SWEBenchProSeedSessionRequest.model_validate(request_body()) + + response = await server.seed_session(request, body) + task_sandbox.start.assert_awaited_once() + task_spec = task_sandbox.start.await_args.args[0] + assert task_spec.provider_options == original_options | { + "network_policy": { + "defaultAction": "deny", + "egress": [{"action": "allow", "target": "model.example"}], + } + } + assert task_spec.image == "docker.io/jefzda/sweap-images:example-tag" + assert server._session_id_to_sandbox["session"] is task_sandbox + if typed: + assert response.sandbox_access.connection.descriptor == {"sandbox_id": "sandbox-id"} + else: + assert response.sandbox_handle == "sandbox-id" + endpoint.assert_called_once_with("policy_model") + + result = await server.verify(request, SWEBenchProVerifyRequest.model_validate(request_body())) + grading_sandbox.start.assert_awaited_once() + grading_spec = grading_sandbox.start.await_args.args[0] + assert grading_spec.provider_options == original_options + assert grading_spec.image == task_spec.image + assert grading_spec.files # Fresh verifier inputs, not the agent workspace. + assert verify.await_args.kwargs["sandbox"] is grading_sandbox + assert result.reward == 1 + assert server.config.sandbox_config["provider_options"] == original_options + endpoint.assert_called_once_with("policy_model") + grading_sandbox.stop.assert_awaited_once() + await server.shutdown() + task_sandbox.stop.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider", ["docker", "opensandbox"]) +@pytest.mark.parametrize("options", [{}, None, {"network_policy": {"defaultAction": "allow", "egress": []}}]) +async def test_default_task_and_grading_networking_is_unchanged( + monkeypatch: MonkeyPatch, provider: str, options: dict[str, object] | None +) -> None: + server = make_server(golden=True, apply_anti_cheating=False) + assert server.config.task_network_policy is None + server.config.sandbox_config["provider_options"] = deepcopy(options) + monkeypatch.setattr("resources_servers.swebench_pro.app.get_global_config_dict", lambda: {"test": {provider: {}}}) + endpoint = MagicMock(side_effect=AssertionError("Unrestricted recipes must not resolve a model endpoint")) + monkeypatch.setattr("nemo_gym.agents.agent_tools.get_server_url", endpoint) + task_sandbox, grading_sandbox = sandbox_double(), sandbox_double() + monkeypatch.setattr( + "resources_servers.swebench_pro.app.AsyncSandbox", MagicMock(side_effect=[task_sandbox, grading_sandbox]) + ) + monkeypatch.setattr( + "resources_servers.swebench_pro.app.run_verification", + AsyncMock( + return_value=VerificationResult( + completed=True, resolved=True, patch_applied=True, test_results={"tests": []} + ) + ), + ) + request = SimpleNamespace(session={SESSION_ID_KEY: "session"}) + + await server.seed_session(request, SWEBenchProSeedSessionRequest.model_validate(request_body())) + await server.verify(request, SWEBenchProVerifyRequest.model_validate(request_body())) + + assert task_sandbox.start.await_args.args[0].provider_options == options + assert grading_sandbox.start.await_args.args[0].provider_options == options + endpoint.assert_not_called() + await server.shutdown() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "provider,endpoint,error", + [ + ("docker", "http://model.example:8000", "requires the OpenSandbox"), + ("e2b", "http://model.example:8000", "requires the OpenSandbox"), + ("opensandbox", "http://localhost:8000", "sandbox-reachable host"), + ("opensandbox", "http://127.0.0.1:8000", "sandbox-reachable host"), + ("opensandbox", "http://0.0.0.0:8000", "sandbox-reachable host"), + ("opensandbox", "http://[::1]:8000", "sandbox-reachable host"), + ], +) +async def test_unsupported_policy_fails_before_creation_or_handoff( + monkeypatch: MonkeyPatch, provider: str, endpoint: str, error: str +) -> None: + server = make_server(golden=False, task_network_policy=MODEL_POLICY) + monkeypatch.setattr("resources_servers.swebench_pro.app.get_global_config_dict", lambda: {"test": {provider: {}}}) + monkeypatch.setattr("nemo_gym.agents.agent_tools.get_server_url", lambda name: endpoint) + create = MagicMock() + monkeypatch.setattr("resources_servers.swebench_pro.app.AsyncSandbox", create) + + with pytest.raises(ValueError, match=error): + await server._seed_session("session", SWEBenchProSeedSessionRequest.model_validate(request_body())) + + create.assert_not_called() + assert not server._session_id_to_sandbox + + +def test_unknown_task_policy_settings_are_rejected() -> None: + with pytest.raises(ValidationError, match="Extra inputs are not permitted"): + SWEBenchProTaskNetworkPolicy.model_validate(MODEL_POLICY | {"allow_internet": True}) + + +@pytest.mark.asyncio +async def test_policy_rejection_from_provider_never_retries_unrestricted(monkeypatch: MonkeyPatch) -> None: + server = make_server(golden=False, task_network_policy=MODEL_POLICY) + monkeypatch.setattr( + "resources_servers.swebench_pro.app.get_global_config_dict", lambda: {"test": {"opensandbox": {}}} + ) + monkeypatch.setattr("nemo_gym.agents.agent_tools.get_server_url", lambda name: "http://model.example:8000") + sandbox = sandbox_double() + sandbox.start.side_effect = RuntimeError("OpenSandbox has no egress image configured") + create = MagicMock(return_value=sandbox) + monkeypatch.setattr("resources_servers.swebench_pro.app.AsyncSandbox", create) + + with pytest.raises(RuntimeError, match="no egress image"): + await server._seed_session("session", SWEBenchProSeedSessionRequest.model_validate(request_body())) + + create.assert_called_once() + sandbox.start.assert_awaited_once() + assert sandbox.start.await_args.args[0].provider_options["network_policy"]["defaultAction"] == "deny" + assert not server._session_id_to_sandbox diff --git a/responses_api_agents/nooa_agent/README.md b/responses_api_agents/nooa_agent/README.md index 3b919db572c..81a95d2a884 100644 --- a/responses_api_agents/nooa_agent/README.md +++ b/responses_api_agents/nooa_agent/README.md @@ -33,6 +33,10 @@ the outer `execute_python` call from nested resource calls. `nooa.execution_mode: sandboxed` runs the complete NOOA agent in the task sandbox supplied by Resources. NOOA stages its own runtime and closes its worker before disconnecting; Resources owns sandbox creation and teardown. +The runtime packages the launching Gym checkout with its unchanged root `pyproject.toml` and full core dependency set. +Tracked package source and assets are copied with their current contents; untracked files, Git metadata, and caches are excluded. +The controller therefore needs a Git checkout and Git installed. The package is installed into NOOA's private Python, +leaving the task's Python environment unchanged. Offline wheelhouses must be rebuilt whenever these packaged bytes change. Agent and resource sessions use Gym's existing `single_agent_turn` environment. Agent close returns the evidence and final Resources cookies before verification; Resources closes its sandbox afterward. diff --git a/responses_api_agents/nooa_agent/app.py b/responses_api_agents/nooa_agent/app.py index 13fc13321da..798163f0863 100644 --- a/responses_api_agents/nooa_agent/app.py +++ b/responses_api_agents/nooa_agent/app.py @@ -90,6 +90,7 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> NOOA server_client=self.server_client, model_server_name=self.config.model_server.name, max_policy_calls=self.config.max_policy_calls, + context_window=self.config.context_window, ) else: if body.sandbox_access is None: @@ -114,10 +115,14 @@ async def _seed_agent_session_state(self, body: AgentSeedSessionRequest) -> NOOA timeout=self.config.sandbox_runner_timeout_seconds, close_timeout=self.config.session_close_timeout_seconds, max_policy_calls=self.config.max_policy_calls, + context_window=self.config.context_window, ) state.runner = runner try: - runner.python = await prepare_nooa_runtime(sandbox) + runner.python = await prepare_nooa_runtime( + sandbox, + wheelhouse_path=self.config.runtime_wheelhouse, + ) await runner.prepare() except BaseException as error: # Main retains this state for close while rejecting activation diff --git a/responses_api_agents/nooa_agent/bench_agent_adapter.py b/responses_api_agents/nooa_agent/bench_agent_adapter.py index 3a69a75d147..97e5f0a5b1f 100644 --- a/responses_api_agents/nooa_agent/bench_agent_adapter.py +++ b/responses_api_agents/nooa_agent/bench_agent_adapter.py @@ -28,7 +28,11 @@ async def invoke_bench_agent(agent: BenchAgent, request: NeMoGymResponseCreatePa if Path(agent.shell.cwd).resolve() != cwd: await agent.shell.close() agent._install_python_tools(str(cwd)) - return await agent._solve_task(_latest_user_text(request)) + task = _latest_user_text(request) + # Native parameter rendering can truncate task instructions; keep a full + # prefix copy under a separate key from the decorated method's "task". + agent.context.set("benchmark_task", task, prefix=True) + return await agent._solve_task(task) finally: # Drain summaries and close the shell before Gym projects the final trace. await agent.aclose() diff --git a/responses_api_agents/nooa_agent/config.py b/responses_api_agents/nooa_agent/config.py index 069590dee17..2cccc9f4e0b 100644 --- a/responses_api_agents/nooa_agent/config.py +++ b/responses_api_agents/nooa_agent/config.py @@ -15,65 +15,27 @@ from __future__ import annotations -import importlib -import inspect -from collections.abc import Awaitable, Callable -from typing import Any, Literal +from pathlib import Path -from nooa import Agent -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import ConfigDict, Field, field_validator, model_validator from nemo_gym.agents import BaseResponsesAPIAgentConfig from nemo_gym.config import ModelServerRef, ResourcesServerRef -from nemo_gym.core import NeMoGymResponseCreateParamsNonStreaming - - -NOOAInvocationAdapter = Callable[ - [Agent, NeMoGymResponseCreateParamsNonStreaming], - Awaitable[object], -] - - -class NOOAInvocationConfig(BaseModel): - """Configuration for constructing and invoking a NOOA agent.""" - - model_config = ConfigDict(extra="forbid") - - agent_class: str - invocation_adapter: str - execution_mode: Literal["embedded", "sandboxed"] = "embedded" - init_kwargs: dict[str, Any] = Field(default_factory=dict) - - @field_validator("agent_class") - @classmethod - def validate_agent_class_path(cls, value: str) -> str: - parts = value.split(":") - if len(parts) != 2: - raise ValueError("agent_class must use the format 'module.path:ClassName'") - - module_name, class_name = parts - if not module_name or not class_name or "." in class_name: - raise ValueError("agent_class must use the format 'module.path:ClassName'") - return value - - @field_validator("invocation_adapter") - @classmethod - def validate_invocation_adapter_path(cls, value: str) -> str: - parts = value.split(":") - if len(parts) != 2: - raise ValueError("invocation_adapter must use the format 'module.path:function_name'") - - module_name, function_name = parts - if not module_name or not function_name or "." in function_name: - raise ValueError("invocation_adapter must use the format 'module.path:function_name'") - return value - - @field_validator("init_kwargs") - @classmethod - def validate_init_kwargs(cls, init_kwargs: dict[str, Any]) -> dict[str, Any]: - if "llm" in init_kwargs: - raise ValueError("init_kwargs.llm is reserved; Gym always injects the rollout LLM") - return init_kwargs +from responses_api_agents.nooa_agent.invocation import ( + NOOAInvocationAdapter as NOOAInvocationAdapter, +) +from responses_api_agents.nooa_agent.invocation import ( + NOOAInvocationConfig as NOOAInvocationConfig, +) +from responses_api_agents.nooa_agent.invocation import ( + load_agent_class as load_agent_class, +) +from responses_api_agents.nooa_agent.invocation import ( + load_invocation_adapter as load_invocation_adapter, +) +from responses_api_agents.nooa_agent.invocation import ( + validate_invocation as validate_invocation, +) class NOOAAgentConfig(BaseResponsesAPIAgentConfig): @@ -86,10 +48,20 @@ class NOOAAgentConfig(BaseResponsesAPIAgentConfig): description="Dataset/schema discovery metadata; runtime access comes from environment session grants.", ) model_server: ModelServerRef + runtime_wheelhouse: Path | None = Field( + default=None, + description="Controller-prepared offline runtime bundle; when set, task setup uses no package-network access.", + ) + runtime_wheelhouse_required: bool = Field( + default=False, description="Require an explicitly prepared offline runtime before starting this agent." + ) nooa: NOOAInvocationConfig max_policy_calls: int | None = Field( default=None, gt=0, description="Optional shared model-call limit per rollout; null disables the limit." ) + context_window: int | None = Field( + default=None, gt=0, description="Known policy context window used by NOOA's context planner." + ) sandbox_runner_timeout_seconds: float = Field( default=21420, @@ -104,56 +76,24 @@ class NOOAAgentConfig(BaseResponsesAPIAgentConfig): description="Budget in seconds for each sandbox close phase and for joining the remaining activation.", ) - -def load_agent_class(path: str) -> type[Agent]: - """Import and validate a ``module:Class`` NOOA agent reference.""" - - module_name, _, class_name = path.partition(":") - try: - module = importlib.import_module(module_name) - except (ImportError, TypeError) as error: - raise ValueError(f"could not import NOOA agent module {module_name!r}") from error - - try: - candidate = getattr(module, class_name) - except AttributeError as error: - raise ValueError(f"module {module_name!r} has no attribute {class_name!r}") from error - - if not inspect.isclass(candidate) or not issubclass(candidate, Agent): - raise ValueError(f"{path!r} must resolve to a subclass of nooa.Agent") - return candidate - - -def load_invocation_adapter(path: str) -> NOOAInvocationAdapter: - """Import and validate an async ``module:function`` invocation adapter.""" - - module_name, _, function_name = path.partition(":") - try: - module = importlib.import_module(module_name) - except (ImportError, TypeError) as error: - raise ValueError(f"could not import NOOA invocation adapter module {module_name!r}") from error - - try: - candidate = getattr(module, function_name) - except AttributeError as error: - raise ValueError(f"module {module_name!r} has no attribute {function_name!r}") from error - - if not callable(candidate) or not inspect.iscoroutinefunction(candidate): - raise ValueError(f"{path!r} must resolve to an async function") - try: - inspect.signature(candidate).bind(object(), object()) - except TypeError as error: - raise ValueError(f"invocation adapter must accept positional agent and request arguments: {error}") from error - return candidate - - -def validate_invocation(config: NOOAInvocationConfig) -> tuple[type[Agent], NOOAInvocationAdapter]: - """Validate agent construction and the Responses invocation adapter at startup.""" - - agent_class = load_agent_class(config.agent_class) - try: - inspect.signature(agent_class).bind(llm=object(), **config.init_kwargs) - except TypeError as error: - raise ValueError(f"init_kwargs do not match {config.agent_class}: {error}") from error - - return agent_class, load_invocation_adapter(config.invocation_adapter) + @field_validator("runtime_wheelhouse") + @classmethod + def normalize_runtime_wheelhouse(cls, value: Path | None) -> Path | None: + if value is None: + return None + expanded = value.expanduser() + if not expanded.is_absolute(): + raise ValueError( + "runtime_wheelhouse must be an absolute path (or ~/path); relative server paths are ambiguous" + ) + return expanded.resolve() + + @model_validator(mode="after") + def require_prepared_runtime(self) -> NOOAAgentConfig: + if self.runtime_wheelhouse_required and self.runtime_wheelhouse is None: + raise ValueError( + "This recipe requires an offline NOOA runtime. Set NOOA_SWE_RUNTIME_WHEELHOUSE to an absolute " + "bundle path and prepare it with python -m responses_api_agents.nooa_agent.offline_runtime " + "--output PATH --target x86_64/gnu before starting servers." + ) + return self diff --git a/responses_api_agents/nooa_agent/gym_llm.py b/responses_api_agents/nooa_agent/gym_llm.py index 95b6c38e151..43a979f5fad 100644 --- a/responses_api_agents/nooa_agent/gym_llm.py +++ b/responses_api_agents/nooa_agent/gym_llm.py @@ -23,6 +23,7 @@ import aiohttp from nooa.unifiedllm import CacheBoundary, LLMResponse, Tool, ToolCall, UnifiedLLM +from nooa.unifiedllm.limits import REPLY_CAP_KEYS, ContextLimits from pydantic import BaseModel from nemo_gym.client import get_response_json, raise_for_status @@ -238,8 +239,9 @@ def __init__( on_call: Callable[[GymModelCall], None] | None = None, on_response: Callable[[], None] | None = None, sampling_overrides: dict[str, Any] | None = None, + context_window: int | None = None, ) -> None: - super().__init__(model=model) + super().__init__(model=model, context_window=context_window) self._server_client = server_client self._model_server_name = model_server_name self._model_url_path = model_url_path @@ -251,6 +253,21 @@ def __init__( self._calls = 0 self._lock = asyncio.Lock() + def get_context_limits( + self, overrides: dict[str, Any] | None = None, *, fallback_reserve: int = 0 + ) -> ContextLimits: + """Reserve the same reply cap that the Gym request will actually send.""" + params = dict(overrides or {}) + cap = self._sampling_overrides.get("max_output_tokens") + if cap is not None: + params = {key: value for key, value in params.items() if key not in REPLY_CAP_KEYS} + if params.get("extra_body"): + params["extra_body"] = { + key: value for key, value in params["extra_body"].items() if key not in REPLY_CAP_KEYS + } + params["max_tokens"] = cap + return super().get_context_limits(params, fallback_reserve=fallback_reserve) + @property def calls(self) -> int: return self._calls @@ -281,6 +298,7 @@ async def _acall( output_model: type[BaseModel] | None = None, **kwargs: Any, ) -> LLMResponse: + limits = self.get_context_limits(kwargs) self._state.charge() self._calls += 1 @@ -303,15 +321,17 @@ async def _acall( } } - aliases = {"max_tokens": "max_output_tokens"} supported = set(NeMoGymResponseCreateParamsNonStreaming.model_fields) - {"model"} for name, value in kwargs.items(): - destination = aliases.get(name, name) - if destination in supported and value is not None: - request[destination] = value + if name not in REPLY_CAP_KEYS and name in supported and value is not None: + request[name] = value # Explicit Gym rollout controls take precedence over NOOA's per-call settings. request.update(self._sampling_overrides) + if limits.reserve_is_fallback: + request.pop("max_output_tokens", None) + else: + request["max_output_tokens"] = limits.reserved_output_tokens body = NeMoGymResponseCreateParamsNonStreaming.model_validate(request) call = GymModelCall( model_ref=ModelServerRef(name=self._model_server_name, type="responses_api_models"), diff --git a/responses_api_agents/nooa_agent/install_nooa_runtime.sh b/responses_api_agents/nooa_agent/install_nooa_runtime.sh index c911864a3fd..060fa61df05 100644 --- a/responses_api_agents/nooa_agent/install_nooa_runtime.sh +++ b/responses_api_agents/nooa_agent/install_nooa_runtime.sh @@ -36,7 +36,7 @@ mode=${1:-} case "$mode" in probe) [[ $# = 1 ]] || fail 'usage: probe'; probe; exit 0 ;; check) [[ $# = 4 ]] || fail 'usage: check ' ;; - install) [[ $# = 6 ]] || fail 'usage: install ' ;; + install) [[ $# = 6 || $# = 7 ]] || fail 'usage: install [wheels sha256]' ;; *) fail 'expected probe, check, or install' ;; esac runtime=$2 @@ -81,6 +81,8 @@ require bash tar gzip sha256sum flock sleep mktemp ln mv rm probe >/dev/null source_sha256=$4 python_sha256=$5 +wheels_sha256=${7:-} +[[ -z "$wheels_sha256" || "$wheels_sha256" =~ ^[0-9a-f]{64}$ ]] || fail 'wheel checksum must be a SHA-256 digest' [[ "$source_sha256" =~ ^[0-9a-f]{64}$ && "$python_sha256" =~ ^[0-9a-f]{64}$ ]] \ || fail 'source and Python checksums must be SHA-256 digests' parent=${runtime%/*} @@ -97,9 +99,12 @@ done if ready; then exit 0; fi [[ ! -e "$runtime" || -L "$runtime" ]] \ || fail 'publication path is a real directory; choose a fresh runtime fingerprint without removing it' -# Neither archive is extracted until both expected digests have been verified. +# No archive is extracted until every expected digest has been verified. printf '%s %s\n' "$source_sha256" "$scratch/source.tar.gz" "$python_sha256" "$scratch/python.tar.gz" \ | sha256sum -c - +if [[ -n "$wheels_sha256" ]]; then + printf '%s %s\n' "$wheels_sha256" "$scratch/wheels.tar.gz" | sha256sum -c - +fi mkdir -p "$parent/.builds" generation=$(mktemp -d "$parent/.builds/$name.XXXXXX") mkdir -p "$generation/python" "$generation/source/cache" @@ -107,8 +112,24 @@ tar -xzf "$scratch/python.tar.gz" -C "$generation/python" --strip-components=1 tar -xzf "$scratch/source.tar.gz" -C "$generation/source" # Installed console scripts reference this permanent path, so never rename the generation. # The installer alone owns the lock; an orphaned pip process cannot retain it after a kill. -"$generation/python/bin/python3" -I -B -m pip --isolated install --no-cache-dir \ - "$generation/source" -r "$generation/source/runtime-requirements.txt" 9>&- +if [[ -n "$wheels_sha256" ]]; then + mkdir -p "$generation/wheels" + tar -xzf "$scratch/wheels.tar.gz" -C "$generation/wheels" + # Every package, including Gym and pinned NOOA, was built outside the task. + # Explicit wheel paths and --no-deps prevent direct-URL metadata from causing downloads. + "$generation/python/bin/python3" -I -B -m pip --isolated install --no-cache-dir \ + --no-index --no-deps --only-binary=:all: "$generation/wheels/"*.whl 9>&- + # Offline health checks and workers must use LiteLLM's bundled model metadata. + # A private-prefix startup hook also applies to later isolated (-I) interpreters, + # without changing the task's Python or process environment. + site_packages="$generation/python/lib/python3.13/site-packages" + mkdir -p "$site_packages" + printf '%s\n' 'import os; os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"' \ + > "$site_packages/nooa_offline_runtime.pth" +else + "$generation/python/bin/python3" -I -B -m pip --isolated install --no-cache-dir \ + "$generation/source" -r "$generation/source/runtime-requirements.txt" 9>&- +fi health "$generation" printf '%s\n' "$fingerprint" > "$generation/ready" publish="$parent/.${generation##*/}.publish" diff --git a/responses_api_agents/nooa_agent/invocation.py b/responses_api_agents/nooa_agent/invocation.py new file mode 100644 index 00000000000..8cafa7a3c31 --- /dev/null +++ b/responses_api_agents/nooa_agent/invocation.py @@ -0,0 +1,128 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import importlib +import inspect +from collections.abc import Awaitable, Callable +from typing import Any, Literal + +from nooa import Agent +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from nemo_gym.core import NeMoGymResponseCreateParamsNonStreaming + + +NOOAInvocationAdapter = Callable[ + [Agent, NeMoGymResponseCreateParamsNonStreaming], + Awaitable[object], +] + + +class NOOAInvocationConfig(BaseModel): + """Configuration for constructing and invoking a NOOA agent.""" + + model_config = ConfigDict(extra="forbid") + + agent_class: str + invocation_adapter: str + execution_mode: Literal["embedded", "sandboxed"] = "embedded" + init_kwargs: dict[str, Any] = Field(default_factory=dict) + + @field_validator("agent_class") + @classmethod + def validate_agent_class_path(cls, value: str) -> str: + parts = value.split(":") + if len(parts) != 2: + raise ValueError("agent_class must use the format 'module.path:ClassName'") + + module_name, class_name = parts + if not module_name or not class_name or "." in class_name: + raise ValueError("agent_class must use the format 'module.path:ClassName'") + return value + + @field_validator("invocation_adapter") + @classmethod + def validate_invocation_adapter_path(cls, value: str) -> str: + parts = value.split(":") + if len(parts) != 2: + raise ValueError("invocation_adapter must use the format 'module.path:function_name'") + + module_name, function_name = parts + if not module_name or not function_name or "." in function_name: + raise ValueError("invocation_adapter must use the format 'module.path:function_name'") + return value + + @field_validator("init_kwargs") + @classmethod + def validate_init_kwargs(cls, init_kwargs: dict[str, Any]) -> dict[str, Any]: + if "llm" in init_kwargs: + raise ValueError("init_kwargs.llm is reserved; Gym always injects the rollout LLM") + return init_kwargs + + +def load_agent_class(path: str) -> type[Agent]: + """Import and validate a ``module:Class`` NOOA agent reference.""" + + module_name, _, class_name = path.partition(":") + try: + module = importlib.import_module(module_name) + except (ImportError, TypeError) as error: + raise ValueError(f"could not import NOOA agent module {module_name!r}") from error + + try: + candidate = getattr(module, class_name) + except AttributeError as error: + raise ValueError(f"module {module_name!r} has no attribute {class_name!r}") from error + + if not inspect.isclass(candidate) or not issubclass(candidate, Agent): + raise ValueError(f"{path!r} must resolve to a subclass of nooa.Agent") + return candidate + + +def load_invocation_adapter(path: str) -> NOOAInvocationAdapter: + """Import and validate an async ``module:function`` invocation adapter.""" + + module_name, _, function_name = path.partition(":") + try: + module = importlib.import_module(module_name) + except (ImportError, TypeError) as error: + raise ValueError(f"could not import NOOA invocation adapter module {module_name!r}") from error + + try: + candidate = getattr(module, function_name) + except AttributeError as error: + raise ValueError(f"module {module_name!r} has no attribute {function_name!r}") from error + + if not callable(candidate) or not inspect.iscoroutinefunction(candidate): + raise ValueError(f"{path!r} must resolve to an async function") + try: + inspect.signature(candidate).bind(object(), object()) + except TypeError as error: + raise ValueError(f"invocation adapter must accept positional agent and request arguments: {error}") from error + return candidate + + +def validate_invocation(config: NOOAInvocationConfig) -> tuple[type[Agent], NOOAInvocationAdapter]: + """Validate agent construction and the Responses invocation adapter at startup.""" + + agent_class = load_agent_class(config.agent_class) + try: + inspect.signature(agent_class).bind(llm=object(), **config.init_kwargs) + except TypeError as error: + raise ValueError(f"init_kwargs do not match {config.agent_class}: {error}") from error + + return agent_class, load_invocation_adapter(config.invocation_adapter) diff --git a/responses_api_agents/nooa_agent/offline_runtime.py b/responses_api_agents/nooa_agent/offline_runtime.py new file mode 100644 index 00000000000..d9017ac2ded --- /dev/null +++ b/responses_api_agents/nooa_agent/offline_runtime.py @@ -0,0 +1,246 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Build a source-bound wheelhouse outside a network-restricted task sandbox.""" + +import argparse +import asyncio +import hashlib +import json +import os +import shutil +import tarfile +import tempfile +from pathlib import Path +from uuid import uuid4 + +from responses_api_agents.nooa_agent.sandbox_runtime import ( + _PYTHON_SHA256, + _PYTHON_VERSION, + _python_download_url, + _runtime_archive, + _runtime_io, +) + + +_TARGETS = {"x86_64/gnu": "amd64", "aarch64/gnu": "arm64"} + + +def file_sha256(path: Path) -> str: + """Hash an artifact without loading large wheel archives into memory.""" + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def load_runtime_wheelhouse(output: Path, *, target: str, source_fingerprint: str) -> dict[str, object]: + """Validate the exact source, platform and artifact bytes before staging.""" + output = output.expanduser().resolve() + if target not in _TARGETS: + raise ValueError(f"Offline NOOA runtime supports only {', '.join(_TARGETS)}; got {target!r}") + try: + manifest = json.loads((output / "manifest.json").read_text()) + except (OSError, ValueError) as error: + raise ValueError(f"Prepare the NOOA runtime wheelhouse first: {output}") from error + if not isinstance(manifest, dict) or any( + manifest.get(key) != expected + for key, expected in { + "version": 1, + "target": target, + "source_fingerprint": source_fingerprint, + "python_version": _PYTHON_VERSION, + "python_sha256": _PYTHON_SHA256[target], + }.items() + ): + raise ValueError( + f"NOOA wheelhouse is incompatible with this source/platform; prepare a fresh directory: {output}" + ) + glibc = manifest.get("glibc") + if not isinstance(glibc, str) or len(glibc.split(".")) != 2 or not all(p.isdigit() for p in glibc.split(".")): + raise ValueError("NOOA wheelhouse manifest requires its builder glibc version") + for name, field in (("python.tar.gz", "python_sha256"), ("wheels.tar.gz", "wheels_sha256")): + try: + actual = file_sha256(output / name) + except OSError as error: + raise ValueError(f"NOOA wheelhouse is missing {name}: {output}") from error + if actual != manifest.get(field): + raise ValueError(f"NOOA wheelhouse checksum mismatch: {output / name}") + with tarfile.open(output / "wheels.tar.gz", "r:gz") as archive: + members = archive.getmembers() + if not members or any( + not m.isfile() or Path(m.name).name != m.name or not m.name.endswith(".whl") for m in members + ): + raise ValueError("NOOA wheelhouse must contain only top-level wheel files") + return manifest + + +def validate_runtime_wheelhouse(output: Path, *, target: str) -> dict[str, object]: + """Check an existing prepared bundle against the current checkout.""" + _, fingerprint = _runtime_archive(Path(__file__).resolve().parents[2]) + return load_runtime_wheelhouse(output, target=target, source_fingerprint=fingerprint) + + +async def _command(*arguments: str, timeout: int = 1800) -> str: + process = await asyncio.create_subprocess_exec( + *arguments, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT + ) + try: + stdout, _ = await asyncio.wait_for(process.communicate(), timeout) + except BaseException: + if process.returncode is None: + process.kill() + await process.wait() + raise + output = stdout.decode(errors="replace") + if process.returncode: + raise RuntimeError( + f"Offline NOOA preparation failed ({arguments[0]}, exit {process.returncode}):\n{output[-12000:]}" + ) + return output + + +async def prepare_runtime_wheelhouse( + output: Path, + *, + target: str, + builder_image: str | None = None, +) -> Path: + """Build every runtime wheel on compatible Linux, using Docker outside the task. + + Requires controller package/GitHub access and Docker. Existing bundles are + reused only when source, platform and checksums match. A new checkout needs + a fresh output directory. No benchmark image or task Python is modified. + """ + if target not in _TARGETS: + raise ValueError(f"Unsupported offline NOOA target {target!r}; expected one of {tuple(_TARGETS)}") + root = Path(__file__).resolve().parents[2] + source, fingerprint = await _runtime_io(_runtime_archive, root) + output = output.expanduser().resolve() + if output.exists() and (not output.is_dir() or any(output.iterdir())): + await _runtime_io(load_runtime_wheelhouse, output, target=target, source_fingerprint=fingerprint) + return output + if shutil.which("docker") is None: + raise RuntimeError( + "Offline NOOA preparation requires Docker on the controller, or a compatible prepared bundle" + ) + from nemo_gym.client import ( + GlobalAIOHTTPAsyncClientConfig, + is_global_aiohttp_client_setup, + set_global_aiohttp_client, + ) + from responses_api_agents.miniswe_sandboxed_agent.bootstrap import cached_download + + # The standalone preparation CLI has argparse flags, not Gym/Hydra overrides. + # Reuse an existing Gym transport, or initialize just its HTTP defaults. + if not is_global_aiohttp_client_setup(): + set_global_aiohttp_client(GlobalAIOHTTPAsyncClientConfig()) + python_archive = await cached_download(_python_download_url(target), sha256=_PYTHON_SHA256[target]) + output.parent.mkdir(parents=True, exist_ok=True) + arch = target.split("/")[0] + builder_image = builder_image or f"quay.io/pypa/manylinux2014_{arch}:latest" + await _command("docker", "pull", "--platform", f"linux/{_TARGETS[target]}", builder_image) + image_id = (await _command("docker", "image", "inspect", "--format", "{{.Id}}", builder_image)).strip() + with tempfile.TemporaryDirectory(prefix=".nooa-wheelhouse-", dir=output.parent) as temporary: + work = Path(temporary) + bundle = work / "bundle" + bundle.mkdir() + shutil.copyfile(python_archive, bundle / "python.tar.gz") + if file_sha256(bundle / "python.tar.gz") != _PYTHON_SHA256[target]: + raise RuntimeError("NOOA Python artifact changed while preparing the offline bundle") + (work / "source.tar.gz").write_bytes(source) + container = f"nooa-wheelhouse-{uuid4().hex}" + # Build scripts use only paths within the fresh bind mount, never the task image. + script = """set -Eeuo pipefail +umask 022 +mkdir -p /build/python /build/source/cache /build/wheels +uname -m > /build/architecture.txt +getconf GNU_LIBC_VERSION > /build/libc.txt +tar -xzf /build/bundle/python.tar.gz -C /build/python --strip-components=1 +tar -xzf /build/source.tar.gz -C /build/source +/build/python/bin/python3 -I -B -m pip --isolated wheel --no-cache-dir --wheel-dir /build/wheels /build/source -r /build/source/runtime-requirements.txt +""" + docker_run = ( + "docker", + "run", + "--rm", + "--name", + container, + "--platform", + f"linux/{_TARGETS[target]}", + "--volume", + f"{work}:/build", + ) + try: + # Use the mount's container-visible owner: rootless Docker remaps host IDs. + # Creating artifacts as this owner also allows cleanup after a killed builder. + owner = (await _command(*docker_run, image_id, "stat", "-c", "%u:%g", "/build")).strip() + log = await _command( + *docker_run, + "--user", + owner, + image_id, + "bash", + "-c", + script, + ) + (bundle / "build.log").write_text(log) + finally: + # Also remove a live builder after cancellation/timeout of the Docker client. + try: + await _command("docker", "rm", "--force", container, timeout=30) + except RuntimeError as error: + if "No such container" not in str(error): + raise + built_arch = (work / "architecture.txt").read_text().strip() + libc = (work / "libc.txt").read_text().strip().split() + if built_arch != arch or len(libc) != 2 or libc[0] != "glibc": + raise RuntimeError(f"Builder does not match {target}: architecture={built_arch}, libc={libc}") + wheels = sorted((work / "wheels").glob("*.whl")) + if not wheels: + raise RuntimeError("NOOA preparation produced no wheels") + with tarfile.open(bundle / "wheels.tar.gz", "w:gz") as archive: + for wheel in wheels: + archive.add(wheel, arcname=wheel.name) + manifest = { + "version": 1, + "target": target, + "glibc": libc[1], + "builder_image": image_id, + "source_fingerprint": fingerprint, + "python_version": _PYTHON_VERSION, + "python_sha256": _PYTHON_SHA256[target], + "wheels_sha256": file_sha256(bundle / "wheels.tar.gz"), + } + (bundle / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + load_runtime_wheelhouse(bundle, target=target, source_fingerprint=fingerprint) + try: + os.rename(bundle, output) + except OSError: + # Concurrent builders may finish first; reuse only a verified compatible result. + if not output.exists(): + raise + load_runtime_wheelhouse(output, target=target, source_fingerprint=fingerprint) + return output + + +def main() -> None: + """Prepare or validate an offline bundle for the selected controller source.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--target", choices=tuple(_TARGETS), default="x86_64/gnu") + parser.add_argument("--builder-image") + parser.add_argument("--validate-only", action="store_true") + args = parser.parse_args() + if args.validate_only: + validate_runtime_wheelhouse(args.output, target=args.target) + else: + asyncio.run( + prepare_runtime_wheelhouse( + args.output, + target=args.target, + builder_image=args.builder_image, + ) + ) + print(f"NOOA runtime wheelhouse ready: {args.output.expanduser().resolve()}") + + +if __name__ == "__main__": + main() diff --git a/responses_api_agents/nooa_agent/requirements.txt b/responses_api_agents/nooa_agent/requirements.txt index 84e61c6b909..adb5f270ff8 100644 --- a/responses_api_agents/nooa_agent/requirements.txt +++ b/responses_api_agents/nooa_agent/requirements.txt @@ -1,4 +1,4 @@ --e nemo-gym[dev] @ ../../ +-e nemo-gym[dev,sandbox] @ ../../ jsonschema>=4.25.1,<5 # Includes the public scoped lifecycle hooks used for Gym observability. nooa @ git+https://github.com/NVIDIA-NeMo/labs-OO-Agents.git@051472343211914222e24ce36d8752f4e86bbe43 diff --git a/responses_api_agents/nooa_agent/runner.py b/responses_api_agents/nooa_agent/runner.py index bc84c23babd..d1d98403d62 100644 --- a/responses_api_agents/nooa_agent/runner.py +++ b/responses_api_agents/nooa_agent/runner.py @@ -34,7 +34,6 @@ from nemo_gym.core import NeMoGymResponseCreateParamsNonStreaming from nemo_gym.server_runtime import DirectHTTPToolAccess from nemo_gym.trajectory import AgentEpisode, TrajectoryRecord -from responses_api_agents.nooa_agent.config import NOOAInvocationConfig, validate_invocation from responses_api_agents.nooa_agent.gym_llm import ( GymModelClient, GymResponsesLLM, @@ -43,6 +42,7 @@ PolicyCallBudgetExceeded, RolloutLLMState, ) +from responses_api_agents.nooa_agent.invocation import NOOAInvocationConfig, validate_invocation from responses_api_agents.nooa_agent.observability import GymTraceHooks from responses_api_agents.nooa_agent.resource_tools import ( ResourceToolDispatcher, @@ -147,12 +147,14 @@ def __init__( model_server_name: str, max_policy_calls: int | None, on_progress: Callable[[NOOARunResult], None] | None = None, + context_window: int | None = None, ) -> None: self._invocation = invocation self._server_client = server_client self._model_server_name = model_server_name self._max_policy_calls = max_policy_calls self._on_progress = on_progress + self._context_window = context_window self._agent_class, self._invocation_adapter = validate_invocation(invocation) _validate_agent_llm_bindings(self._agent_class) @@ -204,6 +206,7 @@ def checkpoint() -> None: on_call=trace.on_model_call, on_response=checkpoint, sampling_overrides=sampling_overrides, + context_window=self._context_window, ) dispatcher = ResourceToolDispatcher( tool_access=request.tool_access, diff --git a/responses_api_agents/nooa_agent/sandbox_entrypoint.py b/responses_api_agents/nooa_agent/sandbox_entrypoint.py index aad7fd3827d..8b08842f012 100644 --- a/responses_api_agents/nooa_agent/sandbox_entrypoint.py +++ b/responses_api_agents/nooa_agent/sandbox_entrypoint.py @@ -37,7 +37,7 @@ ) from nemo_gym.core import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming, rollout_context from nemo_gym.trajectory import AgentEpisode, AgentObservationBundle, ObservationGap, TrajectoryRecord -from responses_api_agents.nooa_agent.config import NOOAInvocationConfig +from responses_api_agents.nooa_agent.invocation import NOOAInvocationConfig from responses_api_agents.nooa_agent.result import finalize_run_result, is_transient_infrastructure_error from responses_api_agents.nooa_agent.runner import InProcessNOOARunner, NOOARunFailure, NOOARunRequest, NOOARunResult from responses_api_agents.nooa_agent.termination import NOOATerminationReason @@ -55,6 +55,7 @@ class SandboxInput(BaseModel): model_base_url: AnyHttpUrl model_server_name: str max_policy_calls: int | None = Field(default=None, gt=0) + context_window: int | None = Field(default=None, gt=0) class RunnerError(BaseModel): @@ -222,6 +223,7 @@ async def execute( model_server_name=payload.model_server_name, max_policy_calls=payload.max_policy_calls, on_progress=on_progress, + context_window=payload.context_window, ) with rollout_context(payload.request.rollout_id): result = await runner.run(payload.request) diff --git a/responses_api_agents/nooa_agent/sandbox_runner.py b/responses_api_agents/nooa_agent/sandbox_runner.py index 5a71b487e8d..e29a608c972 100644 --- a/responses_api_agents/nooa_agent/sandbox_runner.py +++ b/responses_api_agents/nooa_agent/sandbox_runner.py @@ -36,6 +36,7 @@ def __init__( model_base_url: str, model_server_name: str, max_policy_calls: int | None, + context_window: int | None = None, timeout: float = 21600, close_timeout: float = 30, ) -> None: @@ -45,6 +46,7 @@ def __init__( self.model_base_url = model_base_url self.model_server_name = model_server_name self.max_policy_calls = max_policy_calls + self.context_window = context_window self.timeout = timeout self.close_timeout = close_timeout # Construct before prepare so failed setup can release the borrowed client. @@ -78,6 +80,7 @@ async def _stage_activation(self, request: NOOARunRequest) -> SandboxCommand: model_base_url=self.model_base_url, model_server_name=self.model_server_name, max_policy_calls=self.max_policy_calls, + context_window=self.context_window, ) await upload_text(self.sandbox, path=f"{self.directory}/input.json", text=payload.model_dump_json()) return SandboxCommand( diff --git a/responses_api_agents/nooa_agent/sandbox_runtime.py b/responses_api_agents/nooa_agent/sandbox_runtime.py index bbf12430405..c9de7dff03c 100644 --- a/responses_api_agents/nooa_agent/sandbox_runtime.py +++ b/responses_api_agents/nooa_agent/sandbox_runtime.py @@ -10,15 +10,74 @@ import logging import re import shlex +import shutil +import subprocess import tarfile import tempfile +from collections.abc import Callable from pathlib import Path +from typing import TypeVar from uuid import uuid4 +from weakref import WeakKeyDictionary from nemo_gym.sandbox import AsyncSandbox LOGGER = logging.getLogger(__name__) +_T = TypeVar("_T") +_IO_LIMITS: WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Semaphore] = WeakKeyDictionary() + + +async def _runtime_io(function: Callable[..., _T], *args: object, **kwargs: object) -> _T: + """Bound archive I/O and join cancelled workers before removing their files.""" + limit = _IO_LIMITS.setdefault(asyncio.get_running_loop(), asyncio.Semaphore(2)) + async with limit: + worker = asyncio.create_task(asyncio.to_thread(function, *args, **kwargs)) + try: + return await asyncio.shield(worker) + except asyncio.CancelledError: + joined = asyncio.gather(worker, return_exceptions=True) + while not joined.done(): + try: + await asyncio.shield(joined) + except asyncio.CancelledError: + continue + raise + + +def _stage_installer(source: bytes, local: Path) -> Path: + with tarfile.open(fileobj=io.BytesIO(source), mode="r:gz") as archive: + member = archive.extractfile(_INSTALLER_PATH) + assert member is not None + installer = local / "install_nooa_runtime.sh" + installer.write_bytes(member.read()) + return installer + + +def _stage_runtime_artifacts( + local: Path, + *, + source: bytes, + python_source: Path, + python_sha256: str, + wheels_source: Path | None, + wheels_sha256: str | None, +) -> str: + from responses_api_agents.nooa_agent.offline_runtime import file_sha256 + + python_archive = local / "python.tar.gz" + shutil.copyfile(python_source, python_archive) + if file_sha256(python_archive) != python_sha256: + raise RuntimeError("NOOA Python artifact checksum mismatch before sandbox upload") + if wheels_source is not None: + wheels_archive = local / "wheels.tar.gz" + shutil.copyfile(wheels_source, wheels_archive) + if file_sha256(wheels_archive) != wheels_sha256: + raise RuntimeError("NOOA wheelhouse changed before sandbox upload") + (local / "source.tar.gz").write_bytes(source) + return hashlib.sha256(source).hexdigest() + + _RUNTIME_ROOT = "/opt/nemo-gym-nooa" _PYTHON_VERSION = "3.13.14" _PYTHON_RELEASE = "20260805" @@ -32,7 +91,7 @@ _IMPORT_CHECK = ( "import sys; assert sys.version_info[:3] == (3,13,14); " "import nooa, nooa_cli; from nooa_bench.bench_agent import BenchAgent; " - "import responses_api_agents.nooa_agent.runner; " + "import responses_api_agents.nooa_agent.sandbox_entrypoint; " "from responses_api_agents.nooa_agent.resource_tools import ResourceToolDispatcher; " "from responses_api_agents.nooa_agent.observability import GymTraceHooks" ) @@ -40,15 +99,43 @@ def _runtime_archive(root: Path) -> tuple[bytes, str]: """Include current source, never repository metadata, caches, or user artifacts.""" + # Preserve the launching checkout's real package metadata and dependency set. + # Only tracked package files are copied: task datasets, caches, credentials, + # and other untracked controller artifacts must never enter the sandbox. files = {name: (root / name).read_bytes() for name in ("pyproject.toml", "README.md", "LICENSE")} - for path in (root / "nemo_gym").rglob("*.py"): - if not path.is_symlink() and not any(part.startswith(".") for part in path.relative_to(root).parts): - files[path.relative_to(root).as_posix()] = path.read_bytes() + tracked = ( + subprocess.run( + ["git", "-C", str(root), "ls-files", "-z", "--cached"], + check=True, + capture_output=True, + ) + .stdout.decode() + .split("\0") + ) + package_roots = { + "nemo_gym", + "responses_api_agents", + "responses_api_models", + "resources_servers", + "environment_servers", + "benchmarks", + "environments", + } + for name in tracked: + relative = Path(name) + path = root / relative + if ( + relative.parts + and relative.parts[0] in package_roots + and not any(part.startswith(".") for part in relative.parts) + and path.is_file() + and not path.is_symlink() + ): + files[name] = path.read_bytes() + # The snapshot has no .git directory. Supply the tracked asset inventory that + # setuptools-scm normally obtains from Git, including non-Python package data. + files["MANIFEST.in"] = ("\n".join(f"include {name}" for name in sorted(files)) + "\n").encode() agent_dir = root / "responses_api_agents/nooa_agent" - files[_INSTALLER_PATH] = (root / _INSTALLER_PATH).read_bytes() - for path in agent_dir.glob("*.py"): - if not path.is_symlink(): - files[path.relative_to(root).as_posix()] = path.read_bytes() requirements = (agent_dir / "requirements.txt").read_text() runtime_requirements = [] for line in requirements.splitlines(): @@ -77,18 +164,29 @@ def _runtime_archive(root: Path) -> tuple[bytes, str]: return output.getvalue(), digest.hexdigest() -async def prepare_nooa_runtime(sandbox: AsyncSandbox) -> str: +def _python_download_url(target: str) -> str: + arch, libc = target.split("/") + return ( + "https://github.com/astral-sh/python-build-standalone/releases/download/" + f"{_PYTHON_RELEASE}/cpython-{_PYTHON_VERSION}%2B{_PYTHON_RELEASE}-" + f"{arch}-unknown-linux-{libc}-install_only.tar.gz" + ) + + +async def prepare_nooa_runtime(sandbox: AsyncSandbox, *, wheelhouse_path: Path | None = None) -> str: """Stage verified artifacts and return the target's healthy private Python. Cold setup requires Linux x86_64/aarch64 with glibc, bash, tar, gzip, sha256sum, flock, tee, mv with -T support, and writable /opt/nemo-gym-nooa and /tmp. No root or system package manager is invoked. The controller supplies Python; the sandbox still - needs outbound package/build dependency access. This is not an offline install. + needs outbound package/build dependency access unless wheelhouse_path selects + a source-bound bundle prepared outside the task. Offline setup installs only + local wheels, without package indexes, URL requirements or build isolation. A healthy preinstalled or mounted runtime is reused without writing into it. """ root = Path(__file__).resolve().parents[2] - source, fingerprint = await asyncio.to_thread(_runtime_archive, root) + source, fingerprint = await _runtime_io(_runtime_archive, root) scratch = f"/tmp/nemo-gym-nooa-setup-{uuid4().hex}" script = scratch + ".sh" completed = False @@ -96,11 +194,7 @@ async def prepare_nooa_runtime(sandbox: AsyncSandbox) -> str: with tempfile.TemporaryDirectory(prefix="nooa-runtime-") as directory: local = Path(directory) # Execute precisely the installer included in this source fingerprint. - with tarfile.open(fileobj=io.BytesIO(source), mode="r:gz") as archive: - member = archive.extractfile(_INSTALLER_PATH) - assert member is not None - installer = local / "install_nooa_runtime.sh" - installer.write_bytes(member.read()) + installer = await _runtime_io(_stage_installer, source, local) await sandbox.upload(installer, script) probe = await sandbox.exec(shlex.join(["bash", script, "probe"]), cwd="/", timeout_s=30) target = (probe.stdout or "").strip() @@ -110,6 +204,27 @@ async def prepare_nooa_runtime(sandbox: AsyncSandbox) -> str: f"stderr={(probe.stderr or '')[-2000:]}" ) arch, libc = target.split("/") + wheelhouse = None + if wheelhouse_path is not None: + from responses_api_agents.nooa_agent.offline_runtime import load_runtime_wheelhouse + + wheelhouse = await _runtime_io( + load_runtime_wheelhouse, wheelhouse_path, target=target, source_fingerprint=fingerprint + ) + libc_result = await sandbox.exec("getconf GNU_LIBC_VERSION", cwd="/", timeout_s=30) + libc_fields = (libc_result.stdout or "").strip().split() + required_glibc = str(wheelhouse["glibc"]) + if ( + libc_result.return_code != 0 + or len(libc_fields) != 2 + or libc_fields[0] != "glibc" + or re.fullmatch(r"[0-9]+\.[0-9]+", libc_fields[1]) is None + or tuple(map(int, libc_fields[1].split("."))) < tuple(map(int, required_glibc.split("."))) + ): + raise RuntimeError(f"Offline NOOA runtime requires glibc >= {required_glibc}; got {libc_fields}") + # Dependency resolution can change independently of source; never reuse + # a runtime installed from different wheel bytes. + fingerprint = hashlib.sha256(f"{fingerprint}:{wheelhouse['wheels_sha256']}".encode()).hexdigest() runtime = f"{_RUNTIME_ROOT}/{fingerprint}-{arch}-{libc}" python = f"{runtime}/python/bin/python3" check = shlex.join(["bash", script, "check", runtime, fingerprint, _IMPORT_CHECK]) @@ -118,25 +233,32 @@ async def prepare_nooa_runtime(sandbox: AsyncSandbox) -> str: completed = True return python - # Reuse mini-SWE's checksum-aware, atomic controller download cache. - # This controller-only helper is deliberately not needed in the worker. - from responses_api_agents.miniswe_sandboxed_agent.bootstrap import cached_download - expected = _PYTHON_SHA256[target] - url = ( - "https://github.com/astral-sh/python-build-standalone/releases/download/" - f"{_PYTHON_RELEASE}/cpython-{_PYTHON_VERSION}%2B{_PYTHON_RELEASE}-" - f"{arch}-unknown-linux-{libc}-install_only.tar.gz" - ) - cached_archive = await cached_download(url, sha256=expected) python_archive = local / "python.tar.gz" - python_archive.write_bytes(cached_archive.read_bytes()) + wheels_sha256 = None + if wheelhouse is not None: + assert wheelhouse_path is not None + python_source = wheelhouse_path / "python.tar.gz" + wheels_sha256 = str(wheelhouse["wheels_sha256"]) + else: + # Reuse mini-SWE's checksum-aware, atomic controller download cache. + from responses_api_agents.miniswe_sandboxed_agent.bootstrap import cached_download + + python_source = await cached_download(_python_download_url(target), sha256=expected) # Check the private upload snapshot too, so cache replacement cannot # change the bytes between verification and sandbox transfer. - if hashlib.sha256(python_archive.read_bytes()).hexdigest() != expected: - raise RuntimeError(f"NOOA Python artifact checksum mismatch for {target}") + source_sha256 = await _runtime_io( + _stage_runtime_artifacts, + local, + source=source, + python_source=python_source, + python_sha256=expected, + wheels_source=wheelhouse_path / "wheels.tar.gz" if wheelhouse_path is not None else None, + wheels_sha256=wheels_sha256, + ) + if wheels_sha256 is not None: + await sandbox.upload(local / "wheels.tar.gz", f"{scratch}/wheels.tar.gz") source_archive = local / "source.tar.gz" - source_archive.write_bytes(source) await sandbox.upload(source_archive, f"{scratch}/source.tar.gz") await sandbox.upload(python_archive, f"{scratch}/python.tar.gz") command = shlex.join( @@ -146,9 +268,10 @@ async def prepare_nooa_runtime(sandbox: AsyncSandbox) -> str: "install", runtime, fingerprint, - hashlib.sha256(source).hexdigest(), + source_sha256, expected, _IMPORT_CHECK, + *([wheels_sha256] if wheels_sha256 is not None else []), ] ) result = await sandbox.exec(command, cwd="/", timeout_s=1800) diff --git a/responses_api_agents/nooa_agent/tests/test_bench_agent_adapter.py b/responses_api_agents/nooa_agent/tests/test_bench_agent_adapter.py new file mode 100644 index 00000000000..37ffd626a47 --- /dev/null +++ b/responses_api_agents/nooa_agent/tests/test_bench_agent_adapter.py @@ -0,0 +1,195 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +import json +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from nemo_gym.core import NeMoGymResponseCreateParamsNonStreaming, NeMoGymResponseFunctionToolCall +from responses_api_agents.nooa_agent.bench_agent_adapter import invoke_bench_agent + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error", [None, RuntimeError("model failed"), asyncio.CancelledError()]) +async def test_adapter_restores_task_cwd_and_always_drains_cleanup(tmp_path, monkeypatch, error): + monkeypatch.chdir(tmp_path) + value = object() + agent = SimpleNamespace( + shell=SimpleNamespace(cwd="/wrong", close=AsyncMock()), + context=SimpleNamespace(set=MagicMock()), + _install_python_tools=MagicMock(), + _solve_task=AsyncMock(return_value=value, side_effect=error), + aclose=AsyncMock(), + ) + request = NeMoGymResponseCreateParamsNonStreaming(input="unchanged canonical prompt") + if error is None: + assert await invoke_bench_agent(agent, request) is value + else: + with pytest.raises(type(error)): + await invoke_bench_agent(agent, request) + agent.shell.close.assert_awaited_once() + agent._install_python_tools.assert_called_once_with(str(tmp_path)) + agent._solve_task.assert_awaited_once_with("unchanged canonical prompt") + agent.context.set.assert_called_once_with("benchmark_task", "unchanged canonical prompt", prefix=True) + agent.aclose.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("limit", [None, 2]) +async def test_real_benchagent_delegation_uses_gym_trace_and_shared_call_budget( + tmp_path, monkeypatch, limit: int | None +): + pytest.importorskip("nooa_bench.bench_agent") + from responses_api_agents.nooa_agent.invocation import NOOAInvocationConfig + from responses_api_agents.nooa_agent.runner import InProcessNOOARunner, NOOARunRequest + from responses_api_agents.nooa_agent.tests.test_gym_llm import FakeHTTPResponse, model_response + + monkeypatch.chdir(tmp_path) + code = [ + "child = await self.delegate('bounded fixture')\nreturn_result(TaskResult(solution_description='parent', evidence=child.evidence, how_to_verify='fixture'))", + "return_result(TaskResult(solution_description='child', evidence='delegated evidence', how_to_verify='fixture'))", + ] + calls = [] + + async def post(**kwargs): + calls.append(kwargs) + output = NeMoGymResponseFunctionToolCall( + id=f"f{len(calls)}", + call_id=f"c{len(calls)}", + name="python_cell", + arguments=json.dumps({"code": code[len(calls) - 1]}), + ) + return FakeHTTPResponse(model_response(output, response_id=f"r{len(calls)}")) + + client = SimpleNamespace(post=post) + invocation = NOOAInvocationConfig( + agent_class="nooa_bench.bench_agent:BenchAgent", + invocation_adapter="responses_api_agents.nooa_agent.bench_agent_adapter:invoke_bench_agent", + ) + runner = InProcessNOOARunner( + invocation=invocation, + server_client=client, + model_server_name="policy", + max_policy_calls=limit, + context_window=262144, + ) + result = await runner.run( + NOOARunRequest( + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input="fixture", max_output_tokens=32768), + model_url_path="/v1/responses", + ) + ) + assert result.return_value.evidence == "delegated evidence" + assert len(calls) == len(result.trajectory.turns) == 2 + assert len(result.trajectory.invocations) >= 2 + assert all(call["json"].max_output_tokens == 32768 for call in calls) + assert result.termination_reason is None + + +@pytest.mark.asyncio +async def test_default_summarizer_calls_share_gym_budget_and_capture(tmp_path): + module = pytest.importorskip("nooa_bench.bench_agent") + from responses_api_agents.nooa_agent.gym_llm import GymResponsesLLM, PolicyCallBudgetExceeded, RolloutLLMState + from responses_api_agents.nooa_agent.tests.test_gym_llm import FakeHTTPResponse, model_response + + state = RolloutLLMState(max_policy_calls=1) + captured = [] + + async def post(**kwargs): + output = { + "type": "message", + "id": "summary", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": '{"value":"summary"}', "annotations": []}], + } + return FakeHTTPResponse(model_response(output, response_id="summary")) + + llm = GymResponsesLLM( + server_client=SimpleNamespace(post=post), + model_server_name="policy", + model_url_path="/v1/responses", + state=state, + cookies={}, + on_call=captured.append, + context_window=262144, + ) + agent = module.BenchAgent(llm=llm, working_dir=str(tmp_path)) + try: + assert agent._max_delegation_depth == 4 and len(agent._summarizers) == 1 + summarizer = agent._summarizers[0] + assert summarizer.llm is llm + assert await summarizer.summarize("synthetic history", 100) == "summary" + assert len(captured) == len(state.calls) == 1 + with pytest.raises(PolicyCallBudgetExceeded): + await llm.acall([{"role": "user", "content": "second"}]) + assert len(captured) == 1 + finally: + await agent.aclose() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("context_limit", [None, 1]) +async def test_real_benchagent_preserves_full_task_across_model_calls( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, context_limit: int | None +) -> None: + module = pytest.importorskip("nooa_bench.bench_agent") + from nooa.config import TruncationConfig + + from responses_api_agents.nooa_agent.gym_llm import GymResponsesLLM, RolloutLLMState + from responses_api_agents.nooa_agent.tests.test_gym_llm import FakeHTTPResponse, model_response + + monkeypatch.chdir(tmp_path) + middle = "MIDDLE_REQUIREMENT: preserve use_netrc and finish tool instructions." + task = ( + " Original task\n" + + ("context α 'quoted' \\ path\n" * 1500) + + middle + + ("\nremaining context" * 2000) + + "\n " + ) + volatile = "VOLATILE_CONTEXT_SHOULD_BE_EVICTED" + calls = [] + code = [ + "print('continue to the next model call')", + "return_result(TaskResult(solution_description='done', evidence='two calls', how_to_verify='fixture'))", + ] + + async def post(**kwargs): + calls.append(kwargs["json"]) + output = NeMoGymResponseFunctionToolCall( + id=f"f{len(calls)}", + call_id=f"c{len(calls)}", + name="python_cell", + arguments=json.dumps({"code": code[len(calls) - 1]}), + ) + return FakeHTTPResponse(model_response(output, response_id=f"r{len(calls)}")) + + llm = GymResponsesLLM( + server_client=SimpleNamespace(post=post), + model_server_name="policy", + model_url_path="/v1/responses", + state=RolloutLLMState(max_policy_calls=2), + cookies={}, + context_window=262144, + ) + agent = module.BenchAgent( + llm=llm, + working_dir=str(tmp_path), + truncation=TruncationConfig(max_context_tokens=context_limit), + ) + agent.context.set("volatile_probe", volatile) + result = await invoke_bench_agent(agent, NeMoGymResponseCreateParamsNonStreaming(input=task)) + + assert result.evidence == "two calls" + assert len(calls) == 2 + for call in calls: + # These are the actual requests after native argument/context rendering. + assert middle in (call.instructions or "") + assert task in (call.instructions or "") + assert (volatile in call.model_dump_json()) == (context_limit is None) + assert agent.context["benchmark_task"] == task diff --git a/responses_api_agents/nooa_agent/tests/test_config.py b/responses_api_agents/nooa_agent/tests/test_config.py index ea3df4a068c..a22a0e46fff 100644 --- a/responses_api_agents/nooa_agent/tests/test_config.py +++ b/responses_api_agents/nooa_agent/tests/test_config.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from pathlib import Path from typing import Any import pytest @@ -98,6 +99,14 @@ def test_agent_config_rejects_nonpositive_policy_call_limits(limit: int) -> None agent_config(max_policy_calls=limit) +def test_context_window_defaults_to_unknown_and_accepts_known_positive_limit() -> None: + assert agent_config().context_window is None + assert agent_config(context_window=262144).context_window == 262144 + for invalid in (0, -1): + with pytest.raises(ValidationError, match="context_window"): + agent_config(context_window=invalid) + + def test_sandboxed_execution_mode_is_reserved_for_a_future_runner() -> None: assert invocation_config(execution_mode="sandboxed").execution_mode == "sandboxed" @@ -202,3 +211,16 @@ def test_shipped_yaml_declares_worker_and_close_defaults_below_episode_timeout() assert agent["session_close_timeout_seconds"] == defaults.session_close_timeout_seconds # These clocks start at different times; this does not reserve cleanup time within the episode. assert agent["sandbox_runner_timeout_seconds"] < environment["default_episode_timeout_seconds"] + + +def test_runtime_wheelhouse_expands_user_and_resolves_absolute_path( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + assert agent_config(runtime_wheelhouse="~/bundle/../runtime").runtime_wheelhouse == tmp_path / "runtime" + assert agent_config(runtime_wheelhouse=tmp_path / "runtime").runtime_wheelhouse == tmp_path / "runtime" + for relative in ["bundle", "../bundle", Path(".")]: + with pytest.raises(ValidationError, match="must be an absolute path"): + agent_config(runtime_wheelhouse=relative) + with pytest.raises(ValidationError, match="Set NOOA_SWE_RUNTIME_WHEELHOUSE"): + agent_config(runtime_wheelhouse_required=True) diff --git a/responses_api_agents/nooa_agent/tests/test_gym_llm.py b/responses_api_agents/nooa_agent/tests/test_gym_llm.py index 0cca89025bd..b8257775ad1 100644 --- a/responses_api_agents/nooa_agent/tests/test_gym_llm.py +++ b/responses_api_agents/nooa_agent/tests/test_gym_llm.py @@ -82,7 +82,11 @@ def model_response(*outputs: object, response_id: str = "resp-1") -> dict: def make_llm( - payload: dict, *, max_policy_calls: int | None = 2, sampling_overrides: dict | None = None + payload: dict, + *, + max_policy_calls: int | None = 2, + sampling_overrides: dict | None = None, + context_window: int | None = None, ) -> tuple[GymResponsesLLM, MagicMock, RolloutLLMState]: server_client = MagicMock() server_client.post = AsyncMock(return_value=FakeHTTPResponse(payload)) @@ -94,6 +98,7 @@ def make_llm( state=state, cookies={}, sampling_overrides=sampling_overrides, + context_window=context_window, ) return llm, server_client, state @@ -122,6 +127,28 @@ def capture() -> None: assert len(captured) == 1 +@pytest.mark.asyncio +async def test_context_planning_reserves_the_effective_gym_reply_cap() -> None: + llm, client, _ = make_llm(model_response(), context_window=262144, sampling_overrides={"max_output_tokens": 32768}) + overrides = {"max_tokens": 128, "extra_body": {"max_completion_tokens": 64, "custom": True}} + limits = llm.get_context_limits(overrides, fallback_reserve=4096) + assert llm.context_window == limits.context_window == 262144 + assert limits.reserved_output_tokens == 32768 + assert limits.usable_input_tokens == 229376 + assert not limits.reserve_is_fallback + assert overrides == {"max_tokens": 128, "extra_body": {"max_completion_tokens": 64, "custom": True}} + await llm.acall([{"role": "user", "content": "hello"}], max_tokens=128) + assert client.post.await_args.kwargs["json"].max_output_tokens == limits.reserved_output_tokens + + +def test_context_planning_without_gym_reply_override_keeps_nooa_reserve_semantics() -> None: + llm, _, _ = make_llm(model_response(), context_window=262144) + explicit = llm.get_context_limits({"max_tokens": 128}, fallback_reserve=4096) + assert explicit.reserved_output_tokens == 128 and not explicit.reserve_is_fallback + fallback = llm.get_context_limits(fallback_reserve=4096) + assert fallback.reserved_output_tokens == 4096 and fallback.reserve_is_fallback + + @pytest.mark.parametrize( ("schema", "expected"), [ @@ -462,3 +489,47 @@ async def test_successful_response_after_output_limit_recovers_normally() -> Non assert state.used == len(state.calls) == 2 assert state.calls[0].response.incomplete_details.reason == "max_output_tokens" assert state.calls[1].response.incomplete_details is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("key", ["max_tokens", "max_completion_tokens", "max_output_tokens"]) +@pytest.mark.parametrize("nested", [False, True]) +@pytest.mark.parametrize("gym_cap", [None, 256]) +async def test_reply_cap_alias_matches_planning_and_wire_request(key: str, nested: bool, gym_cap: int | None) -> None: + params = {key: 128} + if nested: + params = {"extra_body": params} + llm, client, _ = make_llm(model_response(), context_window=1024, sampling_overrides={"max_output_tokens": gym_cap}) + limits = llm.get_context_limits(params) + await llm.acall([{"role": "user", "content": "hello"}], **params) + body = client.post.await_args.kwargs["json"] + assert body.max_output_tokens == limits.reserved_output_tokens == (gym_cap or 128) + assert limits.usable_input_tokens == 1024 - body.max_output_tokens + + +@pytest.mark.asyncio +async def test_absent_reply_cap_does_not_send_planning_fallback() -> None: + llm, client, _ = make_llm(model_response(), context_window=1024) + assert llm.get_context_limits(fallback_reserve=256).reserved_output_tokens == 256 + await llm.acall([{"role": "user", "content": "hello"}]) + assert client.post.await_args.kwargs["json"].max_output_tokens is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "params", + [ + {"max_tokens": 512, "max_completion_tokens": 64}, + {"extra_body": {"max_tokens": 1024}}, + {"max_completion_tokens": 0}, + {"max_output_tokens": True}, + ], +) +async def test_invalid_reply_caps_fail_planning_and_call_before_model_transport(params: dict) -> None: + llm, client, state = make_llm(model_response(), context_window=1024) + with pytest.raises(ValueError): + llm.get_context_limits(params) + with pytest.raises(ValueError): + await llm.acall([{"role": "user", "content": "hello"}], **params) + client.post.assert_not_awaited() + assert state.used == 0 diff --git a/responses_api_agents/nooa_agent/tests/test_observability.py b/responses_api_agents/nooa_agent/tests/test_observability.py index 7e7d8cac37b..1eea01363f4 100644 --- a/responses_api_agents/nooa_agent/tests/test_observability.py +++ b/responses_api_agents/nooa_agent/tests/test_observability.py @@ -15,13 +15,16 @@ import asyncio import json +import os from types import ModuleType import pytest +from nooa.events import ExecutionResult from pydantic import BaseModel from nemo_gym.config import ModelServerRef from nemo_gym.core import ( + NeMoGymFunctionCallOutput, NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming, NeMoGymResponseFunctionToolCall, @@ -117,6 +120,47 @@ def test_scoped_projection_preserves_model_ownership_and_collector_tool_output_j assert persisted.tool_calls[0].invocation_id == "root" +@pytest.mark.parametrize("has_later_model_call", [False, True]) +def test_code_execution_module_locals_do_not_prevent_trace_projection(has_later_model_call: bool) -> None: + trace = GymTraceHooks() + state = RolloutLLMState(max_policy_calls=1) + params = NeMoGymResponseCreateParamsNonStreaming(input="task") + execution = trace.before_code_execution( + code="import os\nprint(os.name)", execution_id="exec-1", tool_call_id="call-1" + ) + raw_result = ExecutionResult(stdout="posix\n", captured_locals={"os": os}) + trace.after_code_execution(context=execution, result=raw_result, exception=None) + visible_output = "Stdout:\nposix\n" + if has_later_model_call: + call = GymModelCall( + model_ref=ModelServerRef(type="responses_api_models", name="policy"), + request=NeMoGymResponseCreateParamsNonStreaming( + input=[NeMoGymFunctionCallOutput(call_id="call-1", output=visible_output)] + ), + ) + trace.on_model_call(call) + state.calls.append(call) + + episode, trajectory = trace.project(create_params=params, state=state, task_id="task", rollout_id="0-0") + + tool = trajectory.tool_calls[0] + assert tool is not execution.observation + if has_later_model_call: + assert tool.output == visible_output + else: + output = json.loads(tool.output) + assert output["stdout"] == "posix\n" + assert "captured_locals" not in output + assert episode.response.output[-1].output == tool.output + assert NeMoGymResponse.model_validate_json(episode.response.model_dump_json()) == episode.response + assert AgentObservationBundle.model_validate_json(episode.observations.model_dump_json()) == episode.observations + assert TrajectoryRecord.model_validate_json(trajectory.model_dump_json()) == trajectory + snapshot = json.loads(execution.observation.output) + assert snapshot["stdout"] == "posix\n" + assert "captured_locals" not in snapshot + assert raw_result.captured_locals["os"] is os + + def test_no_model_call_preserves_original_responses_input_with_tool_events() -> None: trace = GymTraceHooks() params = NeMoGymResponseCreateParamsNonStreaming(input="original row input") diff --git a/responses_api_agents/nooa_agent/tests/test_offline_runtime.py b/responses_api_agents/nooa_agent/tests/test_offline_runtime.py new file mode 100644 index 00000000000..fdd514f61a6 --- /dev/null +++ b/responses_api_agents/nooa_agent/tests/test_offline_runtime.py @@ -0,0 +1,196 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +import hashlib +import io +import json +import sys +import tarfile +from pathlib import Path +from unittest.mock import AsyncMock, Mock + +import pytest + +from responses_api_agents.nooa_agent import offline_runtime + + +@pytest.fixture(params=["1000:1001", "0:0"]) +def builder(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, request: pytest.FixtureRequest) -> AsyncMock: + from nemo_gym.client import http_client + from responses_api_agents.miniswe_sandboxed_agent import bootstrap + + python_archive = tmp_path / "python.tar.gz" + python_archive.write_bytes(b"portable Python") + monkeypatch.setitem( + offline_runtime._PYTHON_SHA256, "x86_64/gnu", hashlib.sha256(python_archive.read_bytes()).hexdigest() + ) + monkeypatch.setattr(offline_runtime, "_runtime_archive", lambda root: (b"source", "source-fingerprint")) + monkeypatch.setattr(offline_runtime.shutil, "which", lambda tool: "/bin/docker") + monkeypatch.setattr(bootstrap, "cached_download", AsyncMock(return_value=python_archive)) + monkeypatch.setattr(http_client, "is_global_aiohttp_client_setup", lambda: True) + + async def command(*arguments: str, **kwargs: object) -> str: + if arguments[1:3] == ("image", "inspect"): + return "sha256:builder\n" + if arguments[1] == "run": + if arguments[-4:] == ("stat", "-c", "%u:%g", "/build"): + return f"{request.param}\n" + assert arguments[arguments.index("--user") + 1] == request.param + work = Path(arguments[arguments.index("--volume") + 1].removesuffix(":/build")) + assert (work / "source.tar.gz").read_bytes() == b"source" + assert (work / "bundle/python.tar.gz").read_bytes() == b"portable Python" + assert "sha256:builder" in arguments + assert "linux/amd64" in arguments + (work / "architecture.txt").write_text("x86_64\n") + (work / "libc.txt").write_text("glibc 2.28\n") + (work / "wheels").mkdir() + (work / "wheels/fixture-1-py3-none-any.whl").write_bytes(b"fixture wheel") + return "wheel build completed\n" + return "" + + execute = AsyncMock(side_effect=command) + monkeypatch.setattr(offline_runtime, "_command", execute) + return execute + + +@pytest.mark.asyncio +async def test_builder_publishes_validated_bundle_and_reuses_without_docker( + tmp_path: Path, builder: AsyncMock +) -> None: + output = tmp_path / "bundle" + assert await offline_runtime.prepare_runtime_wheelhouse(output, target="x86_64/gnu") == output + manifest = offline_runtime.validate_runtime_wheelhouse(output, target="x86_64/gnu") + assert manifest["builder_image"] == "sha256:builder" + assert manifest["glibc"] == "2.28" + assert (output / "build.log").read_text() == "wheel build completed\n" + assert [call.args[1] for call in builder.await_args_list] == ["pull", "image", "run", "run", "rm"] + owner_probe, build, cleanup = builder.await_args_list[2:] + assert owner_probe.args[:9] == build.args[:9] + assert owner_probe.args[9] == build.args[11] == "sha256:builder" + assert cleanup.args[2] == "--force" + assert cleanup.args[-1] == build.args[build.args.index("--name") + 1] + assert not list(tmp_path.glob(".nooa-wheelhouse-*")) + builder.reset_mock() + assert await offline_runtime.prepare_runtime_wheelhouse(output, target="x86_64/gnu") == output + builder.assert_not_awaited() + (output / "wheels.tar.gz").write_bytes(b"corruption") + with pytest.raises(ValueError, match="checksum mismatch"): + await offline_runtime.prepare_runtime_wheelhouse(output, target="x86_64/gnu") + builder.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", [RuntimeError("build failed"), asyncio.CancelledError()]) +@pytest.mark.parametrize("stage", ["owner", "build"]) +async def test_builder_failure_removes_owned_container_without_publishing( + tmp_path: Path, builder: AsyncMock, failure: BaseException, stage: str +) -> None: + builder.side_effect = ["", "sha256:builder", *(["1000:1001"] if stage == "build" else []), failure, ""] + output = tmp_path / "bundle" + with pytest.raises(type(failure)): + await offline_runtime.prepare_runtime_wheelhouse(output, target="x86_64/gnu") + assert builder.await_args_list[-1].args[1:3] == ("rm", "--force") + assert not output.exists() + assert not list(tmp_path.glob(".nooa-wheelhouse-*")) + + +@pytest.mark.asyncio +async def test_cleanup_failure_is_not_accepted_as_success(tmp_path: Path, builder: AsyncMock) -> None: + create = builder.side_effect + + async def fail_cleanup(*arguments: str, **kwargs: object) -> str: + if arguments[1] == "rm": + raise RuntimeError("Docker daemon disconnected") + return await create(*arguments, **kwargs) + + builder.side_effect = fail_cleanup + output = tmp_path / "bundle" + with pytest.raises(RuntimeError, match="daemon disconnected"): + await offline_runtime.prepare_runtime_wheelhouse(output, target="x86_64/gnu") + assert not output.exists() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("case", ["source", "glibc", "empty", "nested", "symlink"]) +async def test_loader_rejects_incompatible_manifest_or_unsafe_wheels( + tmp_path: Path, builder: AsyncMock, case: str +) -> None: + output = await offline_runtime.prepare_runtime_wheelhouse(tmp_path / "bundle", target="x86_64/gnu") + manifest_path = output / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + if case == "source": + manifest["source_fingerprint"] = "other-source" + elif case == "glibc": + manifest["glibc"] = "unknown" + else: + with tarfile.open(output / "wheels.tar.gz", "w:gz") as archive: + if case != "empty": + member = tarfile.TarInfo("../fixture.whl" if case == "nested" else "fixture.whl") + if case == "symlink": + member.type = tarfile.SYMTYPE + member.linkname = "/outside" + archive.addfile(member, io.BytesIO()) + manifest["wheels_sha256"] = offline_runtime.file_sha256(output / "wheels.tar.gz") + manifest_path.write_text(json.dumps(manifest)) + with pytest.raises(ValueError, match="NOOA wheelhouse"): + offline_runtime.validate_runtime_wheelhouse(output, target="x86_64/gnu") + + +@pytest.mark.asyncio +async def test_missing_docker_fails_before_download(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(offline_runtime, "_runtime_archive", lambda root: (b"source", "fingerprint")) + monkeypatch.setattr(offline_runtime.shutil, "which", lambda tool: None) + with pytest.raises(RuntimeError, match="requires Docker"): + await offline_runtime.prepare_runtime_wheelhouse(tmp_path / "bundle", target="x86_64/gnu") + assert not (tmp_path / "bundle").exists() + + +@pytest.mark.asyncio +async def test_command_decodes_non_utf8_and_reports_failure() -> None: + assert await offline_runtime._command(sys.executable, "-c", "import os; os.write(1, b'\\xff')") == "\ufffd" + with pytest.raises(RuntimeError, match="exit 7"): + await offline_runtime._command(sys.executable, "-c", "raise SystemExit(7)") + + +@pytest.mark.asyncio +async def test_command_timeout_kills_and_joins_process(monkeypatch: pytest.MonkeyPatch) -> None: + async def blocked() -> tuple[bytes, bytes]: + await asyncio.Event().wait() + raise AssertionError("unreachable") + + process = Mock(returncode=None, communicate=blocked, kill=Mock(), wait=AsyncMock()) + monkeypatch.setattr(offline_runtime.asyncio, "create_subprocess_exec", AsyncMock(return_value=process)) + with pytest.raises(TimeoutError): + await offline_runtime._command("fixture", timeout=0) + process.kill.assert_called_once() + process.wait.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_builder_accepts_empty_output_directory(tmp_path: Path, builder: AsyncMock) -> None: + output = tmp_path / "bundle" + output.mkdir() + await offline_runtime.prepare_runtime_wheelhouse(output, target="x86_64/gnu") + assert (output / "manifest.json").is_file() + assert builder.await_args_list[0].args[-1] == "quay.io/pypa/manylinux2014_x86_64:latest" + + +@pytest.mark.asyncio +async def test_builder_does_not_overwrite_nonempty_directory(tmp_path: Path, builder: AsyncMock) -> None: + output = tmp_path / "bundle" + output.mkdir() + (output / "owned.txt").write_text("preserve") + with pytest.raises(ValueError, match="Prepare the NOOA runtime wheelhouse first"): + await offline_runtime.prepare_runtime_wheelhouse(output, target="x86_64/gnu") + assert (output / "owned.txt").read_text() == "preserve" + builder.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_validate_only_expands_tilde( + tmp_path: Path, builder: AsyncMock, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + await offline_runtime.prepare_runtime_wheelhouse(Path("~/bundle"), target="x86_64/gnu") + assert offline_runtime.validate_runtime_wheelhouse(Path("~/bundle"), target="x86_64/gnu")["target"] == "x86_64/gnu" diff --git a/responses_api_agents/nooa_agent/tests/test_recipe.py b/responses_api_agents/nooa_agent/tests/test_recipe.py index 5e710dc3051..a653f20f2d9 100644 --- a/responses_api_agents/nooa_agent/tests/test_recipe.py +++ b/responses_api_agents/nooa_agent/tests/test_recipe.py @@ -83,18 +83,29 @@ def test_swe_shared_preparation_preserves_grading_metadata_through_legacy_adapte assert output.read_bytes() == original -def test_swe_recipe_uses_borrowed_sandbox_and_single_agent_turn_environment() -> None: +def test_swe_recipe_uses_borrowed_sandbox_and_single_agent_turn_environment( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: from omegaconf import OmegaConf from environment_servers.single_agent_turn_legacy.app import SingleAgentTurnLegacyEnvironmentServerConfig + from nemo_gym.cli.env import _configured_servers from nemo_gym.config import GlobalConfigDictParser from nemo_gym.rollouts import RolloutCollectionHelper from nemo_gym.sandbox import AsyncSandbox from resources_servers.swebench_pro.app import SWEBenchProResourcesServerConfig from responses_api_agents.nooa_agent.config import NOOAAgentConfig + for key, value in { + "POLICY_BASE_URL": "http://model.invalid/v1", + "POLICY_API_KEY": "fixture", + "POLICY_MODEL_NAME": "fixture", + "NOOA_SWE_RUNTIME_WHEELHOUSE": str(tmp_path / "wheelhouse"), + }.items(): + monkeypatch.setenv(key, value) parser = GlobalConfigDictParser() - _, configs = parser.load_extra_config_paths([str(ROOT / "benchmarks/swebench/pro/nooa.yaml")]) + paths = [str(ROOT / "benchmarks/swebench/pro/nooa.yaml")] + _, configs = parser.load_extra_config_paths(paths) config = OmegaConf.merge(*configs) parser._recursively_swap_keys(config) environment = SingleAgentTurnLegacyEnvironmentServerConfig( @@ -117,9 +128,14 @@ def test_swe_recipe_uses_borrowed_sandbox_and_single_agent_turn_environment() -> assert not environment.resources_tool_transports resources = config[environment.resources_server.name].resources_servers.swebench_pro assert resources.allowed_agents == ["nooa_agent"] + assert resources.entrypoint == "app.py" + assert resources.sandbox_config.provider_options.platform == {"os": "linux", "arch": "amd64"} + assert resources.task_network_policy.model_server.name == agent.model_server.name + assert "network_policy" not in resources.sandbox_config.provider_options + assert agent.runtime_wheelhouse == tmp_path / "wheelhouse" + assert "sandbox" not in config # The caller explicitly selects a provider overlay. assert resources.datasets[0].jsonl_fpath == "benchmarks/swebench/data/swebench_pro_benchmark.jsonl" assert resources.datasets[0].prepare_script == "benchmarks/swebench/pro/prepare.py" - assert resources.get("task_network_policy") is None resource_config = SWEBenchProResourcesServerConfig( name=environment.resources_server.name, host="localhost", @@ -157,3 +173,68 @@ def test_swe_recipe_uses_borrowed_sandbox_and_single_agent_turn_environment() -> assert [(instance.name, set(instance.environment_servers)) for instance in environments] == [ ("swebench_pro_nooa", {"single_agent_turn_legacy"}) ] + assert {(server.top_level_path, server.server_type, server.name) for server in _configured_servers(config)} == { + ("swebench_pro_nooa_resources_server", "resources_servers", "swebench_pro"), + ("swebench_pro_nooa_agent", "responses_api_agents", "nooa_agent"), + ("swebench_pro_nooa", "environment_servers", "single_agent_turn_legacy"), + } + + +def test_shared_swe_recipe_keeps_default_networking() -> None: + from omegaconf import OmegaConf + + from nemo_gym.config import GlobalConfigDictParser + + parser = GlobalConfigDictParser() + _, configs = parser.load_extra_config_paths( + [str(ROOT / "resources_servers/swebench_pro/configs/swebench_pro.yaml")] + ) + config = OmegaConf.merge(*configs) + parser._recursively_swap_keys(config) + resources = config.swebench_pro_resources_server.resources_servers.swebench_pro + assert resources.get("task_network_policy") is None + assert "network_policy" not in resources.sandbox_config.provider_options + assert resources.entrypoint == "app.py" + + +def test_swe_model_override_drives_the_network_endpoint() -> None: + from omegaconf import OmegaConf + + from nemo_gym.config import GlobalConfigDictParser + + parser = GlobalConfigDictParser() + _, configs = parser.load_extra_config_paths([str(ROOT / "benchmarks/swebench/pro/nooa.yaml")]) + config = OmegaConf.merge(*configs) + parser._recursively_swap_keys(config) + agent = config.swebench_pro_nooa_agent.responses_api_agents.nooa_agent + agent.model_server.name = "alternate_policy" + resources = config.swebench_pro_nooa_resources_server.resources_servers.swebench_pro + assert resources.task_network_policy.model_server.name == "alternate_policy" + assert resources.task_network_policy.model_server.type == "responses_api_models" + + +def test_explicit_unrestricted_docker_config_needs_no_opensandbox_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + from omegaconf import OmegaConf + + from nemo_gym.config import GlobalConfigDictParser + from responses_api_agents.nooa_agent.config import NOOAAgentConfig + + for name in ["OPENSANDBOX_API_KEY", "OPENSANDBOX_DOMAIN", "NOOA_SWE_RUNTIME_WHEELHOUSE"]: + monkeypatch.delenv(name, raising=False) + parser = GlobalConfigDictParser() + _, configs = parser.load_extra_config_paths( + [ + str(ROOT / "benchmarks/swebench/pro/nooa.yaml"), + str(ROOT / "nemo_gym/sandbox/providers/docker/configs/docker.yaml"), + ] + ) + config = OmegaConf.merge(*configs) + parser._recursively_swap_keys(config) + resources = config.swebench_pro_nooa_resources_server.resources_servers.swebench_pro + resources.task_network_policy = None + agent = config.swebench_pro_nooa_agent.responses_api_agents.nooa_agent + agent.runtime_wheelhouse_required = False + parsed = NOOAAgentConfig(name="agent", host="localhost", port=8001, **OmegaConf.to_container(agent, resolve=True)) + assert parsed.runtime_wheelhouse is None + assert resources.task_network_policy is None + assert "docker" in OmegaConf.to_container(config.sandbox, resolve=True) diff --git a/responses_api_agents/nooa_agent/tests/test_runner.py b/responses_api_agents/nooa_agent/tests/test_runner.py index ca8cec4c0d3..4a136b06143 100644 --- a/responses_api_agents/nooa_agent/tests/test_runner.py +++ b/responses_api_agents/nooa_agent/tests/test_runner.py @@ -145,7 +145,9 @@ class FakeResponse: cookies = SimpleCookie() -def make_runner(*, execution_mode: str = "embedded") -> tuple[InProcessNOOARunner, MagicMock]: +def make_runner( + *, execution_mode: str = "embedded", context_window: int | None = None +) -> tuple[InProcessNOOARunner, MagicMock]: invocation = NOOAInvocationConfig.model_validate( { "agent_class": f"{__name__}:ValidAgent", @@ -161,11 +163,34 @@ def make_runner(*, execution_mode: str = "embedded") -> tuple[InProcessNOOARunne server_client=client, model_server_name="policy_model", max_policy_calls=3, + context_window=context_window, ) runner._agent_class = FakeAgent return runner, client +@pytest.mark.asyncio +async def test_runner_passes_context_window_and_request_reply_cap_to_nooa() -> None: + runner, client = make_runner(context_window=262144) + + async def inspect_limits(agent, request): + limits = agent.llm.get_context_limits(fallback_reserve=4096) + assert limits.context_window == 262144 + assert limits.reserved_output_tokens == 32768 + assert limits.usable_input_tokens == 229376 + return "limits checked" + + runner._invocation_adapter = inspect_limits + result = await runner.run( + NOOARunRequest( + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input="hello", max_output_tokens=32768), + model_url_path="/v1/responses", + ) + ) + assert result.return_value == "limits checked" + client.post.assert_not_called() + + def responses_create_params(customer_id: str) -> NeMoGymResponseCreateParamsNonStreaming: return NeMoGymResponseCreateParamsNonStreaming.model_validate( { diff --git a/responses_api_agents/nooa_agent/tests/test_runtime_installer.py b/responses_api_agents/nooa_agent/tests/test_runtime_installer.py index 32555e1293e..c58a8202103 100644 --- a/responses_api_agents/nooa_agent/tests/test_runtime_installer.py +++ b/responses_api_agents/nooa_agent/tests/test_runtime_installer.py @@ -279,3 +279,33 @@ def test_warm_check_uses_private_scratch_without_writing_read_only_runtime(tmp_p for path in paths: if not path.is_symlink(): path.chmod(0o755 if path.is_dir() or path.name == "python3" else 0o644) + + +@pytest.mark.parametrize("corrupt", [False, True]) +def test_offline_install_verifies_wheels_and_never_resolves_remote_requirements(tmp_path: Path, corrupt: bool) -> None: + call = make_call(tmp_path / "offline") + wheels = archive_bytes({"package-1-py3-none-any.whl": (b"fixture", 0o644)}) + (call.scratch / "wheels.tar.gz").write_bytes(wheels + (b"corruption" if corrupt else b"")) + result = subprocess.run( + [*call.command(), hashlib.sha256(wheels).hexdigest()], + env=call.env, + text=True, + capture_output=True, + timeout=20, + ) + if corrupt: + assert result.returncode != 0 + assert not (call.runtime.parent / ".builds").exists() + assert not Path(call.env["NOOA_TEST_EVENTS"]).exists() + else: + assert result.returncode == 0, result.stdout + result.stderr + (record,) = records(Path(call.env["NOOA_TEST_EVENTS"])) + assert "--no-index" in record["argv"] + assert "--no-deps" in record["argv"] + assert "--only-binary=:all:" in record["argv"] + assert not any(arg.startswith(("http:", "https:")) for arg in record["argv"]) + assert "-r" not in record["argv"] + assert record["argv"][-1].endswith("/wheels/package-1-py3-none-any.whl") + hook = call.runtime / "python/lib/python3.13/site-packages/nooa_offline_runtime.pth" + assert hook.read_text() == 'import os; os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"\n' + assert call.runtime.is_symlink() diff --git a/responses_api_agents/nooa_agent/tests/test_sandbox_entrypoint.py b/responses_api_agents/nooa_agent/tests/test_sandbox_entrypoint.py index d702f5fd73a..57ecf1dc315 100644 --- a/responses_api_agents/nooa_agent/tests/test_sandbox_entrypoint.py +++ b/responses_api_agents/nooa_agent/tests/test_sandbox_entrypoint.py @@ -38,6 +38,7 @@ def payload() -> entrypoint.SandboxInput: model_base_url="http://model:8000", model_server_name="policy", max_policy_calls=3, + context_window=262144, ) @@ -118,6 +119,7 @@ async def run(request): monkeypatch.setattr(entrypoint, "InProcessNOOARunner", runner_factory) artifact = await entrypoint.execute(p) assert runner_factory.call_args.kwargs["max_policy_calls"] == limit + assert runner_factory.call_args.kwargs["context_window"] == 262144 assert artifact.observations.gaps[0].code == "test" assert artifact.model_cookies == {"model": "new"} assert artifact.resource_cookies == {"resource": "new"} @@ -185,6 +187,17 @@ def test_launch_policy_call_limit_is_optional_and_survives_json() -> None: entrypoint.SandboxInput.model_validate(data | {"max_policy_calls": invalid}) +def test_sandbox_context_window_is_optional_positive_and_survives_json_boundary() -> None: + p = payload() + assert entrypoint.SandboxInput.model_validate_json(p.model_dump_json()).context_window == 262144 + data = p.model_dump(mode="json") + data.pop("context_window") + assert entrypoint.SandboxInput.model_validate(data).context_window is None + for invalid in (0, -1): + with pytest.raises(ValidationError, match="context_window"): + entrypoint.SandboxInput.model_validate(data | {"context_window": invalid}) + + @pytest.mark.asyncio async def test_child_setup_failure_is_not_empty_success(monkeypatch) -> None: monkeypatch.setattr(entrypoint, "InProcessNOOARunner", MagicMock(side_effect=ValueError("invalid config"))) diff --git a/responses_api_agents/nooa_agent/tests/test_sandbox_runner.py b/responses_api_agents/nooa_agent/tests/test_sandbox_runner.py index 0904eb0d3ad..b116406b434 100644 --- a/responses_api_agents/nooa_agent/tests/test_sandbox_runner.py +++ b/responses_api_agents/nooa_agent/tests/test_sandbox_runner.py @@ -51,6 +51,7 @@ def runner() -> tuple[SandboxNOOARunner, MemorySandbox]: model_base_url=str(p.model_base_url), model_server_name=p.model_server_name, max_policy_calls=3, + context_window=p.context_window, ), sandbox @@ -99,6 +100,7 @@ async def test_launch_quotes_paths_runs_in_task_and_preserves_cookies(limit: int assert launch.max_policy_calls == limit assert launch.request.rollout_id == request.rollout_id assert launch.request.model_url_path == request.model_url_path + assert launch.context_window == 262144 assert request.resource_cookies == result.resource_cookies == {"resource": "new"} assert request.model_cookies == {"model": "new"} assert r.session.cleanup == cleanup_receipt() diff --git a/responses_api_agents/nooa_agent/tests/test_sandbox_runtime.py b/responses_api_agents/nooa_agent/tests/test_sandbox_runtime.py index 2f33797ed7a..3e1c210bb27 100644 --- a/responses_api_agents/nooa_agent/tests/test_sandbox_runtime.py +++ b/responses_api_agents/nooa_agent/tests/test_sandbox_runtime.py @@ -4,11 +4,13 @@ import asyncio import hashlib import io +import json import os import shlex import shutil import subprocess import tarfile +import tomllib from pathlib import Path from unittest.mock import AsyncMock @@ -33,16 +35,28 @@ def source_tree(tmp_path: Path) -> Path: agent.mkdir(parents=True) (agent / "runner.py").write_text("current = True") (agent / "install_nooa_runtime.sh").write_text("#!/bin/bash\necho packaged-installer\n") - (agent / "runtime").mkdir() - (agent / "runtime/pyproject.toml").write_text( - "[project]\ndependencies = []\n[tool.nemo-gym-runtime]\n" - 'core-dependencies = ["openai"]\nadditional-dependencies = ["requests"]\n' - ) (agent / "results.json").write_text("secret") pin = "a" * 40 (agent / "requirements.txt").write_text( f"-e nemo-gym[dev] @ ../../\njsonschema>=4\nnooa @ git+https://github.com/NVIDIA-NeMo/labs-OO-Agents.git@{pin}\n" ) + subprocess.run(["git", "init", str(tmp_path)], check=True, capture_output=True) + subprocess.run( + [ + "git", + "-C", + str(tmp_path), + "add", + "README.md", + "LICENSE", + "pyproject.toml", + "nemo_gym/module.py", + "responses_api_agents/nooa_agent/runner.py", + "responses_api_agents/nooa_agent/install_nooa_runtime.sh", + "responses_api_agents/nooa_agent/requirements.txt", + ], + check=True, + ) return tmp_path @@ -51,6 +65,8 @@ def test_archive_uses_current_source_without_private_files(source_tree: Path) -> with tarfile.open(fileobj=io.BytesIO(blob)) as archive: assert set(archive.getnames()) == { "pyproject.toml", + "MANIFEST.in", + "responses_api_agents/nooa_agent/requirements.txt", "README.md", "LICENSE", "nemo_gym/module.py", @@ -74,6 +90,18 @@ def test_archive_uses_current_source_without_private_files(source_tree: Path) -> assert updated_fingerprint != fingerprint +def test_archive_includes_tracked_package_assets(source_tree: Path) -> None: + asset = source_tree / "nemo_gym/config.yaml" + asset.write_text("current: true\n") + subprocess.run(["git", "-C", str(source_tree), "add", "nemo_gym/config.yaml"], check=True) + blob, fingerprint = sandbox_runtime._runtime_archive(source_tree) + with tarfile.open(fileobj=io.BytesIO(blob)) as archive: + assert archive.extractfile("nemo_gym/config.yaml").read() == asset.read_bytes() + assert b"include nemo_gym/config.yaml" in archive.extractfile("MANIFEST.in").read() + asset.write_text("current: false\n") + assert sandbox_runtime._runtime_archive(source_tree)[1] != fingerprint + + def test_installer_changes_invalidate_runtime_fingerprint(source_tree: Path) -> None: before, fingerprint = sandbox_runtime._runtime_archive(source_tree) installer = source_tree / sandbox_runtime._INSTALLER_PATH @@ -95,6 +123,16 @@ def test_expected_python_checksum_invalidates_runtime_fingerprint( assert updated_fingerprint != fingerprint +def test_archive_preserves_launching_gym_metadata(source_tree: Path) -> None: + blob, before = sandbox_runtime._runtime_archive(source_tree) + with tarfile.open(fileobj=io.BytesIO(blob)) as archive: + assert archive.extractfile("pyproject.toml").read() == (source_tree / "pyproject.toml").read_bytes() + manifest = tomllib.loads(archive.extractfile("pyproject.toml").read().decode()) + assert manifest["project"]["dependencies"] == ["openai==2.44.0", "ray[default]>=2.58.0"] + (source_tree / "pyproject.toml").write_text('[project]\ndependencies = ["openai==2.45.0"]') + assert sandbox_runtime._runtime_archive(source_tree)[1] != before + + @pytest.fixture def portable_download(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> AsyncMock: from responses_api_agents.miniswe_sandboxed_agent import bootstrap @@ -296,7 +334,7 @@ async def test_changed_cached_archive_is_rejected_before_payload_uploads( ) -> None: portable_download.return_value.write_bytes(b"cache replaced after initial verification") sandbox.exec.side_effect = [SandboxExecResult("x86_64/gnu", "", 0), SandboxExecResult("", "", 1)] - with pytest.raises(RuntimeError, match="NOOA Python artifact checksum mismatch for x86_64/gnu"): + with pytest.raises(RuntimeError, match="NOOA Python artifact checksum mismatch before sandbox upload"): await sandbox_runtime.prepare_nooa_runtime(sandbox) sandbox.upload.assert_awaited_once() assert sandbox.exec.await_count == 2 @@ -400,10 +438,133 @@ async def test_cleanup_failure_does_not_discard_healthy_runtime( portable_download.assert_not_awaited() -def test_archive_preserves_full_project_metadata(source_tree: Path) -> None: - blob, _ = sandbox_runtime._runtime_archive(source_tree) - with tarfile.open(fileobj=io.BytesIO(blob)) as archive: - assert archive.extractfile("pyproject.toml").read() == (source_tree / "pyproject.toml").read_bytes() +def test_runtime_entrypoint_import_does_not_load_controller_dependencies() -> None: + import subprocess + import sys + + # A fresh interpreter catches imports hidden by a test process's module cache. + script = """ +import importlib.abc +import sys +class RejectControllerDependencies(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.split('.')[0] in {'ray', 'datasets', 'pyarrow', 'pandas', 'mlflow'}: + raise RuntimeError('task runtime imported controller dependency: ' + fullname) +sys.meta_path.insert(0, RejectControllerDependencies()) +from responses_api_agents.nooa_agent.sandbox_entrypoint import SandboxInput +from responses_api_agents.nooa_agent.invocation import NOOAInvocationConfig, validate_invocation +from responses_api_agents.nooa_agent.task_agent import TaskAgent +config = NOOAInvocationConfig(agent_class='responses_api_agents.nooa_agent.task_agent:TaskAgent', + invocation_adapter='responses_api_agents.nooa_agent.task_agent:invoke_solve_task') +assert validate_invocation(config)[0] is TaskAgent +""" + subprocess.run([sys.executable, "-c", script], cwd=Path(sandbox_runtime.__file__).resolve().parents[2], check=True) + + +@pytest.fixture +def wheelhouse(tmp_path: Path, runtime_package: tuple[bytes, str], portable_download: AsyncMock) -> Path: + path = tmp_path / "wheelhouse" + path.mkdir() + (path / "python.tar.gz").write_bytes(portable_download.return_value.read_bytes()) + with tarfile.open(path / "wheels.tar.gz", "w:gz") as archive: + data = b"fixture wheel bytes" + member = tarfile.TarInfo("fixture-1-py3-none-any.whl") + member.size = len(data) + archive.addfile(member, io.BytesIO(data)) + manifest = { + "version": 1, + "target": "x86_64/gnu", + "source_fingerprint": runtime_package[1], + "python_version": sandbox_runtime._PYTHON_VERSION, + "python_sha256": sandbox_runtime._PYTHON_SHA256["x86_64/gnu"], + "wheels_sha256": hashlib.sha256((path / "wheels.tar.gz").read_bytes()).hexdigest(), + "glibc": "2.28", + } + (path / "manifest.json").write_text(json.dumps(manifest)) + return path + + +@pytest.mark.asyncio +async def test_offline_runtime_stages_only_verified_local_artifacts( + sandbox: AsyncMock, wheelhouse: Path, runtime_package: tuple[bytes, str], portable_download: AsyncMock +) -> None: + sandbox.exec.side_effect = [ + SandboxExecResult("x86_64/gnu", "", 0), + SandboxExecResult("glibc 2.31", "", 0), + SandboxExecResult("", "", 1), + SandboxExecResult("ready", "", 0), + SandboxExecResult("", "", 0), + ] + python = await sandbox_runtime.prepare_nooa_runtime(sandbox, wheelhouse_path=wheelhouse) + portable_download.assert_not_awaited() + script = sandbox.upload.await_args_list[0].args[1] + scratch = script.removesuffix(".sh") + assert sandbox.uploaded[f"{scratch}/wheels.tar.gz"] == (wheelhouse / "wheels.tar.gz").read_bytes() + assert sandbox.uploaded[f"{scratch}/python.tar.gz"] == (wheelhouse / "python.tar.gz").read_bytes() + command = shlex.split(sandbox.exec.await_args_list[3].args[0]) + wheels_sha = json.loads((wheelhouse / "manifest.json").read_text())["wheels_sha256"] + expected_fingerprint = hashlib.sha256(f"{runtime_package[1]}:{wheels_sha}".encode()).hexdigest() + assert command[2] == "install" + assert command[4] == expected_fingerprint + assert command[-1] == wheels_sha + assert python == f"/opt/nemo-gym-nooa/{expected_fingerprint}-x86_64-gnu/python/bin/python3" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("libc", ["glibc 2.17", "unknown", "glibc 2.31 extra"]) +async def test_offline_runtime_rejects_incompatible_libc_without_online_fallback( + sandbox: AsyncMock, wheelhouse: Path, portable_download: AsyncMock, libc: str +) -> None: + sandbox.exec.side_effect = [SandboxExecResult("x86_64/gnu", "", 0), SandboxExecResult(libc, "", 0)] + with pytest.raises(RuntimeError, match="requires glibc >= 2.28"): + await sandbox_runtime.prepare_nooa_runtime(sandbox, wheelhouse_path=wheelhouse) + portable_download.assert_not_awaited() + sandbox.upload.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("changed", ["source", "target", "python", "wheels", "missing"]) +async def test_offline_runtime_rejects_incompatible_or_damaged_bundle_without_online_fallback( + sandbox: AsyncMock, wheelhouse: Path, portable_download: AsyncMock, changed: str +) -> None: + if changed in {"source", "target"}: + manifest = json.loads((wheelhouse / "manifest.json").read_text()) + manifest["source_fingerprint" if changed == "source" else "target"] = "changed" + (wheelhouse / "manifest.json").write_text(json.dumps(manifest)) + elif changed == "missing": + (wheelhouse / "wheels.tar.gz").unlink() + else: + (wheelhouse / f"{changed}.tar.gz").write_bytes(b"replaced") + sandbox.exec.return_value = SandboxExecResult("x86_64/gnu", "", 0) + with pytest.raises(ValueError, match="NOOA wheelhouse"): + await sandbox_runtime.prepare_nooa_runtime(sandbox, wheelhouse_path=wheelhouse) + portable_download.assert_not_awaited() + sandbox.upload.assert_awaited_once() + sandbox.exec.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_offline_runtime_revalidates_private_wheel_snapshot( + monkeypatch: pytest.MonkeyPatch, sandbox: AsyncMock, wheelhouse: Path, portable_download: AsyncMock +) -> None: + copyfile = sandbox_runtime.shutil.copyfile + + def replace_during_copy(source: Path, target: Path) -> Path: + result = copyfile(source, target) + if source.name == "wheels.tar.gz": + target.write_bytes(b"replaced while copying") + return result + + monkeypatch.setattr(sandbox_runtime.shutil, "copyfile", replace_during_copy) + sandbox.exec.side_effect = [ + SandboxExecResult("x86_64/gnu", "", 0), + SandboxExecResult("glibc 2.28", "", 0), + SandboxExecResult("", "", 1), + ] + with pytest.raises(RuntimeError, match="wheelhouse changed before sandbox upload"): + await sandbox_runtime.prepare_nooa_runtime(sandbox, wheelhouse_path=wheelhouse) + portable_download.assert_not_awaited() + sandbox.upload.assert_awaited_once() @pytest.mark.asyncio @@ -438,3 +599,105 @@ def slow_archive(root: Path) -> tuple[bytes, str]: finally: release.set() await preparation + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stage", ["validate", "snapshot"]) +async def test_bundle_work_keeps_loop_responsive_and_joins_cancelled_workers( + monkeypatch: pytest.MonkeyPatch, sandbox: AsyncMock, wheelhouse: Path, stage: str +) -> None: + import threading + + from responses_api_agents.nooa_agent import offline_runtime + + entered = threading.Event() + release = threading.Event() + completed = threading.Event() + snapshot_paths = [] + module = offline_runtime if stage == "validate" else sandbox_runtime + name = "load_runtime_wheelhouse" if stage == "validate" else "_stage_runtime_artifacts" + original = getattr(module, name) + + def blocked(*args, **kwargs): + if stage == "snapshot": + snapshot_paths.append(args[0]) + entered.set() + assert release.wait(5), "Event loop stalled during bundle work" + result = original(*args, **kwargs) + completed.set() + return result + + monkeypatch.setattr(module, name, blocked) + sandbox.exec.side_effect = [ + SandboxExecResult("x86_64/gnu", "", 0), + SandboxExecResult("glibc 2.28", "", 0), + SandboxExecResult("", "", 1), + ] + preparation = asyncio.create_task(sandbox_runtime.prepare_nooa_runtime(sandbox, wheelhouse_path=wheelhouse)) + try: + assert await asyncio.to_thread(entered.wait, 3) + preparation.cancel() + await asyncio.sleep(0.01) + assert not preparation.done() + assert not completed.is_set() + for path in snapshot_paths: + assert path.exists() + preparation.cancel() # Repeated cancellation still cannot race the worker's files. + await asyncio.sleep(0.01) + finally: + release.set() + with pytest.raises(asyncio.CancelledError): + await preparation + assert completed.is_set() + assert all(not path.exists() for path in snapshot_paths) + assert sandbox.upload.await_count == 1 + + +@pytest.mark.asyncio +async def test_runtime_io_bounds_concurrent_bundle_work() -> None: + import threading + + lock = threading.Lock() + release = threading.Event() + two_started = threading.Event() + active = 0 + peak = 0 + + def work() -> None: + nonlocal active, peak + with lock: + active += 1 + peak = max(peak, active) + if active == 2: + two_started.set() + assert release.wait(5) + with lock: + active -= 1 + + tasks = [asyncio.create_task(sandbox_runtime._runtime_io(work)) for _ in range(6)] + try: + assert await asyncio.to_thread(two_started.wait, 3) + await asyncio.sleep(0.01) + assert peak == active == 2 + finally: + release.set() + await asyncio.gather(*tasks) + assert peak == 2 + + +@pytest.mark.asyncio +async def test_glibc_217_bundle_accepts_ubuntu_1804_task_image( + sandbox: AsyncMock, wheelhouse: Path, portable_download: AsyncMock +) -> None: + manifest_path = wheelhouse / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["glibc"] = "2.17" + manifest_path.write_text(json.dumps(manifest)) + sandbox.exec.side_effect = [ + SandboxExecResult("x86_64/gnu", "", 0), + SandboxExecResult("glibc 2.27", "", 0), + SandboxExecResult("", "", 0), + SandboxExecResult("", "", 0), + ] + assert (await sandbox_runtime.prepare_nooa_runtime(sandbox, wheelhouse_path=wheelhouse)).endswith("/bin/python3") + portable_download.assert_not_awaited() diff --git a/responses_api_agents/nooa_agent/tests/test_sandbox_sessions.py b/responses_api_agents/nooa_agent/tests/test_sandbox_sessions.py index 04d3a8f46fa..d08317e5c77 100644 --- a/responses_api_agents/nooa_agent/tests/test_sandbox_sessions.py +++ b/responses_api_agents/nooa_agent/tests/test_sandbox_sessions.py @@ -58,6 +58,36 @@ async def test_missing_resources_sandbox_is_rejected(setup) -> None: sandbox.stop.assert_not_awaited() +async def test_sandbox_session_forwards_configured_context_window(setup) -> None: + instance, sandbox, _, _ = setup + instance.config.context_window = 262144 + instance.config.sandbox_runner_timeout_seconds = 60.5 + instance.config.session_close_timeout_seconds = 1.25 + await instance.seed_agent_session(request(), seed(sandbox_access=access())) + state = instance._require_agent_session("session") + assert state.runner.context_window == 262144 + assert state.runner.timeout == 60.5 + assert state.runner.close_timeout == 1.25 + await close(instance) + sandbox.disconnect.assert_awaited_once() + + +async def test_offline_runtime_failure_never_activates_or_destroys_borrowed_sandbox(setup) -> None: + instance, sandbox, staging, connect = setup + instance.config.runtime_wheelhouse = Path("/prepared/nooa") + staging.side_effect = ValueError("Offline runtime source fingerprint mismatch") + with pytest.raises(ValueError, match="source fingerprint mismatch"): + await instance.seed_agent_session(request(), seed(sandbox_access=access())) + staging.assert_awaited_once_with(sandbox, wheelhouse_path=Path("/prepared/nooa")) + connect.assert_awaited_once() + sandbox.exec.assert_not_awaited() + with pytest.raises(HTTPException, match="closing"): + instance._require_agent_session("session") + await close(instance) + sandbox.disconnect.assert_awaited_once() + sandbox.stop.assert_not_awaited() + + async def test_runtime_staging_failure_keeps_borrowed_connection_for_close(setup) -> None: instance, sandbox, staging, _ = setup staging.side_effect = RuntimeError("unsupported runtime") @@ -278,7 +308,7 @@ async def test_seed_wires_resolved_endpoint_runtime_and_limits_into_one_runner(s assert runner.invocation is instance.config.nooa instance.server_client._resolve_base_url.assert_not_called() instance.server_client._build_server_base_url.assert_called_once_with({"host": "model", "port": 8000}) - staging.assert_awaited_once_with(sandbox) + staging.assert_awaited_once_with(sandbox, wheelhouse_path=None) connect.assert_awaited_once() await instance.seed_agent_session(request(), body) assert instance._require_agent_session("session").runner is runner