feat(nooa): add a canonical GDPval recipe

benchmarks/gdpval/config.yaml

@@ -113,7 +113,7 @@ gdpval_claude48_judge_model:
 gdpval_resources_server:
   resources_servers:
     gdpval:
-      allowed_agents: [stirrup_agent]
+      allowed_agents: [stirrup_agent, pi_agent, codex_agent, nooa_agent]
       entrypoint: app.py
       domain: other
       verified: false

benchmarks/gdpval/nooa.yaml

@@ -0,0 +1,38 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+# Use GDPval's existing preparation, task tools, sandbox sessions, and grading.
+config_paths:
+  - benchmarks/gdpval/config.yaml
+  - responses_api_agents/nooa_agent/configs/nooa_agent.yaml
+
+# The GDPval environment below owns the session pairing.
+gdpval_stirrup_agent: null
+nooa_environment_server: null
+
+gdpval_resources_server:
+  resources_servers:
+    gdpval:
+      datasets:
+        - name: gdpval
+          type: benchmark
+          jsonl_fpath: benchmarks/gdpval/data/gdpval_benchmark.jsonl
+          prompt_config: benchmarks/gdpval/prompts/harness-neutral.yaml
+          prepare_script: benchmarks/gdpval/prepare.py
+          num_repeats: 1
+
+gdpval_nooa_agent:
+  _inherit_from: nooa_agent
+  responses_api_agents:
+    nooa_agent:
+      resources_server:
+        type: resources_servers
+        name: gdpval_resources_server
+      nooa:
+        execution_mode: sandboxed
+
+gdpval_environment_server:
+  environment_servers:
+    single_agent_turn_legacy:
+      agent_server:
+        type: responses_api_agents
+        name: gdpval_nooa_agent

benchmarks/gdpval/tests/test_nooa_gdpval.py

@@ -0,0 +1,158 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+from omegaconf import OmegaConf
+
+from benchmarks.gdpval import prepare as preparation
+from environment_servers.single_agent_turn_legacy.app import (
+    SingleAgentTurnLegacyEnvironmentServer,
+    SingleAgentTurnLegacyEnvironmentServerConfig,
+)
+from nemo_gym.benchmarks import BenchmarkConfig, _benchmark_config_name, _benchmark_config_paths
+from nemo_gym.client import ServerClient
+from nemo_gym.config import GlobalConfigDictParser
+
+
+ROOT = Path(__file__).resolve().parents[3]
+
+
+def test_nooa_recipe_preserves_gdpval_tools_grading_and_dispatch(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    pytest.importorskip("nooa")
+    pytest.importorskip("datasets")
+    from nemo_gym.rollout_collection import (
+        RolloutCollectionHelper,
+        _environment_server_for_agent,
+        _environment_servers_by_agent,
+    )
+    from nemo_gym.train_data_utils import TrainDataProcessor
+    from resources_servers.gdpval.app import GDPValResourcesServerConfig
+    from responses_api_agents.nooa_agent.config import NOOAAgentConfig
+    from responses_api_agents.nooa_agent.task_agent import _latest_user_text
+
+    monkeypatch.setenv("TAVILY_API_KEY", "unused")
+    monkeypatch.setenv("GDPVAL_SANDBOX_IMAGE", "fixture:1")
+    monkeypatch.setenv("PERSIST_DELIVERABLES_DIR", str(tmp_path / "deliverables"))
+    monkeypatch.setattr("nemo_gym.config.global_config._find_open_port_using_range", lambda **_: 12345)
+    recipe = ROOT / "benchmarks/gdpval/nooa.yaml"
+    assert recipe in _benchmark_config_paths(recipe.parent)
+    assert _benchmark_config_name(recipe.relative_to(ROOT / "benchmarks")) == "gdpval/nooa"
+    benchmark = BenchmarkConfig.from_config_path(recipe, strict=False)
+    assert benchmark is not None
+    assert benchmark.agent_name == "gdpval_nooa_agent"
+    assert benchmark.dataset.jsonl_fpath == Path("benchmarks/gdpval/data/gdpval_benchmark.jsonl")
+    assert benchmark.dataset.prepare_script == Path("benchmarks/gdpval/prepare.py")
+    assert benchmark.dataset.prompt_config == Path("benchmarks/gdpval/prompts/harness-neutral.yaml")
+
+    parser = GlobalConfigDictParser()
+    _, configs = parser.load_extra_config_paths([str(recipe)])
+    config = OmegaConf.merge(*configs)
+    parser._recursively_swap_keys(config)
+    _, baseline_configs = parser.load_extra_config_paths([str(recipe.parent / "config.yaml")])
+    baseline = OmegaConf.merge(*baseline_configs)
+    parser._recursively_swap_keys(baseline)
+    resource_block = config.gdpval_resources_server.resources_servers.gdpval
+    # Preserve every canonical resources/judge/provider setting except the dataset declaration.
+    for name, block in baseline.items():
+        if name not in {
+            "config_paths",
+            "gdpval_resources_server",
+            "gdpval_stirrup_agent",
+            "gdpval_environment_server",
+        }:
+            assert config[name] == block
+    expected_resources = OmegaConf.to_container(baseline.gdpval_resources_server.resources_servers.gdpval)
+    actual_resources = OmegaConf.to_container(resource_block)
+    assert len(actual_resources.pop("datasets")) == 1
+    assert actual_resources == expected_resources
+    assert resource_block.allowed_agents == ["stirrup_agent", "pi_agent", "codex_agent", "nooa_agent"]
+    assert (
+        config.rollout_collection_driver == "resources_servers.gdpval.multistage_orchestrator:run_rollout_collection"
+    )
+    environment_block = config.gdpval_environment_server.environment_servers.single_agent_turn_legacy
+    expected_environment = OmegaConf.to_container(
+        baseline.gdpval_environment_server.environment_servers.single_agent_turn_legacy
+    )
+    expected_environment["agent_server"]["name"] = benchmark.agent_name
+    assert OmegaConf.to_container(environment_block) == expected_environment
+    environment = SingleAgentTurnLegacyEnvironmentServerConfig(
+        name="gdpval_environment_server",
+        host="localhost",
+        port=8000,
+        **OmegaConf.to_container(environment_block, resolve=True),
+    )
+    agent = NOOAAgentConfig(
+        name=benchmark.agent_name,
+        host="localhost",
+        port=8001,
+        **OmegaConf.to_container(config.gdpval_nooa_agent.responses_api_agents.nooa_agent, resolve=True),
+    )
+    resources = GDPValResourcesServerConfig(
+        name="gdpval_resources_server",
+        host="localhost",
+        port=8002,
+        **OmegaConf.to_container(resource_block, resolve=True),
+    )
+    assert agent.nooa.execution_mode == "sandboxed"
+    assert agent.resources_server == environment.resources_server
+    assert agent.model_server.name == "policy_model"
+    assert agent.max_policy_calls is None
+    assert resources.entrypoint == "app.py"
+    assert environment.resources_tool_transports == ["direct_http"]
+
+    task = {
+        "task_id": "fixture",
+        "sector": "Finance",
+        "occupation": "Auditor",
+        "prompt": "Audit {this}.",
+        "reference_files": ["reference_files/ledger.xlsx"],
+        "reference_file_urls": ["https://example.invalid/private-download-url"],
+        "rubric_json": [{"criterion": "private grading criterion", "score": 1}],
+        "rubric_pretty": "private grading rubric",
+    }
+    monkeypatch.setattr("datasets.load_dataset", lambda *args, **kwargs: [task])
+    monkeypatch.setattr(preparation, "DATA_DIR", tmp_path / "data")
+    monkeypatch.setattr(preparation, "OUTPUT_FPATH", tmp_path / "data/gdpval.jsonl")
+    source = preparation.prepare()
+    original = source.read_bytes()
+    resource_block.datasets[0].jsonl_fpath = str(source)
+    instances = parser.filter_for_server_instance_configs(config)
+    assert [instance.name for instance in instances if instance.SERVER_TYPE == "environment_servers"] == [
+        environment.name
+    ]
+    config.update(
+        {"mode": "train_preparation", "output_dirpath": str(tmp_path / "collated"), "should_download": False}
+    )
+    TrainDataProcessor().run(config)
+    collated = tmp_path / "collated/benchmark.jsonl"
+    (row,) = RolloutCollectionHelper().preprocess_examples(
+        [json.loads(line) for line in collated.read_text().splitlines()], global_config_dict=config
+    )
+    assert row["task_source"] == resources.name
+    assert row["agent_ref"]["name"] == agent.name
+    assert _environment_server_for_agent(agent.name, _environment_servers_by_agent(config)) == environment.name
+    adapter = SingleAgentTurnLegacyEnvironmentServer(config=environment, server_client=MagicMock(spec=ServerClient))
+    request = adapter._episode_request_from_row(row)
+    task_input = request.task.task_input
+    prompt = _latest_user_text(task_input.responses_create_params)
+    assert "Sector: Finance\nOccupation: Auditor\n\nAudit {this}." in prompt
+    assert "- reference_files/ledger.xlsx" in prompt
+    assert "code_exec" not in prompt
+    for key in ("rubric_json", "rubric_pretty", "reference_file_urls"):
+        assert task_input.task_data[key] == task[key]
+        assert str(task[key]) not in prompt
+    policy_input = task_input.responses_create_params.model_dump_json()
+    assert "private grading" not in policy_input
+    assert "private-download-url" not in policy_input
+    tools = task_input.responses_create_params.model_dump(exclude_unset=True)["tools"]
+    assert tools == json.loads((recipe.parent / "tools.json").read_text())
+    assert {tool["name"] for tool in tools} == {"finish", "abandon_task_finish", "web_search", "fetch_web_page"}
+    finish = next(tool for tool in tools if tool["name"] == "finish")
+    assert set(finish["parameters"]["properties"]) == {"reason", "paths"}
+    assert source.read_bytes() == original

fern/versions/latest/pages/evaluation-tutorials/nooa-gdpval.mdx

@@ -0,0 +1,43 @@
+---
+title: "GDPval with NOOA"
+description: "Run NOOA with GDPval's existing sandbox sessions, task tools, and grading."
+---
+
+# GDPval with NOOA
+
+The [NOOA recipe](https://github.com/NVIDIA-NeMo/Gym/blob/main/benchmarks/gdpval/nooa.yaml)
+uses GDPval's existing environment server, collection driver, resources server, and grading configuration.
+It selects the harness-neutral prompt and runs NOOA inside the resources-owned sandbox.
+
+Use GDPval's existing prerequisites: an image built from
+[the GDPval Dockerfile](https://github.com/NVIDIA-NeMo/Gym/blob/main/resources_servers/gdpval/containers/Dockerfile),
+`GDPVAL_SANDBOX_IMAGE`, `OPENSANDBOX_DOMAIN`, `OPENSANDBOX_API_KEY`, `TAVILY_API_KEY`, and credentials for the
+configured judge endpoints. Set `HF_TOKEN` when needed for reference-file downloads. Set
+`PERSIST_DELIVERABLES_DIR` to a writable directory for collected submissions. See the
+[canonical GDPval instructions](https://github.com/NVIDIA-NeMo/Gym/blob/main/benchmarks/gdpval/README.md)
+for judge-panel configuration and re-judging.
+
+Put your policy endpoint and deployment overrides in a private `private-policy-provider.yaml` overlay.
+Ensure the sandbox can reach the Gym model and resources servers; use `++use_absolute_ip=true` when the
+default addresses are unreachable. From the repository root:
+
+```bash
+gym eval prepare --benchmark gdpval/nooa
+
+gym env resolve \
+    --config benchmarks/gdpval/nooa.yaml \
+    --config responses_api_models/vllm_model/configs/vllm_model.yaml \
+    --config private-policy-provider.yaml
+
+gym eval run \
+    --config benchmarks/gdpval/nooa.yaml \
+    --config responses_api_models/vllm_model/configs/vllm_model.yaml \
+    --config private-policy-provider.yaml \
+    --split benchmark --limit 1 --concurrency 1 --num-repeats 1 \
+    --output results/gdpval-nooa.jsonl
+```
+
+The existing direct task tools are available to NOOA: `finish(reason, paths)`, `abandon_task_finish`,
+`web_search`, and `fetch_web_page`. `finish` validates and records a submission but does not terminate
+NOOA; the agent must still return its native `TaskResult`. GDPval's existing missing-finish fallback and
+grading behavior remain in effect.

nemo_gym/data/dataset_preparation.py

@@ -857,15 +857,20 @@ def _validate_samples_and_aggregate_metrics_single_dataset(
                 state,
                 sample_idx,
                 sample_dict_str,
-                require_responses=dataset_config.taskset is None,
+                # Benchmark prompt templates create the input during collation;
+                # count the original task rows before validating rendered inputs.
+                require_responses=dataset_config.taskset is None
+                and not (dataset_config.type == "benchmark" and dataset_config.prompt_config),
                 dataset_metrics_hook=dataset_metrics_hook,
             )
 
         postprocess_other_metrics(state.metrics, state.other_metrics)
 
         return state
 
-    def _validate_aggregate_metrics(self, aggregate_metrics_dict: Dict, metrics_fpath: Path) -> Optional[Path]:
+    def _validate_aggregate_metrics(
+        self, aggregate_metrics_dict: Dict, metrics_fpath: Path, *, source_metrics: bool = False
+    ) -> Optional[Path]:
         """
         Returns the conflicting metrics fpath if invalid. Else returns None
         """
@@ -943,7 +948,14 @@ def diff_values(prev_v, new_v, path: str, diffs: List[str]) -> None:
                 diffs.append(f"Value differs at {path}: {prev_v} != {new_v}")
 
         diffs: List[str] = []
-        diff_values(previous_aggregate_metrics_dict, aggregate_metrics_dict, path="", diffs=diffs)
+        previous_comparison = previous_aggregate_metrics_dict
+        current_comparison = aggregate_metrics_dict
+        if source_metrics:
+            # A rendering template does not describe the original dataset. Keep it
+            # significant for collated-output metrics, including existing sidecars.
+            previous_comparison = {k: v for k, v in previous_comparison.items() if k != "prompt_config"}
+            current_comparison = {k: v for k, v in current_comparison.items() if k != "prompt_config"}
+        diff_values(previous_comparison, current_comparison, path="", diffs=diffs)
 
         if diffs:
             print("Differences found in aggregate metrics:")
@@ -980,7 +992,7 @@ def validate_samples_and_aggregate_metrics(
                 metrics_fpath = data_fpath.with_name(f"{data_fpath.stem}_metrics.json")
 
                 maybe_conflicting_metrics_fpath = self._validate_aggregate_metrics(
-                    aggregate_metrics_dict, metrics_fpath
+                    aggregate_metrics_dict, metrics_fpath, source_metrics=True
                 )
                 if maybe_conflicting_metrics_fpath is not None:
                     conflicting_fpaths.append(str(maybe_conflicting_metrics_fpath))

resources_servers/gdpval/configs/gdpval.yaml

@@ -1,7 +1,7 @@
 gdpval_resources_server:
   resources_servers:
     gdpval:
-      allowed_agents: [stirrup_agent]
+      allowed_agents: [stirrup_agent, pi_agent, codex_agent, nooa_agent]
       entrypoint: app.py
       domain: other
       verified: false