From af4a32ac815b71eee5007e43d84baf28a699a7f4 Mon Sep 17 00:00:00 2001 From: Nikhil Deshmudre Date: Mon, 2 Nov 2026 12:21:10 -0800 Subject: [PATCH] feat(nooa): add a canonical GDPval recipe Signed-off-by: Nikhil Deshmudre --- benchmarks/gdpval/config.yaml | 2 +- benchmarks/gdpval/nooa.yaml | 38 +++++ benchmarks/gdpval/tests/test_nooa_gdpval.py | 158 ++++++++++++++++++ .../evaluation-tutorials/nooa-gdpval.mdx | 43 +++++ nemo_gym/data/dataset_preparation.py | 20 ++- resources_servers/gdpval/configs/gdpval.yaml | 2 +- .../unit_tests/test_gdpval_prompt_metrics.py | 95 +++++++++++ 7 files changed, 352 insertions(+), 6 deletions(-) create mode 100644 benchmarks/gdpval/nooa.yaml create mode 100644 benchmarks/gdpval/tests/test_nooa_gdpval.py create mode 100644 fern/versions/latest/pages/evaluation-tutorials/nooa-gdpval.mdx create mode 100644 tests/unit_tests/test_gdpval_prompt_metrics.py diff --git a/benchmarks/gdpval/config.yaml b/benchmarks/gdpval/config.yaml index 60f64c0c308..561493c81b4 100644 --- a/benchmarks/gdpval/config.yaml +++ b/benchmarks/gdpval/config.yaml @@ -113,7 +113,7 @@ gdpval_claude48_judge_model: gdpval_resources_server: resources_servers: gdpval: - allowed_agents: [stirrup_agent] + allowed_agents: [stirrup_agent, pi_agent, codex_agent, nooa_agent] entrypoint: app.py domain: other verified: false diff --git a/benchmarks/gdpval/nooa.yaml b/benchmarks/gdpval/nooa.yaml new file mode 100644 index 00000000000..b6879293675 --- /dev/null +++ b/benchmarks/gdpval/nooa.yaml @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Use GDPval's existing preparation, task tools, sandbox sessions, and grading. +config_paths: + - benchmarks/gdpval/config.yaml + - responses_api_agents/nooa_agent/configs/nooa_agent.yaml + +# The GDPval environment below owns the session pairing. +gdpval_stirrup_agent: null +nooa_environment_server: null + +gdpval_resources_server: + resources_servers: + gdpval: + datasets: + - name: gdpval + type: benchmark + jsonl_fpath: benchmarks/gdpval/data/gdpval_benchmark.jsonl + prompt_config: benchmarks/gdpval/prompts/harness-neutral.yaml + prepare_script: benchmarks/gdpval/prepare.py + num_repeats: 1 + +gdpval_nooa_agent: + _inherit_from: nooa_agent + responses_api_agents: + nooa_agent: + resources_server: + type: resources_servers + name: gdpval_resources_server + nooa: + execution_mode: sandboxed + +gdpval_environment_server: + environment_servers: + single_agent_turn_legacy: + agent_server: + type: responses_api_agents + name: gdpval_nooa_agent diff --git a/benchmarks/gdpval/tests/test_nooa_gdpval.py b/benchmarks/gdpval/tests/test_nooa_gdpval.py new file mode 100644 index 00000000000..d0e54176f5d --- /dev/null +++ b/benchmarks/gdpval/tests/test_nooa_gdpval.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from omegaconf import OmegaConf + +from benchmarks.gdpval import prepare as preparation +from environment_servers.single_agent_turn_legacy.app import ( + SingleAgentTurnLegacyEnvironmentServer, + SingleAgentTurnLegacyEnvironmentServerConfig, +) +from nemo_gym.benchmarks import BenchmarkConfig, _benchmark_config_name, _benchmark_config_paths +from nemo_gym.client import ServerClient +from nemo_gym.config import GlobalConfigDictParser + + +ROOT = Path(__file__).resolve().parents[3] + + +def test_nooa_recipe_preserves_gdpval_tools_grading_and_dispatch( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + pytest.importorskip("nooa") + pytest.importorskip("datasets") + from nemo_gym.rollout_collection import ( + RolloutCollectionHelper, + _environment_server_for_agent, + _environment_servers_by_agent, + ) + from nemo_gym.train_data_utils import TrainDataProcessor + from resources_servers.gdpval.app import GDPValResourcesServerConfig + from responses_api_agents.nooa_agent.config import NOOAAgentConfig + from responses_api_agents.nooa_agent.task_agent import _latest_user_text + + monkeypatch.setenv("TAVILY_API_KEY", "unused") + monkeypatch.setenv("GDPVAL_SANDBOX_IMAGE", "fixture:1") + monkeypatch.setenv("PERSIST_DELIVERABLES_DIR", str(tmp_path / "deliverables")) + monkeypatch.setattr("nemo_gym.config.global_config._find_open_port_using_range", lambda **_: 12345) + recipe = ROOT / "benchmarks/gdpval/nooa.yaml" + assert recipe in _benchmark_config_paths(recipe.parent) + assert _benchmark_config_name(recipe.relative_to(ROOT / "benchmarks")) == "gdpval/nooa" + benchmark = BenchmarkConfig.from_config_path(recipe, strict=False) + assert benchmark is not None + assert benchmark.agent_name == "gdpval_nooa_agent" + assert benchmark.dataset.jsonl_fpath == Path("benchmarks/gdpval/data/gdpval_benchmark.jsonl") + assert benchmark.dataset.prepare_script == Path("benchmarks/gdpval/prepare.py") + assert benchmark.dataset.prompt_config == Path("benchmarks/gdpval/prompts/harness-neutral.yaml") + + parser = GlobalConfigDictParser() + _, configs = parser.load_extra_config_paths([str(recipe)]) + config = OmegaConf.merge(*configs) + parser._recursively_swap_keys(config) + _, baseline_configs = parser.load_extra_config_paths([str(recipe.parent / "config.yaml")]) + baseline = OmegaConf.merge(*baseline_configs) + parser._recursively_swap_keys(baseline) + resource_block = config.gdpval_resources_server.resources_servers.gdpval + # Preserve every canonical resources/judge/provider setting except the dataset declaration. + for name, block in baseline.items(): + if name not in { + "config_paths", + "gdpval_resources_server", + "gdpval_stirrup_agent", + "gdpval_environment_server", + }: + assert config[name] == block + expected_resources = OmegaConf.to_container(baseline.gdpval_resources_server.resources_servers.gdpval) + actual_resources = OmegaConf.to_container(resource_block) + assert len(actual_resources.pop("datasets")) == 1 + assert actual_resources == expected_resources + assert resource_block.allowed_agents == ["stirrup_agent", "pi_agent", "codex_agent", "nooa_agent"] + assert ( + config.rollout_collection_driver == "resources_servers.gdpval.multistage_orchestrator:run_rollout_collection" + ) + environment_block = config.gdpval_environment_server.environment_servers.single_agent_turn_legacy + expected_environment = OmegaConf.to_container( + baseline.gdpval_environment_server.environment_servers.single_agent_turn_legacy + ) + expected_environment["agent_server"]["name"] = benchmark.agent_name + assert OmegaConf.to_container(environment_block) == expected_environment + environment = SingleAgentTurnLegacyEnvironmentServerConfig( + name="gdpval_environment_server", + host="localhost", + port=8000, + **OmegaConf.to_container(environment_block, resolve=True), + ) + agent = NOOAAgentConfig( + name=benchmark.agent_name, + host="localhost", + port=8001, + **OmegaConf.to_container(config.gdpval_nooa_agent.responses_api_agents.nooa_agent, resolve=True), + ) + resources = GDPValResourcesServerConfig( + name="gdpval_resources_server", + host="localhost", + port=8002, + **OmegaConf.to_container(resource_block, resolve=True), + ) + assert agent.nooa.execution_mode == "sandboxed" + assert agent.resources_server == environment.resources_server + assert agent.model_server.name == "policy_model" + assert agent.max_policy_calls is None + assert resources.entrypoint == "app.py" + assert environment.resources_tool_transports == ["direct_http"] + + task = { + "task_id": "fixture", + "sector": "Finance", + "occupation": "Auditor", + "prompt": "Audit {this}.", + "reference_files": ["reference_files/ledger.xlsx"], + "reference_file_urls": ["https://example.invalid/private-download-url"], + "rubric_json": [{"criterion": "private grading criterion", "score": 1}], + "rubric_pretty": "private grading rubric", + } + monkeypatch.setattr("datasets.load_dataset", lambda *args, **kwargs: [task]) + monkeypatch.setattr(preparation, "DATA_DIR", tmp_path / "data") + monkeypatch.setattr(preparation, "OUTPUT_FPATH", tmp_path / "data/gdpval.jsonl") + source = preparation.prepare() + original = source.read_bytes() + resource_block.datasets[0].jsonl_fpath = str(source) + instances = parser.filter_for_server_instance_configs(config) + assert [instance.name for instance in instances if instance.SERVER_TYPE == "environment_servers"] == [ + environment.name + ] + config.update( + {"mode": "train_preparation", "output_dirpath": str(tmp_path / "collated"), "should_download": False} + ) + TrainDataProcessor().run(config) + collated = tmp_path / "collated/benchmark.jsonl" + (row,) = RolloutCollectionHelper().preprocess_examples( + [json.loads(line) for line in collated.read_text().splitlines()], global_config_dict=config + ) + assert row["task_source"] == resources.name + assert row["agent_ref"]["name"] == agent.name + assert _environment_server_for_agent(agent.name, _environment_servers_by_agent(config)) == environment.name + adapter = SingleAgentTurnLegacyEnvironmentServer(config=environment, server_client=MagicMock(spec=ServerClient)) + request = adapter._episode_request_from_row(row) + task_input = request.task.task_input + prompt = _latest_user_text(task_input.responses_create_params) + assert "Sector: Finance\nOccupation: Auditor\n\nAudit {this}." in prompt + assert "- reference_files/ledger.xlsx" in prompt + assert "code_exec" not in prompt + for key in ("rubric_json", "rubric_pretty", "reference_file_urls"): + assert task_input.task_data[key] == task[key] + assert str(task[key]) not in prompt + policy_input = task_input.responses_create_params.model_dump_json() + assert "private grading" not in policy_input + assert "private-download-url" not in policy_input + tools = task_input.responses_create_params.model_dump(exclude_unset=True)["tools"] + assert tools == json.loads((recipe.parent / "tools.json").read_text()) + assert {tool["name"] for tool in tools} == {"finish", "abandon_task_finish", "web_search", "fetch_web_page"} + finish = next(tool for tool in tools if tool["name"] == "finish") + assert set(finish["parameters"]["properties"]) == {"reason", "paths"} + assert source.read_bytes() == original diff --git a/fern/versions/latest/pages/evaluation-tutorials/nooa-gdpval.mdx b/fern/versions/latest/pages/evaluation-tutorials/nooa-gdpval.mdx new file mode 100644 index 00000000000..92acf59ea85 --- /dev/null +++ b/fern/versions/latest/pages/evaluation-tutorials/nooa-gdpval.mdx @@ -0,0 +1,43 @@ +--- +title: "GDPval with NOOA" +description: "Run NOOA with GDPval's existing sandbox sessions, task tools, and grading." +--- + +# GDPval with NOOA + +The [NOOA recipe](https://github.com/NVIDIA-NeMo/Gym/blob/main/benchmarks/gdpval/nooa.yaml) +uses GDPval's existing environment server, collection driver, resources server, and grading configuration. +It selects the harness-neutral prompt and runs NOOA inside the resources-owned sandbox. + +Use GDPval's existing prerequisites: an image built from +[the GDPval Dockerfile](https://github.com/NVIDIA-NeMo/Gym/blob/main/resources_servers/gdpval/containers/Dockerfile), +`GDPVAL_SANDBOX_IMAGE`, `OPENSANDBOX_DOMAIN`, `OPENSANDBOX_API_KEY`, `TAVILY_API_KEY`, and credentials for the +configured judge endpoints. Set `HF_TOKEN` when needed for reference-file downloads. Set +`PERSIST_DELIVERABLES_DIR` to a writable directory for collected submissions. See the +[canonical GDPval instructions](https://github.com/NVIDIA-NeMo/Gym/blob/main/benchmarks/gdpval/README.md) +for judge-panel configuration and re-judging. + +Put your policy endpoint and deployment overrides in a private `private-policy-provider.yaml` overlay. +Ensure the sandbox can reach the Gym model and resources servers; use `++use_absolute_ip=true` when the +default addresses are unreachable. From the repository root: + +```bash +gym eval prepare --benchmark gdpval/nooa + +gym env resolve \ + --config benchmarks/gdpval/nooa.yaml \ + --config responses_api_models/vllm_model/configs/vllm_model.yaml \ + --config private-policy-provider.yaml + +gym eval run \ + --config benchmarks/gdpval/nooa.yaml \ + --config responses_api_models/vllm_model/configs/vllm_model.yaml \ + --config private-policy-provider.yaml \ + --split benchmark --limit 1 --concurrency 1 --num-repeats 1 \ + --output results/gdpval-nooa.jsonl +``` + +The existing direct task tools are available to NOOA: `finish(reason, paths)`, `abandon_task_finish`, +`web_search`, and `fetch_web_page`. `finish` validates and records a submission but does not terminate +NOOA; the agent must still return its native `TaskResult`. GDPval's existing missing-finish fallback and +grading behavior remain in effect. diff --git a/nemo_gym/data/dataset_preparation.py b/nemo_gym/data/dataset_preparation.py index 5ade5204d76..3ce064efc7b 100644 --- a/nemo_gym/data/dataset_preparation.py +++ b/nemo_gym/data/dataset_preparation.py @@ -857,7 +857,10 @@ def _validate_samples_and_aggregate_metrics_single_dataset( state, sample_idx, sample_dict_str, - require_responses=dataset_config.taskset is None, + # Benchmark prompt templates create the input during collation; + # count the original task rows before validating rendered inputs. + require_responses=dataset_config.taskset is None + and not (dataset_config.type == "benchmark" and dataset_config.prompt_config), dataset_metrics_hook=dataset_metrics_hook, ) @@ -865,7 +868,9 @@ def _validate_samples_and_aggregate_metrics_single_dataset( return state - def _validate_aggregate_metrics(self, aggregate_metrics_dict: Dict, metrics_fpath: Path) -> Optional[Path]: + def _validate_aggregate_metrics( + self, aggregate_metrics_dict: Dict, metrics_fpath: Path, *, source_metrics: bool = False + ) -> Optional[Path]: """ Returns the conflicting metrics fpath if invalid. Else returns None """ @@ -943,7 +948,14 @@ def diff_values(prev_v, new_v, path: str, diffs: List[str]) -> None: diffs.append(f"Value differs at {path}: {prev_v} != {new_v}") diffs: List[str] = [] - diff_values(previous_aggregate_metrics_dict, aggregate_metrics_dict, path="", diffs=diffs) + previous_comparison = previous_aggregate_metrics_dict + current_comparison = aggregate_metrics_dict + if source_metrics: + # A rendering template does not describe the original dataset. Keep it + # significant for collated-output metrics, including existing sidecars. + previous_comparison = {k: v for k, v in previous_comparison.items() if k != "prompt_config"} + current_comparison = {k: v for k, v in current_comparison.items() if k != "prompt_config"} + diff_values(previous_comparison, current_comparison, path="", diffs=diffs) if diffs: print("Differences found in aggregate metrics:") @@ -980,7 +992,7 @@ def validate_samples_and_aggregate_metrics( metrics_fpath = data_fpath.with_name(f"{data_fpath.stem}_metrics.json") maybe_conflicting_metrics_fpath = self._validate_aggregate_metrics( - aggregate_metrics_dict, metrics_fpath + aggregate_metrics_dict, metrics_fpath, source_metrics=True ) if maybe_conflicting_metrics_fpath is not None: conflicting_fpaths.append(str(maybe_conflicting_metrics_fpath)) diff --git a/resources_servers/gdpval/configs/gdpval.yaml b/resources_servers/gdpval/configs/gdpval.yaml index 9ae3b3ef531..fba33fe7444 100644 --- a/resources_servers/gdpval/configs/gdpval.yaml +++ b/resources_servers/gdpval/configs/gdpval.yaml @@ -1,7 +1,7 @@ gdpval_resources_server: resources_servers: gdpval: - allowed_agents: [stirrup_agent] + allowed_agents: [stirrup_agent, pi_agent, codex_agent, nooa_agent] entrypoint: app.py domain: other verified: false diff --git a/tests/unit_tests/test_gdpval_prompt_metrics.py b/tests/unit_tests/test_gdpval_prompt_metrics.py new file mode 100644 index 00000000000..a778d9f1272 --- /dev/null +++ b/tests/unit_tests/test_gdpval_prompt_metrics.py @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GDPval harnesses share source metrics, while rendered outputs stay distinct.""" + +import json +from pathlib import Path + +import pytest +from omegaconf import DictConfig, OmegaConf + +from benchmarks.gdpval import prepare +from nemo_gym.config.global_config import GlobalConfigDictParser +from nemo_gym.train_data_utils import TrainDataProcessor + + +def _config(harness: str, source: Path, output: Path) -> DictConfig: + parser = GlobalConfigDictParser() + filename = "config.yaml" if harness == "stirrup" else "nooa.yaml" + _, parts = parser.load_extra_config_paths([f"benchmarks/gdpval/{filename}"]) + config = OmegaConf.merge(*parts) + parser._recursively_swap_keys(config) + if harness == "stirrup": + dataset = config.gdpval_stirrup_agent.responses_api_agents.stirrup_agent.datasets[0] + else: + dataset = config.gdpval_resources_server.resources_servers.gdpval.datasets[0] + dataset.jsonl_fpath = str(source) + config.update({"mode": "train_preparation", "output_dirpath": str(output), "should_download": False}) + return config + + +@pytest.mark.parametrize("order", [("stirrup", "nooa"), ("nooa", "stirrup")]) +def test_harness_reuse_preserves_source_and_output_guards( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, order: tuple[str, str] +) -> None: + row = { + "task_id": "fixture", + "sector": "Finance", + "occupation": "Auditor", + "prompt": "Audit {this}.", + "reference_files": ["reference_files/ledger.xlsx"], + "reference_file_urls": ["https://example.invalid/private-download-url"], + "rubric_json": [{"criterion": "private grading criterion", "score": 1}], + "rubric_pretty": "private grading rubric", + } + monkeypatch.setenv("TAVILY_API_KEY", "unused") + monkeypatch.setenv("GDPVAL_SANDBOX_IMAGE", "fixture:1") + monkeypatch.setattr("datasets.load_dataset", lambda *args, **kwargs: [row]) + monkeypatch.setattr("nemo_gym.config.global_config._find_open_port_using_range", lambda *args, **kwargs: 12345) + monkeypatch.setattr(prepare, "DATA_DIR", tmp_path / "data") + monkeypatch.setattr(prepare, "OUTPUT_FPATH", tmp_path / "data" / "gdpval_benchmark.jsonl") + source = prepare.prepare() + original = source.read_bytes() + source_row = json.loads(original) + processor = TrainDataProcessor() + sidecar = source.with_name("gdpval_benchmark_metrics.json") + sidecar_bytes = None + + for harness in order: + output = tmp_path / harness + processor.run(_config(harness, source, output)) + rendered = json.loads((output / "benchmark.jsonl").read_text()) + prompt = json.dumps(rendered["responses_create_params"]["input"]) + assert ("code_exec" in prompt) == (harness == "stirrup") + assert rendered["responses_create_params"]["tools"] == source_row["responses_create_params"]["tools"] + assert row["reference_file_urls"][0] not in prompt + assert row["rubric_pretty"] not in prompt + assert row["rubric_json"][0]["criterion"] not in prompt + assert source.read_bytes() == original + if sidecar_bytes is None: + sidecar_bytes = sidecar.read_bytes() + assert sidecar.read_bytes() == sidecar_bytes + assert json.loads(sidecar_bytes)["Number of examples"] == 1 + metrics = json.loads((output / "benchmark_metrics.json").read_text()) + assert metrics["Number of examples"] == 1 + expected_template = "default-stirrup.yaml" if harness == "stirrup" else "harness-neutral.yaml" + assert metrics["prompt_config"].endswith(expected_template) + + # The same source is reusable, but an existing rendered output must not be + # silently relabeled with the other harness's template. + with pytest.raises(ValueError, match="conflicting aggregate metrics"): + processor.run(_config(order[1], source, tmp_path / order[0])) + assert (tmp_path / order[0] / "benchmark_metrics_conflict.json").exists() + + # Changing actual source statistics remains a conflict even with a new output. + source.write_bytes(original + original) + with pytest.raises(ValueError, match="conflicting aggregate metrics"): + processor.run(_config(order[1], source, tmp_path / "changed-source")) + assert source.with_name("gdpval_benchmark_metrics_conflict.json").exists() + + # Source counting must not bypass the existing rendering compatibility guard. + incompatible_source = tmp_path / "already-rendered.jsonl" + source_row["responses_create_params"]["input"] = "An existing prompt" + incompatible_source.write_text(json.dumps(source_row) + "\n") + with pytest.raises(ValueError, match="mutually exclusive"): + processor.run(_config(order[1], incompatible_source, tmp_path / "invalid-rendering"))