diff --git a/benchmarks/gdpval/README.md b/benchmarks/gdpval/README.md index 7b7be3e4c0a..3372ec3d7c2 100644 --- a/benchmarks/gdpval/README.md +++ b/benchmarks/gdpval/README.md @@ -451,6 +451,47 @@ gym eval run \ `--resume` reruns; Pi does not set it yet. So a timeout counts against Pi's score, but not against Stirrup's. - The prompt and the harness differ from the certified Stirrup setup, so these are not AA v2 scores. +`benchmarks/gdpval/codex.yaml` runs [Codex](../../responses_api_agents/codex_agent/README.md) the same way. Codex sends +no sampling or output limit to a Gym model server, so set them on the model server, for example on `openai_model` in +front of vLLM: + +```bash +export PERSIST_DELIVERABLES_DIR=$PWD/output/gdpval_codex/deliverables +gym eval prepare --benchmark gdpval/codex +gym eval run \ + --model-type openai_model \ + --benchmark gdpval/codex \ + --split benchmark \ + --output results/gdpval_codex.jsonl \ + --model-url \ + --model-api-key \ + --model \ + '++policy_model.responses_api_models.openai_model.extra_body={temperature: 1.0, top_p: 0.95, max_output_tokens: 32768}' \ + '++policy_model.responses_api_models.openai_model.propagate_upstream_http_status_codes=[400]' +``` + +- The rows, the task tools and the deliverables directory work as for Pi, and these are not AA v2 scores either. + Codex installs Node.js and Codex in the task sandbox at the start of each episode, from nodejs.org and the npm + registry (about 190 MB). The model sees the tools as `mcp__gdpval_resources_server__`, and rollouts record + the calls by the tool name alone, such as `finish`. Codex keeps about 12 KB of each tool result, so the model sees + only the start and end of a long `fetch_web_page` result. +- `extra_body` pins the Stirrup agent's temperature and top_p and limits each model call to 32,768 output tokens. +- Codex checks `model_auto_compact_token_limit` after each reply that calls a tool, so the context can pass it by + about a reply and its tool output before Codex compacts, and the compaction call reserves 32,768 tokens for its own + reply. So keep the limit at about the window minus twice the output limit or lower: `codex.yaml` uses 196,608 for + a 262,144-token window. A near-full reply with a large tool result can still overflow the compaction call, and the + run then fails. +- Codex does not compact on a context-length error; the run fails. `propagate_upstream_http_status_codes: [400]` + makes it fail within seconds with vLLM's message. By default `openai_model` returns HTTP 500, and Codex sends the + request 30 times before failing with a "high demand" error. +- `finish` does not end a Codex run: Codex continues until the model replies without a tool call, and a later + `finish` or `abandon_task_finish` replaces the recorded call. The files are collected when the run ends, so a run + that fails after `finish`, for example on a context-length error, is not judged. +- The Codex agent rejects sampling and output limits in the request, so every row fails if `gym eval run` gets + `--temperature`, `--top-p` or `--max-output-tokens`. Change Codex's own settings, such as `model_context_window` + and `model_auto_compact_token_limit` for another window, with + `++gdpval_codex_agent.responses_api_agents.codex_agent.=`. + ## Multi-judge panel By default every GDPVal deliverable is graded by a **panel** of frontier LLM diff --git a/benchmarks/gdpval/codex.yaml b/benchmarks/gdpval/codex.yaml new file mode 100644 index 00000000000..62015ec64e4 --- /dev/null +++ b/benchmarks/gdpval/codex.yaml @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# GDPVal with Codex in place of Stirrup: same resources server, judges and multi-stage driver as config.yaml. +# Codex runs in the task sandbox the resources server lends it and reaches the task tools +# (finish, abandon_task_finish, web_search, fetch_web_page) over MCP. The rows use the harness-neutral prompt. +# See "Run another harness" in README.md for the run command and the model server settings Codex needs. +# Codex has no finish_tool_names: finish does not end its run. Codex continues until the model replies without a +# tool call, a later finish or abandon_task_finish replaces the recorded call, and the resources server collects +# the files when the run ends. +config_paths: + - benchmarks/gdpval/config.yaml + +# Drop the Stirrup agent; the environment server below runs Codex instead. +gdpval_stirrup_agent: null + +gdpval_resources_server: + resources_servers: + gdpval: + allowed_agents: [codex_agent] + expose_tools_over_mcp: true + +gdpval_codex_agent: + responses_api_agents: + codex_agent: + entrypoint: app.py + description: GDPVal professional deliverable tasks with Codex in place of Stirrup + model_server: + type: responses_api_models + name: policy_model + model: ${policy_model_name} + codex_version: 0.144.4 + # Concurrent Codex runs, as Stirrup's concurrency. Keep it at least the environment server's + # max_concurrent_episodes: an episode beyond it waits with its sandbox started and its deadline running. + concurrency: 32 + model_context_window: 262144 + # Codex checks this after each reply that calls a tool, so the context can pass it by about a reply and its + # tool output. The compaction call re-sends the context without the tool definitions, and vLLM reserves + # 32,768 tokens for its reply: 196,608 + 32,768 + 32,768 = 262,144 fits a full reply with a short tool + # result. At 229,376 the compaction call would usually overflow the window. + model_auto_compact_token_limit: 196608 + # The Codex process, inside the environment server's default_episode_timeout_seconds. + timeout: 10800 + session_close_timeout_seconds: 90 + # A Codex MCP call that times out keeps running on the server and holds back the next call to it, so keep + # this well above a slow web call: httpx gives each step of a search or fetch up to 180 s. + mcp_tool_timeout_seconds: 900 + datasets: + - name: gdpval + type: benchmark + jsonl_fpath: benchmarks/gdpval/data/gdpval_benchmark_harness_neutral.jsonl + prompt_config: benchmarks/gdpval/prompts/harness-neutral.yaml + prepare_script: benchmarks/gdpval/prepare_harness_neutral.py + num_repeats: 1 + +gdpval_environment_server: + environment_servers: + single_agent_turn_legacy: + agent_server: + type: responses_api_agents + name: gdpval_codex_agent + resources_tool_transports: [mcp] diff --git a/benchmarks/gdpval/prepare_harness_neutral.py b/benchmarks/gdpval/prepare_harness_neutral.py index 543791be3aa..774e1b0ea55 100644 --- a/benchmarks/gdpval/prepare_harness_neutral.py +++ b/benchmarks/gdpval/prepare_harness_neutral.py @@ -3,8 +3,8 @@ """Prepare the GDPVal rows for harnesses that get the task tools from the resources server. The rows match ``prepare.py``'s but carry no tool list, which session harnesses such as Pi and Codex reject in the -request. Pi reaches finish, abandon_task_finish, web_search and fetch_web_page through the resources server's MCP -grant. Use these rows with ``prompts/harness-neutral.yaml``. +request. Pi and Codex reach finish, abandon_task_finish, web_search and fetch_web_page through the resources server's +MCP grant. Use these rows with ``prompts/harness-neutral.yaml``. """ from __future__ import annotations diff --git a/resources_servers/gdpval/tests/test_harness_configs.py b/resources_servers/gdpval/tests/test_harness_configs.py index 47331a7862c..c6cb18fa24a 100644 --- a/resources_servers/gdpval/tests/test_harness_configs.py +++ b/resources_servers/gdpval/tests/test_harness_configs.py @@ -9,12 +9,13 @@ from nemo_gym.benchmarks import BenchmarkConfig from nemo_gym.global_config import GlobalConfigDictParser, GlobalConfigDictParserConfig +from responses_api_agents.codex_agent.app import CodexAgentConfig from responses_api_agents.pi_agent.app import PiAgentConfig _BENCHMARK_DIR = Path(__file__).resolve().parents[3] / "benchmarks" / "gdpval" # Each harness config here (.yaml), with its agent's config class. -_HARNESS_CONFIGS = {"pi": PiAgentConfig} +_HARNESS_CONFIGS = {"pi": PiAgentConfig, "codex": CodexAgentConfig} @pytest.fixture(autouse=True) @@ -49,6 +50,8 @@ def test_harness_replaces_stirrup_and_reaches_the_task_tools_over_mcp(harness): # The agent ignores keys its config class does not define, so a misspelled setting would be dropped silently. # Gym reads datasets and description itself. assert agent.keys() - {"datasets", "description"} - _HARNESS_CONFIGS[harness].model_fields.keys() == set() + # Building the config also runs its validators, such as the limits on timeouts and compaction. + _HARNESS_CONFIGS[harness].model_validate({**agent, "name": benchmark.agent_name}) # Only the agent, the agents the resources server allows and the tool transport differ from config.yaml, so the # judges, scoring, sandbox and multi-stage driver are the ones Stirrup runs with. overrides = OmegaConf.to_container(OmegaConf.load(path))