diff --git a/README.md b/README.md
index af969264cf8..1b8fbff4949 100644
--- a/README.md
+++ b/README.md
@@ -326,6 +326,7 @@ The Dataset column links to publicly available datasets (e.g., on HuggingFace).
| Math With Autograder | math | Math QA verified by a Skills-style autograder LLM judge with math-verify symbolic fallback | Score hard-math benchmarks (e.g. IMO AnswerBench) where the judge is a unidirectional Correct/Incorrect grader | - | - | - | math_with_autograder.yaml | - |
| Math With Code | math | Model solves competitive math problems using simple calculator tools | Improve math and simple tool use capabilities | ✓ | - | Apache 2.0 | math_with_code.yaml | - |
| Math With Judge | math | DAPO17k math dataset with math-verify | Improve math capabilities including AIME 24 / 25 | ✓ | ✓ | Apache 2.0 | dapo17k.yaml | - |
+| Math With Judge | math | DAPO17k train problems repeated for trajectory collection with math-verify | Collect rollouts on DAPO17k without training (NeMo RL is_trajectory_collection) | - | ✓ | - | dapo17k_trajectory_collection.yaml | - |
| Math With Judge | math | MathStackOverflow math dataset with math-verify | Improve math capabilities including AIME 24 / 25 | ✓ | ✓ | Creative Commons Attribution-ShareAlike 4.0 International | math_stack_overflow.yaml | Nemotron-RL-math-stack_overflow |
| Math With Judge | math | OpenMathReasoning math dataset with math-verify and LLM-as-a-judge | Improve math capabilities including AIME 24 / 25 | ✓ | ✓ | Creative Commons Attribution 4.0 International | math_with_judge.yaml | Nemotron-RL-math-OpenMathReasoning |
| Mcqa | knowledge | Multi-choice question answering problems | Improve benchmarks like MMLU / GPQA / HLE | ✓ | ✓ | Apache 2.0 | mcqa.yaml | Nemotron-RL-knowledge-mcqa |
diff --git a/nemo_gym/gitlab_utils.py b/nemo_gym/gitlab_utils.py
index 54f63005b66..a95cef99b45 100644
--- a/nemo_gym/gitlab_utils.py
+++ b/nemo_gym/gitlab_utils.py
@@ -108,7 +108,10 @@ def download_jsonl_dataset(
download_link,
headers={"Authorization": f"Bearer {MLFLOW_TRACKING_TOKEN.get()}"},
)
- with open(config.output_fpath, "w") as f:
+ # Relative outputs such as `data/x.jsonl` only exist in the Gym root; create the directory like the HF download.
+ output_fpath = Path(config.output_fpath)
+ output_fpath.parent.mkdir(parents=True, exist_ok=True)
+ with open(output_fpath, "w") as f:
f.write(response.content.decode())
diff --git a/resources_servers/math_with_judge/README.md b/resources_servers/math_with_judge/README.md
index 97d1cd66ca9..be874e995b4 100644
--- a/resources_servers/math_with_judge/README.md
+++ b/resources_servers/math_with_judge/README.md
@@ -17,6 +17,10 @@ gym env start \
+math_with_judge.resources_servers.math_with_judge.judge_model_server.name=policy_model
```
+The OpenMathReasoning download below and `--download` under "Prepare for trajectory collection" fetch from the
+GitLab model registry. They need the `mlflow` extra (`uv sync --extra mlflow`) and `mlflow_tracking_uri` and
+`mlflow_tracking_token` in `env.yaml`.
+
To download the OpenMathReasoning dataset, the following command can be run:
```bash
gym dataset download --storage gitlab \
@@ -36,6 +40,9 @@ gym eval run --no-serve \
```
## Prepare for trajectory collection
+This writes `validation.jsonl` with each DAPO17k train problem repeated 16 times. NeMo RL reads only the
+validation split when `is_trajectory_collection: true`. `--download` fetches `dapo17k` 0.0.1 from the GitLab
+model registry.
```bash
gym dataset collate \
--config resources_servers/math_with_judge/configs/dapo17k_trajectory_collection.yaml \
diff --git a/resources_servers/math_with_judge/configs/dapo17k_trajectory_collection.yaml b/resources_servers/math_with_judge/configs/dapo17k_trajectory_collection.yaml
new file mode 100644
index 00000000000..d7ec58e793c
--- /dev/null
+++ b/resources_servers/math_with_judge/configs/dapo17k_trajectory_collection.yaml
@@ -0,0 +1,48 @@
+math_with_judge:
+ resources_servers:
+ math_with_judge:
+ entrypoint: app.py
+ # Grading uses math-verify only (should_use_judge: false), so the judge reference just needs to resolve.
+ judge_model_server:
+ type: responses_api_models
+ name: policy_model
+ judge_responses_create_params: {
+ input: []
+ }
+ should_use_judge: false
+ domain: math
+ verified: false
+ description: DAPO17k train problems repeated for trajectory collection with math-verify
+ value: Collect rollouts on DAPO17k without training (NeMo RL is_trajectory_collection)
+math_with_judge_simple_agent:
+ responses_api_agents:
+ simple_agent:
+ entrypoint: app.py
+ resources_server:
+ type: resources_servers
+ name: math_with_judge
+ model_server:
+ type: responses_api_models
+ name: policy_model
+ datasets:
+ # NeMo RL trajectory collection reads only the validation split, so the train problems go there.
+ # Own local file: collate checks each dataset against `_metrics.json`, and
+ # dapo17k_train_metrics.json describes dapo17k.yaml's train split.
+ - name: validation
+ type: validation
+ jsonl_fpath: resources_servers/math_with_judge/data/dapo17k_trajectory_collection_train.jsonl
+ # From YouJiacheng/DAPO-Math-17k-dedup
+ gitlab_identifier:
+ dataset_name: dapo17k
+ version: 0.0.1
+ artifact_fpath: dapo17k_train.jsonl
+ num_repeats: 16
+ license: Apache 2.0
+
+math_with_judge_environment_server:
+ environment_servers:
+ legacy_agent:
+ entrypoint: app.py
+ agent_server:
+ type: responses_api_agents
+ name: math_with_judge_simple_agent
diff --git a/resources_servers/math_with_judge/data/dapo17k_trajectory_collection_train_metrics.json b/resources_servers/math_with_judge/data/dapo17k_trajectory_collection_train_metrics.json
new file mode 100644
index 00000000000..0e05d58eef2
--- /dev/null
+++ b/resources_servers/math_with_judge/data/dapo17k_trajectory_collection_train_metrics.json
@@ -0,0 +1,56 @@
+{
+ "name": "validation",
+ "type": "validation",
+ "jsonl_fpath": "resources_servers/math_with_judge/data/dapo17k_trajectory_collection_train.jsonl",
+ "num_repeats": 16,
+ "source": {
+ "type": "gitlab",
+ "dataset_name": "dapo17k",
+ "version": "0.0.1",
+ "artifact_fpath": "dapo17k_train.jsonl"
+ },
+ "gitlab_identifier": {
+ "dataset_name": "dapo17k",
+ "version": "0.0.1",
+ "artifact_fpath": "dapo17k_train.jsonl"
+ },
+ "huggingface_identifier": null,
+ "license": "Apache 2.0",
+ "Number of examples": 278368,
+ "Number of tools": {
+ "Total # non-null values": 0,
+ "Average": 0.0,
+ "Min": 0.0,
+ "Max": 0.0,
+ "Standard deviation": 0.0
+ },
+ "Json-dumped number of words (proxy for token count)": {
+ "Total # non-null values": 278368,
+ "Average": 85.63,
+ "Min": 45.0,
+ "Max": 322.0,
+ "Standard deviation": 26.94
+ },
+ "Number of turns": {
+ "Total # non-null values": 278368,
+ "Average": 1.0,
+ "Min": 1.0,
+ "Max": 1.0,
+ "Standard deviation": 0.0
+ },
+ "Temperature": {
+ "Total # non-null values": 0,
+ "Average": 0.0,
+ "Min": 0.0,
+ "Max": 0.0,
+ "Standard deviation": 0.0
+ },
+ "question": {
+ "unique_count": 17398,
+ "total_count": 278368
+ },
+ "expected_answer": {
+ "unique_count": 2251,
+ "total_count": 278368
+ }
+}
\ No newline at end of file
diff --git a/tests/unit_tests/test_gitlab_utils.py b/tests/unit_tests/test_gitlab_utils.py
index ba10c120ae6..bba8669393b 100644
--- a/tests/unit_tests/test_gitlab_utils.py
+++ b/tests/unit_tests/test_gitlab_utils.py
@@ -13,12 +13,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
+from pathlib import Path
+from types import ModuleType, SimpleNamespace
import pytest
from pytest import MonkeyPatch
-from nemo_gym.config_types import ConfigError, MLFlowConfig
-from nemo_gym.gitlab_utils import create_mlflow_client
+from nemo_gym.config_types import ConfigError, DownloadJsonlDatasetGitlabConfig, MLFlowConfig
+from nemo_gym.gitlab_utils import create_mlflow_client, download_jsonl_dataset
# TODO: Eventually we want to add more tests to ensure that the Gitlab flow does not break
@@ -40,3 +42,39 @@ def test_create_mlflow_client_gives_an_actionable_error_when_mlflow_is_missing(
with pytest.raises(ConfigError, match=r"pip install nemo-gym\[mlflow\]"):
create_mlflow_client()
+
+ def test_download_creates_the_output_directory(self, monkeypatch: MonkeyPatch, tmp_path: Path) -> None:
+ # README commands download to a relative `data/...` path, which exists only in the Gym root.
+ # Fake the registry lookup and the HTTP fetch; mlflow itself is an optional extra.
+ client = SimpleNamespace(
+ tracking_uri="https://gitlab.invalid/api/v4/projects/1/ml/mlflow",
+ get_model_version=lambda name, version: SimpleNamespace(run_id="run"),
+ )
+ repository = SimpleNamespace(repo=SimpleNamespace(artifact_uri="https://gitlab.invalid/artifacts/"))
+ monkeypatch.setattr("nemo_gym.gitlab_utils.create_mlflow_client", lambda: client)
+ monkeypatch.setitem(sys.modules, "mlflow", ModuleType("mlflow"))
+ monkeypatch.setitem(
+ sys.modules, "mlflow.artifacts", SimpleNamespace(get_artifact_repository=lambda **_: repository)
+ )
+ monkeypatch.setitem(
+ sys.modules,
+ "mlflow.environment_variables",
+ SimpleNamespace(MLFLOW_TRACKING_TOKEN=SimpleNamespace(get=lambda: "token")),
+ )
+ requested = []
+
+ def fake_get(url: str, headers: dict) -> SimpleNamespace:
+ requested.append(url)
+ return SimpleNamespace(content=b'{"question": "q"}\n')
+
+ monkeypatch.setattr("nemo_gym.gitlab_utils.requests.get", fake_get)
+ output = tmp_path / "data" / "nested" / "rows.jsonl"
+
+ download_jsonl_dataset(
+ DownloadJsonlDatasetGitlabConfig(
+ dataset_name="dataset", version="0.0.1", artifact_fpath="rows.jsonl", output_fpath=str(output)
+ )
+ )
+
+ assert requested == ["https://gitlab.invalid/artifacts/rows.jsonl"]
+ assert output.read_text() == '{"question": "q"}\n'
diff --git a/tests/unit_tests/test_math_with_judge_readme_trajectory_config.py b/tests/unit_tests/test_math_with_judge_readme_trajectory_config.py
new file mode 100644
index 00000000000..28ec96d82a1
--- /dev/null
+++ b/tests/unit_tests/test_math_with_judge_readme_trajectory_config.py
@@ -0,0 +1,104 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Replay the math_with_judge README trajectory collation through the real CLI and collator, offline."""
+
+import json
+import re
+import shlex
+import sys
+from pathlib import Path
+
+from omegaconf import OmegaConf
+from pytest import MonkeyPatch
+
+import nemo_gym.cli.main as cli_main
+from nemo_gym import NEMO_GYM_EXTRA_ROOTS_ENV_VAR_NAME
+from nemo_gym.global_config import GlobalConfigDictParser, GlobalConfigDictParserConfig
+from nemo_gym.train_data_utils import TrainDataProcessor
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+SERVER_ROOT = REPO_ROOT / "resources_servers/math_with_judge"
+README = SERVER_ROOT / "README.md"
+TRAJECTORY_REPEATS = 16
+
+
+def _readme_collate_overrides(monkeypatch: MonkeyPatch) -> list[str]:
+ """Return the Hydra overrides the README's `gym dataset collate` block hands to dispatch."""
+ blocks = re.findall(r"```bash\n(.*?)```", README.read_text(), flags=re.DOTALL)
+ commands = [block for block in blocks if block.lstrip().startswith("gym dataset collate")]
+ assert len(commands) == 1, "Expected one `gym dataset collate` block in the README"
+ captured: dict = {}
+
+ def fake_dispatch(target: str, overrides: list[str]) -> None:
+ captured.update(target=target, overrides=overrides)
+
+ monkeypatch.delenv(NEMO_GYM_EXTRA_ROOTS_ENV_VAR_NAME, raising=False)
+ monkeypatch.setattr(cli_main, "dispatch", fake_dispatch)
+ monkeypatch.setattr(sys, "argv", shlex.split(commands[0].replace("\\\n", " ")))
+ cli_main.main()
+ assert captured["target"] == "nemo_gym.cli.dataset:prepare_data"
+ return captured["overrides"]
+
+
+def test_readme_trajectory_collation_repeats_dapo17k_train_as_validation(
+ monkeypatch: MonkeyPatch, tmp_path: Path
+) -> None:
+ # NeMo RL's `is_trajectory_collection` reads only the validation split, so the README step must put the
+ # DAPO17k train problems there, each repeated. The dataset file is a local fixture, so nothing is
+ # downloaded, and env.yaml is skipped, so no developer credentials are read.
+ monkeypatch.chdir(REPO_ROOT)
+ overrides = _readme_collate_overrides(monkeypatch)
+ initial = OmegaConf.merge(
+ GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT,
+ OmegaConf.from_dotlist([override.lstrip("+") for override in overrides]),
+ )
+ config = GlobalConfigDictParser().parse(
+ GlobalConfigDictParserConfig(
+ initial_global_config_dict=initial, skip_load_from_cli=True, skip_load_from_dotenv=True
+ )
+ )
+
+ datasets = config.math_with_judge_simple_agent.responses_api_agents.simple_agent.datasets
+ assert [(d.type, d.get("num_repeats", 1)) for d in datasets] == [("validation", TRAJECTORY_REPEATS)]
+ (dataset,) = datasets
+ maintained = OmegaConf.load(SERVER_ROOT / "configs/dapo17k.yaml")
+ (maintained_train,) = [
+ d
+ for d in maintained.math_with_judge_simple_agent.responses_api_agents.simple_agent.datasets
+ if d.type == "train"
+ ]
+ assert dataset.gitlab_identifier == maintained_train.gitlab_identifier
+ # Collate checks the real data against `_metrics.json`, so the committed sidecar must describe
+ # this split. Sharing dapo17k.yaml's local file would compare against its train sidecar and fail.
+ data_fpath = Path(dataset.jsonl_fpath)
+ sidecar = json.loads((REPO_ROOT / data_fpath.with_name(f"{data_fpath.stem}_metrics.json")).read_text())
+ assert (sidecar["type"], sidecar["num_repeats"]) == ("validation", TRAJECTORY_REPEATS)
+ assert sidecar["gitlab_identifier"] == OmegaConf.to_container(dataset.gitlab_identifier)
+
+ source_rows = [json.loads(line) for line in (SERVER_ROOT / "data/example.jsonl").read_text().splitlines()[:3]]
+ fixture = tmp_path / dataset.jsonl_fpath
+ fixture.parent.mkdir(parents=True)
+ fixture.write_text("".join(json.dumps(row) + "\n" for row in source_rows))
+ monkeypatch.chdir(tmp_path)
+ config.output_dirpath = str(tmp_path / "collated")
+
+ TrainDataProcessor().run(config)
+
+ collated = tmp_path / "collated"
+ assert sorted(p.name for p in collated.iterdir()) == ["validation.jsonl", "validation_metrics.json"]
+ rows = [json.loads(line) for line in (collated / "validation.jsonl").read_text().splitlines()]
+ expected_questions = [row["question"] for row in source_rows for _ in range(TRAJECTORY_REPEATS)]
+ assert [row["question"] for row in rows] == expected_questions