From d861b6f66da349106052e6e33f3c4d8b103ac7fb Mon Sep 17 00:00:00 2001 From: Wojciech Prazuch Date: Mon, 15 Jun 2026 13:18:06 +0200 Subject: [PATCH 1/3] feat(cli): document ng_init_resources_server generated config inline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generated server configs shipped with zero comments — new authors couldn't tell what each field meant or which values were valid without consulting external docs (#1205 friction point #7). Add inline comments to every field in the generated template (server type, domain values, cross-references, dataset splits, license). Also emit 'verified: false' in the generated resources_servers block so the add-verified-flag pre-commit hook is a no-op: its yaml.safe_load + yaml.dump round-trip would otherwise rewrite the file and strip the new comments on the author's first commit. Test asserts the hook leaves the generated config byte -for-byte unchanged. Addresses #1205 friction point #7. Signed-off-by: Wojciech Prazuch --- nemo_gym/cli.py | 36 +++++++++++++++++++++--------------- tests/unit_tests/test_cli.py | 15 +++++++++++++++ 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/nemo_gym/cli.py b/nemo_gym/cli.py index b38c029f361..9788ae67422 100644 --- a/nemo_gym/cli.py +++ b/nemo_gym/cli.py @@ -805,31 +805,37 @@ def init_resources_server(): # pragma: no cover config_fpath = configs_dirpath / f"{server_type_name}.yaml" with open(config_fpath, "w") as f: - f.write(f"""{server_type_name}_resources_server: - {server_type}: - {server_type_name}: - entrypoint: app.py - domain: other -{server_type_name}_simple_agent: + f.write(f"""# Resources server: owns this environment's task verification (verify()) and reward. +{server_type_name}_resources_server: # instance name — how agents/CLI refer to this server + {server_type}: # server type: resources_servers | responses_api_agents | responses_api_models + {server_type_name}: # implementation directory under {server_type}/ + entrypoint: app.py # server entry module + domain: other # capability under test; one of: math, coding, agent, knowledge, + # instruction_following, long_context, safety, games, translation, + # e2e, rlhf, other. Change 'other' to the closest fit. + verified: false # set true once the benchmark has been baselined and reviewed + +# Agent pairing: drives the resources server above with a model via the single-turn simple_agent. +{server_type_name}_simple_agent: # agent instance name (pass as +agent_name= to ng_collect_rollouts) responses_api_agents: - simple_agent: + simple_agent: # built-in single-turn agent; replace with your own agent dir if needed entrypoint: app.py - resources_server: + resources_server: # the resources server this agent drives type: resources_servers name: {server_type_name}_resources_server - model_server: + model_server: # the model that answers; 'policy_model' is resolved from a model config type: responses_api_models name: policy_model - datasets: + datasets: # one block per split: train | validation | example - name: train type: train - jsonl_fpath: resources_servers/{server_type_name}/data/train.jsonl - num_repeats: 1 - gitlab_identifier: + jsonl_fpath: resources_servers/{server_type_name}/data/train.jsonl # local path / download destination + num_repeats: 1 # rollouts per task (raise for pass@k / variance reduction) + gitlab_identifier: # where to fetch this split from the GitLab dataset registry dataset_name: {server_type_name} version: 0.0.1 artifact_fpath: train.jsonl - license: Apache 2.0 + license: Apache 2.0 # required for train/validation; must be an allowed license string - name: validation type: validation jsonl_fpath: resources_servers/{server_type_name}/data/validation.jsonl @@ -839,7 +845,7 @@ def init_resources_server(): # pragma: no cover version: 0.0.1 artifact_fpath: validation.jsonl license: Apache 2.0 - - name: example + - name: example # 5 rows committed to git for quick smoke tests type: example jsonl_fpath: resources_servers/{server_type_name}/data/example.jsonl num_repeats: 1 diff --git a/tests/unit_tests/test_cli.py b/tests/unit_tests/test_cli.py index d6208b5de42..32361858369 100644 --- a/tests/unit_tests/test_cli.py +++ b/tests/unit_tests/test_cli.py @@ -158,6 +158,21 @@ def test_init_resources_server_includes_domain(self) -> None: assert "domain" in server_config, "Domain field missing from resources server config" assert server_config["domain"] == "other", f"Expected domain 'other', got '{server_config['domain']}'" + # Generated config ships `verified: false` so the add-verified-flag pre-commit hook + # is a no-op and does not rewrite (and strip the comments from) the file on commit. + assert server_config["verified"] is False + + # The generated config is documented with inline comments (friction #7). + config_text = config_file.read_text() + assert "# Resources server:" in config_text + assert config_text.count("#") >= 10, "expected inline field documentation comments" + + # The add-verified-flag hook must NOT modify the generated config (would strip comments). + from scripts.add_verified_flag import ensure_verified_flag + + assert ensure_verified_flag(config_file) is False + assert config_file.read_text() == config_text, "verified-flag hook altered the generated config" + # Verify that the config can be validated (this would have failed before the fix) full_config_dict = OmegaConf.create( { From e6a0f7aa7df77baf426b6cc8a28f9acc0830b7d1 Mon Sep 17 00:00:00 2001 From: Wojciech Prazuch Date: Mon, 15 Jun 2026 13:40:31 +0200 Subject: [PATCH 2/3] docs(cli): tighten generated-config comment accuracy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fact-checked the generated comments against the code: - simple_agent isn't single-turn (it runs a tool-calling loop bounded by max_steps) — drop the 'single-turn' wording. - num_repeats repeats each example N times (pass@k/mean@k), not 'rollouts per task'. - note huggingface_identifier as an alternative to gitlab_identifier (both are first-class). Signed-off-by: Wojciech Prazuch --- nemo_gym/cli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nemo_gym/cli.py b/nemo_gym/cli.py index 9788ae67422..694f219d33e 100644 --- a/nemo_gym/cli.py +++ b/nemo_gym/cli.py @@ -815,10 +815,10 @@ def init_resources_server(): # pragma: no cover # e2e, rlhf, other. Change 'other' to the closest fit. verified: false # set true once the benchmark has been baselined and reviewed -# Agent pairing: drives the resources server above with a model via the single-turn simple_agent. -{server_type_name}_simple_agent: # agent instance name (pass as +agent_name= to ng_collect_rollouts) +# Agent pairing: connects the resources server above to a model via the built-in simple_agent. +{server_type_name}_simple_agent: # this agent instance's name — pass as +agent_name= to ng_collect_rollouts responses_api_agents: - simple_agent: # built-in single-turn agent; replace with your own agent dir if needed + simple_agent: # built-in agent: runs the model with tool calls (up to max_steps); swap for your own agent dir entrypoint: app.py resources_server: # the resources server this agent drives type: resources_servers @@ -830,8 +830,8 @@ def init_resources_server(): # pragma: no cover - name: train type: train jsonl_fpath: resources_servers/{server_type_name}/data/train.jsonl # local path / download destination - num_repeats: 1 # rollouts per task (raise for pass@k / variance reduction) - gitlab_identifier: # where to fetch this split from the GitLab dataset registry + num_repeats: 1 # times to repeat each example (e.g. for pass@k / mean@k) + gitlab_identifier: # where to fetch this split — GitLab registry (or use huggingface_identifier for HF Hub) dataset_name: {server_type_name} version: 0.0.1 artifact_fpath: train.jsonl From c720c42778f3773a782d75abd23c62611d6ae804 Mon Sep 17 00:00:00 2001 From: Wojciech Prazuch Date: Mon, 15 Jun 2026 15:45:56 +0200 Subject: [PATCH 3/3] docs(cli): address review on generated-config comments - reword resources_server comment (tools, state and verification) - rephrase agent server config header - drop 'capability under test' framing on domain (eval-only connotation) - remove gitlab_identifier from generated train/validation splits; note remote source as opt-in (gitlab/huggingface are Optional) Signed-off-by: Wojciech Prazuch --- nemo_gym/cli.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/nemo_gym/cli.py b/nemo_gym/cli.py index 694f219d33e..ec0ce66c0fa 100644 --- a/nemo_gym/cli.py +++ b/nemo_gym/cli.py @@ -810,17 +810,17 @@ def init_resources_server(): # pragma: no cover {server_type}: # server type: resources_servers | responses_api_agents | responses_api_models {server_type_name}: # implementation directory under {server_type}/ entrypoint: app.py # server entry module - domain: other # capability under test; one of: math, coding, agent, knowledge, + domain: other # task domain; one of: math, coding, agent, knowledge, # instruction_following, long_context, safety, games, translation, # e2e, rlhf, other. Change 'other' to the closest fit. verified: false # set true once the benchmark has been baselined and reviewed -# Agent pairing: connects the resources server above to a model via the built-in simple_agent. +# Agent server config specifies the agent server to run and any additional components of the environment such as resources servers {server_type_name}_simple_agent: # this agent instance's name — pass as +agent_name= to ng_collect_rollouts responses_api_agents: simple_agent: # built-in agent: runs the model with tool calls (up to max_steps); swap for your own agent dir entrypoint: app.py - resources_server: # the resources server this agent drives + resources_server: # the resources server this agent interacts with for tools, state and verification type: resources_servers name: {server_type_name}_resources_server model_server: # the model that answers; 'policy_model' is resolved from a model config @@ -829,21 +829,14 @@ def init_resources_server(): # pragma: no cover datasets: # one block per split: train | validation | example - name: train type: train - jsonl_fpath: resources_servers/{server_type_name}/data/train.jsonl # local path / download destination + jsonl_fpath: resources_servers/{server_type_name}/data/train.jsonl # local data file for this split num_repeats: 1 # times to repeat each example (e.g. for pass@k / mean@k) - gitlab_identifier: # where to fetch this split — GitLab registry (or use huggingface_identifier for HF Hub) - dataset_name: {server_type_name} - version: 0.0.1 - artifact_fpath: train.jsonl license: Apache 2.0 # required for train/validation; must be an allowed license string + # to fetch this split from a registry instead, add gitlab_identifier: or huggingface_identifier: - name: validation type: validation jsonl_fpath: resources_servers/{server_type_name}/data/validation.jsonl num_repeats: 1 - gitlab_identifier: - dataset_name: {server_type_name} - version: 0.0.1 - artifact_fpath: validation.jsonl license: Apache 2.0 - name: example # 5 rows committed to git for quick smoke tests type: example