feat: environment registry + 'gym list environments' (#1205 friction #8 / M2)

nemo_gym/cli/env.py

@@ -51,6 +51,7 @@
     GlobalConfigDictParserConfig,
     get_global_config_dict,
 )
+from nemo_gym.registry import discover_environments
 from nemo_gym.server_status import StatusCommand
 from nemo_gym.server_utils import (
     HEAD_SERVER_KEY_NAME,
@@ -863,6 +864,50 @@ def dump_config():  # pragma: no cover
     print(OmegaConf.to_yaml(global_config_dict, resolve=True))
 
 
+def list_environments() -> None:
+    """List the environments available under environments/, by short name.
+
+    Examples:
+
+    ```bash
+    gym list environments
+    gym list environments --json
+    ```
+    """
+    global_config_dict = get_global_config_dict(
+        global_config_dict_parser_config=GlobalConfigDictParserConfig(
+            initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT,
+        )
+    )
+    BaseNeMoGymCLIConfig.model_validate(global_config_dict)
+
+    environments = discover_environments()
+
+    if global_config_dict.get(JSON_OUTPUT_KEY_NAME, False):
+        print(
+            json.dumps(
+                [
+                    {"name": name, "domain": env.domain, "description": env.description}
+                    for name, env in environments.items()
+                ]
+            )
+        )
+        return
+
+    if not environments:
+        rich.print("[yellow]No environments found.[/yellow]")
+        return
+
+    table = Table(title=f"Available environments in NeMo Gym ({len(environments)})")
+    table.add_column("Environment")
+    table.add_column("Domain")
+    table.add_column("Description")
+    for name, environment in environments.items():
+        table.add_row(name, environment.domain or "", environment.description or "")
+
+    rich.print(table)
+
+
 def status():  # pragma: no cover
     global_config_dict = get_global_config_dict()
     BaseNeMoGymCLIConfig.model_validate(global_config_dict)

nemo_gym/cli/main.py

@@ -285,7 +285,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None:
 
 # One-line help for each command group, shown in `gym --help`.
 GROUPS = {
-    "list": "List available components. As of now, only benchmarks are available.",
+    "list": "List available components (benchmarks, environments).",
     "dataset": "Manage datasets.",
     "env": "Develop and run environments.",
     "eval": "Run evaluations.",
@@ -296,6 +296,9 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None:
     "list benchmarks": Command(
         target="nemo_gym.cli.eval:list_benchmarks", summary="List available benchmarks.", flags=(JSON,)
     ),
+    "list environments": Command(
+        target="nemo_gym.cli.env:list_environments", summary="List available environments by name.", flags=(JSON,)
+    ),
     "search": Command(
         target="nemo_gym.cli.eval:list_benchmarks",
         summary="Search available components (currently benchmarks) by name; like `list` filtered to a query.",

nemo_gym/registry.py

@@ -0,0 +1,105 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Registry of co-located environments under ``environments/<name>/``.
+
+An *environment* is a directory ``environments/<name>/`` whose ``config.yaml`` wires together a
+resources server, an agent, and datasets (and references a model server). This module maps an
+environment's short ``<name>`` to its config so it can be enumerated by name — the foundation for
+``gym list environments``. Resolving a name to a config path for *running* is handled by the CLI's
+generic ``--environment`` asset selector, so this module is intentionally discovery-only.
+
+Discovery only reads config files; it never resolves interpolations or starts servers, so it is
+safe to call even when secrets/API keys referenced by a config are not set in the environment.
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Optional, Tuple
+
+from omegaconf import OmegaConf
+
+from nemo_gym import PARENT_DIR
+
+
+ENVIRONMENTS_DIR = PARENT_DIR / "environments"
+ENVIRONMENT_CONFIG_FILENAME = "config.yaml"
+
+
+@dataclass(frozen=True)
+class EnvironmentEntry:
+    """A discovered environment: its name, where it lives, and lightweight metadata."""
+
+    name: str
+    config_path: Path
+    path: Path
+    description: Optional[str] = None
+    domain: Optional[str] = None
+
+
+def _read_metadata(config_path: Path) -> Tuple[Optional[str], Optional[str]]:
+    """Best-effort ``(description, domain)`` from the config's resources_servers entry.
+
+    Reads without resolving interpolations or missing values so a config that references an unset
+    key (e.g. an API key) still yields metadata instead of raising.
+    """
+    try:
+        container = OmegaConf.to_container(OmegaConf.load(config_path), resolve=False, throw_on_missing=False)
+    except Exception:
+        return None, None
+
+    if not isinstance(container, dict):
+        return None, None
+
+    for top_level_value in container.values():
+        if not isinstance(top_level_value, dict):
+            continue
+        resources_servers = top_level_value.get("resources_servers")
+        if not isinstance(resources_servers, dict):
+            continue
+        for server_config in resources_servers.values():
+            if isinstance(server_config, dict):
+                description = server_config.get("description")
+                domain = server_config.get("domain")
+                return (
+                    description if isinstance(description, str) else None,
+                    domain if isinstance(domain, str) else None,
+                )
+    return None, None
+
+
+def discover_environments(environments_dir: Path = ENVIRONMENTS_DIR) -> Dict[str, EnvironmentEntry]:
+    """Map environment name -> :class:`EnvironmentEntry` for every ``<name>/config.yaml``.
+
+    The name is the directory name. Returns an empty dict if the directory is missing.
+    """
+    environments: Dict[str, EnvironmentEntry] = {}
+    if not environments_dir.is_dir():
+        return environments
+
+    for child in sorted(environments_dir.iterdir()):
+        config_path = child / ENVIRONMENT_CONFIG_FILENAME
+        if not (child.is_dir() and config_path.is_file()):
+            continue
+
+        description, domain = _read_metadata(config_path)
+        environments[child.name] = EnvironmentEntry(
+            name=child.name,
+            config_path=config_path,
+            path=child,
+            description=description,
+            domain=domain,
+        )
+
+    return environments

tests/unit_tests/test_cli.py

@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import json
 import shutil
 import tomllib
 from importlib import import_module
@@ -22,6 +23,7 @@
 from omegaconf import OmegaConf
 from pytest import MonkeyPatch, raises
 
+import nemo_gym.cli.env
 import nemo_gym.global_config
 from nemo_gym import PARENT_DIR
 from nemo_gym.cli.env import (
@@ -32,8 +34,10 @@
     _select_shard,
     exit_cleanly_on_config_error,
     init_resources_server,
+    list_environments,
 )
 from nemo_gym.config_types import ConfigError, NoServerInstancesError, ResourcesServerInstanceConfig
+from nemo_gym.registry import EnvironmentEntry
 
 
 class TestSelectShard:
@@ -284,3 +288,41 @@ def ok():
 
     def test_config_error_base_catches_subclasses(self) -> None:
         assert issubclass(NoServerInstancesError, ConfigError)
+
+
+class TestListEnvironments:
+    _ALPHA = EnvironmentEntry(
+        name="alpha",
+        config_path=Path("environments/alpha/config.yaml"),
+        path=Path("environments/alpha"),
+        description="Alpha env",
+        domain="agent",
+    )
+
+    def test_lists_discovered_environments(self, monkeypatch: MonkeyPatch, capsys) -> None:
+        monkeypatch.setattr(nemo_gym.cli.env, "get_global_config_dict", lambda **k: OmegaConf.create({}))
+        monkeypatch.setattr(nemo_gym.cli.env, "discover_environments", lambda *a, **k: {"alpha": self._ALPHA})
+
+        list_environments()
+
+        out = capsys.readouterr().out
+        assert "alpha" in out
+        assert "agent" in out
+
+    def test_no_environments(self, monkeypatch: MonkeyPatch, capsys) -> None:
+        monkeypatch.setattr(nemo_gym.cli.env, "get_global_config_dict", lambda **k: OmegaConf.create({}))
+        monkeypatch.setattr(nemo_gym.cli.env, "discover_environments", lambda *a, **k: {})
+
+        list_environments()
+
+        assert "No environments found" in capsys.readouterr().out
+
+    def test_json_output(self, monkeypatch: MonkeyPatch, capsys) -> None:
+        monkeypatch.setattr(nemo_gym.cli.env, "get_global_config_dict", lambda **k: OmegaConf.create({"json": True}))
+        monkeypatch.setattr(nemo_gym.cli.env, "discover_environments", lambda *a, **k: {"alpha": self._ALPHA})
+
+        list_environments()
+
+        assert json.loads(capsys.readouterr().out) == [
+            {"name": "alpha", "domain": "agent", "description": "Alpha env"}
+        ]

tests/unit_tests/test_cli_main.py

@@ -933,3 +933,15 @@ def test_did_you_mean_spans_user_dir(self, monkeypatch: MonkeyPatch, tmp_path, c
         with pytest.raises(SystemExit):
             main()
         assert "Did you mean `mybench`?" in capsys.readouterr().err
+
+
+class TestListEnvironmentsRouting:
+    def test_list_environments_dispatches(self, monkeypatch: MonkeyPatch) -> None:
+        target, overrides = _dispatch_for(monkeypatch, ["list", "environments"])
+        assert target == "nemo_gym.cli.env:list_environments"
+        assert overrides == []
+
+    def test_list_environments_json_dispatches(self, monkeypatch: MonkeyPatch) -> None:
+        target, overrides = _dispatch_for(monkeypatch, ["list", "environments", "--json"])
+        assert target == "nemo_gym.cli.env:list_environments"
+        assert overrides == ["+json=true"]

tests/unit_tests/test_registry.py

@@ -0,0 +1,115 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from pathlib import Path
+
+from nemo_gym.registry import discover_environments
+
+
+def _make_env(environments_dir: Path, name: str, config_body: str) -> Path:
+    env_dir = environments_dir / name
+    env_dir.mkdir(parents=True)
+    config_path = env_dir / "config.yaml"
+    config_path.write_text(config_body)
+    return config_path
+
+
+_ENV_CONFIG = """{name}:
+  resources_servers:
+    {name}:
+      entrypoint: app.py
+      domain: agent
+      description: {name} test environment
+{name}_simple_agent:
+  responses_api_agents:
+    simple_agent:
+      entrypoint: app.py
+      resources_server:
+        type: resources_servers
+        name: {name}
+      model_server:
+        type: responses_api_models
+        name: policy_model
+"""
+
+
+class TestDiscoverEnvironments:
+    def test_discovers_by_directory_name_with_metadata(self, tmp_path: Path) -> None:
+        envs_dir = tmp_path / "environments"
+        _make_env(envs_dir, "alpha", _ENV_CONFIG.format(name="alpha"))
+        _make_env(envs_dir, "beta", _ENV_CONFIG.format(name="beta"))
+
+        environments = discover_environments(envs_dir)
+
+        assert set(environments) == {"alpha", "beta"}
+        alpha = environments["alpha"]
+        assert alpha.name == "alpha"
+        assert alpha.config_path == envs_dir / "alpha" / "config.yaml"
+        assert alpha.path == envs_dir / "alpha"
+        assert alpha.description == "alpha test environment"
+        assert alpha.domain == "agent"
+
+    def test_missing_directory_returns_empty(self, tmp_path: Path) -> None:
+        assert discover_environments(tmp_path / "does_not_exist") == {}
+
+    def test_ignores_dirs_without_config_and_loose_files(self, tmp_path: Path) -> None:
+        envs_dir = tmp_path / "environments"
+        _make_env(envs_dir, "real", _ENV_CONFIG.format(name="real"))
+        (envs_dir / "not_an_env").mkdir()  # dir without a config.yaml
+        (envs_dir / "__init__.py").write_text("")  # loose file
+
+        assert set(discover_environments(envs_dir)) == {"real"}
+
+    def test_unparseable_or_metadataless_configs_still_discovered(self, tmp_path: Path) -> None:
+        # Configs without a parseable resources_servers block (or malformed YAML) must still be
+        # discovered by name, just with no description/domain — never crash discovery.
+        envs_dir = tmp_path / "environments"
+        _make_env(envs_dir, "no_rs", "agent_only:\n  responses_api_agents:\n    a: {}\n")  # no resources_servers
+        _make_env(envs_dir, "top_list", "- x\n- y\n")  # top-level not a mapping
+        _make_env(envs_dir, "scalar_top", "top: just_a_string\n")  # top-level value not a dict
+        _make_env(envs_dir, "rs_not_dict", "top:\n  resources_servers: not_a_mapping\n")  # rs not a dict
+        _make_env(envs_dir, "broken", "key: [unclosed\n")  # malformed YAML -> load raises
+
+        environments = discover_environments(envs_dir)
+
+        assert set(environments) == {"no_rs", "top_list", "scalar_top", "rs_not_dict", "broken"}
+        for entry in environments.values():
+            assert entry.description is None
+            assert entry.domain is None
+
+    def test_metadata_does_not_resolve_interpolations(self, tmp_path: Path) -> None:
+        # A config referencing an unset interpolation must still be discoverable (no resolution).
+        envs_dir = tmp_path / "environments"
+        _make_env(
+            envs_dir,
+            "needs_key",
+            "needs_key:\n"
+            "  resources_servers:\n"
+            "    needs_key:\n"
+            "      entrypoint: app.py\n"
+            "      domain: other\n"
+            "      api_key: ${some_unset_key}\n",
+        )
+
+        environments = discover_environments(envs_dir)
+        assert "needs_key" in environments
+        assert environments["needs_key"].domain == "other"
+
+
+class TestRealEnvironments:
+    def test_workplace_assistant_is_discoverable(self) -> None:
+        # The repo ships environments/workplace_assistant/ — the registry must find it by name.
+        environments = discover_environments()
+        assert "workplace_assistant" in environments
+        assert environments["workplace_assistant"].config_path.name == "config.yaml"