yuhezhang-ai · Sep 30, 2026 · Sep 24, 2026
diff --git a/fern/versions/latest/pages/reference/configuration.mdx b/fern/versions/latest/pages/reference/configuration.mdx
@@ -30,6 +30,14 @@ On multi-node deployments, the `swe_agents` server embeds these paths in command
 
 Servers that pre-staged setup trees next to the package (the layout before `cache_dir` existed, for example baked container images) keep using those trees **only while `cache_dir` is left at its default** — explicitly configuring `cache_dir` opts out, and removing the pre-staged trees migrates a default-config deployment. Contents accumulate across runs; both directories are safe to delete when no run is active.
 
+### Server Virtual Environments
+
+`gym env start` sets up one uv virtual environment per server and starts the server with that environment's own interpreter (`<venv>/bin/python`), then logs which venv each server runs from. A venv that was copied or moved after it was built still runs its own interpreter, but its `bin/activate` still puts the original location first on `PATH`, so commands a server runs by name (such as `python` or `uv`) are looked up there first. Rebuild a moved venv, or create it with `uv venv --relocatable`, if its servers start such subprocesses.
+
+| Key | Default | Description |
+|-----|---------|-------------|
+| `uv_lock_timeout_seconds` | `UV_LOCK_TIMEOUT` if already exported, else `1800` | Seconds a server install waits for uv's cache lock. Servers start together and share one `uv_cache_dir`, so the rest wait while one of them does a cold install; uv's own default of 300 can be too short for that. Exported as `UV_LOCK_TIMEOUT` to the Gym process and to every server, test, and venv-setup subprocess it launches. Set it to `null` to leave `UV_LOCK_TIMEOUT` as inherited. |
+
 ---
 
 ## Path Resolution and External Roots

diff --git a/nemo_gym/cli/env.py b/nemo_gym/cli/env.py
@@ -313,6 +313,26 @@ def _resolve_server_dir(rel_path: Path) -> Path:
     )
 
 
+def _server_launch_command(
+    dir_path: Path,
+    global_config_dict: DictConfig,
+    server_name: str,
+    entrypoint_fpath: Path,
+    escaped_config_dict_yaml_str: str,
+) -> str:
+    """Shell command that sets up a server's venv and starts its entrypoint with that venv's own interpreter.
+
+    The interpreter is named explicitly instead of relying on the bare `python` that `source bin/activate`
+    puts on PATH: a venv copied or moved after creation still names its original prefix in `bin/activate`,
+    so activating it would silently run a different interpreter than `uv_venv_dir` selected.
+    """
+    venv_python_fpath = get_venv_path(dir_path, global_config_dict) / "bin" / "python"
+    return f"""{setup_env_command(dir_path, global_config_dict, server_name)} \\
+    && {NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME}={escaped_config_dict_yaml_str} \\
+    {NEMO_GYM_CONFIG_PATH_ENV_VAR_NAME}={shlex.quote(server_name)} \\
+    {shlex.quote(str(venv_python_fpath))} {shlex.quote(str(entrypoint_fpath))}"""
+
+
 class RunConfig(BaseNeMoGymCLIConfig):
     """
     Start NeMo Gym servers for agents, models, and resources.
@@ -441,10 +461,11 @@ def start(self, global_config_dict_parser_config: GlobalConfigDictParserConfig)
             # Resolve cwd-first (a local server), else the install location for built-ins.
             dir_path = _resolve_server_dir(Path(first_key, second_key))
 
-            command = f"""{setup_env_command(dir_path, global_config_dict, top_level_path)} \\
-    && {NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME}={escaped_config_dict_yaml_str} \\
-    {NEMO_GYM_CONFIG_PATH_ENV_VAR_NAME}={shlex.quote(top_level_path)} \\
-    python {str(entrypoint_fpath)}"""
+            # Name the venv in the logs: the server runs that venv's interpreter (see _server_launch_command).
+            print(f"Starting `{top_level_path}` from venv {get_venv_path(dir_path, global_config_dict)}")
+            command = _server_launch_command(
+                dir_path, global_config_dict, top_level_path, entrypoint_fpath, escaped_config_dict_yaml_str
+            )
 
             process = run_command(command, dir_path, server_name=top_level_path)
             self._processes[top_level_path] = process

diff --git a/nemo_gym/cli/setup_command.py b/nemo_gym/cli/setup_command.py
@@ -32,6 +32,7 @@
     PYTHON_VERSION_KEY_NAME,
     SKIP_VENV_IF_PRESENT_KEY_NAME,
     UV_CACHE_DIR_KEY_NAME,
+    UV_LOCK_TIMEOUT_KEY_NAME,
     UV_PIP_SET_PYTHON_KEY_NAME,
     UV_VENV_DIR_KEY_NAME,
     get_global_config_dict,
@@ -104,7 +105,13 @@ def _get_nemo_gym_version_spec(is_editable_install: bool) -> str:
 
 
 def get_venv_path(dir_path: Path, global_config_dict: DictConfig) -> Path:
-    """Return the server venv path for the configured venv root."""
+    """Resolve the venv a server runs from: ``uv_venv_dir/<type>/<name>/.venv``, else ``<server dir>/.venv``.
+
+    Callers need this to launch a server with the venv's own interpreter rather than trusting whatever
+    ``bin/activate`` puts on PATH. A venv copied or moved after creation keeps the *original* prefix
+    hard-coded in ``bin/activate``, so sourcing it silently hands the server a different interpreter than
+    the one ``uv_venv_dir`` asked for.
+    """
     root_venv_path = Path(global_config_dict[UV_VENV_DIR_KEY_NAME])
     if root_venv_path.resolve() != PARENT_DIR.resolve():
         return Path(root_venv_path, *dir_path.parts[-2:], ".venv").absolute()
@@ -233,6 +240,11 @@ def run_command(
     custom_env["PYTHONPATH"] = ":".join(py_path_entries)
 
     custom_env["UV_CACHE_DIR"] = global_config_dict[UV_CACHE_DIR_KEY_NAME]
+    # Servers start concurrently and contend for the lock on that shared cache, so the wait has to
+    # cover a cold install of the slowest one rather than uv's 300s default.
+    uv_lock_timeout = global_config_dict.get(UV_LOCK_TIMEOUT_KEY_NAME)
+    if uv_lock_timeout is not None:
+        custom_env["UV_LOCK_TIMEOUT"] = str(uv_lock_timeout)
 
     log_dir = global_config_dict.get(NEMO_GYM_LOG_DIR_KEY_NAME)
     if log_dir:

diff --git a/nemo_gym/global_config.py b/nemo_gym/global_config.py
@@ -92,6 +92,7 @@
 MODEL_ENDPOINT_READINESS_TIMEOUT_KEY_NAME = "model_endpoint_readiness_timeout_seconds"
 ALLOW_OPENAI_VERSION_SKEW_KEY_NAME = "allow_openai_version_skew"
 UV_CACHE_DIR_KEY_NAME = "uv_cache_dir"
+UV_LOCK_TIMEOUT_KEY_NAME = "uv_lock_timeout_seconds"
 UV_VENV_DIR_KEY_NAME = "uv_venv_dir"
 RESULTS_DIR_KEY_NAME = "results_dir"
 CACHE_DIR_KEY_NAME = "cache_dir"
@@ -141,6 +142,7 @@
     MODEL_ENDPOINT_READINESS_TIMEOUT_KEY_NAME,
     ALLOW_OPENAI_VERSION_SKEW_KEY_NAME,
     UV_CACHE_DIR_KEY_NAME,
+    UV_LOCK_TIMEOUT_KEY_NAME,
     UV_VENV_DIR_KEY_NAME,
     RESULTS_DIR_KEY_NAME,
     CACHE_DIR_KEY_NAME,
@@ -1396,6 +1398,20 @@ def parse(self, parse_config: Optional[GlobalConfigDictParserConfig] = None) ->
             # Runtime subprocesses inherit the configured cache directory.
             if not parse_config.offline:
                 environ["UV_CACHE_DIR"] = global_config_dict[UV_CACHE_DIR_KEY_NAME]
+            # Every server installs into that one shared cache, and they all start at once, so
+            # exactly one holds uv's distribution-cache lock while the rest wait out its cold
+            # resolve and download. uv's own default is 300s, which is shorter than a cold install
+            # of a large dependency set - the waiters then abort and the run dies during spinup.
+            # An explicit key wins, then a UV_LOCK_TIMEOUT the user already exported, then 1800.
+            # A null key exports nothing, so the inherited environment (or uv's default) applies.
+            exported_uv_lock_timeout = environ.get("UV_LOCK_TIMEOUT", "").strip()
+            global_config_dict.setdefault(
+                UV_LOCK_TIMEOUT_KEY_NAME,
+                int(exported_uv_lock_timeout) if exported_uv_lock_timeout.isdecimal() else 1800,
+            )
+            uv_lock_timeout = global_config_dict[UV_LOCK_TIMEOUT_KEY_NAME]
+            if not parse_config.offline and uv_lock_timeout is not None:
+                environ["UV_LOCK_TIMEOUT"] = str(uv_lock_timeout)
             # By default, build the directories in their individual folders using the root repository
             # e.g. WORKING_DIR/responses_api_models/my_server
             # Deliberately anchored at WORKING_DIR rather than the cache root: venv

diff --git a/scripts/wait_for_servers.sh b/scripts/wait_for_servers.sh
@@ -7,6 +7,9 @@
 #   gym_env_start_pid   PID of the background gym env start process
 #   head_server_port    Port of the head server (default: 11000)
 #   max_wait_seconds    Max seconds to wait for child servers (default: 180)
+#
+# Environment:
+#   HEAD_MAX_WAIT       Max seconds to wait for the head server (default: 180)
 
 set -euo pipefail
 
@@ -24,11 +27,14 @@ check_pid() {
   fi
 }
 
-# Phase 1: Wait for head server to respond on /server_instances (max 60s)
+# Phase 1: Wait for head server to respond on /server_instances.
+# The head server starts Ray first, which can take close to a minute on a
+# shared network filesystem, so the old fixed 60s was too tight.
+HEAD_MAX_WAIT="${HEAD_MAX_WAIT:-180}"
 # The head server has no root route, but /server_instances returns 200.
 echo "Waiting for head server on port ${HEAD_PORT}..."
 HEAD_READY="false"
-for i in $(seq 1 $((60 / POLL_INTERVAL))); do
+for i in $(seq 1 $((HEAD_MAX_WAIT / POLL_INTERVAL))); do
   if curl -sf "${HEAD_URL}/server_instances" > /dev/null 2>&1; then
     echo "Head server up after $((i * POLL_INTERVAL))s"
     HEAD_READY="true"
@@ -38,7 +44,7 @@ for i in $(seq 1 $((60 / POLL_INTERVAL))); do
   sleep "$POLL_INTERVAL"
 done
 if [ "$HEAD_READY" != "true" ]; then
-  echo "Head server did not respond within 60s"
+  echo "Head server did not respond within ${HEAD_MAX_WAIT}s"
   exit 1
 fi
 

diff --git a/tests/unit_tests/test_cli_server_launch.py b/tests/unit_tests/test_cli_server_launch.py
@@ -0,0 +1,63 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import shlex
+from pathlib import Path
+
+import pytest
+
+from nemo_gym import PARENT_DIR
+from nemo_gym.cli.env import _server_launch_command
+from nemo_gym.global_config import (
+    NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME,
+    NEMO_GYM_CONFIG_PATH_ENV_VAR_NAME,
+    UV_VENV_DIR_KEY_NAME,
+)
+from tests.unit_tests.test_global_config import TestGlobalConfig as _TestGlobalConfig
+
+
+class TestServerLaunchCommand:
+    """`gym env start` runs each server with its venv's own interpreter, not the `python` on PATH after
+    `source bin/activate`, so a relocated venv whose `bin/activate` names its original prefix cannot win."""
+
+    def _setup_server_dir(self, tmp_path: Path) -> Path:
+        # A space in the path catches any unquoted venv or server path in the generated command.
+        server_dir = tmp_path / "my checkout" / "resources_servers" / "my_server"
+        server_dir.mkdir(parents=True)
+        (server_dir / "requirements.txt").write_text("pytest\n")
+        (tmp_path / "my checkout" / "pyproject.toml").write_text("")
+        return server_dir.absolute()
+
+    @pytest.mark.parametrize("uv_venv_dir", [None, "shared venvs"])
+    def test_runs_the_entrypoint_with_the_venv_interpreter(self, tmp_path: Path, uv_venv_dir: str | None) -> None:
+        server_dir = self._setup_server_dir(tmp_path)
+        global_config_dict = _TestGlobalConfig._default_global_config_dict_values.fget(None) | {
+            UV_VENV_DIR_KEY_NAME: str(tmp_path / uv_venv_dir) if uv_venv_dir else str(PARENT_DIR)
+        }
+        expected_venv = (
+            tmp_path / uv_venv_dir / "resources_servers" / "my_server" / ".venv"
+            if uv_venv_dir
+            else server_dir / ".venv"
+        )
+
+        command = _server_launch_command(
+            server_dir, global_config_dict, "my_server", Path("app.py"), shlex.quote("config: yaml")
+        )
+
+        *setup_lines, launch_line = command.splitlines()
+        assert shlex.split(launch_line) == [str(expected_venv / "bin" / "python"), "app.py"]
+        # The interpreter comes from the same venv that the setup step builds and activates.
+        assert f"source {shlex.quote(str(expected_venv / 'bin' / 'activate'))}" in setup_lines[0]
+        assert f"{NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME}='config: yaml'" in command
+        assert f"{NEMO_GYM_CONFIG_PATH_ENV_VAR_NAME}=my_server" in command
diff --git a/tests/unit_tests/test_cli_setup_command.py b/tests/unit_tests/test_cli_setup_command.py
@@ -503,6 +503,28 @@ def test_supplied_config_and_streams_avoid_global_config(self, monkeypatch: Monk
         assert popen.call_args.kwargs["stdout"] == "isolated stdout"
         assert popen.call_args.kwargs["stderr"] == "isolated stderr"
 
+    def test_uv_lock_timeout_is_propagated_to_server_processes(self, monkeypatch: MonkeyPatch) -> None:
+        popen, _ = self._setup(monkeypatch)
+
+        run_command(
+            command="my command",
+            working_dir_path=Path("/my path"),
+            global_config_dict={"uv_cache_dir": "shared cache", "uv_lock_timeout_seconds": 1800},
+        )
+
+        assert popen.call_args.kwargs["env"]["UV_LOCK_TIMEOUT"] == "1800"
+
+    def test_uv_lock_timeout_absent_when_unconfigured(self, monkeypatch: MonkeyPatch) -> None:
+        popen, _ = self._setup(monkeypatch)
+
+        run_command(
+            command="my command",
+            working_dir_path=Path("/my path"),
+            global_config_dict={"uv_cache_dir": "shared cache"},
+        )
+
+        assert "UV_LOCK_TIMEOUT" not in popen.call_args.kwargs["env"]
+
 
 class TestGetNemoGymInstallFlags:
     """Test _get_nemo_gym_install_flags helper function."""

diff --git a/tests/unit_tests/test_global_config.py b/tests/unit_tests/test_global_config.py
@@ -47,6 +47,7 @@
     DEFAULT_HEAD_SERVER_PORT,
     NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME,
     USE_ABSOLUTE_IP,
+    UV_LOCK_TIMEOUT_KEY_NAME,
     GlobalConfigDictParser,
     GlobalConfigDictParserConfig,
     _openai_version_matches_nemo_gym_constraint,
@@ -62,6 +63,8 @@
 
 class TestGlobalConfig:
     def _mock_versions_for_testing(self, monkeypatch: MonkeyPatch) -> None:
+        # An exported UV_LOCK_TIMEOUT becomes the resolved default; keep snapshots hermetic.
+        monkeypatch.delenv("UV_LOCK_TIMEOUT", raising=False)
         monkeypatch.setattr(nemo_gym.global_config, "openai_version", "test openai version")
         monkeypatch.setattr(nemo_gym.global_config, "ray_version", "test ray version")
 
@@ -84,6 +87,7 @@ def _default_global_config_dict_values(self) -> dict:
             "model_endpoint_readiness_timeout_seconds": 600,
             "allow_openai_version_skew": False,
             "uv_cache_dir": str(CACHE_DIR.expanduser().resolve() / "uv"),
+            "uv_lock_timeout_seconds": 1800,
             "uv_venv_dir": str(WORKING_DIR),
             "results_dir": str(RESULTS_DIR.expanduser().resolve()),
             "cache_dir": str(CACHE_DIR.expanduser().resolve()),
@@ -94,6 +98,7 @@ def test_get_global_config_dict_sanity(self, monkeypatch: MonkeyPatch) -> None:
 
         # Clear any lingering env vars.
         monkeypatch.delenv(NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME, raising=False)
+        monkeypatch.delenv("UV_LOCK_TIMEOUT", raising=False)
         monkeypatch.setattr(nemo_gym.global_config, "_GLOBAL_CONFIG_DICT", None)
 
         # Explicitly handle any local .env.yaml files. Either read or don't read.
@@ -118,6 +123,7 @@ def hydra_main_wrapper(fn):
     def test_offline_resolution_uses_invalid_port_without_probing(self, monkeypatch: MonkeyPatch) -> None:
         self._mock_versions_for_testing(monkeypatch)
         monkeypatch.delenv("UV_CACHE_DIR", raising=False)
+        monkeypatch.delenv("UV_LOCK_TIMEOUT", raising=False)
         probe = MagicMock(side_effect=AssertionError("offline resolution must not probe sockets"))
         hostname = MagicMock(side_effect=AssertionError("offline resolution must not resolve hostnames"))
         setup_exporters = MagicMock(side_effect=AssertionError("offline resolution must not start exporters"))
@@ -146,6 +152,56 @@ def test_offline_resolution_uses_invalid_port_without_probing(self, monkeypatch:
         hostname.assert_not_called()
         setup_exporters.assert_not_called()
         assert "UV_CACHE_DIR" not in nemo_gym.global_config.environ
+        assert "UV_LOCK_TIMEOUT" not in nemo_gym.global_config.environ
+        assert config[UV_LOCK_TIMEOUT_KEY_NAME] == 1800
+
+    def test_uv_lock_timeout_is_exported_and_overridable(self, monkeypatch: MonkeyPatch) -> None:
+        self._mock_versions_for_testing(monkeypatch)
+        monkeypatch.setattr(nemo_gym.global_config, "environ", dict())
+        self._mock_parse_environment(monkeypatch, DictConfig({}))
+
+        default_config = get_global_config_dict()
+
+        assert default_config[UV_LOCK_TIMEOUT_KEY_NAME] == 1800
+        assert nemo_gym.global_config.environ["UV_LOCK_TIMEOUT"] == "1800"
+
+        monkeypatch.setattr(nemo_gym.global_config, "environ", dict())
+        self._mock_parse_environment(monkeypatch, DictConfig({UV_LOCK_TIMEOUT_KEY_NAME: 60}))
+
+        overridden_config = get_global_config_dict()
+
+        assert overridden_config[UV_LOCK_TIMEOUT_KEY_NAME] == 60
+        assert nemo_gym.global_config.environ["UV_LOCK_TIMEOUT"] == "60"
+
+    @mark.parametrize(
+        ("exported", "configured", "expected"),
+        [
+            ("3600", {}, 3600),  # a timeout the user already exported survives
+            ("3600", {UV_LOCK_TIMEOUT_KEY_NAME: 60}, 60),  # the config key wins over the export
+            ("not a number", {}, 1800),  # uv cannot use a non-integer value either
+        ],
+    )
+    def test_uv_lock_timeout_precedence(
+        self, monkeypatch: MonkeyPatch, exported: str, configured: dict, expected: int
+    ) -> None:
+        self._mock_versions_for_testing(monkeypatch)
+        monkeypatch.setattr(nemo_gym.global_config, "environ", {"UV_LOCK_TIMEOUT": exported})
+        self._mock_parse_environment(monkeypatch, DictConfig(configured))
+
+        config = get_global_config_dict()
+
+        assert config[UV_LOCK_TIMEOUT_KEY_NAME] == expected
+        assert nemo_gym.global_config.environ["UV_LOCK_TIMEOUT"] == str(expected)
+
+    def test_null_uv_lock_timeout_leaves_the_environment_alone(self, monkeypatch: MonkeyPatch) -> None:
+        self._mock_versions_for_testing(monkeypatch)
+        monkeypatch.setattr(nemo_gym.global_config, "environ", dict())
+        self._mock_parse_environment(monkeypatch, DictConfig({UV_LOCK_TIMEOUT_KEY_NAME: None}))
+
+        config = get_global_config_dict()
+
+        assert config[UV_LOCK_TIMEOUT_KEY_NAME] is None
+        assert "UV_LOCK_TIMEOUT" not in nemo_gym.global_config.environ
 
     def _mock_parse_environment(self, monkeypatch: MonkeyPatch, config_dict: "DictConfig") -> None:
         """Standard parser mocks (no env var, no .env.yaml, fixed hydra config)."""