From d9aeabc246b483e991d9a8902bf9bdaca0162165 Mon Sep 17 00:00:00 2001 From: Teodor-Dumitru Ene Date: Mon, 3 Aug 2026 15:44:05 -0500 Subject: [PATCH 1/2] fix: validate an OpenHands setup before using it Signed-off-by: Teodor-Dumitru Ene --- responses_api_agents/swe_agents/app.py | 41 ++++++++++++++++++- .../swe_agents/setup_scripts/openhands.sh | 4 +- .../swe_agents/tests/test_app.py | 35 ++++++++++++++++ 3 files changed, 77 insertions(+), 3 deletions(-) diff --git a/responses_api_agents/swe_agents/app.py b/responses_api_agents/swe_agents/app.py index a2f11ff822e..da5922e95ae 100644 --- a/responses_api_agents/swe_agents/app.py +++ b/responses_api_agents/swe_agents/app.py @@ -1728,6 +1728,45 @@ def _openhands_setup_target(self, commit: str) -> Path: return legacy_dir return self.setup_root / "swe_openhands_setup" / _repo_slug(self.config.agent_framework_repo) / commit + def _probe_openhands_venv(self, openhands_dir: Path) -> Optional[str]: + """Probe an existing OpenHands venv's deps; return None if healthy, else the failure.""" + venv_python = openhands_dir / ".venv" / "bin" / "python" + probe = subprocess_run( + [ + str(venv_python), + "-c", + ( + "from importlib.metadata import version; " + "from packaging.version import Version; " + "assert Version(version('jinja2')) >= Version('3.1.3'); " + "assert Version(version('pyjwt')) >= Version('2.9'); " + "assert Version(version('sqlalchemy')) >= Version('2.0.40'); " + "assert Version(version('flask')) >= Version('2.2'); " + "import datasets, wandb" + ), + ], + capture_output=True, + text=True, + env={**os.environ, "PYTHONNOUSERSITE": "1"}, + ) + if probe.returncode == 0: + return None + return " ".join(probe.stderr.strip().splitlines()[-1:]) or f"exit code {probe.returncode}" + + def _existing_setup_is_reusable(self, openhands_dir: Path) -> bool: + """Whether a pre-existing OpenHands checkout can be reused as-is.""" + if not (openhands_dir.exists() and Path(openhands_dir / ".venv" / "bin" / "python").exists()): + return False + probe_failure = self._probe_openhands_venv(openhands_dir) + if probe_failure is not None: + print( + f"OpenHands venv at {openhands_dir} failed its runtime probe ({probe_failure}); " + "rebuilding the setup instead of reusing it", + flush=True, + ) + return False + return True + def setup(self) -> Path: repo = self.config.agent_framework_repo ref = self.config.agent_framework_commit @@ -1751,7 +1790,7 @@ def setup(self) -> Path: openhands_dir = setup_dir / "OpenHands" miniforge_dir = setup_dir / "miniforge3" - if self._openhands_tree_valid(setup_dir): + if self._existing_setup_is_reusable(openhands_dir): print(f"OpenHands already set up at {setup_dir}", flush=True) self._sync_openhands_to_commit(openhands_dir, commit) return setup_dir diff --git a/responses_api_agents/swe_agents/setup_scripts/openhands.sh b/responses_api_agents/swe_agents/setup_scripts/openhands.sh index 836c161c8ec..6a8dc77d28b 100755 --- a/responses_api_agents/swe_agents/setup_scripts/openhands.sh +++ b/responses_api_agents/swe_agents/setup_scripts/openhands.sh @@ -163,10 +163,10 @@ done echo "Installing Python dependencies (creating .venv in OpenHands directory)..." poetry install --no-interaction --no-root -# Install datasets package +# Install datasets package. echo "Installing datasets package..." -poetry run python -m pip install datasets huggingface_hub packaging==26.0 +poetry run python -m pip install datasets huggingface_hub packaging==26.0 wandb mkdir -p evaluation/oh mkdir -p logs diff --git a/responses_api_agents/swe_agents/tests/test_app.py b/responses_api_agents/swe_agents/tests/test_app.py index b834b054cd5..d69cd70ae7b 100644 --- a/responses_api_agents/swe_agents/tests/test_app.py +++ b/responses_api_agents/swe_agents/tests/test_app.py @@ -1211,6 +1211,41 @@ def test_get_run_command_no_replay_messages_omits_replay_file(self) -> None: assert not (config.persistent_dir / "replay_messages.json").exists() assert "replay_messages.json" not in self._read_agent_script(config) + def _reuse_processor(self) -> OpenHandsHarnessProcessor: + return OpenHandsHarnessProcessor(config=_minimal_server_config()) + + def _fake_openhands_dir(self, tmp_path: Path, probe_script: str) -> Path: + openhands_dir = tmp_path / "OpenHands" + bin_dir = openhands_dir / ".venv" / "bin" + bin_dir.mkdir(parents=True) + python = bin_dir / "python" + python.write_text(probe_script) + python.chmod(0o755) + return openhands_dir + + @pytest.mark.parametrize( + ("probe_script", "reusable"), + [ + (None, False), # no setup at all + ("#!/bin/sh\nexit 0\n", True), # healthy venv + ("#!/bin/sh\nexit 1\n", False), # interpreter exists but the probe fails -> rebuild + ], + ) + def test_setup_reuse_gate(self, tmp_path, probe_script, reusable) -> None: + """A pre-existing setup is reused only when its venv passes the + runtime probe: publishing a corpse makes every subsequent episode + fail after the agent has done its real work.""" + openhands_dir = self._fake_openhands_dir(tmp_path, probe_script) if probe_script else tmp_path / "OpenHands" + + assert self._reuse_processor()._existing_setup_is_reusable(openhands_dir) is reusable + + def test_probe_reports_the_last_stderr_line(self, tmp_path) -> None: + openhands_dir = self._fake_openhands_dir( + tmp_path, "#!/bin/sh\necho 'ModuleNotFoundError: No module named wandb' >&2\nexit 1\n" + ) + failure = self._reuse_processor()._probe_openhands_venv(openhands_dir) + assert failure is not None and "ModuleNotFoundError" in failure + ######################################## # Workspace path + user-message resolver tests From 6bc4eb72f01391dcb691ec4094ae16bf01583ccf Mon Sep 17 00:00:00 2001 From: Teodor-Dumitru Ene Date: Thu, 1 Oct 2026 21:08:51 -0500 Subject: [PATCH 2/2] Don't rebuild a pre-staged OpenHands tree Signed-off-by: Teodor-Dumitru Ene --- responses_api_agents/swe_agents/app.py | 16 +++++++++++++--- .../swe_agents/tests/test_app.py | 6 ++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/responses_api_agents/swe_agents/app.py b/responses_api_agents/swe_agents/app.py index da5922e95ae..3eb63b7b006 100644 --- a/responses_api_agents/swe_agents/app.py +++ b/responses_api_agents/swe_agents/app.py @@ -1723,7 +1723,11 @@ def _openhands_setup_target(self, commit: str) -> Path: # run configured with a different ref could `reset --hard` the shared # tree while containers still execute from it. legacy_dir = self.parent_dir / "swe_openhands_setup" - if legacy_dir.exists() and self._cache_dir_is_default() and self._openhands_tree_valid(legacy_dir): + if ( + legacy_dir.exists() + and self._cache_dir_is_default() + and self._existing_setup_is_reusable(legacy_dir / "OpenHands") + ): print(f"Using pre-staged setup tree at {legacy_dir}", flush=True) return legacy_dir return self.setup_root / "swe_openhands_setup" / _repo_slug(self.config.agent_framework_repo) / commit @@ -1760,8 +1764,7 @@ def _existing_setup_is_reusable(self, openhands_dir: Path) -> bool: probe_failure = self._probe_openhands_venv(openhands_dir) if probe_failure is not None: print( - f"OpenHands venv at {openhands_dir} failed its runtime probe ({probe_failure}); " - "rebuilding the setup instead of reusing it", + f"OpenHands venv at {openhands_dir} failed its runtime probe ({probe_failure}); not reusing it", flush=True, ) return False @@ -1780,6 +1783,13 @@ def setup(self) -> Path: ): # Baked tree with no remote to resolve `ref` against: use it as # shipped (the pre-resolution local-HEAD sync was a no-op here). + probe_failure = self._probe_openhands_venv(legacy_dir / "OpenHands") + if probe_failure is not None: + print( + f"WARNING: pre-staged OpenHands venv at {legacy_dir} failed its runtime probe " + f"({probe_failure}); using it anyway because there is no remote to rebuild from", + flush=True, + ) print(f"Using pre-staged setup tree at {legacy_dir} (no resolvable remote)", flush=True) return legacy_dir diff --git a/responses_api_agents/swe_agents/tests/test_app.py b/responses_api_agents/swe_agents/tests/test_app.py index d69cd70ae7b..b0d89676634 100644 --- a/responses_api_agents/swe_agents/tests/test_app.py +++ b/responses_api_agents/swe_agents/tests/test_app.py @@ -605,10 +605,12 @@ def _patched_roots(self, cache_root: Path, legacy_root: Path, config_dict=None) stack.enter_context(patch.object(swe_app, "maybe_get_global_config_dict", return_value=config_dict)) return stack - def _make_valid_legacy_tree(self, legacy_root: Path) -> Path: + def _make_valid_legacy_tree(self, legacy_root: Path, probe_script: str = "#!/bin/sh\nexit 0\n") -> Path: legacy_setup = legacy_root / "swe_openhands_setup" (legacy_setup / "OpenHands" / ".venv" / "bin").mkdir(parents=True) - (legacy_setup / "OpenHands" / ".venv" / "bin" / "python").touch() + python = legacy_setup / "OpenHands" / ".venv" / "bin" / "python" + python.write_text(probe_script) + python.chmod(0o755) return legacy_setup def test_target_keys_by_repo_identity_and_commit(self, tmp_path: Path) -> None: