From 0d459c08889d0fa7893270fb11226363bed55f65 Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Fri, 9 Oct 2026 19:04:40 -0400 Subject: [PATCH 1/4] An author session that stops with nothing staged is resumed, not ended A session that ended without a syscall (an empty reply after long reasoning, or narration with no command) ended its run as 'ended without a submit', discarding the open run and any unsubmitted result. The kernel now resumes the session with a note that nothing ran, up to three times per leg. A stop after a kernel note, a gate verdict, or with a PR open is still the agent's answer, and end still ends the run. --- CHANGELOG.md | 6 +++++ src/outerloop/orchestrator.py | 46 ++++++++++++++++++++++++++++++-- tests/test_attempt.py | 6 ++++- tests/test_orchestrator.py | 49 ++++++++++++++++++++++++++++++++++- 4 files changed, 103 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96ae9bcc..d4fb9046 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ Versions follow [SemVer](https://semver.org). ### Fixed +- An author session that stops without staging anything (an empty reply after + long reasoning, or a line of narration with no command) no longer ends its + run as "ended without a submit". The kernel resumes the session with a note + that nothing ran and the run is still open, up to three times per leg. A stop + after a kernel note, a gate verdict, or with a PR open is still taken as the + agent's answer, and `end` still ends the run. - A Codex session that recovers from a transient error no longer ends its run as aborted. Codex reports a dropped or timed-out model stream as an `error` event ("Reconnecting... 1/5") and carries on; the kernel now counts an error diff --git a/src/outerloop/orchestrator.py b/src/outerloop/orchestrator.py index 40af5291..69d9e4a7 100644 --- a/src/outerloop/orchestrator.py +++ b/src/outerloop/orchestrator.py @@ -74,6 +74,8 @@ EVAL_TIMEOUT_S = 1800 MAX_REPORT_BODY = 20_000 +# times one leg of a session is resumed after stopping with nothing staged +SILENT_STOP_RESUMES = 3 # Environment keys the evaluator manages itself; a contract's seed_env may @@ -1633,6 +1635,11 @@ def _ack(messages: list[Message]) -> None: baseline_note = "" measured: tuple[str, ...] = () refused_once = False + # A session that stops without staging anything (an empty reply, a line of + # narration) is resumed with a note rather than taken as the end of the run. + # Bounded so a model that keeps stopping still ends. + silent_stops = 0 + kernel_spoke = False # a kernel note this leg makes the next stop an answer # Rejection protects stop/error paths; it is never a refusal limit. tree_rejected = False scope_refused = any( @@ -1651,7 +1658,8 @@ def _ack(messages: list[Message]) -> None: def _resume(message: Message, *, allow_checkpoint: bool = True) -> AttemptResult | None: """Deliver pending messages; return an ending only if the resume fails.""" - nonlocal session + nonlocal session, kernel_spoke + kernel_spoke = kernel_spoke or not message.key.startswith("silent-stop:") if on_meter is not None: on_meter(launches_used, sleeps_used, gpu_hours_used) append(inbox_dir, message) @@ -1769,7 +1777,41 @@ def _not_run_note(request: SyscallRequest | None) -> str: read_error = f"unhonorable syscall request: {exc}" request = SyscallRequest(launches=(), problem=read_error) if request is None: - break + # After a kernel note, a verdict or with a PR open, stopping is + # a considered answer; only a plain silent stop is resumed. + if ( + launcher is None + or on_stop is not None + or kernel_spoke + or scope_refused + or failed_gate is not None + or silent_stops >= SILENT_STOP_RESUMES + or not _can_resume() + ): + break + silent_stops += 1 + failed = _resume( + Message( + 0, + "note", + "kernel", + inbox_thread, + time.time(), + f"silent-stop:{session.session_id}:{inbox_seq}:{silent_stops}", + { + "text": ( + "Your session ended without a syscall: nothing was staged " + "(no launch, sleep, submit or end), so nothing ran and your " + "run is still open. Continue the work. If you are done, " + "stage `end --report FILE` and end your turn." + ), + }, + origin=inbox_dir.name, + ) + ) + if failed is not None: + return failed + continue # Refresh ancestry before deriving the paths admitted below. no_backend = ( "sleep is not available here: this run has no compute backend for " diff --git a/tests/test_attempt.py b/tests/test_attempt.py index 1d0be644..17671869 100644 --- a/tests/test_attempt.py +++ b/tests/test_attempt.py @@ -7538,7 +7538,11 @@ def successful_resume(self, brief, workspace, resume_session_id=None): resume_run( state, run_id, harness=HermesHarness(api_key="key", repo_dir=tmp_path / "hermes"), **kwargs ) - assert resumed == ["s1"] + from outerloop.orchestrator import SILENT_STOP_RESUMES + + # with nothing staged and no PR, a stop is resumed with a note (bounded) + silent = not (pr or sleep_again) + assert resumed == ["s1"] * (1 + (SILENT_STOP_RESUMES if silent else 0)) saved = load_record(state, run_id) assert "hermes_resume_required_chars" not in saved.stage if sleep_again: diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 068b5493..a7ba8c8d 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -3145,7 +3145,8 @@ def run(self, text, workspace, resume_session_id=None): if self.retry_launch: _write_syscall(workspace, {"launches": [{"name": "probe", "command": "true"}]}) else: - assert "Capacity may now be free" in text + # the capacity wake, then the kernel's notes after each silent stop + assert "Capacity may now be free" in text or "ended without a syscall" in text return ok_session() def launch(sha, request): @@ -3289,3 +3290,49 @@ def test_legacy_suite_row_and_rule_reports(): ) assert rule in result.report(CONFIG) assert rule in pr_body(result, CONFIG, redact_secrets=()) + + +def test_silent_stop_is_resumed_and_an_explicit_end_ends_the_run(tmp_path): + from outerloop.syscall_cli import main + + class Author: + supports_resume = True + + def __init__(self): + self.calls = [] + + def run(self, brief_text, workspace, resume_session_id=None): + self.calls.append((brief_text, resume_session_id)) + if len(self.calls) == 2: + assert "ended without a syscall" in brief_text + assert main(["end"], root=workspace) == 0 + return ok_session() + + author = Author() + result, _, evaluator = run_climb(tmp_path, [], harness=author, launcher=_fake_launcher([])) + assert len(author.calls) == 2 and author.calls[1][1] == "s1" + assert result.outcome == "no-improvement" + assert not evaluator.calls + + +def test_silent_stops_are_bounded(tmp_path): + from outerloop.orchestrator import SILENT_STOP_RESUMES + + result, harness, _ = run_climb(tmp_path, [], launcher=_fake_launcher([])) + assert len(harness.calls) == 1 + SILENT_STOP_RESUMES + assert all("ended without a syscall" in call[0] for call in harness.calls[1:]) + assert result.outcome == "no-improvement" and result.note == "ended without a submit" + + +def test_silent_stop_with_an_open_pr_still_parks_for_review(tmp_path): + from outerloop.orchestrator import AttemptResult + + stopped = [] + + def on_stop(session): + stopped.append(session) + return AttemptResult(outcome="review", session=session) + + result, harness, _ = run_climb(tmp_path, [], launcher=_fake_launcher([]), on_stop=on_stop) + assert len(harness.calls) == 1 and len(stopped) == 1 + assert result.outcome == "review" From 1eb0f21f4059c862738364af1fd91cdc6fed731a Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Fri, 9 Oct 2026 19:29:17 -0400 Subject: [PATCH 2/4] A refusal delivered with the wake (such as a capacity note) makes a stop the author's answer The silent-stop guard only saw notes sent during the leg; a capacity refusal delivered with the wake was missed, so a stop answering it was nudged. The guard now reads refusals pending from the leg's starting cursor; pacing notes and launch results still allow the nudge. The capacity-wake test pins that no nudge follows. --- src/outerloop/orchestrator.py | 8 +++++++- tests/test_orchestrator.py | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/outerloop/orchestrator.py b/src/outerloop/orchestrator.py index 69d9e4a7..6dbf6ac8 100644 --- a/src/outerloop/orchestrator.py +++ b/src/outerloop/orchestrator.py @@ -1379,6 +1379,7 @@ def attempt_once( contract = load_contract(contract_text, config.target) bench = contract.benchmark(config.benchmark) spec = spec or author_spec() + leg_start_seq = inbox_seq # the cursor before this leg delivers anything if not spec.execution.can_execute: raise ValueError("attempt_once runs an editing role; the spec must allow execution") if not spec.scope: @@ -1639,7 +1640,12 @@ def _ack(messages: list[Message]) -> None: # narration) is resumed with a note rather than taken as the end of the run. # Bounded so a model that keeps stopping still ends. silent_stops = 0 - kernel_spoke = False # a kernel note this leg makes the next stop an answer + # A refusal delivered with this leg (or sent during it) makes the next stop + # an answer; pacing notes and launch results do not. + kernel_spoke = any( + m.source == "kernel" and m.key.startswith(("refusal:", "message-refused:")) + for m in pending_messages(inbox_dir, leg_start_seq) + ) # Rejection protects stop/error paths; it is never a refusal limit. tree_rejected = False scope_refused = any( diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index a7ba8c8d..ee4171b1 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -3145,8 +3145,8 @@ def run(self, text, workspace, resume_session_id=None): if self.retry_launch: _write_syscall(workspace, {"launches": [{"name": "probe", "command": "true"}]}) else: - # the capacity wake, then the kernel's notes after each silent stop - assert "Capacity may now be free" in text or "ended without a syscall" in text + # a stop after the capacity note is the author's answer: no nudge + assert "Capacity may now be free" in text return ok_session() def launch(sha, request): From 278d48a73cebdcc691145db75c4eb17c66212c7f Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Fri, 9 Oct 2026 22:13:39 -0400 Subject: [PATCH 3/4] Every kernel refusal key makes a silent stop the author's answer refused: (stale submit), capacity-refusal: and publish-refused: notes were not recognized, so a stop answering them was nudged. The keys live in one tuple, and a test covers each. --- src/outerloop/orchestrator.py | 4 +++- tests/test_orchestrator.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/outerloop/orchestrator.py b/src/outerloop/orchestrator.py index 6dbf6ac8..bc9d89b7 100644 --- a/src/outerloop/orchestrator.py +++ b/src/outerloop/orchestrator.py @@ -76,6 +76,8 @@ MAX_REPORT_BODY = 20_000 # times one leg of a session is resumed after stopping with nothing staged SILENT_STOP_RESUMES = 3 +# inbox keys of kernel notes that refuse something; a stop after one is an answer +REFUSAL_KEYS = ("refusal:", "refused:", "capacity-refusal:", "message-refused:", "publish-refused:") # Environment keys the evaluator manages itself; a contract's seed_env may @@ -1643,7 +1645,7 @@ def _ack(messages: list[Message]) -> None: # A refusal delivered with this leg (or sent during it) makes the next stop # an answer; pacing notes and launch results do not. kernel_spoke = any( - m.source == "kernel" and m.key.startswith(("refusal:", "message-refused:")) + m.source == "kernel" and m.key.startswith(REFUSAL_KEYS) for m in pending_messages(inbox_dir, leg_start_seq) ) # Rejection protects stop/error paths; it is never a refusal limit. diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index ee4171b1..b0a7d430 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -3336,3 +3336,19 @@ def on_stop(session): result, harness, _ = run_climb(tmp_path, [], launcher=_fake_launcher([]), on_stop=on_stop) assert len(harness.calls) == 1 and len(stopped) == 1 assert result.outcome == "review" + + +@pytest.mark.parametrize( + "key", ["refusal:", "refused:", "capacity-refusal:", "message-refused:", "publish-refused:"] +) +def test_a_stop_after_a_delivered_refusal_is_not_nudged(tmp_path, key): + import time + + from outerloop.inbox import Message, append + + inbox = tmp_path.parent / (tmp_path.name + "-run") + inbox.mkdir(exist_ok=True) + append(inbox, Message(0, "note", "kernel", "", time.time(), key + "x", {"text": "refused"})) + result, harness, _ = run_climb(tmp_path, [], launcher=_fake_launcher([])) + assert len(harness.calls) == 1 + assert result.outcome == "no-improvement" From 2a47e7499faa001dd746c58b97690749b8108d91 Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Fri, 9 Oct 2026 22:27:30 -0400 Subject: [PATCH 4/4] Silent-stop recovery is not disabled by a scope refusal from an earlier leg scope_refused reads the whole inbox, so one answered scope refusal turned the nudge off for the rest of the run. Refusals delivered with or during the current leg already make a stop an answer; the history-wide flag is dropped from the condition. --- src/outerloop/orchestrator.py | 1 - tests/test_orchestrator.py | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/outerloop/orchestrator.py b/src/outerloop/orchestrator.py index bc9d89b7..ddf91033 100644 --- a/src/outerloop/orchestrator.py +++ b/src/outerloop/orchestrator.py @@ -1791,7 +1791,6 @@ def _not_run_note(request: SyscallRequest | None) -> str: launcher is None or on_stop is not None or kernel_spoke - or scope_refused or failed_gate is not None or silent_stops >= SILENT_STOP_RESUMES or not _can_resume() diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index b0a7d430..92bfad71 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -3352,3 +3352,19 @@ def test_a_stop_after_a_delivered_refusal_is_not_nudged(tmp_path, key): result, harness, _ = run_climb(tmp_path, [], launcher=_fake_launcher([])) assert len(harness.calls) == 1 assert result.outcome == "no-improvement" + + +def test_a_refusal_from_an_earlier_leg_does_not_disable_the_nudge(tmp_path): + import time + + from outerloop.inbox import Message, append + from outerloop.orchestrator import SILENT_STOP_RESUMES + + inbox = tmp_path.parent / (tmp_path.name + "-run") + inbox.mkdir(exist_ok=True) + note = {"text": "Refused: out-of-scope", "quoted_text": "out-of-scope paths: x"} + append(inbox, Message(0, "note", "kernel", "", time.time(), "refusal:old", note)) + # the refusal was delivered on an earlier leg: this leg starts past it + result, harness, _ = run_climb(tmp_path, [], launcher=_fake_launcher([]), inbox_seq=1) + assert len(harness.calls) == 1 + SILENT_STOP_RESUMES + assert result.outcome == "no-improvement"