Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@ Versions follow [SemVer](https://semver.org).

### Fixed

- An author session that stops without staging anything (an empty reply after
long reasoning, or a line of narration with no command) no longer ends its
run as "ended without a submit". The kernel resumes the session with a note
that nothing ran and the run is still open, up to three times per leg. A stop
after a kernel note, a gate verdict, or with a PR open is still taken as the
agent's answer, and `end` still ends the run.
- A Codex session that recovers from a transient error no longer ends its run
as aborted. Codex reports a dropped or timed-out model stream as an `error`
event ("Reconnecting... 1/5") and carries on; the kernel now counts an error
Expand Down
53 changes: 51 additions & 2 deletions src/outerloop/orchestrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,10 @@

EVAL_TIMEOUT_S = 1800
MAX_REPORT_BODY = 20_000
# times one leg of a session is resumed after stopping with nothing staged
SILENT_STOP_RESUMES = 3
# inbox keys of kernel notes that refuse something; a stop after one is an answer
REFUSAL_KEYS = ("refusal:", "refused:", "capacity-refusal:", "message-refused:", "publish-refused:")


# Environment keys the evaluator manages itself; a contract's seed_env may
Expand Down Expand Up @@ -1377,6 +1381,7 @@ def attempt_once(
contract = load_contract(contract_text, config.target)
bench = contract.benchmark(config.benchmark)
spec = spec or author_spec()
leg_start_seq = inbox_seq # the cursor before this leg delivers anything
if not spec.execution.can_execute:
raise ValueError("attempt_once runs an editing role; the spec must allow execution")
if not spec.scope:
Expand Down Expand Up @@ -1633,6 +1638,16 @@ def _ack(messages: list[Message]) -> None:
baseline_note = ""
measured: tuple[str, ...] = ()
refused_once = False
# A session that stops without staging anything (an empty reply, a line of
# narration) is resumed with a note rather than taken as the end of the run.
# Bounded so a model that keeps stopping still ends.
silent_stops = 0
# A refusal delivered with this leg (or sent during it) makes the next stop
# an answer; pacing notes and launch results do not.
kernel_spoke = any(
m.source == "kernel" and m.key.startswith(REFUSAL_KEYS)
for m in pending_messages(inbox_dir, leg_start_seq)
)
# Rejection protects stop/error paths; it is never a refusal limit.
tree_rejected = False
scope_refused = any(
Expand All @@ -1651,7 +1666,8 @@ def _ack(messages: list[Message]) -> None:

def _resume(message: Message, *, allow_checkpoint: bool = True) -> AttemptResult | None:
"""Deliver pending messages; return an ending only if the resume fails."""
nonlocal session
nonlocal session, kernel_spoke
kernel_spoke = kernel_spoke or not message.key.startswith("silent-stop:")
if on_meter is not None:
on_meter(launches_used, sleeps_used, gpu_hours_used)
append(inbox_dir, message)
Expand Down Expand Up @@ -1769,7 +1785,40 @@ def _not_run_note(request: SyscallRequest | None) -> str:
read_error = f"unhonorable syscall request: {exc}"
request = SyscallRequest(launches=(), problem=read_error)
if request is None:
break
# After a kernel note, a verdict or with a PR open, stopping is
# a considered answer; only a plain silent stop is resumed.
if (
launcher is None
or on_stop is not None
or kernel_spoke
or failed_gate is not None
or silent_stops >= SILENT_STOP_RESUMES
or not _can_resume()
):
break
silent_stops += 1
failed = _resume(
Message(
0,
"note",
"kernel",
inbox_thread,
time.time(),
f"silent-stop:{session.session_id}:{inbox_seq}:{silent_stops}",
{
"text": (
"Your session ended without a syscall: nothing was staged "
"(no launch, sleep, submit or end), so nothing ran and your "
"run is still open. Continue the work. If you are done, "
"stage `end --report FILE` and end your turn."
),
},
origin=inbox_dir.name,
)
)
if failed is not None:
return failed
continue
# Refresh ancestry before deriving the paths admitted below.
no_backend = (
"sleep is not available here: this run has no compute backend for "
Expand Down
6 changes: 5 additions & 1 deletion tests/test_attempt.py
Original file line number Diff line number Diff line change
Expand Up @@ -7538,7 +7538,11 @@ def successful_resume(self, brief, workspace, resume_session_id=None):
resume_run(
state, run_id, harness=HermesHarness(api_key="key", repo_dir=tmp_path / "hermes"), **kwargs
)
assert resumed == ["s1"]
from outerloop.orchestrator import SILENT_STOP_RESUMES

# with nothing staged and no PR, a stop is resumed with a note (bounded)
Comment thread
renmengye marked this conversation as resolved.
silent = not (pr or sleep_again)
assert resumed == ["s1"] * (1 + (SILENT_STOP_RESUMES if silent else 0))
saved = load_record(state, run_id)
assert "hermes_resume_required_chars" not in saved.stage
if sleep_again:
Expand Down
79 changes: 79 additions & 0 deletions tests/test_orchestrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -3145,6 +3145,7 @@ def run(self, text, workspace, resume_session_id=None):
if self.retry_launch:
_write_syscall(workspace, {"launches": [{"name": "probe", "command": "true"}]})
else:
# a stop after the capacity note is the author's answer: no nudge
assert "Capacity may now be free" in text
return ok_session()

Expand Down Expand Up @@ -3289,3 +3290,81 @@ def test_legacy_suite_row_and_rule_reports():
)
assert rule in result.report(CONFIG)
assert rule in pr_body(result, CONFIG, redact_secrets=())


def test_silent_stop_is_resumed_and_an_explicit_end_ends_the_run(tmp_path):
from outerloop.syscall_cli import main

class Author:
supports_resume = True

def __init__(self):
self.calls = []

def run(self, brief_text, workspace, resume_session_id=None):
self.calls.append((brief_text, resume_session_id))
if len(self.calls) == 2:
assert "ended without a syscall" in brief_text
assert main(["end"], root=workspace) == 0
return ok_session()

author = Author()
result, _, evaluator = run_climb(tmp_path, [], harness=author, launcher=_fake_launcher([]))
assert len(author.calls) == 2 and author.calls[1][1] == "s1"
assert result.outcome == "no-improvement"
assert not evaluator.calls


def test_silent_stops_are_bounded(tmp_path):
from outerloop.orchestrator import SILENT_STOP_RESUMES

result, harness, _ = run_climb(tmp_path, [], launcher=_fake_launcher([]))
assert len(harness.calls) == 1 + SILENT_STOP_RESUMES
assert all("ended without a syscall" in call[0] for call in harness.calls[1:])
assert result.outcome == "no-improvement" and result.note == "ended without a submit"


def test_silent_stop_with_an_open_pr_still_parks_for_review(tmp_path):
from outerloop.orchestrator import AttemptResult

stopped = []

def on_stop(session):
stopped.append(session)
return AttemptResult(outcome="review", session=session)

result, harness, _ = run_climb(tmp_path, [], launcher=_fake_launcher([]), on_stop=on_stop)
assert len(harness.calls) == 1 and len(stopped) == 1
assert result.outcome == "review"


@pytest.mark.parametrize(
"key", ["refusal:", "refused:", "capacity-refusal:", "message-refused:", "publish-refused:"]
)
def test_a_stop_after_a_delivered_refusal_is_not_nudged(tmp_path, key):
import time

from outerloop.inbox import Message, append

inbox = tmp_path.parent / (tmp_path.name + "-run")
inbox.mkdir(exist_ok=True)
append(inbox, Message(0, "note", "kernel", "", time.time(), key + "x", {"text": "refused"}))
result, harness, _ = run_climb(tmp_path, [], launcher=_fake_launcher([]))
assert len(harness.calls) == 1
assert result.outcome == "no-improvement"


def test_a_refusal_from_an_earlier_leg_does_not_disable_the_nudge(tmp_path):
import time

from outerloop.inbox import Message, append
from outerloop.orchestrator import SILENT_STOP_RESUMES

inbox = tmp_path.parent / (tmp_path.name + "-run")
inbox.mkdir(exist_ok=True)
note = {"text": "Refused: out-of-scope", "quoted_text": "out-of-scope paths: x"}
append(inbox, Message(0, "note", "kernel", "", time.time(), "refusal:old", note))
# the refusal was delivered on an earlier leg: this leg starts past it
result, harness, _ = run_climb(tmp_path, [], launcher=_fake_launcher([]), inbox_seq=1)
assert len(harness.calls) == 1 + SILENT_STOP_RESUMES
assert result.outcome == "no-improvement"
Loading