Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -476,8 +476,11 @@ explains it.
pauses the continuation turns it into an ordinary limit resume. Cut off a
second time, even after such a pause, the task is retired. A
task deleted meanwhile is not continued.
Recurring tasks start their next occurrence fresh as before, and scripts
are not continued.
This needs a session the harness itself named: one it reported before the
cut, or a Claude Code transcript on disk. Without one (a Codex or opencode
run taken back or ended while DispatchQ was down, a harness stopped before
it started), the task is retired as before. Recurring tasks start their
next occurrence fresh as before, and scripts are not continued.
- **Runs survive a daemon restart.** A run's process does not stop when the
daemon does (an update, a crash). The next daemon takes the run back: it
shows as *running* with its process id, its task is not started a second
Expand Down
29 changes: 21 additions & 8 deletions internal/engine/adopt.go
Original file line number Diff line number Diff line change
Expand Up @@ -222,8 +222,8 @@ func (e *Engine) Recover() (adopted []store.Run, interrupted int, err error) {
// recorded and notified, but must not retire the task or plan a resume
// under the run that came after it.
current := !tasks[rec.TaskID] && !rec.StartedAt.Before(latest[rec.TaskID])
e.settleEnded(rec, current)
if _, ok := goneRuns[rec.RunID]; ok {
recorded := e.settleEnded(rec, current)
if _, ok := goneRuns[rec.RunID]; ok && recorded {
e.clearLive(rec.RunID)
}
interrupted++
Expand Down Expand Up @@ -286,18 +286,31 @@ func latestStarts(history, ended []store.Run) map[string]time.Time {
//
// current says whether the run is its task's latest, the one whose end the
// task's schedule follows; for any other only the record is written.
func (e *Engine) settleEnded(rec store.Run, current bool) {
//
// It reports whether the outcome was written or kept (see conclude).
func (e *Engine) settleEnded(rec store.Run, current bool) bool {
t := e.taskOf(rec)
rec.PID = 0
how := outcome{keepSchedule: !current}
res, ok := e.sessionOutcome(rec)
if !ok {
res = provider.Result{Status: store.StatusFailed, SessionID: rec.SessionID, Message: store.InterruptedError}
res = unknownEnd(res, rec, store.InterruptedError)
how.interrupted = true
}
fmt.Fprintf(os.Stdout, "dispatchqd: run %s (%s) ended while no daemon watched it: %s\n",
rec.RunID, rec.TaskName, endedText(res))
e.conclude(t, rec.Provider.ID, rec, res, nil, how)
return e.conclude(t, rec.Provider.ID, rec, res, nil, how)
}

// unknownEnd is the outcome of a run whose end its transcript could not tell:
// a failure with reason, keeping what the reader did find out (that the
// harness knows the session, say).
func unknownEnd(read provider.Result, rec store.Run, reason string) provider.Result {
read.Status, read.Message = store.StatusFailed, reason
if read.SessionID == "" {
read.SessionID = rec.SessionID
}
return read
}

// endedText is how the daemon log names the outcome of a run that ended
Expand Down Expand Up @@ -492,12 +505,12 @@ func (e *Engine) watchAdopted(ctx context.Context, t task.Task, providerID strin
// Without an outcome to read, the run counts as cut off, as a run that
// ended before the restart does (see settleEnded): its one-shot task
// continues the session once instead of being retired half done.
if res, ok := e.sessionOutcome(rec); ok {
res, ok := e.sessionOutcome(rec)
if ok {
e.finish(t, providerID, rec, res, nil)
return
}
res := provider.Result{Status: store.StatusFailed, SessionID: rec.SessionID, Message: adoptedEndMessage}
e.end(t, providerID, rec, res, nil, true)
e.end(t, providerID, rec, unknownEnd(res, rec, adoptedEndMessage), nil, true)
}

// letGo gives back a taken-back run's slot without recording an outcome, for a
Expand Down
4 changes: 4 additions & 0 deletions internal/engine/adopt_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,10 @@ func daemonDies(t *testing.T) (*Engine, *store.Store, *fakeProcs, *stub) {
// four seconds later. It must take the run back instead.
func TestRecoverTakesBackRunThatOutlivedTheDaemon(t *testing.T) {
e, st, procs, second := daemonDies(t)
// Its transcript exists but cannot say how the turn ended.
e.SetSessionReader(func(rec store.Run) (provider.Result, bool) {
return provider.Result{SessionID: rec.SessionID, SessionConfirmed: true}, false
})

adopted, interrupted, err := e.Recover()
if err != nil {
Expand Down
37 changes: 29 additions & 8 deletions internal/engine/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -1362,12 +1362,17 @@ func (e *Engine) end(t task.Task, providerID string, rec store.Run, res provider
}
e.mu.Unlock()
e.awake.release()
e.clearLive(rec.RunID)

// A run that failed because the daemon stopped it on its way down (the
// shutdown grace ran out) was cut off by DispatchQ, not by its own work.
interrupted := runErr == nil && !wasCanceled && res.Status == store.StatusFailed && (cutOff || e.runCtx.Err() != nil)
e.conclude(t, providerID, rec, res, runErr, outcome{canceled: wasCanceled, takeover: takeover, interrupted: interrupted})
// The process record goes only once the outcome is written or kept: for a
// quiet-history run it is the only trace, and a daemon killed in between,
// or a history that took nothing, must leave the run for the next start
// to record (see Recover).
if e.conclude(t, providerID, rec, res, runErr, outcome{canceled: wasCanceled, takeover: takeover, interrupted: interrupted}) {
e.clearLive(rec.RunID)
}
}

// outcome is what finish knows about how a run ended beyond its result.
Expand All @@ -1389,7 +1394,10 @@ type outcome struct {
// conclude records how a run ended, updates its task's scheduling state and
// sends the notification. The run holds no slot any more: finish has given it
// back, or the run was never this daemon's (see Recover).
func (e *Engine) conclude(t task.Task, providerID string, rec store.Run, res provider.Result, runErr error, how outcome) {
//
// It reports whether the outcome reached history or was kept for it (see
// store.ErrRunKept): only then may the run's process record go.
func (e *Engine) conclude(t task.Task, providerID string, rec store.Run, res provider.Result, runErr error, how outcome) (recorded bool) {
wasCanceled, takeover := how.canceled, how.takeover
finished := e.clock.Now()
if !res.EndedAt.IsZero() && res.EndedAt.Before(finished) {
Expand Down Expand Up @@ -1469,7 +1477,7 @@ func (e *Engine) conclude(t task.Task, providerID string, rec store.Run, res pro
return nil
})
default:
if how.interrupted && e.resumeAfterInterrupt(t, rec, providerID) {
if how.interrupted && e.resumeAfterInterrupt(t, rec, providerID, res.SessionConfirmed) {
rec.Error += "; " + resumeAfterInterruptNote
} else {
e.retire(t.ID, oneShotTrigger(t.Trigger))
Expand All @@ -1489,8 +1497,10 @@ func (e *Engine) conclude(t task.Task, providerID string, rec store.Run, res pro
// the default Log and never unread (see store.Run.Quiet). Only a success
// is; every other outcome is recorded like any other run's.
rec.Quiet = quietSuccess(t, rec.Status)
recorded = true
if err := e.store.AppendRun(rec); err != nil {
fmt.Fprintf(os.Stderr, "dispatchqd: run %s: record outcome %s: %v\n", rec.RunID, rec.Status, err)
recorded = errors.Is(err, store.ErrRunKept)
}

// Bound disk usage: drop expired quiet runs first, then prune old
Expand Down Expand Up @@ -1525,6 +1535,7 @@ func (e *Engine) conclude(t task.Task, providerID string, rec store.Run, res pro

// Notify outside any lock so channel I/O never blocks other finishing runs.
e.notifyOutcome(t, rec, res.FinalOutput)
return recorded
}

// resumeAfterInterruptNote is added to the reason of a run whose session
Expand All @@ -1538,9 +1549,15 @@ const resumeAfterInterruptNote = "DispatchQ continues its session once when it r
// own record of what it did, which is what a rate-limit pause does too. A
// session that was already continuing after an interruption is not given a
// third go: whatever keeps cutting it off is not going to stop, and the task
// is retired like any other ended run. Recurring tasks, scripts and runs
// without a session keep their old behaviour.
func (e *Engine) resumeAfterInterrupt(t task.Task, rec store.Run, providerID string) bool {
// is retired like any other ended run. Recurring tasks and scripts keep their
// old behaviour.
//
// Only a session the harness knows can be continued: one it named on this
// run (named), or the waiting session this run resumed. An id DispatchQ
// assigned that the harness never took up (Codex picks its own; a harness
// killed before it started has none) would resume nothing, so without one the
// task is retired as before.
func (e *Engine) resumeAfterInterrupt(t task.Task, rec store.Run, providerID string, named bool) bool {
if !oneShotTrigger(t.Trigger) || t.IsScript() || rec.SessionID == "" {
return false
}
Expand All @@ -1560,7 +1577,11 @@ func (e *Engine) resumeAfterInterrupt(t task.Task, rec store.Run, providerID str
if st.IsCompletedOnce(t.ID) && st.IsRemovalOwed(t.ID) {
return nil
}
if p, ok := st.PendingResume(t.ID); ok && p.InterruptedRun != "" {
p, waiting := st.PendingResume(t.ID)
if !named && (!waiting || p.SessionID != rec.SessionID) {
return nil
}
if waiting && p.InterruptedRun != "" {
// The same run met again (its end was not recorded before the
// daemon stopped) is still the first interruption; any other run
// is the second.
Expand Down
5 changes: 4 additions & 1 deletion internal/engine/engine_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -457,6 +457,9 @@ func TestGracefulShutdownLetsRunFinish(t *testing.T) {
type ctxStub struct {
mu sync.Mutex
started int
// unconfirmed reports the run cut off before the harness named its
// session.
unconfirmed bool
}

func (s *ctxStub) Run(ctx context.Context, req executor.Request) (provider.Result, error) {
Expand All @@ -465,7 +468,7 @@ func (s *ctxStub) Run(ctx context.Context, req executor.Request) (provider.Resul
s.mu.Unlock()
<-ctx.Done()
return provider.Result{
Status: store.StatusFailed, SessionID: req.SessionID, ExitCode: -1,
Status: store.StatusFailed, SessionID: req.SessionID, SessionConfirmed: !s.unconfirmed, ExitCode: -1,
Message: "run was interrupted before completing (the process was terminated — e.g. the daemon stopped)",
}, nil
}
Expand Down
74 changes: 74 additions & 0 deletions internal/engine/records_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ func TestARefusedStartRecordLeavesNoPhantomRun(t *testing.T) {
fc := clock.NewFake(time.Date(2026, 11, 11, 11, 13, 2, 0, time.UTC))
r := &stub{}
e, st := newTestEngine(t, r, fc)
st.SetRetrySleep(func(time.Duration) {})
saveTasks(t, st, asapTask("a", false))
unblock := refuseHistoryWrites(t, st)

Expand Down Expand Up @@ -154,6 +155,7 @@ func TestARefusedStartRecordLeavesNoPhantomRun(t *testing.T) {
func TestCancelResumeSucceedsWhenItsRecordIsKept(t *testing.T) {
fc := clock.NewFake(time.Date(2026, 7, 17, 22, 0, 0, 0, time.UTC))
e, st, _ := rateLimitedTask(t, asapTask("a", false), fc)
st.SetRetrySleep(func(time.Duration) {})
unblock := refuseHistoryWrites(t, st)
defer unblock()

Expand All @@ -164,3 +166,75 @@ func TestCancelResumeSucceedsWhenItsRecordIsKept(t *testing.T) {
t.Fatal("the resume is still planned")
}
}

// The review finding: a run's process record was cleared before its outcome
// was written, so a quiet-history run whose outcome history could not take
// (or a daemon killed in between) left no trace at all. The process record
// stays until the outcome is written or kept: a start that cannot write it
// either keeps it too, and the first start that can records the run.
func TestAQuietRunWhoseOutcomeCannotBeWrittenIsRecordedAtTheNextStart(t *testing.T) {
fc := clock.NewFake(time.Date(2026, 11, 11, 11, 13, 2, 0, time.UTC))
procs := newFakeProcs()
first := &startedStub{pid: 50311, procs: procs, block: make(chan struct{})}
e, st := newTestEngine(t, first, fc)
st.SetRetrySleep(func(time.Duration) {})
e.procs = procs
tk := asapTask("watch", false)
tk.QuietHistory = true
saveTasks(t, st, tk)
if err := e.Tick(context.Background()); err != nil {
t.Fatal(err)
}
waitFor(t, func() bool {
s, _ := st.LoadState()
return len(s.LiveRuns) == 1
})

// History refuses the outcome, and there is no room to keep it either.
unblock := refuseHistoryWrites(t, st)
runs := filepath.Join(st.Home(), "runs")
if err := os.Chmod(runs, 0o555); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.Chmod(runs, 0o755) })
close(first.block)
e.WaitIdle()
if s, _ := st.LoadState(); len(s.LiveRuns) != 1 {
t.Fatal("the process record went although the outcome was written nowhere")
}
procs.exit(50311)

// restart is a daemon start in a new process: a store with nothing in
// memory, on the same data directory.
restart := func() *store.Store {
t.Helper()
next, err := store.Open(st.Home())
if err != nil {
t.Fatal(err)
}
next.SetRetrySleep(func(time.Duration) {})
d, _ := newTestEngine(t, &stub{}, fc)
d.store = next
d.procs = procs
if _, _, err := d.Recover(); err != nil {
t.Fatal(err)
}
return next
}
// A start that cannot write the outcome either keeps the process record.
if s, _ := restart().LoadState(); len(s.LiveRuns) != 1 {
t.Fatal("a start that could not record the run forgot its process record")
}

unblock()
if err := os.Chmod(runs, 0o755); err != nil {
t.Fatal(err)
}
next := restart()
if run := latestRun(t, next, "run-1"); run.Status != store.StatusFailed || run.Quiet {
t.Fatalf("run = %s quiet=%v, want a visible failed row", run.Status, run.Quiet)
}
if s, _ := next.LoadState(); len(s.LiveRuns) != 0 {
t.Fatal("the process record stayed after the run was recorded")
}
}
Loading
Loading