From b12d00df86979752873e1e5d203704227ba0851e Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Thu, 23 Apr 2026 22:28:02 +0000 Subject: [PATCH 1/2] fix(session): prevent amnesia on crash by deferring cursor advance to TurnCompleted Two fixes for session state loss after daemon crashes: 1. Slack cursor race: the thread binding actor persisted its cursor (marking messages as "seen") at enqueue time, before the session actor persisted the completed turn. On crash between those two writes, the message was permanently lost. Now the cursor advances only when TurnCompleted confirms the turn is durably recorded. 2. System prompt eviction: when identity files were missing on recovery, SetSystemPrompt() actively deleted the last-known prompt from state. Now it retains the recovered prompt as a fallback and logs a warning. --- .../Sessions/LlmSessionActor.cs | 8 +++--- .../SlackThreadBindingActor.cs | 27 ++++++++++++++++--- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs index 484409c3f..7827bed49 100644 --- a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs @@ -2197,11 +2197,13 @@ private void SetSystemPrompt() var content = _promptProvider.GetSystemPrompt(); if (string.IsNullOrWhiteSpace(content)) { - // Clear stale prompt from snapshot recovery rather than silently keeping it + // Retain the last-known prompt from recovery if we have one. + // Deleting the prompt strips the agent of all identity and project context, + // which is worse than a potentially stale prompt. The proper fix for + // stale project context is CWD tracking (#595). if (_state.History.Count > 0 && _state.History[0].Role == Protocol.ChatRole.System) { - _state = _state with { History = _state.History.RemoveAt(0) }; - _log.Warning("Identity files missing — cleared stale system prompt from snapshot"); + _log.Warning("Identity files missing — retaining last-known system prompt from recovery"); } else { diff --git a/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs b/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs index c2328dc0d..ba7467219 100644 --- a/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs +++ b/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs @@ -34,6 +34,7 @@ internal sealed class SlackThreadBindingActor : ReceivePersistentActor, IWithTim private readonly SessionPipelineHandle _handle; private bool _threadHistoryFetchAttempted; private SlackEventTs? _cursorTs; + private SlackEventTs? _pendingCursorTs; private static readonly object ReinitializeTimerKey = new(); private static readonly TimeSpan InboundProcessingTimeout = TimeSpan.FromSeconds(30); private static readonly TimeSpan OperationTimeout = TimeSpan.FromSeconds(10); @@ -323,8 +324,15 @@ await ProcessInboundAttachmentsAsync( await writer.WriteAsync(input, queueWriteCts.Token); + // Track the highest enqueued timestamp but do NOT persist the cursor yet. + // The cursor advances only when TurnCompleted confirms the session actor + // has durably recorded the turn — see HandleOutputAsync. This prevents + // message loss when the daemon crashes between cursor persist and turn persist. if (currentTs is { } ts) - AdvanceCursor(ts); + { + if (_pendingCursorTs is not { } pending || ts.CompareTo(pending) > 0) + _pendingCursorTs = ts; + } } catch (OperationCanceledException ex) { @@ -877,10 +885,16 @@ private void AdvanceCursor(SlackEventTs candidateTs) private bool IsStaleInboundEvent(SlackEventTs? eventTs) { - if (_cursorTs is not { } c || eventTs is not { } ts) + if (eventTs is not { } ts) return false; - return ts.CompareTo(c) <= 0; + if (_cursorTs is { } c && ts.CompareTo(c) <= 0) + return true; + + if (_pendingCursorTs is { } p && ts.CompareTo(p) <= 0) + return true; + + return false; } private void ApplyCursorAdvanced(CursorAdvanced advanced) @@ -1066,6 +1080,13 @@ private async Task HandleOutputAsync(ThreadOutput threadOutput) break; case TurnCompleted completed: + // Advance the cursor now that the session actor has durably persisted the turn. + if (completed.Outcome == TurnOutcome.Completed && _pendingCursorTs is { } pendingTs) + { + AdvanceCursor(pendingTs); + _pendingCursorTs = null; + } + if (!string.IsNullOrWhiteSpace(completed.SourceReminderId) && (_postedThisTurn || _uploadedFileThisTurn)) { Context.System.EventStream.Publish(new ReminderDeliveryObserved( From 0e394653277350c75187ed222c0d1777bc3e7342 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Thu, 23 Apr 2026 22:33:50 +0000 Subject: [PATCH 2/2] refactor: simplify pending cursor lifecycle and trim comments - Clear _pendingCursorTs unconditionally on TurnCompleted (not just on Completed outcome) so failed/skipped turns don't leave stale state that blocks message retries - Clear _pendingCursorTs on pipeline reinitialization - Condense enqueue and system prompt comments --- src/Netclaw.Actors/Sessions/LlmSessionActor.cs | 6 ++---- .../SlackThreadBindingActor.cs | 12 ++++-------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs index 7827bed49..e9b7dee01 100644 --- a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs @@ -2197,10 +2197,8 @@ private void SetSystemPrompt() var content = _promptProvider.GetSystemPrompt(); if (string.IsNullOrWhiteSpace(content)) { - // Retain the last-known prompt from recovery if we have one. - // Deleting the prompt strips the agent of all identity and project context, - // which is worse than a potentially stale prompt. The proper fix for - // stale project context is CWD tracking (#595). + // Retain the last-known prompt from recovery — deleting it strips the agent + // of all identity and project context, which is worse than a stale prompt. if (_state.History.Count > 0 && _state.History[0].Role == Protocol.ChatRole.System) { _log.Warning("Identity files missing — retaining last-known system prompt from recovery"); diff --git a/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs b/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs index ba7467219..cefec6c20 100644 --- a/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs +++ b/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs @@ -324,10 +324,8 @@ await ProcessInboundAttachmentsAsync( await writer.WriteAsync(input, queueWriteCts.Token); - // Track the highest enqueued timestamp but do NOT persist the cursor yet. - // The cursor advances only when TurnCompleted confirms the session actor - // has durably recorded the turn — see HandleOutputAsync. This prevents - // message loss when the daemon crashes between cursor persist and turn persist. + // Defer cursor persistence until TurnCompleted confirms durable turn recording, + // otherwise a crash between cursor and turn persist loses messages. if (currentTs is { } ts) { if (_pendingCursorTs is not { } pending || ts.CompareTo(pending) > 0) @@ -1016,6 +1014,7 @@ private static List MergeGapWithLiveContents( private async Task ReinitializePipelineAsync(string reason) { + _pendingCursorTs = null; await _handle.ReinitializeAsync( reason, () => Timers.StartSingleTimer( @@ -1080,12 +1079,9 @@ private async Task HandleOutputAsync(ThreadOutput threadOutput) break; case TurnCompleted completed: - // Advance the cursor now that the session actor has durably persisted the turn. if (completed.Outcome == TurnOutcome.Completed && _pendingCursorTs is { } pendingTs) - { AdvanceCursor(pendingTs); - _pendingCursorTs = null; - } + _pendingCursorTs = null; if (!string.IsNullOrWhiteSpace(completed.SourceReminderId) && (_postedThisTurn || _uploadedFileThisTurn)) {