diff --git a/src/MeshWeaver.Fixture/TeardownStragglerCapturer.cs b/src/MeshWeaver.Fixture/TeardownStragglerCapturer.cs new file mode 100644 index 0000000000..dfeed48634 --- /dev/null +++ b/src/MeshWeaver.Fixture/TeardownStragglerCapturer.cs @@ -0,0 +1,172 @@ +using System.Runtime.CompilerServices; + +namespace MeshWeaver.Fixture; + +/// +/// Names the teardown straggler behind the "green-but-catastrophic" CI shard failures: +/// Catastrophic failure: System.ObjectDisposedException : Instances cannot be resolved and +/// nested lifetimes cannot be created from this LifetimeScope … with 123/123 passed yet exit 1. +/// +/// The escape channel is NOT the unobserved-task path. xunit v3 (3.2.2) never hooks +/// — verified against the shipped binaries +/// (xunit.v3.runner.inproc.console.dll references add_UnhandledException only, no +/// UnobservedTask) — and .NET Core does not escalate unobserved task exceptions +/// (ThrowUnobservedTaskExceptions is off). The producer of the runner ErrorMessage +/// reported as "Catastrophic failure: {0}" is xunit's +/// hook in the in-proc console runner (Xunit.Runner.InProc.SystemConsole.ConsoleRunner: +/// AppDomain.CurrentDomain.UnhandledException += … logger.WriteMessage(ErrorMessage.FromException(ex))). +/// It records message only, NO stack — hence "anonymous". That is why the two +/// UnobservedTaskException-based suppressors (, the +/// Orleans-local one) never stopped it: wrong event. +/// +/// The named root (task #20), by this capturer's first-chance stack. The disposed-scope +/// ODE is thrown from Orleans' own connection message pump — +/// Orleans.Runtime.Messaging.Connection.ProcessIncoming → MessageSerializer.TryRead → +/// CodecProvider.TryCreateCodec → ActivatorUtilities.CreateInstance → +/// AutofacServiceProvider.GetService on a disposed LifetimeScope. A silo/client +/// connection is still deserializing an in-flight message while its Autofac container is torn down +/// during TestCluster teardown. Purely Orleans-internal (the #231 mesh drain never covered it); +/// fixed at the disposal seam — OrleansClusterDisposal now runs the graceful +/// StopAllSilosAsync() (which drains the pumps) BEFORE DisposeAsync(). This capturer +/// stays as the permanent net so any future straggler fails with a named stack, not folklore. +/// +/// What this capturer does — record, never suppress: +/// +/// , filtered STRICTLY to the teardown +/// disposed-resource shapes (Autofac LifetimeScope + +/// mesh-owned MemoryCache — see ): records the full +/// throw-site stack ( — at first-chance time the exception's own +/// StackTrace has only the throw frame) + thread/task context. This fires even when the runtime +/// later CATCHES the throw (Orleans caught all 137/137 in a green run), so the throw-site is named +/// without needing the rare unobserved escape. Bounded (cap, cheap filter), so it can live on +/// permanently. +/// — the channel that actually reds the shard: +/// records the COMPLETE exception (full stack, by now unwound to the callback root) to the +/// test-logs file AND stderr, right next to xunit's anonymous catastrophic line. Future stragglers +/// are named failures, not folklore. The run still fails — nothing is swallowed here. +/// , same strict filter — diagnostics +/// only; observation policy stays in . +/// +/// +/// Output: {BaseDirectory}/test-logs/teardown-stragglers.log — the CI log-collection +/// step already globs */bin/*/test-logs/*.log into the run artifacts, so every capture ships +/// automatically. +/// +/// NoStaticState note: this is process-lifetime diagnostics infrastructure hooked on +/// process-wide CLR events — there is no mesh to scope it to. It holds no collections; the log FILE +/// is the record, the only mutable statics are bounded counters. +/// +internal static class TeardownStragglerCapturer +{ + private const int FirstChanceCap = 200; + + private static int firstChanceCount; + private static int unhandledCount; + private static int unobservedCount; + private static readonly object WriteLock = new(); + + private static string LogPath => + Path.Combine(AppContext.BaseDirectory, "test-logs", "teardown-stragglers.log"); + + [ModuleInitializer] + public static void Init() + { + AppDomain.CurrentDomain.FirstChanceException += static (_, e) => + { + // Strict filter — only the teardown disposed-resource shapes (Autofac LifetimeScope + + // MemoryCache); everything else is untouched. Recursion safety: nothing our own + // recording code can throw matches this filter. + if (!IsTeardownDisposedStraggler(e.Exception)) + return; + var n = Interlocked.Increment(ref firstChanceCount); + if (n > FirstChanceCap) + return; + // Environment.StackTrace at first-chance time = throw site + callers (the exception's + // own StackTrace is not yet unwound past the throw frame). + Append( + $"FIRST-CHANCE #{n} {Header()}\n{e.Exception.GetType().FullName}: {e.Exception.Message}\n{Environment.StackTrace}\n" + + (n == FirstChanceCap ? "── first-chance capture cap reached; further matches counted but not dumped ──\n" : "")); + }; + + AppDomain.CurrentDomain.UnhandledException += static (_, e) => + { + // THIS is the escape that reds a green shard as an anonymous "Catastrophic failure": + // xunit's own hook forwards only the MESSAGE. Name it — full stack, thread context — + // in the CI-collected log AND on stderr. Never swallows; the run still fails. + var n = Interlocked.Increment(ref unhandledCount); + var text = + $"UNHANDLED #{n} (terminating={e.IsTerminating}) {Header()}\n" + + "── this exception is what xunit reports as the anonymous 'Catastrophic failure' ──\n" + + (e.ExceptionObject as Exception)?.ToString() + + $"\n(first-chance disposed-resource captures so far: {Volatile.Read(ref firstChanceCount)})\n"; + Append(text); + try + { + Console.Error.WriteLine($"[TeardownStragglerCapturer] {text}"); + } + catch + { + // stderr may be gone at process teardown — the file capture above is the record + } + }; + + TaskScheduler.UnobservedTaskException += static (_, e) => + { + // Diagnostics only — whether a benign shape gets observed stays the business of + // TestTeardownExceptionSuppressor. Unobserved task exceptions never terminate on + // .NET Core, so these captures explain history, not shard exit codes. + var inners = e.Exception?.Flatten().InnerExceptions; + if (inners is not { Count: > 0 } || !inners.Any(IsTeardownDisposedStraggler)) + return; + var n = Interlocked.Increment(ref unobservedCount); + Append($"UNOBSERVED-TASK #{n} {Header()}\n{e.Exception}\n"); + }; + } + + /// + /// The teardown disposed-resource shapes this capturer names: a late continuation touching a + /// resource whose owning scope has already been torn down. Two known shapes, both + /// : (1) the Autofac root container — the Orleans connection + /// message pump resolving a codec from a disposed LifetimeScope (task #20, fixed at + /// OrleansClusterDisposal); (2) a mesh-owned MemoryCache — a late write/read hitting + /// e.g. MeshNodeStreamCache._updateQueues after the cache hub disposed. Both are the same + /// "graceful terminal, never an unobserved throw" defect class (#228/#231 doctrine). + /// + private static bool IsTeardownDisposedStraggler(Exception? ex) => + IsAutofacDisposedScope(ex) || IsDisposedMemoryCache(ex); + + private static bool IsAutofacDisposedScope(Exception? ex) => + ex is ObjectDisposedException + && (ex.Message?.Contains("LifetimeScope", StringComparison.OrdinalIgnoreCase) == true + || ex.Message?.Contains("nested lifetimes cannot be created", StringComparison.OrdinalIgnoreCase) == true); + + private static bool IsDisposedMemoryCache(Exception? ex) => + ex is ObjectDisposedException ode + && (string.Equals(ode.ObjectName, "Microsoft.Extensions.Caching.Memory.MemoryCache", StringComparison.Ordinal) + || ode.Message?.Contains("MemoryCache", StringComparison.OrdinalIgnoreCase) == true); + + private static string Header() + { + var t = Thread.CurrentThread; + return $"utc={DateTime.UtcNow:O} thread={t.ManagedThreadId} name='{t.Name}' pool={t.IsThreadPoolThread} " + + $"taskId={Task.CurrentId?.ToString() ?? "-"} syncCtx={SynchronizationContext.Current?.GetType().Name ?? "-"}"; + } + + private static void Append(string text) + { + // A capturer must never take the run down: swallow ONLY our own IO faults here (this is + // diagnostics hardening, not suppression of product errors — nothing product-side is caught). + try + { + lock (WriteLock) + { + Directory.CreateDirectory(Path.GetDirectoryName(LogPath)!); + File.AppendAllText(LogPath, $"════ {text}\n"); + } + } + catch + { + // disk unavailable at teardown — nothing further we can do + } + } +} diff --git a/src/MeshWeaver.Hosting/MeshNodeStreamCache.cs b/src/MeshWeaver.Hosting/MeshNodeStreamCache.cs index d20dcf510e..b7b29cebcd 100644 --- a/src/MeshWeaver.Hosting/MeshNodeStreamCache.cs +++ b/src/MeshWeaver.Hosting/MeshNodeStreamCache.cs @@ -1015,6 +1015,20 @@ private IObservable UpdateRaw(string path, Func up // patches that the RFC 7396 owner-side merge cannot resolve (lists // collapse to the last writer). Serializing per path makes each // lambda observe its predecessor's effect. + // DISPOSED-CACHE GUARD (write side): a late write after this cache's Dispose() — the + // canonical case is an agent round whose ThreadExecution.PushToResponseMessage is still + // streaming when its mesh's cacheHub tore down (State captured by TeardownStragglerCapturer: + // UpdateRaw → _updateQueues.TryGetValue → MemoryCache.CheckDisposed) — must NOT touch the + // disposed _updateQueues MemoryCache, whose TryGetValue throws a synchronous + // ObjectDisposedException on the caller's ThreadPool continuation. Unobserved, that reaches + // AppDomain.UnhandledException and xUnit escalates it to a "Catastrophic failure" that reds + // an otherwise-green shard. Dispose() sets _disposed=1 BEFORE it disposes _updateQueues, so + // this flag check (same Volatile idiom as ReleaseIdleReadStreams) fully covers the straggler. + // Return the same graceful Observable.Throw terminal the negative-cache breaker below uses — + // observed by the caller's Subscribe (a benign teardown write), never an unobserved throw. + if (System.Threading.Volatile.Read(ref _disposed) != 0) + return Observable.Throw(new ObjectDisposedException(nameof(MeshNodeStreamCache))); + // STORM BREAKER (write side): if this path's owner is in a known missing-node failure // window, fast-fail rather than enqueue another PatchDataRequest the owner can't // activate. Same negative cache as the read-side GetStreamRaw breaker, so a missing-node @@ -1261,6 +1275,11 @@ public IObservable Overwrite(string path, MeshNode node, JsonSerialize // Handle.Overwrite (ChangeType.Full) instead of Handle.Update. private IObservable OverwriteRaw(string path, MeshNode node) { + // Disposed-cache guard — see UpdateRaw. A late overwrite after Dispose() must not hit the + // disposed _updateQueues MemoryCache; return a graceful observed terminal, never a throw. + if (System.Threading.Volatile.Read(ref _disposed) != 0) + return Observable.Throw(new ObjectDisposedException(nameof(MeshNodeStreamCache))); + var queue = GetOrCreateUpdateQueue(path); var result = new ReplaySubject(); var seq = System.Threading.Interlocked.Increment(ref _updateSeq); diff --git a/test/MeshWeaver.Hosting.Orleans.Test/OrleansClusterDisposal.cs b/test/MeshWeaver.Hosting.Orleans.Test/OrleansClusterDisposal.cs index ede2de4740..ee7bf4b0d9 100644 --- a/test/MeshWeaver.Hosting.Orleans.Test/OrleansClusterDisposal.cs +++ b/test/MeshWeaver.Hosting.Orleans.Test/OrleansClusterDisposal.cs @@ -31,6 +31,30 @@ internal static class OrleansClusterDisposal /// Hand a cluster's disposal to a background pool thread — NEVER awaited on the /// teardown thread. Null-safe and best-effort (the cluster is on its way down; a /// shutdown-race exception is benign and swallowed). + /// + /// Graceful stop BEFORE dispose — the root fix for the Autofac + /// disposed-LifetimeScope straggler (task #20). + /// disposes the Orleans CLIENT host through the generic IHost.DisposeAsync(), which + /// only disposes the service provider and NEVER runs StopAsync(). So the client's + /// connection message pump (Orleans.Runtime.Messaging.Connection.ProcessIncoming) keeps + /// deserializing in-flight messages while the client's Autofac container is being torn down; + /// CodecProvider then lazily resolves a serialization codec from the already-disposed + /// LifetimeScope and throws ("Instances cannot be + /// resolved and nested lifetimes cannot be created from this LifetimeScope … already disposed"). + /// Unobserved under CI load, that reaches — the ONLY + /// channel xUnit v3's in-proc console runner hooks — and is reported as an anonymous + /// "Catastrophic failure" that reds a 123/123-green shard. (The silos are already graceful: + /// InProcessSiloHandle.DisposeAsync runs StopSiloAsync(graceful) first; only the + /// client host skips its StopAsync.) The pump is purely Orleans-internal, so the mesh + /// drain added in #231 (MeshTeardownHostedService) never covered it — and because the + /// client's StopAsync is skipped, that drain never even ran on the client. + /// + /// stops the client and every silo through + /// their StopAsync lifecycle (client → secondaries → primary), which drains those + /// connection pumps (AND runs MeshTeardownHostedService) so that by the time the + /// containers are disposed nothing is left resolving. Named + measured by + /// TeardownStragglerCapturer: the disposed-scope first-chance throw count per Orleans.Test + /// run drops from ~130 to ~0 once the graceful stop precedes disposal. /// public static void DisposeInBackground(TestCluster? cluster) { @@ -38,6 +62,12 @@ public static void DisposeInBackground(TestCluster? cluster) return; Pending.Add(Task.Run(async () => { + // Graceful ordered stop FIRST (drains client + silo connection pumps before any + // Autofac container is disposed), THEN the dispose. Separate guards so a stop-race + // can't skip the dispose that finishes teardown. Both benign here — the cluster is + // going down; a genuinely surviving straggler is now NAMED by TeardownStragglerCapturer. + try { await cluster.StopAllSilosAsync(); } + catch { /* stop-race on a cluster that's going down — DisposeAsync still completes teardown */ } try { await cluster.DisposeAsync(); } catch { /* shutdown-race / zombie silo — benign, the cluster is going down */ } }));