From a531e432d41ea78ac07dedd8eb650261644f0127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Wed, 12 Aug 2026 10:58:25 +0200 Subject: [PATCH] fix: an unowned AiSettings seed outlived its reader and stranded a create MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `AiSettingsNodeType.Observe` opened with a bare statement: EnsureExists(hub, services, user); `EnsureExists` was `void` and subscribed its own chain, so the create-on-absent write had NO owner. Two consequences, both real: * it fired at COMPOSITION time — building the observable was enough, even if nobody ever subscribed to the settings stream; and * nothing could cancel it, so it could still be in flight — or start — after the last reader had gone away, including on a hub that was already tearing down. That is what reddened SkillAutocompleteTest.Autocomplete_ListsBuiltInSkills_- FromCatalog on CI (3 occurrences on main in one morning: runs 31569022680, 31574636901, 31577766913). The test body PASSES — start 07:45:18.862, end 07:45:19.083 — and then teardown's quiesce guard throws, reporting two pending CreateNodeRequests for Roland/_Memex/AiSettings: cb1 ... CREATE_CHAIN_SUBSCRIBED path=Roland/_Memex/AiSettings -> BOOTSTRAP_PERM_AWAIT partition=Roland user=system-security (+189ms) -> BOOTSTRAP_PERM_VERDICT authorized=True (+512ms) [no reply, ever] cb2 ... RECEIVED runLevel=Quiescing -> DEFERRED gates=[DataContextInit] cb1 is the seed. Because the test mesh has no `Roland` partition root, EnsurePartitionBootstrap heals it, and the heal issues cb2 — a NESTED root create. By then the mesh is Quiescing, so cb2 parks behind `DataContextInit`, a gate shutdown never reopens; cb1's chain therefore neither emits, faults nor completes, and no response is ever posted. The 500 ms quiesce budget expires and the guard fails the class. On a loaded CI runner the 512 ms permission fold alone overruns that budget, which is why it only reproduces there. Root fix: the seed is MERGED INTO the stream it serves, so its lifetime is the consumer's subscription — it starts on Subscribe (not at composition) and is disposed when the consumer unsubscribes. `EnsureExists` now returns a cold `IObservable` and no longer self-subscribes, matching its sibling `NotificationSettingsNodeType.EnsureExists`. Errors stay contained exactly as before (log + swallow) so a failed seed can never break the settings read. This does not paper over the two deeper holes the trace also exposes — a gate that can never reopen after shutdown, and nested creates inside EnsurePartitionBootstrap that ride an unbounded `meshService.CreateNode` whose advertised `MeshOperationOptions.Timeout` is dead code. Both are filed separately; neither is reachable from here once the seed cannot outlive its reader. Co-Authored-By: Claude Opus 5 (1M context) --- src/MeshWeaver.AI/AiSettingsNodeType.cs | 55 ++++++++++++++++++++----- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/src/MeshWeaver.AI/AiSettingsNodeType.cs b/src/MeshWeaver.AI/AiSettingsNodeType.cs index ec41f6b62b..9f5af77350 100644 --- a/src/MeshWeaver.AI/AiSettingsNodeType.cs +++ b/src/MeshWeaver.AI/AiSettingsNodeType.cs @@ -346,22 +346,59 @@ public static IObservable Observe( IWorkspace workspace, IMessageHub hub, IServiceProvider services, string user) { var defaults = BuildDefaults(services); - EnsureExists(hub, services, user); - return workspace + var live = workspace .GetQuery($"{NodeType}|{user}", $"path:{PathFor(user)} nodeType:{NodeType} select:path,id,name,nodeType,content") .Select(nodes => Effective( nodes.FirstOrDefault(n => string.Equals(n.NodeType, NodeType, StringComparison.OrdinalIgnoreCase)), defaults, hub.JsonSerializerOptions)) .StartWith(defaults) .DistinctUntilChanged(); + + // 🚨 The create-on-absent is MERGED INTO THIS STREAM, so its lifetime is the + // consumer's subscription — it starts on Subscribe (not at composition time) and is + // DISPOSED when the consumer unsubscribes. + // + // It used to be a bare `EnsureExists(hub, services, user);` statement: a void method + // that subscribed its own chain. Nobody owned that subscription, so the create could + // still be in flight — or start — after the last reader had gone away, and on a hub + // that was already tearing down it stranded a CreateNodeRequest that nothing would + // ever answer. That is what reddened + // MeshWeaver.AI.Test.SkillAutocompleteTest.Autocomplete_ListsBuiltInSkills_FromCatalog + // on CI: the test body passed, then teardown's quiesce guard reported two pending + // CreateNodeRequests for Roland/_Memex/AiSettings — the outer seed plus the partition + // bootstrap's nested root create, the latter deferred behind a DataContextInit gate + // that shutdown never reopens. Owned by the subscription, both go away with it. + // + // Errors stay contained exactly as before (log + swallow) so a failed seed can never + // break the settings read that consumers actually subscribe for. + var seed = EnsureExists(hub, services, user) + .Select(_ => defaults) + .IgnoreElements() + .Catch(ex => + { + services.GetService() + ?.CreateLogger(typeof(AiSettingsNodeType)) + .LogWarning(ex, "EnsureExists: AiSettings create-on-absent failed for {Path}", PathFor(user)); + return Observable.Empty(); + }); + + return live.Merge(seed); } - /// Create-on-absent (with defaults); existing node untouched. - public static void EnsureExists(IMessageHub hub, IServiceProvider services, string user) + /// + /// Create-on-absent (with defaults); existing node untouched. + /// + /// COLD — the create runs on Subscribe and is cancelled on unsubscribe, so the + /// caller owns its lifetime. Never re-introduce a self-subscribing void variant: + /// an unowned write outlives its reader and can strand a request on a hub that is + /// already shutting down. Mirrors NotificationSettingsNodeType.EnsureExists. + /// + public static IObservable EnsureExists( + IMessageHub hub, IServiceProvider services, string user) { var meshService = services.GetService(); if (meshService is null) - return; + return Observable.Empty(); var path = PathFor(user); // 🚨 Create-on-absent must NEVER point-read/patch the node via // GetMeshNodeStream(path).Update. On an ABSENT node that opens a cross-hub @@ -374,7 +411,7 @@ public static void EnsureExists(IMessageHub hub, IServiceProvider services, stri // only when genuinely absent through the node-lifecycle CreateNode (create-only: // it does not clobber an existing customised node). See // feedback_optional_node_query_not_access / Doc/Architecture/AsynchronousCalls.md. - hub.GetWorkspace() + return hub.GetWorkspace() .GetQuery($"{NodeType}|{user}", $"path:{path} nodeType:{NodeType} select:path,id,name,nodeType,content") .Take(1) .Where(nodes => !nodes.Any(n => @@ -386,11 +423,7 @@ public static void EnsureExists(IMessageHub hub, IServiceProvider services, stri Name = "AI Settings", Content = BuildDefaults(services) })) - .Subscribe( - _ => { }, - ex => services.GetService() - ?.CreateLogger(typeof(AiSettingsNodeType)) - .LogWarning(ex, "EnsureExists: AiSettings create-on-absent failed for {Path}", path)); + .Select(_ => System.Reactive.Unit.Default); } ///