From 321ac1e20c7292f3d39d60335da31a15958d18d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= <6334612+rbuergi@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:26:53 +0200 Subject: [PATCH 1/2] fix(compiler): never alias a read to a same-MVID load context that can no longer load its build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FindLiveContextOfSameBuild matched contexts by MVID alone. The MVID is read once from the file header and outlives the file, so a context over an evicted store name (v14766-…) that never loaded its assembly was handed to a read of the present copy (v18979-…) and every load answered "No file at" the deleted path. Each recompile re-adopted identical bytes under a new node version and was aliased back to the same dead context: a permanent self-heal/recompile loop. On memex.systemorph.com 2026-10-07 Store/Plugin instances failed 604 times in 25 minutes on one pod, the type node was written ~6,900 times in 15 hours, and the replica sat at an 11+ GiB heap with GC above half of wall time (#5555, #6161). NodeAssemblyLoadContext.CanLoadItsBuild (assembly loaded, or its file present) now gates the alias. ScanPinSupersessionTest.AReadIsNeverAliasedToASameBuildContextThatCanNoLongerLoadIt is red on the old resolver; its control keeps the in-memory reuse. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../CompilationCacheService.cs | 19 +++++- .../Data/Architecture/NodeTypeCompilation.md | 15 +++++ .../ScanPinSupersessionTest.cs | 62 +++++++++++++++++++ 3 files changed, 95 insertions(+), 1 deletion(-) diff --git a/src/MeshWeaver.Compiler.Pipeline/CompilationCacheService.cs b/src/MeshWeaver.Compiler.Pipeline/CompilationCacheService.cs index e67ce71d33..8a468f05db 100644 --- a/src/MeshWeaver.Compiler.Pipeline/CompilationCacheService.cs +++ b/src/MeshWeaver.Compiler.Pipeline/CompilationCacheService.cs @@ -452,6 +452,15 @@ public string? BuildMvid private readonly Lazy _buildMvidFromFile; + /// + /// Whether this context can still serve its build: the assembly is loaded (the bytes are in + /// memory), or the file it would load from still exists. alone cannot + /// say so — it is read once from the header and outlives the file (#5555). + /// + public bool CanLoadItsBuild + => LoadedAssembly is not null + || (!string.IsNullOrEmpty(_dllPath) && File.Exists(_dllPath)); + /// /// Pins the context for the duration of a SCAN of its loaded assembly (GetTypes + attribute /// reflection + Activator). While any pin is held, waits before @@ -1468,7 +1477,15 @@ private NodeAssemblyLoadContext ResolveLoadContextForPath( foreach (var context in _loadContexts.Values) { if (!string.Equals(context.NodeName, nodeName, StringComparison.Ordinal) - || context.IsRetired) + || context.IsRetired + // 🚨 The same MVID is the same build only while the context can still LOAD it + // (#5555). Re-adopting identical bytes lands them under a new store name + // (v{version}-…), the retention sweep deletes the old one, and a context over the + // old name that never loaded its assembly keeps reporting the MVID it read from + // the header. Aliasing a read of the present copy to it made every load answer + // "No file at" the deleted path — permanently, since each recompile's identical + // bytes were aliased back to the same dead context. + || !context.CanLoadItsBuild) continue; if (string.Equals(context.BuildMvid, mvid, StringComparison.OrdinalIgnoreCase)) return context; diff --git a/src/MeshWeaver.Documentation/Data/Architecture/NodeTypeCompilation.md b/src/MeshWeaver.Documentation/Data/Architecture/NodeTypeCompilation.md index c0f0d5ecdb..62364c1696 100644 --- a/src/MeshWeaver.Documentation/Data/Architecture/NodeTypeCompilation.md +++ b/src/MeshWeaver.Documentation/Data/Architecture/NodeTypeCompilation.md @@ -2540,6 +2540,21 @@ safe: alias and creates a fresh context, exactly as a first read of that path always did. - **One lease per context.** `LeaseNodeContexts` leases each context once however many keys name it, so releasing the hub's one lease performs the deferred unload. +- **A context that can no longer LOAD its build is never aliased (#5555).** The MVID is read once, + from the header, and outlives the file. Re-adopting identical bytes lands them under a new store + name (`v{node version}-{tag}-{hash}.dll`) and the retention sweep deletes the old name, so a context + over the old name that never loaded its assembly still reports the build's MVID while it can load + nothing. Aliasing a read of the present copy to it made every load answer *"No file at"* the deleted + path, and every recompile's identical bytes were aliased back to the same dead context — a + permanent loop. Measured on memex.systemorph.com 2026-10-07: `Store/Plugin` instances failed 604 + times in 25 minutes on one pod (*"Failed to load assembly at …/v18979-c003e001-1db01ff9e178.dll — + No file at …/v14766-c003e001-1db01ff9e178.dll"*), the type node was written ~6,900 times in 15 hours + (v12083 → v18986), the recycle-on-new-build rule read each re-adoption as a new build (2,126 + recycle jobs for that type alone), and the replica held an 11+ GiB heap with GC taking more than + half of wall time. `NodeAssemblyLoadContext.CanLoadItsBuild` — loaded, or its file still present — + now gates the alias. Pinned by + `ScanPinSupersessionTest.AReadIsNeverAliasedToASameBuildContextThatCanNoLongerLoadIt` (red on the + old resolver) and its control `ASameBuildContextThatHasLoadedItsAssemblyIsStillReusedAfterItsFileIsGone`. No identity ⇒ no alias: an unreadable file, or an in-memory context, resolves a fresh context as before. Pinned in core by `ScanPinSupersessionTest.AReadOfTheStoreCopyOfThePublishedBuildIsThatBuildsContext`, diff --git a/test/MeshWeaver.Compiler.Pipeline.Test/ScanPinSupersessionTest.cs b/test/MeshWeaver.Compiler.Pipeline.Test/ScanPinSupersessionTest.cs index 34defc0ef1..b18f87962d 100644 --- a/test/MeshWeaver.Compiler.Pipeline.Test/ScanPinSupersessionTest.cs +++ b/test/MeshWeaver.Compiler.Pipeline.Test/ScanPinSupersessionTest.cs @@ -277,6 +277,68 @@ public void AReadOfTheStoreCopyOfThePublishedBuildIsThatBuildsContext() read.LoadNodeAssembly().Should().NotBeNull(); } + /// + /// 🚨 THE SAME MVID IS NOT THE SAME LOADABLE BUILD WHEN THE BYTES ARE GONE (#5555). The + /// assembly store names a build v{node version}-{tag}-{hash}.dll, and re-adopting + /// identical bytes lands them under a NEW version, so one generation gathers many store names + /// over time while the retention sweep deletes the old ones. A context created over an old name + /// that never loaded its assembly still reports that name's MVID (read once, from the header), + /// so the same-build alias handed it to a read of the NEW, present name — and every load then + /// answered "No file at" the deleted path. Measured on memex.systemorph.com 2026-10-07: + /// Store/Plugin instances failed 604 times in 25 minutes on one pod with "Failed to load + /// assembly at …/v18979-c003e001-1db01ff9e178.dll — No file at …/v14766-c003e001-1db01ff9e178.dll", + /// each failure re-driving a recompile whose identical bytes were aliased to the same dead context. + /// + [Fact] + public void AReadIsNeverAliasedToASameBuildContextThatCanNoLongerLoadIt() + { + var emitted = EmitAssembly("emit"); + var evicted = CopyToStore(emitted, "v14766-tag-hash.dll"); + var current = CopyToStore(emitted, "v18979-tag-hash.dll"); + + // An instance read the old store name and never loaded it; its identity was read from the + // header while the file existed. + var stale = _service.GetOrCreateLoadContextForPath(NodeName, evicted); + stale.BuildMvid.Should().NotBeNull("premise: the old context knows the build's identity"); + stale.LoadedAssembly.Should().BeNull("premise: the old context never loaded its assembly"); + + // The retention sweep deletes the old name. + File.Delete(evicted); + + var read = _service.GetOrCreateLoadContextForPath(NodeName, current); + + read.Should().NotBeSameAs(stale, + "a context whose bytes are neither loaded nor on disk cannot serve the build, so a read " + + "of a present copy must not be aliased to it"); + read.LoadNodeAssembly().Should().NotBeNull( + $"the file at the requested path exists; the load failed with: {read.LastLoadFailure}"); + } + + /// + /// The control for the test above: when the old context HAS loaded its assembly, the bytes are + /// in memory and the alias is the correct reuse — dropping it would mint a second collectible + /// context over identical bytes (the duplicate AReadOfTheStoreCopyOfThePublishedBuildIsThatBuildsContext + /// forbids), even after the old file is gone. + /// + [Fact] + public void ASameBuildContextThatHasLoadedItsAssemblyIsStillReusedAfterItsFileIsGone() + { + var emitted = EmitAssembly("emit"); + var evicted = CopyToStore(emitted, "v14766-tag-hash.dll"); + var current = CopyToStore(emitted, "v18979-tag-hash.dll"); + + var live = _service.GetOrCreateLoadContextForPath(NodeName, evicted); + var assembly = live.LoadNodeAssembly(); + assembly.Should().NotBeNull(); + File.Delete(evicted); + + var read = _service.GetOrCreateLoadContextForPath(NodeName, current); + + read.Should().BeSameAs(live, "the loaded generation serves its build from memory"); + read.LoadNodeAssembly().Should().BeSameAs(assembly); + GC.KeepAlive(assembly); + } + /// /// The reuse must be a REUSE and nothing else: once a publish has superseded the build, a read of /// its store copy must NOT be handed the retired context (that would resurrect a generation the From ea1ef08fed1cc51b0b2c38515d129bce256f3937 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= <6334612+rbuergi@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:36:07 +0200 Subject: [PATCH 2/2] =?UTF-8?q?fix(compiler):=20make=20the=20same-build=20?= =?UTF-8?q?candidate=20resident=20before=20aliasing=20=E2=80=94=20an=20exi?= =?UTF-8?q?stence=20check=20keeps=20the=20deletion=20race?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review on #6261: File.Exists then return leaves a TOCTOU window against the retention sweep. TryMakeResident loads the candidate's assembly (the durable guarantee) and skips it when the load loses the race (file gone, context retired). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../CompilationCacheService.cs | 54 +++++++++++++------ .../Data/Architecture/NodeTypeCompilation.md | 6 ++- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/src/MeshWeaver.Compiler.Pipeline/CompilationCacheService.cs b/src/MeshWeaver.Compiler.Pipeline/CompilationCacheService.cs index 8a468f05db..07b0e6a2a7 100644 --- a/src/MeshWeaver.Compiler.Pipeline/CompilationCacheService.cs +++ b/src/MeshWeaver.Compiler.Pipeline/CompilationCacheService.cs @@ -453,13 +453,33 @@ public string? BuildMvid private readonly Lazy _buildMvidFromFile; /// - /// Whether this context can still serve its build: the assembly is loaded (the bytes are in - /// memory), or the file it would load from still exists. alone cannot - /// say so — it is read once from the header and outlives the file (#5555). + /// Makes this context's build RESIDENT — its assembly loaded, so the bytes are in memory and + /// no later deletion of its file can take them away — and says whether that succeeded. + /// alone cannot answer "can this context serve its build": it is read + /// once from the header and outlives the file (#5555). An existence check would not answer it + /// either: the retention sweep can delete the file between the check and the caller's load. + /// Loading IS the durable guarantee, and a load that loses that race — the file already gone, + /// or the context retired meanwhile — answers false. /// - public bool CanLoadItsBuild - => LoadedAssembly is not null - || (!string.IsNullOrEmpty(_dllPath) && File.Exists(_dllPath)); + public bool TryMakeResident() + { + try + { + return LoadNodeAssembly() is not null; + } + catch (ObjectDisposedException) + { + // Retired between the caller's IsRetired check and this load — not a fault, the + // outcome of the race: this context can no longer be handed to a new caller. + return false; + } + catch (FileNotFoundException) + { + // The sweep deleted the file between LoadNodeAssembly's existence check and its load — + // the very race this method closes; the bytes are not resident, so no reuse. + return false; + } + } /// /// Pins the context for the duration of a SCAN of its loaded assembly (GetTypes + attribute @@ -1477,17 +1497,19 @@ private NodeAssemblyLoadContext ResolveLoadContextForPath( foreach (var context in _loadContexts.Values) { if (!string.Equals(context.NodeName, nodeName, StringComparison.Ordinal) - || context.IsRetired - // 🚨 The same MVID is the same build only while the context can still LOAD it - // (#5555). Re-adopting identical bytes lands them under a new store name - // (v{version}-…), the retention sweep deletes the old one, and a context over the - // old name that never loaded its assembly keeps reporting the MVID it read from - // the header. Aliasing a read of the present copy to it made every load answer - // "No file at" the deleted path — permanently, since each recompile's identical - // bytes were aliased back to the same dead context. - || !context.CanLoadItsBuild) + || context.IsRetired) continue; - if (string.Equals(context.BuildMvid, mvid, StringComparison.OrdinalIgnoreCase)) + // 🚨 The same MVID is the same build only while the context can still LOAD it + // (#5555). Re-adopting identical bytes lands them under a new store name + // (v{version}-…), the retention sweep deletes the old one, and a context over the old + // name that never loaded its assembly keeps reporting the MVID it read from the + // header. Aliasing a read of the present copy to it made every load answer "No file + // at" the deleted path — permanently, since each recompile's identical bytes were + // aliased back to the same dead context. So the candidate's bytes are made RESIDENT + // before it is returned; a candidate that loses the deletion race is skipped and the + // read resolves a context over its own, present path. + if (string.Equals(context.BuildMvid, mvid, StringComparison.OrdinalIgnoreCase) + && context.TryMakeResident()) return context; } return null; diff --git a/src/MeshWeaver.Documentation/Data/Architecture/NodeTypeCompilation.md b/src/MeshWeaver.Documentation/Data/Architecture/NodeTypeCompilation.md index 62364c1696..991c3f68f0 100644 --- a/src/MeshWeaver.Documentation/Data/Architecture/NodeTypeCompilation.md +++ b/src/MeshWeaver.Documentation/Data/Architecture/NodeTypeCompilation.md @@ -2551,8 +2551,10 @@ safe: No file at …/v14766-c003e001-1db01ff9e178.dll"*), the type node was written ~6,900 times in 15 hours (v12083 → v18986), the recycle-on-new-build rule read each re-adoption as a new build (2,126 recycle jobs for that type alone), and the replica held an 11+ GiB heap with GC taking more than - half of wall time. `NodeAssemblyLoadContext.CanLoadItsBuild` — loaded, or its file still present — - now gates the alias. Pinned by + half of wall time. The resolver now makes the candidate's bytes RESIDENT before aliasing to it + (`NodeAssemblyLoadContext.TryMakeResident` — load the assembly; a load that loses the deletion race + skips the candidate). An existence check would not do: the sweep can delete the file between the + check and the caller's load, so loading is the only durable guarantee. Pinned by `ScanPinSupersessionTest.AReadIsNeverAliasedToASameBuildContextThatCanNoLongerLoadIt` (red on the old resolver) and its control `ASameBuildContextThatHasLoadedItsAssemblyIsStillReusedAfterItsFileIsGone`.