Pooled I/O leaf now observes cancellation: script-completion's first-in-process reference build (#2480, #2578)

src/MeshWeaver.Documentation/Data/WhatsNew/2026-08-28-script-completions-no-longer-stall-portal-shutdown.md

@@ -0,0 +1,20 @@
+---
+Name: Script completions no longer stall portal shutdown
+Category: Fix
+Description: The first code-completion or script run in a portal's lifetime could hold up a restart for up to 30 seconds; it no longer does.
+Icon: Sparkle
+Order: -20260828
+---
+
+# Script completions no longer stall portal shutdown
+
+The very first time a portal process ran a script cell or offered code completions in a
+script-flavored editor, it had to build a shared list of reference assemblies — a one-time cost
+that is normally quick, but occasionally ran long under memory pressure. That build used to run
+inline while holding a slot in the pool a shutdown needs to hand back cleanly, so on the rare
+occasion it was slow, a restart or redeploy could sit waiting on it for up to 30 seconds instead of
+completing promptly.
+
+The build now runs independently of any one request: a request waits for it but no longer blocks a
+shutdown from proceeding, even on that first, slower call. Restarts and redeploys are consistently
+quick regardless of when they land relative to the first script or completion request.

src/MeshWeaver.Graph/Configuration/MeshNodeLanguageService.cs

@@ -55,8 +55,18 @@ internal sealed class MeshNodeLanguageService(
     // lifetime, so a single lazily-built project serves every request: each suggest forks the
     // immutable solution with the in-flight text, nothing is applied back, and concurrent
     // requests never contend. Bounded: one AdhocWorkspace total, not one per cell.
-    private readonly Lazy<ScriptWorkspace> _scriptWorkspace = new(
-        () => BuildScriptWorkspace(hub.ServiceProvider),
+    //
+    // 🚨 Task, not a bare value (issues #2480/#2578). Building it touches
+    // MeshScriptEnvironment.ReferencesAsync, whose FIRST-ever call in the process can be
+    // genuinely slow (materializing ~350 assemblies' metadata) — it used to run synchronously,
+    // inline, inside this leaf's Compile-pool gate, with no cancellation participation at all,
+    // so a silo/mesh teardown racing the first caller could not reclaim that gate permit within
+    // the drain budget. Every caller now races its OWN `ct` against this shared Task via
+    // Task.WaitAsync (see GetScriptCompletionsAsync/GetScriptDiagnosticsAsync) — the FIRST
+    // caller's own token never cancels the build (it is shared by every request against this
+    // mesh instance), it only governs how long THAT caller waits.
+    private readonly Lazy<Task<ScriptWorkspace>> _scriptWorkspace = new(
+        () => BuildScriptWorkspaceAsync(hub.ServiceProvider),
         LazyThreadSafetyMode.ExecutionAndPublication);
 
     // Path used as the script document's FilePath — never a real MeshNode path.
@@ -469,7 +479,7 @@ private async Task<IReadOnlyList<CompletionEntry>> GetScriptCompletionsAsync(
     {
         var usage = usageIndex;
         var memory = CurrentMemory();
-        var script = _scriptWorkspace.Value;
+        var script = await _scriptWorkspace.Value.WaitAsync(ct).ConfigureAwait(false);
         var document = script.Workspace.CurrentSolution
             .WithDocumentText(script.DocumentId, SourceText.From(StripKernelDirectives(proposedCode)))
             .GetDocument(script.DocumentId);
@@ -484,7 +494,7 @@ private async Task<IReadOnlyList<CompletionEntry>> GetScriptCompletionsAsync(
     private async Task<IReadOnlyList<DiagnosticInfo>> GetScriptDiagnosticsAsync(
         string proposedCode, CancellationToken ct)
     {
-        var script = _scriptWorkspace.Value;
+        var script = await _scriptWorkspace.Value.WaitAsync(ct).ConfigureAwait(false);
         var document = script.Workspace.CurrentSolution
             .WithDocumentText(script.DocumentId, SourceText.From(StripKernelDirectives(proposedCode)))
             .GetDocument(script.DocumentId);
@@ -781,12 +791,20 @@ private static CompletionKind MapTagsToKind(ImmutableArray<string> tags)
     /// compilation's usings, the process-shared reference set, the shared metadata resolver,
     /// and the script globals as the submission's host object — so <c>Mesh</c>, <c>Log</c>,
     /// <c>Ct</c> and <c>Inputs</c> complete as bare identifiers exactly as they run.
+    ///
+    /// <para>🚨 Awaits the shared reference snapshot with <see cref="CancellationToken.None"/> —
+    /// this build is cached (once) for every future request against THIS mesh instance's
+    /// <c>_scriptWorkspace</c>, so it must never be torn down by whichever caller happens to be
+    /// first; each caller's own cancellation is applied at the `_scriptWorkspace.Value.WaitAsync`
+    /// call site instead (see the field's doc comment).</para>
     /// </summary>
-    private static ScriptWorkspace BuildScriptWorkspace(IServiceProvider serviceProvider)
+    private static async Task<ScriptWorkspace> BuildScriptWorkspaceAsync(IServiceProvider serviceProvider)
     {
         var workspace = new AdhocWorkspace();
         var projectId = ProjectId.CreateNewId();
         var docId = DocumentId.CreateNewId(projectId);
+        var references = await Kernel.Hub.MeshScriptEnvironment
+            .ReferencesAsync(serviceProvider, CancellationToken.None).ConfigureAwait(false);
         var projectInfo = ProjectInfo
             .Create(
                 projectId,
@@ -799,7 +817,7 @@ private static ScriptWorkspace BuildScriptWorkspace(IServiceProvider serviceProv
                         metadataReferenceResolver: Kernel.Hub.MeshScriptEnvironment.MetadataResolver)
                     .WithUsings(Kernel.Hub.MeshScriptEnvironment.Imports),
                 parseOptions: new CSharpParseOptions(kind: SourceCodeKind.Script),
-                metadataReferences: Kernel.Hub.MeshScriptEnvironment.References(serviceProvider),
+                metadataReferences: references,
                 isSubmission: true,
                 hostObjectType: Kernel.Hub.MeshScriptEnvironment.GlobalsType)
             .WithDocuments(

src/MeshWeaver.Kernel.Hub/KernelExecutor.cs

@@ -349,17 +349,17 @@ private void ClearCancellationIf(CancellationTokenSource cts)
         // No lock and no gate here: REPL ordering across submissions is owned by the serial
         // Concat pump (see `submissions`), which subscribes this observable only after the
         // previous submission completed. So Execute is just: init → resolve NuGet refs → run
-        // the Roslyn pass. SubscribeOn moves the WHOLE subscribe — including
-        // EnsureInitialized's AppDomain scan — onto the ThreadPool, so the executor action
-        // block is never blocked by a compile. (The Roslyn compile itself runs on the bounded
-        // Compile IoPool inside RunOnePass; NuGet restore on the Http pool.)
+        // the Roslyn pass. SubscribeOn moves the WHOLE subscribe onto the ThreadPool, so the
+        // executor action block is never blocked by a compile. EnsureInitializedAsync's
+        // AppDomain scan is ADDITIONALLY routed through the bounded Compile IoPool (issues
+        // #2480/#2578) — it used to run as a bare synchronous call here, off any pool, which
+        // meant it could neither be tracked nor cancelled by silo/mesh teardown; now it is the
+        // same tracked, drainable leaf shape as the Roslyn compile itself in RunOnePass.
         return Observable.Defer(() =>
-            {
-                EnsureInitialized();
-                return EnsureCellSurfaceReferences()
+                compilePool.Invoke(t => EnsureInitializedAsync(t))
+                    .SelectMany(_ => EnsureCellSurfaceReferences())
                     .SelectMany(_ => nugetPool.Invoke(t => ResolveNuGetReferencesAsync(code, t)))
-                    .SelectMany(cleaned => RunOnePass(cleaned, viewId, scriptOutputLogger, inputs, accessContext, ct));
-            })
+                    .SelectMany(cleaned => RunOnePass(cleaned, viewId, scriptOutputLogger, inputs, accessContext, ct)))
             .Catch<object?, Exception>(ex =>
             {
                 if (ex is OperationCanceledException) return Observable.Throw<object?>(ex);
@@ -479,16 +479,24 @@ public void Dispose()
     /// Build script options once on first submission. Pulls the IMessageHub for
     /// the globals from the public hub (the parent), so scripts see the SAME hub
     /// external callers do — they can post messages, subscribe streams, etc.
+    ///
+    /// <para>🚨 Runs INSIDE the Compile IoPool leaf <see cref="Execute"/> wraps it in — never bare
+    /// on the ThreadPool as before (issues #2480/#2578). The reference materialization behind
+    /// <see cref="MeshScriptEnvironment.ReferencesAsync"/> can be genuinely slow the first time
+    /// any mesh in the process touches it, and used to hold no pool gate at all while doing so —
+    /// invisible to <c>IoPoolRegistry.DrainAll</c> and therefore never cancelled or joined by
+    /// teardown. Now the wait is a real, trackable, cancellable leaf; <paramref name="ct"/>
+    /// governs only how long THIS session waits, never the shared snapshot build itself (see
+    /// <see cref="KernelScriptReferences.GetReferencesAsync"/>).</para>
     /// </summary>
-    private void EnsureInitialized()
+    private async Task<Unit> EnsureInitializedAsync(CancellationToken ct)
     {
-        if (initialized) return;
-        initialized = true;
+        if (initialized) return Unit.Default;
 
         var defaultLogger = publicHub.ServiceProvider.GetRequiredService<ILoggerFactory>()
             .CreateLogger("MeshWeaver.Kernel.Script");
-        scriptLogger = new ScriptLogger(defaultLogger);
-        scriptGlobals = new MeshScriptGlobals { Mesh = publicHub, Log = scriptLogger };
+        var localScriptLogger = new ScriptLogger(defaultLogger);
+        var localScriptGlobals = new MeshScriptGlobals { Mesh = publicHub, Log = localScriptLogger };
 
         // Curated anchors + DI-contributed module assemblies + default imports all come
         // from MeshScriptEnvironment — the ONE description of the script environment,
@@ -498,15 +506,33 @@ private void EnsureInitialized()
         // metadata block per reference PER SESSION (~350 refs ≈ 150-200 MiB of native
         // memory per kernel session, never reclaimed — the CI memory-pressure leak; see
         // KernelScriptReferences docs).
-        scriptOptions = ScriptOptions.Default
-            .WithReferences(MeshScriptEnvironment.References(publicHub.ServiceProvider))
+        //
+        // 🚨 `initialized` flips to true ONLY after every step below has succeeded (Copilot
+        // review, PR #2598). It used to be set FIRST — harmless when the reference build was a
+        // synchronous call that either finished or threw synchronously right there, but this
+        // method is now `await`-based and genuinely cancellable, so a cancellation or a
+        // transient fault partway through used to leave `initialized == true` with
+        // `scriptOptions` still null FOREVER: the guard at the top would then skip every future
+        // submission's attempt to build it, permanently wedging the session. Local variables
+        // here (rather than writing straight into the instance fields) keep a failed attempt
+        // from leaving scriptLogger/scriptGlobals half-set either.
+        var references = await MeshScriptEnvironment.ReferencesAsync(publicHub.ServiceProvider, ct)
+            .ConfigureAwait(false);
+        var options = ScriptOptions.Default
+            .WithReferences(references)
             // 🚨 Required for the sharing to be complete: the compilation resolves
             // the globals type's transitive closure via ResolveMissingAssembly —
             // with the default resolver that re-materializes every assembly's
             // native metadata PER SESSION (the other half of the leak).
             .WithMetadataResolver(MeshScriptEnvironment.MetadataResolver)
             .WithImports(MeshScriptEnvironment.Imports)
             .WithEmitDebugInformation(true);
+
+        scriptLogger = localScriptLogger;
+        scriptGlobals = localScriptGlobals;
+        scriptOptions = options;
+        initialized = true;
+        return Unit.Default;
     }
 
     /// <summary>

src/MeshWeaver.Kernel.Hub/KernelScriptAssembly.cs

@@ -8,7 +8,7 @@ namespace MeshWeaver.Kernel.Hub;
 /// MeshNodes (export, import, NodeType compile, …) registers one
 /// <see cref="KernelScriptAssembly"/> per assembly that scripts need to reference.
 ///
-/// <para>The kernel's <see cref="KernelExecutor.EnsureInitialized"/> resolves all
+/// <para>The kernel's <see cref="KernelExecutor.EnsureInitializedAsync"/> resolves all
 /// instances via <c>IServiceProvider.GetServices&lt;KernelScriptAssembly&gt;</c>
 /// and adds each <see cref="Assembly"/> to the Roslyn script options. That way
 /// the script's <c>using MyNamespace;</c> directive resolves even if Roslyn's

src/MeshWeaver.Kernel.Hub/KernelScriptReferences.cs

@@ -31,7 +31,7 @@ namespace MeshWeaver.Kernel.Hub;
 ///
 /// <para><b>The fix:</b> one <see cref="PortableExecutableReference"/> per assembly
 /// file, once per process, shared by every session — both for the declared
-/// reference set (<see cref="GetReferences"/>) and for resolver-driven resolution
+/// reference set (<see cref="GetReferencesAsync"/>) and for resolver-driven resolution
 /// (<see cref="SharedScriptMetadataResolver"/> wraps
 /// <see cref="ScriptMetadataResolver.Default"/> and memoizes per path). Roslyn
 /// shares the underlying <c>AssemblyMetadata</c> across compilations that use the
@@ -54,8 +54,15 @@ internal static class KernelScriptReferences
     private static readonly ConcurrentDictionary<string, PortableExecutableReference> Materialized =
         new(StringComparer.OrdinalIgnoreCase);
 
-    private static readonly Lazy<ImmutableArray<PortableExecutableReference>> SharedSnapshot =
-        new(CreateSharedSnapshot, LazyThreadSafetyMode.ExecutionAndPublication);
+    // 🚨 Task, not a bare value — see GetReferencesAsync. The materialization below is
+    // synchronous, uncancellable (no CancellationToken overload exists for
+    // MetadataReference.CreateFromFile) CPU/disk work over ~350 assemblies, run via Task.Run so
+    // it executes on its OWN detached ThreadPool work item, decoupled from whichever caller's
+    // pooled leaf happens to trigger the FIRST-ever build. A caller's own cancellation must never
+    // abort this — it is shared by every mesh in the process — so GetReferencesAsync races the
+    // CALLER's token against this shared Task with Task.WaitAsync, never against the work itself.
+    private static readonly Lazy<Task<ImmutableArray<PortableExecutableReference>>> SharedSnapshot =
+        new(() => Task.Run(CreateSharedSnapshot), LazyThreadSafetyMode.ExecutionAndPublication);
 
     private static ImmutableArray<PortableExecutableReference> CreateSharedSnapshot()
         => AppDomain.CurrentDomain.GetAssemblies()
@@ -165,12 +172,27 @@ private static bool IsTestScaffolding(string assemblyName)
     /// References for one kernel session: the process-shared snapshot plus shared
     /// references for any assembly in <paramref name="sessionAssemblies"/> (curated
     /// core set + DI-contributed module assemblies) that was loaded after the
-    /// snapshot was taken. The per-session cost is ~zero — everything resolves to
-    /// the same process-wide instances.
+    /// snapshot was taken. The per-session cost is ~zero once the snapshot is warm —
+    /// everything resolves to the same process-wide instances.
+    ///
+    /// <para>🚨 <paramref name="ct"/> governs only how long THIS caller waits — it is raced
+    /// against the shared snapshot Task via <c>Task.WaitAsync</c>, never used to cancel the
+    /// snapshot build itself. The build is a process-wide, cache-forever memo shared by every
+    /// mesh in the process (issues #2480 /
+    /// #2578: it used to run inline inside a per-request pooled leaf with no cancellation
+    /// participation at all — the FIRST caller in the process paid its full, sometimes
+    /// multi-second cold cost while holding that leaf's IIoPool gate permit, and a silo/mesh
+    /// teardown racing that first caller had no way to reclaim the permit within the drain
+    /// budget). Racing the WAIT (not the work) means: this caller's own pooled leaf settles
+    /// promptly on cancellation and releases its gate permit, while the shared build keeps
+    /// running to completion in the background for whichever mesh/request needs it next — safe,
+    /// because it never touches a collectible NodeType ALC or any one mesh's disposed DI scope
+    /// (<see cref="CreateSharedSnapshot"/> explicitly excludes collectible-ALC assemblies).</para>
     /// </summary>
-    public static ImmutableArray<MetadataReference> GetReferences(IEnumerable<Assembly> sessionAssemblies)
+    public static async Task<ImmutableArray<MetadataReference>> GetReferencesAsync(
+        IEnumerable<Assembly> sessionAssemblies, CancellationToken ct)
     {
-        var snapshot = SharedSnapshot.Value;
+        var snapshot = await SharedSnapshot.Value.WaitAsync(ct).ConfigureAwait(false);
         var result = ImmutableArray.CreateBuilder<MetadataReference>(snapshot.Length + 4);
         var seen = new HashSet<PortableExecutableReference>();
         foreach (var reference in snapshot)

src/MeshWeaver.Kernel.Hub/MeshScriptEnvironment.cs

@@ -68,7 +68,7 @@ public static class MeshScriptEnvironment
     /// free — only its metadata reference has to be guaranteed here, immune to the process-wide
     /// snapshot freeze in <see cref="KernelScriptReferences"/>. De-duplication against the
     /// snapshot and the anchors happens by file path in
-    /// <see cref="KernelScriptReferences.GetReferences"/>.
+    /// <see cref="KernelScriptReferences.GetReferencesAsync"/>.
     /// </summary>
     public static IReadOnlyCollection<Assembly> SessionAssemblies(IServiceProvider serviceProvider)
     {
@@ -98,9 +98,15 @@ public static IReadOnlyCollection<Assembly> SessionAssemblies(IServiceProvider s
     /// The full, process-shared metadata reference set for a script session — the shared
     /// snapshot of every referenceable loaded assembly plus the session anchors. Safe to
     /// hand to a Roslyn workspace: every instance is the one shared materialization.
+    ///
+    /// <para>🚨 <paramref name="ct"/> governs only how long the CALLER waits for the shared,
+    /// process-wide snapshot — never used to abort the snapshot build itself, which keeps
+    /// running for the next caller regardless of this one's cancellation. See
+    /// <see cref="KernelScriptReferences.GetReferencesAsync"/>.</para>
     /// </summary>
-    public static ImmutableArray<MetadataReference> References(IServiceProvider serviceProvider)
-        => KernelScriptReferences.GetReferences(SessionAssemblies(serviceProvider));
+    public static Task<ImmutableArray<MetadataReference>> ReferencesAsync(
+        IServiceProvider serviceProvider, CancellationToken ct)
+        => KernelScriptReferences.GetReferencesAsync(SessionAssemblies(serviceProvider), ct);
 
     /// <summary>
     /// Whether an assembly is TEST SCAFFOLDING and must therefore be kept out of the script