fix(compiler): bound compile CPU on a dedicated-thread IIoPool lane

src/MeshWeaver.Compiler.Pipeline/MeshNodeCompilationService.cs

@@ -52,6 +52,12 @@ internal class MeshNodeCompilationService(
         hub.ServiceProvider.GetService<IoPoolRegistry>()?.Get(IoPoolNames.Compile)
         ?? IoPool.Unbounded;
 
+    // The CPU lane (IoPoolNames.CompileCpu): the Roslyn half of every compile, on a bounded set of
+    // DEDICATED threads. Unbounded fallback only when no registry is wired (DI-less tests).
+    private readonly IIoPool _cpuLane =
+        hub.ServiceProvider.GetService<IoPoolRegistry>()?.Get(IoPoolNames.CompileCpu)
+        ?? IoPool.Unbounded;
+
     // 🚨 Offload the compile to the ThreadPool via Task.Run — NOT the IoPool. Two reasons the
     // IoPool ("the correct abstraction") fails for this leaf:
     //   1. Its SemaphoreSlim gate serialises the compile against itself (activity-driven +
@@ -1224,7 +1230,8 @@ private IObservable<Unit> ValidateCellSurfaceSingleHome(
             .Take(1)
             .SelectMany(inputs => inputs is null
                 ? Observable.Return<IReadOnlyList<Lsp.DiagnosticInfo>>(Array.Empty<Lsp.DiagnosticInfo>())
-                : OnThreadPool(() => CompileDiagnostics.DiagnoseInputs(inputs)))
+                // Pure Roslyn compute -> the bounded CPU lane, off the ThreadPool (CompileOffTheThreadPool).
+                : _cpuLane.InvokeBlocking(ct => CompileDiagnostics.DiagnoseInputs(inputs, ct)))
             // 🚨 BOUNDED with the same clock as the emit leg — this runs on the FAILURE path,
             // where a hang is worst: the compile has already failed and this is what stands
             // between that failure and its terminal Error write. Unbounded, a stalled
@@ -2048,24 +2055,31 @@ private async Task<CompileEmit> CompileAsyncCore(
         logger.LogInformation("Compiling assembly for {NodeName} ({Mode}, {NuGetRefs} NuGet refs)",
             nodeName, cacheService.IsDiskCacheEnabled ? "disk" : "in-memory", nugetRefs.Length);
 
-        // 🚨 THE CPU-BOUND HALF RUNS ON A DEDICATED THREAD — never on the ThreadPool this async
-        // method happens to be on. Everything above is IO (NuGet restore, the debug source write) and
-        // may resume on any pool thread; everything in EmitCompiled is synchronous Roslyn work
-        // (parse, source generators, bind, emit) that holds its thread for the whole compile. This
-        // leaf is reached through OnThreadPool(Func<Task<T>>) — Task.Run — and the debug source write
-        // above is ON by default (EnableSourceDebugging), so before this hop EVERY production emit ran
-        // on a ThreadPool worker even though CompileThread's own doc said the compile leaf did not:
-        // the sync overload that uses CompileThread is not the one `() => CompileAsync(...)` binds to.
-        // With ConcurrentBuild also off (EmitPipeline.CreateRunCompilationOptions) a compile now costs
-        // one dedicated thread and zero pool workers. Doc/Architecture/CompileOffTheThreadPool.
-        return await CompileThread.Run(() => EmitCompiled(
-                node, nodeName, source, references, nugetAssemblyPaths, sourcePath, ct))
+        // 🚨 THE CPU-BOUND HALF RUNS ON THE CPU LANE — a dedicated thread, BOUNDED. Everything above
+        // is IO (NuGet restore, the debug source write) and may resume on any pool thread; everything
+        // in EmitCompiled is synchronous Roslyn work (generators, bind, emit) that holds its thread
+        // for the whole compile. Before #5327 it ran on whatever ThreadPool worker the debug write's
+        // await resumed on; #5327 moved it to CompileThread, which is off the pool but UNBOUNDED — one
+        // new thread per distinct NodeType compiling at once, hundreds on a roll. The CompileCpu lane
+        // keeps it off the pool AND caps it at IoPoolOptions.CompileCpu (processor count); the rest
+        // queue. EmitCompiled does nothing but compute and write its own artifact — no pool call, no
+        // mesh read — so it can never wait on another lane leaf (the nested-gate deadlock the Compile
+        // pool hit). Doc/Architecture/CompileOffTheThreadPool.
+        return await _cpuLane
+            .InvokeBlocking(laneCt =>
+            {
+                // The lane's token (pool drain) AND the compile's own bound (BoundLeg) both stop Roslyn.
+                using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct, laneCt);
+                return EmitCompiled(
+                    node, nodeName, source, references, nugetAssemblyPaths, sourcePath, linked.Token);
+            })
+            .Await(ct)
             .ConfigureAwait(false);
     }
 
     /// <summary>
     /// The synchronous, CPU-bound half of <see cref="CompileAsyncCore"/>: parse, source generators,
-    /// emit and the disk/in-memory publish. Called ONLY on a <see cref="CompileThread"/> thread.
+    /// emit and the disk/in-memory publish. Called ONLY on a CPU-lane thread (<see cref="IoPoolNames.CompileCpu"/>).
     /// </summary>
     private CompileEmit EmitCompiled(
         MeshNode node,

src/MeshWeaver.Compiler.Pipeline/MeshNodeLanguageService.cs

@@ -62,7 +62,7 @@ internal sealed class MeshNodeLanguageService : IMeshLanguageService
     // inline, inside this leaf's Compile-pool gate, with no cancellation participation at all,
     // so a silo/mesh teardown racing the first caller could not reclaim that gate permit within
     // the drain budget. Every caller now races its OWN `ct` against this shared Task via
-    // Task.WaitAsync (see GetScriptCompletionsAsync/GetScriptDiagnosticsAsync) — the FIRST
+    // Task.WaitAsync (see GetScriptCompletionsAsync/ScriptCompilationAsync) — the FIRST
     // caller's own token never cancels the build (it is shared by every request against this
     // mesh instance), it only governs how long THAT caller waits.
     private readonly Lazy<Task<ScriptWorkspace>> _scriptWorkspace;
@@ -86,6 +86,11 @@ internal sealed class MeshNodeLanguageService : IMeshLanguageService
     // with ConfigureAwait(false) behind a gate. Falls back to the unbounded pool
     // when no registry is wired (e.g. tests constructing the service outside DI).
     private readonly IIoPool _ioPool;
+    private readonly IIoPool _cpuLane;
+
+    /// <summary>Test seam (InternalsVisibleTo): runs on the thread that binds a diagnostics compilation.
+    /// Null in production.</summary>
+    internal Action? OnDiagnoseStarting { get; set; }
 
     public MeshNodeLanguageService(
         MeshNodeCompilationService compilationService,
@@ -104,6 +109,11 @@ public MeshNodeLanguageService(
         memoryStore = new CompletionMemoryStore(hub, logger);
         _ioPool = hub.ServiceProvider.GetService<IoPoolRegistry>()?.Get(IoPoolNames.Compile)
             ?? IoPool.Unbounded;
+        // The CPU lane: every GetDiagnostics (Roslyn's bind — the expensive part of a squiggle
+        // request) runs on a bounded set of DEDICATED threads, never a ThreadPool worker
+        // (Doc/Architecture/CompileOffTheThreadPool). The async workspace legs stay on _ioPool.
+        _cpuLane = hub.ServiceProvider.GetService<IoPoolRegistry>()?.Get(IoPoolNames.CompileCpu)
+            ?? IoPool.Unbounded;
 
         // 🚨 The two helpers own subscriptions that outlive the request that started them — the
         // usage index's mesh-wide corpus scan (its own 30 s Timeout, a 1 s Throttle) and the memory
@@ -140,7 +150,8 @@ public IObservable<NodeDiagnosticsOutcome> GetDiagnostics(string nodeTypePath)
                         // no diagnostics, which is outside #1592's failure (the mandated sweep
                         // enumerates nodeType:NodeType) but is not what this branch means.
                         ? Observable.Return(NodeDiagnosticsOutcome.NotCompilable)
-                        : _ioPool.Run(ct => GetDiagnosticsAsync(cached, ct))
+                        : _ioPool.Run(ct => WorkspaceCompilationAsync(cached, ct))
+                            .SelectMany(compilation => OnCpuLane(compilation, DiagnoseWorkspace))
                             .Select(NodeDiagnosticsOutcome.Compiled)),
 
                 // Genuinely not there, or its delete is in flight — either way nothing was checked.
@@ -280,13 +291,15 @@ private IObservable<NodeDiagnosticsOutcome> SpeculativeFor(
                     // this says "nothing was compiled" rather than inventing a clean bill.
                     ? Observable.Return(NodeDiagnosticsOutcome.NotCompilable)
                     : _ioPool.Run(ct =>
-                            speculativeCompilation.GetDiagnosticsAsync(inputs, sourcePath, proposedCode, ct))
+                            speculativeCompilation.CreateCompilationAsync(inputs, sourcePath, proposedCode, ct))
+                        .SelectMany(compilation => OnCpuLane(compilation, SpeculativeCompilation.Diagnose))
                         .Select(NodeDiagnosticsOutcome.Compiled)),
             // A non-NodeType owner that EXISTS → a standalone script Code node: diagnose in
             // the script environment so lesson cells get live squiggles too (this used to
             // fall through to a compilation that parses a cell as REGULAR C#, reporting the
             // spurious "top-level statements must be in an executable" on every cell).
-            _ => _ioPool.Run(ct => GetScriptDiagnosticsAsync(proposedCode, ct))
+            _ => _ioPool.Run(ct => ScriptCompilationAsync(proposedCode, ct))
+                .SelectMany(compilation => OnCpuLane(compilation, DiagnoseScript))
                 .Select(NodeDiagnosticsOutcome.Compiled),
         };
 
@@ -536,14 +549,32 @@ private static bool VersionsEqual(
         return true;
     }
 
-    private static async Task<IReadOnlyList<DiagnosticInfo>> GetDiagnosticsAsync(
-        CachedWorkspace cached, CancellationToken ct)
+    /// <summary>
+    /// Hands <paramref name="compilation"/> to the CPU lane for its bind. A null compilation (no
+    /// project / document) answers empty without taking a lane slot.
+    /// </summary>
+    private IObservable<IReadOnlyList<DiagnosticInfo>> OnCpuLane<TCompilation>(
+        TCompilation? compilation,
+        Func<TCompilation, CancellationToken, IReadOnlyList<DiagnosticInfo>> diagnose)
+        where TCompilation : class
+        => compilation is null
+            ? Observable.Return<IReadOnlyList<DiagnosticInfo>>(Array.Empty<DiagnosticInfo>())
+            : _cpuLane.InvokeBlocking(ct =>
+            {
+                OnDiagnoseStarting?.Invoke();
+                return diagnose(compilation, ct);
+            });
+
+    /// <summary>The IO half of a node workspace's diagnostics: the project's compilation (not yet bound).</summary>
+    private static async Task<Compilation?> WorkspaceCompilationAsync(CachedWorkspace cached, CancellationToken ct)
     {
         var project = cached.Workspace.CurrentSolution.GetProject(cached.ProjectId);
-        if (project is null) return Array.Empty<DiagnosticInfo>();
+        return project is null ? null : await project.GetCompilationAsync(ct).ConfigureAwait(false);
+    }
 
-        var compilation = await project.GetCompilationAsync(ct).ConfigureAwait(false);
-        if (compilation is null) return Array.Empty<DiagnosticInfo>();
+    /// <summary>The CPU half: bind and report — a CPU-lane leaf (no IO, no pool call).</summary>
+    private static IReadOnlyList<DiagnosticInfo> DiagnoseWorkspace(Compilation compilation, CancellationToken ct)
+    {
         var diags = compilation.GetDiagnostics(ct);
         if (diags.IsDefaultOrEmpty) return Array.Empty<DiagnosticInfo>();
 
@@ -645,19 +676,20 @@ private async Task<IReadOnlyList<CompletionEntry>> GetScriptCompletionsAsync(
     /// Diagnostics for a standalone SCRIPT Code node — the script-environment sibling of
     /// <see cref="SpeculativeCompilation"/>, so lesson cells get live squiggles.
     /// </summary>
-    private async Task<IReadOnlyList<DiagnosticInfo>> GetScriptDiagnosticsAsync(
-        string proposedCode, CancellationToken ct)
+    private async Task<Compilation?> ScriptCompilationAsync(string proposedCode, CancellationToken ct)
     {
         var script = await _scriptWorkspace.Value.WaitAsync(ct).ConfigureAwait(false);
         var document = script.Workspace.CurrentSolution
             .WithDocumentText(script.DocumentId, SourceText.From(StripKernelDirectives(proposedCode)))
             .GetDocument(script.DocumentId);
-        var compilation = document is null
+        return document is null
             ? null
             : await document.Project.GetCompilationAsync(ct).ConfigureAwait(false);
-        if (compilation is null) return Array.Empty<DiagnosticInfo>();
-
+    }
 
+    /// <summary>The CPU half of a script's diagnostics — a CPU-lane leaf.</summary>
+    private static IReadOnlyList<DiagnosticInfo> DiagnoseScript(Compilation compilation, CancellationToken ct)
+    {
         var diags = compilation.GetDiagnostics(ct);
         if (diags.IsDefaultOrEmpty) return Array.Empty<DiagnosticInfo>();
         var result = new List<DiagnosticInfo>();
@@ -968,7 +1000,10 @@ private static async Task<ScriptWorkspace> BuildScriptWorkspaceAsync(IServicePro
                 language: LanguageNames.CSharp,
                 compilationOptions: new CSharpCompilationOptions(
                         OutputKind.DynamicallyLinkedLibrary,
-                        metadataReferenceResolver: Kernel.Hub.MeshScriptEnvironment.MetadataResolver)
+                        metadataReferenceResolver: Kernel.Hub.MeshScriptEnvironment.MetadataResolver,
+                        // Roslyn's default fans every bind out onto the ThreadPool — see
+                        // EmitPipeline.CreateRunCompilationOptions / CompileOffTheThreadPool.
+                        concurrentBuild: false)
                     .WithUsings(Kernel.Hub.MeshScriptEnvironment.Imports),
                 parseOptions: new CSharpParseOptions(kind: SourceCodeKind.Script),
                 metadataReferences: references,

src/MeshWeaver.Compiler/CompileDiagnostics.cs

@@ -61,7 +61,10 @@ internal static string FormatCompileFailure(string nodePath, IEnumerable<Diagnos
     /// GUI mark each error at its exact line/column in a Monaco editor and link to the source.
     /// Synchronous, CPU-bound — the caller schedules it off the hub.
     /// </summary>
-    internal static IReadOnlyList<Lsp.DiagnosticInfo> DiagnoseInputs(CompilationInputs inputs)
+    /// <param name="inputs">The compilation inputs to diagnose.</param>
+    /// <param name="ct">Stops the bind when the caller stops waiting — on the CPU lane an abandoned
+    /// diagnose must release its slot, not run to the end holding it.</param>
+    internal static IReadOnlyList<Lsp.DiagnosticInfo> DiagnoseInputs(CompilationInputs inputs, CancellationToken ct = default)
     {
         var trees = new List<SyntaxTree>(inputs.Sources.Length + 2)
         {
@@ -86,9 +89,9 @@ internal static string FormatCompileFailure(string nodePath, IEnumerable<Diagnos
         // compile already reflects generation). Avoids loading any generator on every failed compile.
         var compilation = GeneratorPipeline.RunSourceGenerators(
             CSharpCompilation.Create(inputs.AssemblyName, trees, inputs.References, inputs.CompilationOptions),
-            Array.Empty<string>(), Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance, CancellationToken.None);
+            Array.Empty<string>(), Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance, ct);
 
-        var diags = compilation.GetDiagnostics();
+        var diags = compilation.GetDiagnostics(ct);
         if (diags.IsDefaultOrEmpty) return Array.Empty<Lsp.DiagnosticInfo>();
 
         var result = new List<Lsp.DiagnosticInfo>(diags.Length);

src/MeshWeaver.Compiler/SpeculativeCompilation.cs

@@ -41,6 +41,19 @@ public async Task<IReadOnlyList<DiagnosticInfo>> GetDiagnosticsAsync(
         string sourcePath,
         string proposedCode,
         CancellationToken ct)
+        => Diagnose(await CreateCompilationAsync(inputs, sourcePath, proposedCode, ct).ConfigureAwait(false), ct);
+
+    /// <summary>
+    /// The IO half of <see cref="GetDiagnosticsAsync"/>: resolves any proposed NuGet references and
+    /// builds the speculative compilation (parse only — nothing is bound yet). Split from
+    /// <see cref="Diagnose"/> so a caller can run the CPU-bound half on the dedicated CPU lane
+    /// (<c>IoPoolNames.CompileCpu</c>, <c>Doc/Architecture/CompileOffTheThreadPool</c>).
+    /// </summary>
+    public async Task<CSharpCompilation> CreateCompilationAsync(
+        CompilationInputs inputs,
+        string sourcePath,
+        string proposedCode,
+        CancellationToken ct)
     {
         // Strip #r from the proposed source so Roslyn doesn't reject it with CS7011.
         // Resolve any new package refs and append them to the cached reference set —
@@ -106,12 +119,20 @@ public async Task<IReadOnlyList<DiagnosticInfo>> GetDiagnosticsAsync(
                 SourceText.From(code), inputs.ParseOptions, path: path));
         }
 
-        var compilation = CSharpCompilation.Create(
+        return CSharpCompilation.Create(
             inputs.AssemblyName,
             syntaxTrees: trees,
             references: effectiveReferences,
             options: inputs.CompilationOptions);
+    }
 
+    /// <summary>
+    /// The CPU-bound half of <see cref="GetDiagnosticsAsync"/>: binds <paramref name="compilation"/>
+    /// and reports its user-actionable diagnostics (framework-generated trees filtered out).
+    /// Synchronous and pure — no IO, no pool call — so it is a valid CPU-lane leaf.
+    /// </summary>
+    public static IReadOnlyList<DiagnosticInfo> Diagnose(CSharpCompilation compilation, CancellationToken ct)
+    {
         var diags = compilation.GetDiagnostics(ct);
         if (diags.IsDefaultOrEmpty) return Array.Empty<DiagnosticInfo>();