Don't disable the MSBuild server for /mt builds when node reuse is off

documentation/MSBuild-Server.md

@@ -11,6 +11,16 @@ To re-enable MSBuild Server, remove the variable or set its value to `0`.
 
 When a build is multithreaded (`/mt`), the server node is launched with [Server GC](https://learn.microsoft.com/dotnet/standard/garbage-collection/workstation-server-gc) enabled. Under `/mt` the server runs all project work on threads in this single process, so Server GC's higher throughput is beneficial; without `/mt` the server only orchestrates and delegates project work to separate worker nodes, so it keeps the default Workstation GC. GC mode is fixed at CLR startup, so it is set via the `DOTNET_gcServer` environment variable in the server's launch environment (decided from the launching invocation's command line). An explicit user-set `DOTNET_gcServer` is honored (e.g. set `DOTNET_gcServer=0` to keep Workstation GC in a memory-constrained environment). This is scoped to the server process only: sidecar TaskHosts and worker nodes keep the default Workstation GC.
 
+## Node reuse and server lifetime
+
+MSBuild Server is a form of node reuse: the whole point of the server is to stay resident between builds so later builds reuse its warmed-up process and caches. Consequently:
+
+- **Node reuse on (the default).** The server is eligible and, after a build, returns to listening so the next compatible client reuses it.
+- **Node reuse off (`-nodeReuse:false` / `-nr:false`) without `/mt`.** Keeping a process resident contradicts the no-reuse intent, so the build does not use the server at all (it runs entirely in the launching process). See `ServerShouldNotRunWhenNodeReuseEqualsFalse`.
+- **Node reuse off *with* `/mt`.** A `/mt` build needs the server for a different reason: multithreaded project execution runs inside the server process, which is where Server GC is applied (see [Garbage collection](#garbage-collection)). So a `/mt` build still engages the server even when node reuse is off - but it must honor the no-reuse request by **not** leaving the server resident afterwards.
+
+The client makes a single, response-file-aware determination and sets the `ShutdownAfterBuild` flag on the `ServerNodeBuildCommand` packet if server needs shutdown.
+
 ## Communication protocol
 
 The server node uses same IPC approach as current worker nodes - named pipes. This solution allows to reuse existing code. When process starts, pipe with deterministic name is opened and waiting for commands. Client has following worfklow:

src/Build.UnitTests/BackEnd/ServerNodeBuildCommand_Tests.cs

@@ -0,0 +1,63 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using Microsoft.Build.BackEnd;
+using Microsoft.Build.BackEnd.Logging;
+using Shouldly;
+using Xunit;
+
+namespace Microsoft.Build.UnitTests.BackEnd
+{
+    /// <summary>
+    /// Unit tests for the <see cref="ServerNodeBuildCommand"/> packet.
+    /// </summary>
+    public class ServerNodeBuildCommand_Tests
+    {
+        /// <summary>
+        /// Round-trips a <see cref="ServerNodeBuildCommand"/> through the binary translator and verifies that all
+        /// fields - including the <see cref="ServerNodeBuildCommand.ShutdownAfterBuild"/> flag that drives the
+        /// server's self-teardown - survive serialization for both values of the flag.
+        /// </summary>
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public void RoundTripSerializationPreservesFields(bool shutdownAfterBuild)
+        {
+            string[] commandLine = ["msbuild.exe", "project.proj", "-mt", "-nr:false"];
+            string startupDirectory = "C:\\some\\startup\\dir";
+            Dictionary<string, string> buildProcessEnvironment = new(StringComparer.OrdinalIgnoreCase)
+            {
+                ["VAR1"] = "value1",
+                ["VAR2"] = "value2",
+            };
+            CultureInfo culture = new("en-US");
+            CultureInfo uiCulture = new("en-GB");
+            TargetConsoleConfiguration consoleConfiguration = new(bufferWidth: 80, acceptAnsiColorCodes: true, outputIsScreen: false, backgroundColor: ConsoleColor.Black);
+
+            ServerNodeBuildCommand command = new(
+                commandLine,
+                startupDirectory,
+                buildProcessEnvironment,
+                culture,
+                uiCulture,
+                consoleConfiguration,
+                partialBuildTelemetry: null,
+                shutdownAfterBuild);
+
+            ((INodePacket)command).Translate(TranslationHelpers.GetWriteTranslator());
+            INodePacket packet = ServerNodeBuildCommand.FactoryForDeserialization(TranslationHelpers.GetReadTranslator());
+
+            ServerNodeBuildCommand deserialized = packet.ShouldBeOfType<ServerNodeBuildCommand>();
+
+            deserialized.ShutdownAfterBuild.ShouldBe(shutdownAfterBuild);
+            deserialized.CommandLine.ShouldBe(commandLine);
+            deserialized.StartupDirectory.ShouldBe(startupDirectory);
+            deserialized.BuildProcessEnvironment.ShouldBe(buildProcessEnvironment);
+            deserialized.Culture.ShouldBe(culture);
+            deserialized.UICulture.ShouldBe(uiCulture);
+        }
+    }
+}

src/Build/BackEnd/Client/MSBuildClient.cs

@@ -113,6 +113,11 @@ public sealed class MSBuildClient
         /// </summary>
         private readonly bool _multiThreaded;
 
+        /// <summary>
+        /// Whether the server should shut itself down once this build completes instead of staying resident for reuse. 
+        /// </summary>
+        private readonly bool _shutdownServerAfterBuild;
+
         /// <summary>
         /// Public constructor with parameters.
         /// </summary>
@@ -135,6 +140,22 @@ public MSBuildClient(string[] commandLine, string msbuildLocation)
         /// <param name="multiThreaded">Whether this build is multithreaded (/mt). When true, the launched
         /// server process is started with Server GC.</param>
         public MSBuildClient(string[] commandLine, string msbuildLocation, bool multiThreaded)
+            : this(commandLine, msbuildLocation, multiThreaded, shutdownServerAfterBuild: false)
+        {
+        }
+
+        /// <summary>
+        /// Public constructor with parameters.
+        /// </summary>
+        /// <param name="commandLine">The command line to process. The first argument
+        /// on the command line is assumed to be the name/path of the executable, and is ignored</param>
+        /// <param name="msbuildLocation"> Full path to current MSBuild.exe if executable is MSBuild.exe,
+        /// or to version of MSBuild.dll found to be associated with the current process.</param>
+        /// <param name="multiThreaded">Whether this build is multithreaded (/mt). When true, the launched
+        /// server process is started with Server GC.</param>
+        /// <param name="shutdownServerAfterBuild">Whether the server should shut itself down once this build
+        /// completes instead of staying resident for reuse (e.g. a /mt build with -nodeReuse:false).</param>
+        public MSBuildClient(string[] commandLine, string msbuildLocation, bool multiThreaded, bool shutdownServerAfterBuild)
         {
             _serverEnvironmentVariables = new();
             _exitResult = new();
@@ -143,6 +164,7 @@ public MSBuildClient(string[] commandLine, string msbuildLocation, bool multiThr
             _commandLine = commandLine;
             _msbuildLocation = msbuildLocation;
             _multiThreaded = multiThreaded;
+            _shutdownServerAfterBuild = shutdownServerAfterBuild;
 
             // Client <-> Server communication stream
             _handshake = GetHandshake();
@@ -587,7 +609,8 @@ private ServerNodeBuildCommand GetServerNodeBuildCommand()
                         CultureInfo.CurrentCulture,
                         CultureInfo.CurrentUICulture,
                         _consoleConfiguration!,
-                        partialBuildTelemetry);
+                        partialBuildTelemetry,
+                        _shutdownServerAfterBuild);
         }
 
         private ServerNodeHandshake GetHandshake() => new(CommunicationsUtilities.GetHandshakeOptions(

src/Build/BackEnd/Node/OutOfProcServerNode.cs

@@ -456,8 +456,8 @@ private void HandleServerNodeBuildCommand(ServerNodeBuildCommand command)
             var response = new ServerNodeBuildResult(buildResult.exitCode, buildResult.exitType);
             SendPacket(response);
 
-            // Shutdown server if cancel was requested. This is consistent with nodes behavior.
-            _shutdownReason = _cancelRequested ? NodeEngineShutdownReason.BuildComplete : NodeEngineShutdownReason.BuildCompleteReuse;
+            // Shutdown server after this build if a cancel was requested, or if the client asked for no reuse. This is consistent with nodes behavior.
+            _shutdownReason = (_cancelRequested || command.ShutdownAfterBuild) ? NodeEngineShutdownReason.BuildComplete : NodeEngineShutdownReason.BuildCompleteReuse;
             _shutdownEvent.Set();
         }
 

src/Build/BackEnd/Node/ServerNodeBuildCommand.cs

@@ -20,6 +20,7 @@ internal sealed class ServerNodeBuildCommand : INodePacket
         private CultureInfo _uiCulture = default!;
         private TargetConsoleConfiguration _consoleConfiguration = default!;
         private PartialBuildTelemetry? _partialBuildTelemetry = default;
+        private bool _shutdownAfterBuild;
 
         /// <summary>
         /// Retrieves the packet type.
@@ -62,6 +63,11 @@ internal sealed class ServerNodeBuildCommand : INodePacket
         /// </summary>
         public PartialBuildTelemetry? PartialBuildTelemetry => _partialBuildTelemetry;
 
+        /// <summary>
+        /// Whether the server should shut itself down once this build completes instead of staying resident for reuse.
+        /// </summary>
+        public bool ShutdownAfterBuild => _shutdownAfterBuild;
+
         /// <summary>
         /// Private constructor for deserialization
         /// </summary>
@@ -75,7 +81,8 @@ public ServerNodeBuildCommand(
             Dictionary<string, string> buildProcessEnvironment,
             CultureInfo culture, CultureInfo uiCulture,
             TargetConsoleConfiguration consoleConfiguration,
-            PartialBuildTelemetry? partialBuildTelemetry)
+            PartialBuildTelemetry? partialBuildTelemetry,
+            bool shutdownAfterBuild)
         {
             Assumed.NotNull(consoleConfiguration);
 
@@ -86,6 +93,7 @@ public ServerNodeBuildCommand(
             _uiCulture = uiCulture;
             _consoleConfiguration = consoleConfiguration;
             _partialBuildTelemetry = partialBuildTelemetry;
+            _shutdownAfterBuild = shutdownAfterBuild;
         }
 
         /// <summary>
@@ -101,6 +109,7 @@ public void Translate(ITranslator translator)
             translator.TranslateCulture(ref _uiCulture);
             translator.Translate(ref _consoleConfiguration, TargetConsoleConfiguration.FactoryForDeserialization);
             translator.Translate(ref _partialBuildTelemetry, PartialBuildTelemetry.FactoryForDeserialization);
+            translator.Translate(ref _shutdownAfterBuild);
         }
 
         /// <summary>

src/MSBuild.UnitTests/MSBuildServer_Tests.cs

@@ -386,6 +386,73 @@ public void ServerStartsWhenMtInResponseFileEvenWithoutEnvVar()
             MSBuildClient.ShutdownServer(CancellationToken.None);
         }
 
+#if NET
+        /// <summary>
+        /// Disabling node reuse (e.g. <c>-nr:false</c>, as <c>dotnet restore</c> does) must NOT prevent a
+        /// multithreaded (/mt) build from using the server. Instead of skipping the server, the no-reuse intent is
+        /// honored by shutting the server down after the build. This test verifies both halves: the build runs in a
+        /// separate server process, and that process does not survive the build (so a subsequent build gets a fresh server).
+        /// </summary>
+        [Fact]
+        public void MultiThreadedServerIsUsedButShutDownWhenNodeReuseDisabled()
+        {
+            // Clear MSBUILDUSESERVER so we exercise the -mt-implies-server path, and isolate this test's server
+            // with a unique handshake salt and a clean environment.
+            PrepareIsolatedServerEnv(useServer: false);
+            TransientTestFile project = _env.CreateFile("mtNoReuseProbe.proj", GetServerGCProbeProjectContents(useTaskHostFactory: false));
+
+            // Make sure we start with no server running.
+            MSBuildClient.ShutdownServer(CancellationToken.None);
+
+            try
+            {
+                // -mt forces the server on even though node reuse is disabled.
+                string output1 = RunnerUtilities.ExecMSBuild(BuildEnvironmentHelper.Instance.CurrentMSBuildExePath, $"{project.Path} -mt -nr:false", out bool success1, false, _output);
+                success1.ShouldBeTrue();
+                int clientPid1 = ParseNumber(output1, "Process ID is ");
+                int serverPid1 = ParseNumber(output1, "TaskRanInPID=");
+                // Register cleanup before any assertion so the server does not leak if an assertion throws.
+                _env.WithTransientProcess(serverPid1);
+
+                // The build ran in a separate server process: proof the server was engaged despite -nr:false.
+                serverPid1.ShouldNotBe(clientPid1, "Even with node reuse disabled, -mt must run the build in the server node, not the entry process.");
+
+                // Because node reuse is disabled, the server must not persist past the build: its process should exit.
+                WaitForProcessExit(serverPid1).ShouldBeTrue($"Server process {serverPid1} should have been shut down after the build when node reuse is disabled.");
+
+                // A second build cannot reuse the (now gone) server, so it must launch a fresh server process.
+                string output2 = RunnerUtilities.ExecMSBuild(BuildEnvironmentHelper.Instance.CurrentMSBuildExePath, $"{project.Path} -mt -nr:false", out bool success2, false, _output);
+                success2.ShouldBeTrue();
+                int serverPid2 = ParseNumber(output2, "TaskRanInPID=");
+                _env.WithTransientProcess(serverPid2);
+                serverPid2.ShouldNotBe(serverPid1, "With node reuse disabled, each -mt build should get a fresh, non-persistent server process.");
+            }
+            finally
+            {
+                // Ensure any server we spun up is torn down even if an assertion above fails.
+                MSBuildClient.ShutdownServer(CancellationToken.None);
+            }
+        }
+
+        /// <summary>
+        /// Waits up to <paramref name="timeoutMs"/> for the process with the given PID to exit. Returns true if
+        /// the process exited (or was already gone), false if it was still running when the timeout elapsed.
+        /// </summary>
+        private static bool WaitForProcessExit(int pid, int timeoutMs = 10000)
+        {
+            try
+            {
+                using Process process = Process.GetProcessById(pid);
+                return process.WaitForExit(timeoutMs);
+            }
+            catch (ArgumentException)
+            {
+                // No process with that PID is running - it has already exited.
+                return true;
+            }
+        }
+#endif
+
         [Fact]
         public void PropertyMSBuildStartupDirectoryOnServer()
         {

src/MSBuild/MSBuildClientApp.cs

@@ -30,20 +30,23 @@ internal static class MSBuildClientApp
         /// on the command line is assumed to be the name/path of the executable, and
         /// is ignored.</param>
         /// <param name="multiThreaded">Whether this build is multithreaded (/mt).</param>
+        /// <param name="shutdownServerAfterBuild">Whether the server should shut itself down once the build
+        /// completes instead of staying resident for reuse.</param>
         /// <param name="cancellationToken">Cancellation token.</param>
         /// <returns>A value of type <see cref="MSBuildApp.ExitType"/> that indicates whether the build succeeded,
         /// or the manner in which it failed.</returns>
         /// <remarks>
         /// The locations of msbuild exe/dll and dotnet.exe would be automatically detected if called from dotnet or msbuild cli. Calling this function from other executables might not work.
         /// </remarks>
-        public static MSBuildApp.ExitType Execute(string[] commandLineArgs, bool multiThreaded, CancellationToken cancellationToken)
+        public static MSBuildApp.ExitType Execute(string[] commandLineArgs, bool multiThreaded, bool shutdownServerAfterBuild, CancellationToken cancellationToken)
         {
             string msbuildLocation = BuildEnvironmentHelper.Instance.CurrentMSBuildExePath;
 
             return Execute(
                 commandLineArgs,
                 msbuildLocation,
                 multiThreaded,
+                shutdownServerAfterBuild,
                 cancellationToken);
         }
 
@@ -56,12 +59,14 @@ public static MSBuildApp.ExitType Execute(string[] commandLineArgs, bool multiTh
         /// <param name="msbuildLocation"> Full path to current MSBuild.exe if executable is MSBuild.exe,
         /// or to version of MSBuild.dll found to be associated with the current process.</param>
         /// <param name="multiThreaded">Whether this build is multithreaded (/mt).</param>
+        /// <param name="shutdownServerAfterBuild">Whether the server should shut itself down once the build
+        /// completes instead of staying resident for reuse.</param>
         /// <param name="cancellationToken">Cancellation token.</param>
         /// <returns>A value of type <see cref="MSBuildApp.ExitType"/> that indicates whether the build succeeded,
         /// or the manner in which it failed.</returns>
-        public static MSBuildApp.ExitType Execute(string[] commandLineArgs, string msbuildLocation, bool multiThreaded, CancellationToken cancellationToken)
+        public static MSBuildApp.ExitType Execute(string[] commandLineArgs, string msbuildLocation, bool multiThreaded, bool shutdownServerAfterBuild, CancellationToken cancellationToken)
         {
-            MSBuildClient msbuildClient = new MSBuildClient(commandLineArgs, msbuildLocation, multiThreaded);
+            MSBuildClient msbuildClient = new MSBuildClient(commandLineArgs, msbuildLocation, multiThreaded, shutdownServerAfterBuild);
             MSBuildClientExitResult exitResult = msbuildClient.Execute(cancellationToken);
 
             if (exitResult.MSBuildClientExitType == MSBuildClientExitType.ServerBusy ||