When sharing the terminal with child nodes, wait for the children to terminate before exiting ourselves.

scripts/Deploy-MSBuild.ps1

@@ -73,11 +73,11 @@ $filesToCopyToBin = @(
     FileToCopy "$bootstrapBinDirectory\Microsoft.Managed.targets"
     FileToCopy "$bootstrapBinDirectory\Microsoft.Managed.Before.targets"
     FileToCopy "$bootstrapBinDirectory\Microsoft.Managed.After.targets"
-    FileToCopy "$bootstrapBinDirectory\Microsoft.Net.props"
-    FileToCopy "$bootstrapBinDirectory\Microsoft.NetFramework.CurrentVersion.props"
-    FileToCopy "$bootstrapBinDirectory\Microsoft.NetFramework.CurrentVersion.targets"
-    FileToCopy "$bootstrapBinDirectory\Microsoft.NetFramework.props"
-    FileToCopy "$bootstrapBinDirectory\Microsoft.NetFramework.targets"
+    FileToCopy "$bootstrapBinDirectory\Microsoft.NET.props"
+    FileToCopy "$bootstrapBinDirectory\Microsoft.NETFramework.CurrentVersion.props"
+    FileToCopy "$bootstrapBinDirectory\Microsoft.NETFramework.CurrentVersion.targets"
+    FileToCopy "$bootstrapBinDirectory\Microsoft.NETFramework.props"
+    FileToCopy "$bootstrapBinDirectory\Microsoft.NETFramework.targets"
     FileToCopy "$bootstrapBinDirectory\Microsoft.VisualBasic.CrossTargeting.targets"
     FileToCopy "$bootstrapBinDirectory\Microsoft.VisualBasic.CurrentVersion.targets"
     FileToCopy "$bootstrapBinDirectory\Microsoft.VisualBasic.targets"

src/Build/BackEnd/Components/Communications/NodeProviderOutOfProcBase.cs

@@ -50,6 +50,11 @@ internal abstract class NodeProviderOutOfProcBase
         /// </summary>
         private const int TimeoutForNewNodeCreation = 30000;
 
+        /// <summary>
+        /// The amount of time to wait for an out-of-proc node to exit.
+        /// </summary>
+        private const int TimeoutForWaitForExit = 30000;
+
         /// <summary>
         /// The build component host.
         /// </summary>
@@ -96,9 +101,29 @@ protected void ShutdownConnectedNodes(List<NodeContext> contextsToShutDown, bool
             // Send the build completion message to the nodes, causing them to shutdown or reset.
             _processesToIgnore.Clear();
 
+            // We wait for child nodes to exit to avoid them changing the terminal
+            // after this process terminates.
+            bool waitForExit =  !enableReuse &&
+                                !Console.IsInputRedirected &&
+                                Traits.Instance.EscapeHatches.EnsureStdOutForChildNodesIsPrimaryStdout;
+
+            Task[] waitForExitTasks = waitForExit && contextsToShutDown.Count > 0? new Task[contextsToShutDown.Count] : null;
+            int i = 0;
             foreach (NodeContext nodeContext in contextsToShutDown)
             {
-                nodeContext?.SendData(new NodeBuildComplete(enableReuse));
+                if (nodeContext is null)
+                {
+                    continue;
+                }
+                nodeContext.SendData(new NodeBuildComplete(enableReuse));
+                if (waitForExit)
+                {
+                    waitForExitTasks[i++] = nodeContext.WaitForExitAsync();
+                }
+            }
+            if (waitForExitTasks != null)
+            {
+                Task.WaitAll(waitForExitTasks);
             }
         }
 
@@ -613,14 +638,14 @@ internal class NodeContext
             private byte[] _smallReadBuffer;
 
             /// <summary>
-            /// Event indicating the node has terminated.
+            /// Delegate called when the context terminates.
             /// </summary>
-            private ManualResetEvent _nodeTerminated;
+            private NodeContextTerminateDelegate _terminateDelegate;
 
             /// <summary>
-            /// Delegate called when the context terminates.
+            /// Node was requested to terminate.
             /// </summary>
-            private NodeContextTerminateDelegate _terminateDelegate;
+            private bool _closeSent;
 
             /// <summary>
             /// Per node read buffers
@@ -641,7 +666,6 @@ public NodeContext(int nodeId, int processId,
                 _packetFactory = factory;
                 _headerByte = new byte[5]; // 1 for the packet type, 4 for the body length
                 _smallReadBuffer = new byte[1000]; // 1000 was just an average seen on one profile run.
-                _nodeTerminated = new ManualResetEvent(false);
                 _terminateDelegate = terminateDelegate;
                 _sharedReadBuffer = InterningBinaryReader.CreateSharedBuffer();
             }
@@ -777,6 +801,7 @@ public void SendData(INodePacket packet)
 #endif
                         }
                     }
+                    _closeSent = packet is NodeBuildComplete buildCompletePacket && !buildCompletePacket.PrepareForReuse;
                 }
                 catch (IOException e)
                 {
@@ -792,8 +817,9 @@ public void SendData(INodePacket packet)
             /// <summary>
             /// Closes the node's context, disconnecting it from the node.
             /// </summary>
-            public void Close()
+            private void Close()
             {
+                _processId = -1;
                 _clientToServerStream.Dispose();
                 if (!object.ReferenceEquals(_clientToServerStream, _serverToClientStream))
                 {
@@ -802,6 +828,56 @@ public void Close()
                 _terminateDelegate(_nodeId);
             }
 
+            /// <summary>
+            /// Waits for the child node process to exit.
+            /// </summary>
+            public async Task WaitForExitAsync()
+            {
+                int processId = _processId;
+                if (processId != -1)
+                {
+                    Process childProcess;
+                    try
+                    {
+                        childProcess = Process.GetProcessById(processId);
+                    }
+                    catch (System.ArgumentException)
+                    {
+                        // The process has terminated already.
+                        return;
+                    }
+
+                    // Wait for the process to terminate.
+                    CommunicationsUtilities.Trace("Waiting for node with pid = {0} to terminate", processId);
+
+                    if (_closeSent)
+                    {
+                        // .NET 5 introduces a real WaitForExitAsyc.
+                        // This is a poor man's implementation that uses polling.
+                        int timeout = TimeoutForWaitForExit;
+                        int delay = 5;
+                        while (timeout > 0)
+                        {
+                            bool exited = childProcess.WaitForExit(milliseconds: 0);
+                            if (exited)
+                            {
+                                return;
+                            }
+                            timeout -= delay;
+                            await Task.Delay(delay).ConfigureAwait(false);
+
+                            // Double delay up to 500ms.
+                            delay = Math.Min(delay * 2, 500);
+                        }
+                    }
+
+                    // Kill the child and do a blocking wait.
+                    CommunicationsUtilities.Trace("Killing node with pid = {0}", processId);
+                    childProcess.Kill();
+                    childProcess.WaitForExit();
+                }
+            }
+
 #if FEATURE_APM
             /// <summary>
             /// Completes the asynchronous packet write to the node.