[6.1] Fix flaky code coverage job: pin tool versions and harden thread-job handling (#4611)

Author: paulmedynski

eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml

@@ -49,6 +49,9 @@ jobs:
     variables:
       netFxDir: $(Build.SourcesDirectory)\coverageNetFx
       netCoreDir: $(Build.SourcesDirectory)\coverageNetCore
+      # The first-run telemetry notice is written to stderr, which PowerShell
+      # surfaces as an error record and fails the step.
+      DOTNET_COVERAGE_TELEMETRY_OPTOUT: 1
 
     steps:
       - ${{if eq(parameters.debug, true)}}:
@@ -68,10 +71,30 @@ jobs:
           debug: ${{ parameters.debug }}
 
       # Install additional dotnet tools.
-      - pwsh: |
-          dotnet tool install --global dotnet-coverage
-          dotnet tool install --global dotnet-reportgenerator-globaltool
-        displayName: Install dotnet tools
+      #
+      # The dotnet CLI tool tasks need a working directory that contains at most
+      # one project file, but they must still obey the repo's NuGet.config.  We
+      # copy the NuGet.config to a temp dir and run the tasks from there; the
+      # tools are still installed globally for subsequent steps to use.
+      #
+      - pwsh: Copy-Item "$(Build.SourcesDirectory)\NuGet.config" "$(Agent.TempDirectory)\"
+        displayName: Copy NuGet.config for tool installs
+
+      - task: DotNetCoreCLI@2
+        displayName: Install dotnet-coverage tool
+        inputs:
+          command: custom
+          custom: tool
+          workingDirectory: $(Agent.TempDirectory)
+          arguments: install --global dotnet-coverage --version 18.9.0
+
+      - task: DotNetCoreCLI@2
+        displayName: Install dotnet-reportgenerator tool
+        inputs:
+          command: custom
+          custom: tool
+          workingDirectory: $(Agent.TempDirectory)
+          arguments: install --global dotnet-reportgenerator-globaltool --version 5.5.11
 
       - ${{ each targetFramework in parameters.targetFrameworks }}:
         - task: DownloadPipelineArtifact@2
@@ -120,20 +143,33 @@ jobs:
             foreach ($file in $toProcess){
                 $jobs += Start-ThreadJob -ScriptBlock {
                     $params = $using:file
-                    & dotnet-coverage merge $($params.File) --output $($params.OutputFile) --output-format xml
+                    # Redirect stderr into stdout so informational tool output
+                    # isn't surfaced as a PowerShell error record.
+                    $out = & dotnet-coverage merge $($params.File) --output $($params.OutputFile) --output-format xml 2>&1
+                    $exitCode = $LASTEXITCODE
+                    $out | Out-String | Write-Output
+                    if ($exitCode -ne 0) {
+                        throw "dotnet-coverage merge failed with exit code $exitCode for $($params.File)."
+                    }
                 }
             }
 
             if ($jobs.Count -eq 0) {
-                Write-Error "No .coverage files found in $InputDirectoryPath. Cannot merge."
-                exit 1
+                throw "No .coverage files found in $InputDirectoryPath. Cannot merge."
             }
 
             Write-Host "Merging started..."
-            Wait-Job -Job $jobs
+            $null = Wait-Job -Job $jobs
+
+            $failed = @($jobs | Where-Object { $_.State -ne 'Completed' })
 
             foreach ($job in $jobs) {
-                Receive-Job -Job $job -Wait -AutoRemoveJob
+                Receive-Job -Job $job -ErrorAction Continue 2>&1 | Out-String | Write-Host
+                Remove-Job -Job $job -Force
+            }
+
+            if ($failed.Count -gt 0) {
+                throw "$($failed.Count) of $($jobs.Count) merge jobs failed for $InputDirectoryPath."
             }
           }
 
@@ -155,24 +191,40 @@ jobs:
           displayName: '[Debug] List converted files'
 
       - pwsh: |
-          $jobs = @()
-          $jobs += Start-ThreadJob -ScriptBlock {
-              & reportgenerator "-reports:coverageNetFxXml\*.coveragexml" "-targetdir:coveragereportNetFx" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\netfx\src;$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\src" "-classfilters:+Microsoft.Data.*"
-          }
-
-          $jobs += Start-ThreadJob -ScriptBlock {
-              & reportgenerator "-reports:coverageNetCoreXml\*.coveragexml" "-targetdir:coveragereportNetCore" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\netcore\src;$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\src" "-classfilters:+Microsoft.Data.*"
+          function StartReportGenerator {
+            param([string[]]$Arguments)
+
+            Start-ThreadJob -ArgumentList @(,$Arguments) -ScriptBlock {
+              param([string[]]$reportGeneratorArgs)
+
+              # Redirect stderr into stdout so informational tool output isn't
+              # surfaced as a PowerShell error record.
+              $out = & reportgenerator @reportGeneratorArgs 2>&1
+              $exitCode = $LASTEXITCODE
+              $out | Out-String | Write-Output
+              if ($exitCode -ne 0) {
+                  throw "reportgenerator failed with exit code $exitCode."
+              }
+            }
           }
 
-          $jobs += Start-ThreadJob -ScriptBlock {
-              & reportgenerator "-reports:coverageNetCoreXml\*.coveragexml" "-targetdir:coveragereportAddOns" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.alwaysencrypted.azurekeyvaultprovider.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\add-ons\AzureKeyVaultProvider" "-classfilters:+Microsoft.Data.*"
-          }
+          $jobs = @()
+          $jobs += StartReportGenerator @("-reports:coverageNetFxXml\*.coveragexml", "-targetdir:coveragereportNetFx", "-reporttypes:Cobertura;", "-assemblyfilters:+microsoft.data.sqlclient.dll", "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\netfx\src;$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\src", "-classfilters:+Microsoft.Data.*")
+          $jobs += StartReportGenerator @("-reports:coverageNetCoreXml\*.coveragexml", "-targetdir:coveragereportNetCore", "-reporttypes:Cobertura;", "-assemblyfilters:+microsoft.data.sqlclient.dll", "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\netcore\src;$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\src", "-classfilters:+Microsoft.Data.*")
+          $jobs += StartReportGenerator @("-reports:coverageNetCoreXml\*.coveragexml", "-targetdir:coveragereportAddOns", "-reporttypes:Cobertura;", "-assemblyfilters:+microsoft.data.sqlclient.alwaysencrypted.azurekeyvaultprovider.dll", "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\add-ons\AzureKeyVaultProvider", "-classfilters:+Microsoft.Data.*")
 
           Write-Host "Running ReportGenerator..."
-          Wait-Job -Job $jobs
+          $null = Wait-Job -Job $jobs
+
+          $failed = @($jobs | Where-Object { $_.State -ne 'Completed' })
 
           foreach ($job in $jobs) {
-              Receive-Job -Job $job -Wait -AutoRemoveJob
+              Receive-Job -Job $job -ErrorAction Continue 2>&1 | Out-String | Write-Host
+              Remove-Job -Job $job -Force
+          }
+
+          if ($failed.Count -gt 0) {
+              throw "$($failed.Count) of $($jobs.Count) ReportGenerator jobs failed."
           }
 
           Write-Host "Removing merged XML files..."