The benchmark leg seems to reliably fail (#16600)

Co-authored-by: Vlad Zarytovskii <vzaritovsky@hotmail.com>

Author: KevinRansom

Directory.Build.props

@@ -16,10 +16,6 @@
       <BUILDING_USING_DOTNET>true</BUILDING_USING_DOTNET>
   </PropertyGroup>
 
-  <PropertyGroup Condition="'$(SolutionName)' == 'FSharp.Benchmarks' and '$(BUILDING_USING_DOTNET)' != 'false'">
-      <BUILDING_USING_DOTNET>true</BUILDING_USING_DOTNET>
-  </PropertyGroup>
-
   <!-- Temporary workaround if proj-info is used, allows Ionide's evaluation to work correctly. -->
   <PropertyGroup Condition="'$(BUILDING_USING_DOTNET)' != 'false' and '$(DotnetProjInfo)' == 'true'">
       <BUILDING_USING_DOTNET>true</BUILDING_USING_DOTNET>

azure-pipelines.yml

@@ -1,4 +1,4 @@
-# CI and PR triggers
+# CI and PR triggers 
 trigger:
   branches:
     include:
@@ -718,25 +718,10 @@ stages:
           steps:
           - checkout: self
             clean: true
-          - script: dotnet --list-sdks
-            displayName: Report dotnet SDK versions
-          - task: UseDotNet@2
-            displayName: install SDK
-            inputs:
-              packageType: sdk
-              useGlobalJson: true
-              includePreviewVersions: true
-              workingDirectory: $(Build.SourcesDirectory)
-              installationPath: $(Agent.ToolsDirectory)/dotnet
-          - script: dotnet build -c $(_BuildConfig) .\FSharp.Benchmarks.sln /bl:\"artifacts/log/$(_BuildConfig)/BenchmarkBuild.binlog\"
-            workingDirectory: $(Build.SourcesDirectory)
-            displayName: Plain build of FSharp.Benchmarks.sln
+          - script: eng\CIBuild.cmd -testBenchmarks
+            displayName: Smoke test fast benchmarks
             continueOnError: true
             condition: always()
-          - script: .\Build.cmd -c $(_BuildConfig)
-          - pwsh: ./SmokeTestBenchmarks.ps1
-            workingDirectory: $(Build.SourcesDirectory)/tests/benchmarks
-            displayName: Smoke test fast benchmarks
 
         # Test trimming on Windows
         - job: Build_And_Test_Trimming_Windows

eng/Build.ps1

@@ -62,6 +62,7 @@ param (
     [switch]$testAllButIntegration,
     [switch]$testpack,
     [switch]$testAOT,
+    [switch]$testBenchmarks,
     [string]$officialSkipTests = "false",
     [switch]$noVisualStudio,
     [switch]$sourceBuild,
@@ -111,6 +112,7 @@ function Print-Usage() {
     Write-Host "  -testVs                       Run F# editor unit tests"
     Write-Host "  -testpack                     Verify built packages"
     Write-Host "  -testAOT                      Run AOT/Trimming tests"
+    Write-Host "  -testBenchmarks               Build and Run Benchmark suite"
     Write-Host "  -officialSkipTests <bool>     Set to 'true' to skip running tests"
     Write-Host ""
     Write-Host "Advanced settings:"
@@ -176,6 +178,7 @@ function Process-Arguments() {
         $script:testVs = $False
         $script:testpack = $False
         $script:testAOT = $False
+        $script:testBenchmarks = $False
         $script:verifypackageshipstatus = $True
     }
 
@@ -211,6 +214,10 @@ function Process-Arguments() {
         $script:pack = $True;
     }
 
+    if ($testBenchmarks) {
+        $script:testBenchmarks = $True
+    }
+
     foreach ($property in $properties) {
         if (!$property.StartsWith("/p:", "InvariantCultureIgnoreCase")) {
             Write-Host "Invalid argument: $property"
@@ -339,7 +346,7 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str
 
     if ($env:RunningAsPullRequest -ne "true" -and $noTestFilter -eq $false) {
         $args += " --filter TestCategory!=PullRequest"
-    }
+    }`
 
     if ($asBackgroundJob) {
         Write-Host("Starting on the background: $args")
@@ -541,12 +548,21 @@ try {
         }
     }
 
+    if ($testBenchmarks) {
+        $properties_storage = $properties
+        $properties += "/p:RuntimeIdentifier=win-x64"
+        $properties += "/p:Configuration=Release" # Always run in release.
+        BuildSolution "FSharp.Benchmarks.sln" $False
+        $properties = $properties_storage
+    }
+
     if ($pack) {
         $properties_storage = $properties
         $properties += "/p:GenerateSbom=false"
         BuildSolution "Microsoft.FSharp.Compiler.sln" $True
         $properties = $properties_storage
     }
+
     if ($build) {
         VerifyAssemblyVersionsAndSymbols
     }
@@ -662,6 +678,12 @@ try {
         Pop-Location
     }
 
+    if ($testBenchmarks) {
+        Push-Location "$RepoRoot\tests\benchmarks"
+        ./SmokeTestBenchmarks.ps1
+        Pop-Location
+    }
+
     # verify nupkgs have access to the source code
     $nupkgtestFailed = $false
     if ($testpack) {