Batch CI tests based on namespace

azure-pipelines-PR.yml

@@ -267,10 +267,10 @@ stages:
           - task: PublishTestResults@2
             displayName: Publish Test Results
             inputs:
-              testResultsFormat: 'VSTest'
+              testResultsFormat: 'XUnit'
               testRunTitle: WindowsNoRealsig_testCoreclr
               mergeTestResults: true
-              testResultsFiles: '*.trx'
+              testResultsFiles: '*.xml'
               searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release'
             condition: succeededOrFailed()
 
@@ -314,10 +314,10 @@ stages:
           - task: PublishTestResults@2
             displayName: Publish Test Results
             inputs:
-              testResultsFormat: 'VSTest'
+              testResultsFormat: 'XUnit'
               testRunTitle: WindowsNoRealsig_testDesktop
               mergeTestResults: true
-              testResultsFiles: '*.trx'
+              testResultsFiles: '*.xml'
               searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release'
             condition: succeededOrFailed()
             continueOnError: true
@@ -462,10 +462,10 @@ stages:
           - task: PublishTestResults@2
             displayName: Publish Test Results
             inputs:
-              testResultsFormat: 'VSTest'
+              testResultsFormat: 'XUnit'
               testRunTitle: WindowsCompressedMetadata $(_testKind) $(transparentCompiler)
               mergeTestResults: true
-              testResultsFiles: '*.trx'
+              testResultsFiles: '*.xml'
               searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_configuration)'
             continueOnError: true
             condition: succeededOrFailed()
@@ -507,8 +507,16 @@ stages:
             continueOnError: true
             condition: failed()
 
-          # Windows With Compressed Metadata Desktop
+          # Windows With Compressed Metadata Desktop (split into 3 batches)
         - job: WindowsCompressedMetadata_Desktop
+          strategy:
+            matrix:
+              Batch1:
+                batchNumber: 1
+              Batch2:
+                batchNumber: 2
+              Batch3:
+                batchNumber: 3
           variables:
           - name: XUNIT_LOGS
             value: $(Build.SourcesDirectory)\artifacts\TestResults\Release
@@ -523,7 +531,7 @@ stages:
           - checkout: self
             clean: true
 
-          - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktop
+          - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch $(batchNumber)
             env:
               FSharp_CacheEvictionImmediate: true
               DOTNET_DbgEnableMiniDump: 1
@@ -535,10 +543,10 @@ stages:
           - task: PublishTestResults@2
             displayName: Publish Test Results
             inputs:
-              testResultsFormat: 'VSTest'
-              testRunTitle: WindowsCompressedMetadata testDesktop
+              testResultsFormat: 'XUnit'
+              testRunTitle: WindowsCompressedMetadata testDesktop Batch$(batchNumber)
               mergeTestResults: true
-              testResultsFiles: '*.trx'
+              testResultsFiles: '*.xml'
               searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release'
             continueOnError: true
             condition: succeededOrFailed()
@@ -548,7 +556,7 @@ stages:
             continueOnError: true
             inputs:
                 PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog'
-                ArtifactName: 'Windows testDesktop binlogs'
+                ArtifactName: 'Windows testDesktop Batch$(batchNumber) binlogs'
                 ArtifactType: Container
                 parallel: true
           - task: PublishBuildArtifacts@1
@@ -557,14 +565,14 @@ stages:
             continueOnError: true
             inputs:
                 PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release'
-                ArtifactName: 'Windows testDesktop process dumps'
+                ArtifactName: 'Windows testDesktop Batch$(batchNumber) process dumps'
                 ArtifactType: Container
                 parallel: true
           - task: PublishBuildArtifacts@1
             displayName: Publish Test Logs
             inputs:
               PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release'
-              ArtifactName: 'Windows testDesktop test logs'
+              ArtifactName: 'Windows testDesktop Batch$(batchNumber) test logs'
               publishLocation: Container
             continueOnError: true
             condition: always()
@@ -575,7 +583,7 @@ stages:
             displayName: Publish NuGet cache contents
             inputs:
               PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents'
-              ArtifactName: 'NuGetPackageContents Windows testDesktop'
+              ArtifactName: 'NuGetPackageContents Windows testDesktop Batch$(batchNumber)'
               publishLocation: Container
             continueOnError: true
             condition: failed()
@@ -608,9 +616,9 @@ stages:
           - task: PublishTestResults@2
             displayName: Publish Test Results
             inputs:
-              testResultsFormat: 'VSTest'
+              testResultsFormat: 'XUnit'
               testRunTitle: Linux
-              testResultsFiles: '*.trx'
+              testResultsFiles: '*.xml'
               mergeTestResults: true
               searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
             continueOnError: true
@@ -653,8 +661,8 @@ stages:
           - task: PublishTestResults@2
             displayName: Publish Test Results
             inputs:
-              testResultsFormat: 'VSTest'
-              testResultsFiles: '*.trx'
+              testResultsFormat: 'XUnit'
+              testResultsFiles: '*.xml'
               testRunTitle: MacOS
               mergeTestResults: true
               searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'

azure-pipelines.yml

@@ -152,8 +152,8 @@ extends:
             - task: PublishTestResults@2
               displayName: Publish Test Results
               inputs:
-                testResultsFormat: 'VSTest'
-                testResultsFiles: '*.trx'
+                testResultsFormat: 'XUnit'
+                testResultsFiles: '*.xml'
                 searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
               continueOnError: true
               condition: ne(variables['SkipTests'], 'true')

eng/Build.ps1

@@ -49,6 +49,7 @@ param (
     [switch]$dontUseGlobalNuGetCache = $false,
     [switch]$warnAsError = $true,
     [switch][Alias('test')]$testDesktop,
+    [string]$testDesktopBatch = "",
     [switch]$testCoreClr,
     [switch]$testCambridge,
     [switch]$testCompiler,
@@ -121,6 +122,7 @@ function Print-Usage() {
     Write-Host "  -testCompilerService          Run FSharpCompilerService unit tests"
     Write-Host "  -testCompilerComponentTests   Run FSharpCompilerService component tests"
     Write-Host "  -testDesktop                  Run tests against full .NET Framework"
+    Write-Host "  -testDesktopBatch <1|2|3>       Run a specific batch of the desktop test split (implies -testDesktop)"
     Write-Host "  -testCoreClr                  Run tests against CoreCLR"
     Write-Host "  -testFSharpCore               Run FSharpCore unit tests"
     Write-Host "  -testIntegration              Run F# integration tests"
@@ -193,6 +195,10 @@ function Process-Arguments() {
         $script:testEditor = $True
     }
 
+    if ($script:testDesktopBatch -ne "") {
+        $script:testDesktop = $True
+    }
+
     if ([System.Boolean]::Parse($script:officialSkipTests)) {
         $script:testAll = $False
         $script:testAllButIntegration = $False
@@ -374,14 +380,10 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str
     # MTP requires --solution flag for .sln files
     $testTarget = if ($testProject.EndsWith('.sln')) { "--solution ""$testProject""" } else { "--project ""$testProject""" }
 
-    # For solutions, omit --report-xunit-trx-filename so each test assembly generates a unique .trx file.
-    # With a static filename, all assemblies overwrite the same file and only the last one's results survive.
-    if ($testProject.EndsWith('.sln')) {
-        $reportArgs = "--report-xunit-trx"
-    } else {
-        $testLogFileName = "${projectName}_${targetFramework}.trx"
-        $reportArgs = "--report-xunit-trx --report-xunit-trx-filename ""$testLogFileName"""
-    }
+    # Xunit XML report via XunitXml.TestLogger with CI-friendly filenames
+    $jobName = if ($env:SYSTEM_JOBNAME) { $env:SYSTEM_JOBNAME } else { "local" }
+    $xunitLogFileName = "{assembly}.{framework}.${jobName}.xml"
+    $reportArgs = "--report-spekt-xunit --report-spekt-xunit-filename ""$xunitLogFileName"""
 
     $test_args = "test $testTarget -c $configuration -f $targetFramework $reportArgs --results-directory ""$testResultsDir"" /bl:$testBinLogPath"
     # MTP HangDump extension replaces VSTest --blame-hang-timeout
@@ -605,7 +607,23 @@ try {
     }
 
     if ($testDesktop) {
-        TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework
+        if ($testDesktopBatch -ne "") {
+            $dotnetPath = InitializeDotNetCli
+            $dotnetExe = Join-Path $dotnetPath "dotnet.exe"
+            $splitScript = Join-Path $RepoRoot "eng\tests\TestSplit.fsx"
+            $splitOutput = & $dotnetExe fsi $splitScript $testDesktopBatch
+            if ($LASTEXITCODE -ne 0) { throw "TestSplit.fsx failed with exit code $LASTEXITCODE" }
+            foreach ($line in $splitOutput) {
+                if ($line -match '^dotnet test (\S+) --no-build -c Release\s*(.*)$') {
+                    $proj = $Matches[1] -replace '/', '\'
+                    $projPath = Join-Path $RepoRoot $proj
+                    $settings = $Matches[2].Trim()
+                    TestUsingMSBuild -testProject $projPath -targetFramework $script:desktopTargetFramework -settings $settings
+                }
+            }
+        } else {
+            TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework
+        }
     }
 
     if ($testFSharpCore) {

eng/Versions.props

@@ -170,6 +170,7 @@
     <NewtonsoftJsonVersion>13.0.3</NewtonsoftJsonVersion>
     <XunitVersion>3.2.2</XunitVersion>
     <XunitRunnerConsoleVersion>3.2.2</XunitRunnerConsoleVersion>
+    <XunitXmlTestLoggerVersion>8.0.0</XunitXmlTestLoggerVersion>
     <!-- -->
   </PropertyGroup>
 

eng/build.sh

@@ -233,17 +233,17 @@ function Test() {
   testresultsdir="$artifacts_dir/TestResults/$configuration"
 
   # MTP requires --solution flag for .sln files
-  # For solutions, omit --report-xunit-trx-filename so each test assembly generates a unique .trx file.
-  # With a static filename, all assemblies overwrite the same file and only the last one's results survive.
   if [[ "$testproject" == *.sln ]]; then
     testtarget="--solution"
-    reportargs="--report-xunit-trx"
   else
     testtarget="--project"
-    testlogfilename="${projectname}_${targetframework}.trx"
-    reportargs="--report-xunit-trx --report-xunit-trx-filename $testlogfilename"
   fi
 
+  # Xunit XML report via XunitXml.TestLogger with CI-friendly filenames
+  jobname="${SYSTEM_JOBNAME:-local}"
+  xunitlogfilename="{assembly}.{framework}.${jobname}.xml"
+  reportargs="--report-spekt-xunit --report-spekt-xunit-filename $xunitlogfilename"
+
   args=(test $testtarget "$testproject" --no-build -c "$configuration" -f "$targetframework" $reportargs --results-directory "$testresultsdir" --hangdump --hangdump-timeout 5m --hangdump-type Full)
 
   "$DOTNET_INSTALL_DIR/dotnet" "${args[@]}" || exit $?

eng/templates/batched-test-job.yml

@@ -0,0 +1,93 @@
+# Template for running test jobs split into parallel batches via strategy:matrix.
+# Place this alongside (not inside) the Arcade jobs.yml template reference,
+# since Arcade's ${{ each job }} loop cannot expand nested template references.
+
+parameters:
+  jobName: ''
+  pool: {}
+  timeoutInMinutes: 120
+  buildCommand: ''          # Full build/test command including $(batchNumber)
+  buildEnv: {}
+  batches: [1, 2, 3]
+  variables: []
+  testRunTitlePrefix: ''    # e.g. 'WindowsCompressedMetadata testDesktop'
+  artifactNamePrefix: ''    # e.g. 'Windows testDesktop'
+  configuration: 'Release'
+  publishBinLog: false
+  binLogPath: ''
+  publishDumps: false
+
+jobs:
+- job: ${{ parameters.jobName }}
+  strategy:
+    matrix:
+      ${{ each batch in parameters.batches }}:
+        Batch${{ batch }}:
+          batchNumber: ${{ batch }}
+  pool: ${{ parameters.pool }}
+  timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+  variables:
+  - ${{ each var in parameters.variables }}:
+    - name: ${{ var.name }}
+      value: ${{ var.value }}
+  steps:
+  - checkout: self
+    clean: true
+
+  - script: ${{ parameters.buildCommand }}
+    env: ${{ parameters.buildEnv }}
+    displayName: Build / Test
+
+  - task: PublishTestResults@2
+    displayName: Publish Test Results
+    inputs:
+      testResultsFormat: 'VSTest'
+      testRunTitle: ${{ parameters.testRunTitlePrefix }} Batch$(batchNumber)
+      mergeTestResults: true
+      testResultsFiles: '*.trx'
+      searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.configuration }}'
+    continueOnError: true
+    condition: succeededOrFailed()
+
+  - ${{ if parameters.publishBinLog }}:
+    - task: PublishBuildArtifacts@1
+      displayName: Publish BinLog
+      continueOnError: true
+      inputs:
+        PathToPublish: ${{ parameters.binLogPath }}
+        ArtifactName: '${{ parameters.artifactNamePrefix }} Batch$(batchNumber) binlogs'
+        ArtifactType: Container
+        parallel: true
+
+  - ${{ if parameters.publishDumps }}:
+    - task: PublishBuildArtifacts@1
+      displayName: Publish Dumps
+      condition: failed()
+      continueOnError: true
+      inputs:
+        PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/${{ parameters.configuration }}'
+        ArtifactName: '${{ parameters.artifactNamePrefix }} Batch$(batchNumber) process dumps'
+        ArtifactType: Container
+        parallel: true
+
+  - task: PublishBuildArtifacts@1
+    displayName: Publish Test Logs
+    inputs:
+      PathtoPublish: '$(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.configuration }}'
+      ArtifactName: '${{ parameters.artifactNamePrefix }} Batch$(batchNumber) test logs'
+      publishLocation: Container
+    continueOnError: true
+    condition: always()
+
+  - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj
+    displayName: Dump NuGet cache contents
+    condition: failed()
+
+  - task: PublishBuildArtifacts@1
+    displayName: Publish NuGet cache contents
+    inputs:
+      PathtoPublish: '$(Build.SourcesDirectory)/artifacts/NugetPackageRootContents'
+      ArtifactName: 'NuGetPackageContents ${{ parameters.artifactNamePrefix }} Batch$(batchNumber)'
+      publishLocation: Container
+    continueOnError: true
+    condition: failed()