From 628834abaed4c291881768fc6d84b0029f0b1c20 Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Mon, 23 Feb 2026 19:12:59 +0100 Subject: [PATCH 1/6] Batch CI tests based on namespace --- azure-pipelines-PR.yml | 160 ++++++++++++++++++++++++++++++++++++++-- eng/Build.ps1 | 24 +++++- eng/tests/TestSplit.fsx | 73 ++++++++++++++++++ 3 files changed, 248 insertions(+), 9 deletions(-) create mode 100644 eng/tests/TestSplit.fsx diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index d775d9ed907..fe75f070cf1 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -507,8 +507,8 @@ stages: continueOnError: true condition: failed() - # Windows With Compressed Metadata Desktop - - job: WindowsCompressedMetadata_Desktop + # Windows With Compressed Metadata Desktop (split into 3 batches) + - job: WindowsCompressedMetadata_Desktop_Batch1 variables: - name: XUNIT_LOGS value: $(Build.SourcesDirectory)\artifacts\TestResults\Release @@ -523,7 +523,7 @@ stages: - checkout: self clean: true - - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktop + - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch 1 env: FSharp_CacheEvictionImmediate: true DOTNET_DbgEnableMiniDump: 1 @@ -536,7 +536,7 @@ stages: displayName: Publish Test Results inputs: testResultsFormat: 'VSTest' - testRunTitle: WindowsCompressedMetadata testDesktop + testRunTitle: WindowsCompressedMetadata testDesktop Batch1 mergeTestResults: true testResultsFiles: '*.trx' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' @@ -548,7 +548,7 @@ stages: continueOnError: true inputs: PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows testDesktop binlogs' + ArtifactName: 'Windows testDesktop Batch1 binlogs' ArtifactType: Container parallel: true - task: PublishBuildArtifacts@1 @@ -557,14 +557,14 @@ stages: continueOnError: true inputs: PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' - ArtifactName: 'Windows testDesktop process dumps' + ArtifactName: 'Windows testDesktop Batch1 process dumps' ArtifactType: Container parallel: true - task: PublishBuildArtifacts@1 displayName: Publish Test Logs inputs: PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release' - ArtifactName: 'Windows testDesktop test logs' + ArtifactName: 'Windows testDesktop Batch1 test logs' publishLocation: Container continueOnError: true condition: always() @@ -575,7 +575,151 @@ stages: displayName: Publish NuGet cache contents inputs: PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' - ArtifactName: 'NuGetPackageContents Windows testDesktop' + ArtifactName: 'NuGetPackageContents Windows testDesktop Batch1' + publishLocation: Container + continueOnError: true + condition: failed() + + - job: WindowsCompressedMetadata_Desktop_Batch2 + variables: + - name: XUNIT_LOGS + value: $(Build.SourcesDirectory)\artifacts\TestResults\Release + - name: __VSNeverShowWhatsNew + value: 1 + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(_WindowsMachineQueueName) + timeoutInMinutes: 120 + + steps: + - checkout: self + clean: true + + - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch 2 + env: + FSharp_CacheEvictionImmediate: true + DOTNET_DbgEnableMiniDump: 1 + DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. + DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp + NativeToolsOnMachine: true + displayName: Build / Test + + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'VSTest' + testRunTitle: WindowsCompressedMetadata testDesktop Batch2 + mergeTestResults: true + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' + continueOnError: true + condition: succeededOrFailed() + + - task: PublishBuildArtifacts@1 + displayName: Publish BinLog + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' + ArtifactName: 'Windows testDesktop Batch2 binlogs' + ArtifactType: Container + parallel: true + - task: PublishBuildArtifacts@1 + displayName: Publish Dumps + condition: failed() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' + ArtifactName: 'Windows testDesktop Batch2 process dumps' + ArtifactType: Container + parallel: true + - task: PublishBuildArtifacts@1 + displayName: Publish Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release' + ArtifactName: 'Windows testDesktop Batch2 test logs' + publishLocation: Container + continueOnError: true + condition: always() + - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj + displayName: Dump NuGet cache contents + condition: failed() + - task: PublishBuildArtifacts@1 + displayName: Publish NuGet cache contents + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' + ArtifactName: 'NuGetPackageContents Windows testDesktop Batch2' + publishLocation: Container + continueOnError: true + condition: failed() + + - job: WindowsCompressedMetadata_Desktop_Batch3 + variables: + - name: XUNIT_LOGS + value: $(Build.SourcesDirectory)\artifacts\TestResults\Release + - name: __VSNeverShowWhatsNew + value: 1 + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(_WindowsMachineQueueName) + timeoutInMinutes: 120 + + steps: + - checkout: self + clean: true + + - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch 3 + env: + FSharp_CacheEvictionImmediate: true + DOTNET_DbgEnableMiniDump: 1 + DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. + DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp + NativeToolsOnMachine: true + displayName: Build / Test + + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'VSTest' + testRunTitle: WindowsCompressedMetadata testDesktop Batch3 + mergeTestResults: true + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' + continueOnError: true + condition: succeededOrFailed() + + - task: PublishBuildArtifacts@1 + displayName: Publish BinLog + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' + ArtifactName: 'Windows testDesktop Batch3 binlogs' + ArtifactType: Container + parallel: true + - task: PublishBuildArtifacts@1 + displayName: Publish Dumps + condition: failed() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' + ArtifactName: 'Windows testDesktop Batch3 process dumps' + ArtifactType: Container + parallel: true + - task: PublishBuildArtifacts@1 + displayName: Publish Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release' + ArtifactName: 'Windows testDesktop Batch3 test logs' + publishLocation: Container + continueOnError: true + condition: always() + - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj + displayName: Dump NuGet cache contents + condition: failed() + - task: PublishBuildArtifacts@1 + displayName: Publish NuGet cache contents + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' + ArtifactName: 'NuGetPackageContents Windows testDesktop Batch3' publishLocation: Container continueOnError: true condition: failed() diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 770e9e88c8a..8a8c4afdab4 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -49,6 +49,7 @@ param ( [switch]$dontUseGlobalNuGetCache = $false, [switch]$warnAsError = $true, [switch][Alias('test')]$testDesktop, + [string]$testDesktopBatch = "", [switch]$testCoreClr, [switch]$testCambridge, [switch]$testCompiler, @@ -121,6 +122,7 @@ function Print-Usage() { Write-Host " -testCompilerService Run FSharpCompilerService unit tests" Write-Host " -testCompilerComponentTests Run FSharpCompilerService component tests" Write-Host " -testDesktop Run tests against full .NET Framework" + Write-Host " -testDesktopBatch <1|2|3> Run a specific batch of the desktop test split (implies -testDesktop)" Write-Host " -testCoreClr Run tests against CoreCLR" Write-Host " -testFSharpCore Run FSharpCore unit tests" Write-Host " -testIntegration Run F# integration tests" @@ -193,6 +195,10 @@ function Process-Arguments() { $script:testEditor = $True } + if ($script:testDesktopBatch -ne "") { + $script:testDesktop = $True + } + if ([System.Boolean]::Parse($script:officialSkipTests)) { $script:testAll = $False $script:testAllButIntegration = $False @@ -605,7 +611,23 @@ try { } if ($testDesktop) { - TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework + if ($testDesktopBatch -ne "") { + $dotnetPath = InitializeDotNetCli + $dotnetExe = Join-Path $dotnetPath "dotnet.exe" + $splitScript = Join-Path $RepoRoot "eng\tests\TestSplit.fsx" + $splitOutput = & $dotnetExe fsi $splitScript $testDesktopBatch + if ($LASTEXITCODE -ne 0) { throw "TestSplit.fsx failed with exit code $LASTEXITCODE" } + foreach ($line in $splitOutput) { + if ($line -match '^dotnet test (\S+) --no-build -c Release\s*(.*)$') { + $proj = $Matches[1] -replace '/', '\' + $projPath = Join-Path $RepoRoot $proj + $settings = $Matches[2].Trim() + TestUsingMSBuild -testProject $projPath -targetFramework $script:desktopTargetFramework -settings $settings + } + } + } else { + TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework + } } if ($testFSharpCore) { diff --git a/eng/tests/TestSplit.fsx b/eng/tests/TestSplit.fsx new file mode 100644 index 00000000000..42980e1f162 --- /dev/null +++ b/eng/tests/TestSplit.fsx @@ -0,0 +1,73 @@ +/// Test split table for parallel CI. +/// Edit the batch assignments below, then run: +/// dotnet fsi eng/tests/TestSplit.fsx +/// to get the dotnet test commands for that batch. + +let totalBatches = 3 +let residualBatch = 2 // uses negation filter; catches unlisted atoms + future namespaces + +// MTP --filter-namespace uses starts-with matching on the test namespace. +// Unlisted atoms go to the residual batch automatically via --filter-not-namespace. + +let componentTestsAtoms = + [// atom batch + "CompilerDirectives", 1 + "CompilerService", 1 + "ErrorMessages", 1 + "FSharpChecker", 1 + "Import", 1 + "Language", 1 + "Miscellaneous", 1 + "XmlComments", 1 + + "Libraries", 2 + "Globalization", 2 + + "EmittedIL", 3 + "Interop", 3 + "InteractiveSession", 3 + "CompilerOptions", 3 + "Conformance", 3 + "Diagnostics", 3 + "Signatures", 3 + "ConstraintSolver", 3 + "Debugger", 3 + "Scripting", 3 + "TypeChecks", 3 + ] + +let otherProjects = + [// project path batch + "tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj", 1 + "tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj", 1 + "tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj", 3 + "tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj", 3 + ] + +// ── filter generation ── + +let batch = + match fsi.CommandLineArgs with + | [| _; n |] -> + let v = int n + if v < 1 || v > totalBatches then failwith $"Batch number must be between 1 and {totalBatches}, got {v}" + v + | _ -> failwith "Usage: dotnet fsi eng/tests/TestSplit.fsx " + +let atomsForBatch b = componentTestsAtoms |> List.filter (fun (_, ba) -> ba = b) |> List.map fst |> List.sort +let otherBatchesAtoms = componentTestsAtoms |> List.filter (fun (_, b) -> b <> batch) |> List.map fst |> List.sort + +let filterArgs = + if batch = residualBatch then + let atoms = otherBatchesAtoms |> String.concat " " + $"--filter-not-namespace {atoms}" + else + let atoms = atomsForBatch batch |> String.concat " " + $"--filter-namespace {atoms}" + +let componentTests = "tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj" + +printfn $"dotnet test {componentTests} --no-build -c Release {filterArgs}" + +for (proj, _) in otherProjects |> List.filter (fun (_, b) -> b = batch) do + printfn $"dotnet test {proj} --no-build -c Release" From 94be73700106652e3a7997493c0bcc5edd7fdafc Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Tue, 24 Feb 2026 07:41:05 +0100 Subject: [PATCH 2/6] Use test matrix for batching --- azure-pipelines-PR.yml | 166 ++++------------------------------------- 1 file changed, 15 insertions(+), 151 deletions(-) diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index fe75f070cf1..b3abd12e12e 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -508,151 +508,15 @@ stages: condition: failed() # Windows With Compressed Metadata Desktop (split into 3 batches) - - job: WindowsCompressedMetadata_Desktop_Batch1 - variables: - - name: XUNIT_LOGS - value: $(Build.SourcesDirectory)\artifacts\TestResults\Release - - name: __VSNeverShowWhatsNew - value: 1 - pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(_WindowsMachineQueueName) - timeoutInMinutes: 120 - - steps: - - checkout: self - clean: true - - - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch 1 - env: - FSharp_CacheEvictionImmediate: true - DOTNET_DbgEnableMiniDump: 1 - DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. - DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp - NativeToolsOnMachine: true - displayName: Build / Test - - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'VSTest' - testRunTitle: WindowsCompressedMetadata testDesktop Batch1 - mergeTestResults: true - testResultsFiles: '*.trx' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' - continueOnError: true - condition: succeededOrFailed() - - - task: PublishBuildArtifacts@1 - displayName: Publish BinLog - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows testDesktop Batch1 binlogs' - ArtifactType: Container - parallel: true - - task: PublishBuildArtifacts@1 - displayName: Publish Dumps - condition: failed() - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' - ArtifactName: 'Windows testDesktop Batch1 process dumps' - ArtifactType: Container - parallel: true - - task: PublishBuildArtifacts@1 - displayName: Publish Test Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release' - ArtifactName: 'Windows testDesktop Batch1 test logs' - publishLocation: Container - continueOnError: true - condition: always() - - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj - displayName: Dump NuGet cache contents - condition: failed() - - task: PublishBuildArtifacts@1 - displayName: Publish NuGet cache contents - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' - ArtifactName: 'NuGetPackageContents Windows testDesktop Batch1' - publishLocation: Container - continueOnError: true - condition: failed() - - - job: WindowsCompressedMetadata_Desktop_Batch2 - variables: - - name: XUNIT_LOGS - value: $(Build.SourcesDirectory)\artifacts\TestResults\Release - - name: __VSNeverShowWhatsNew - value: 1 - pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(_WindowsMachineQueueName) - timeoutInMinutes: 120 - - steps: - - checkout: self - clean: true - - - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch 2 - env: - FSharp_CacheEvictionImmediate: true - DOTNET_DbgEnableMiniDump: 1 - DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. - DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp - NativeToolsOnMachine: true - displayName: Build / Test - - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'VSTest' - testRunTitle: WindowsCompressedMetadata testDesktop Batch2 - mergeTestResults: true - testResultsFiles: '*.trx' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' - continueOnError: true - condition: succeededOrFailed() - - - task: PublishBuildArtifacts@1 - displayName: Publish BinLog - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows testDesktop Batch2 binlogs' - ArtifactType: Container - parallel: true - - task: PublishBuildArtifacts@1 - displayName: Publish Dumps - condition: failed() - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' - ArtifactName: 'Windows testDesktop Batch2 process dumps' - ArtifactType: Container - parallel: true - - task: PublishBuildArtifacts@1 - displayName: Publish Test Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release' - ArtifactName: 'Windows testDesktop Batch2 test logs' - publishLocation: Container - continueOnError: true - condition: always() - - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj - displayName: Dump NuGet cache contents - condition: failed() - - task: PublishBuildArtifacts@1 - displayName: Publish NuGet cache contents - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' - ArtifactName: 'NuGetPackageContents Windows testDesktop Batch2' - publishLocation: Container - continueOnError: true - condition: failed() - - - job: WindowsCompressedMetadata_Desktop_Batch3 + - job: WindowsCompressedMetadata_Desktop + strategy: + matrix: + Batch1: + batchNumber: 1 + Batch2: + batchNumber: 2 + Batch3: + batchNumber: 3 variables: - name: XUNIT_LOGS value: $(Build.SourcesDirectory)\artifacts\TestResults\Release @@ -667,7 +531,7 @@ stages: - checkout: self clean: true - - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch 3 + - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch $(batchNumber) env: FSharp_CacheEvictionImmediate: true DOTNET_DbgEnableMiniDump: 1 @@ -680,7 +544,7 @@ stages: displayName: Publish Test Results inputs: testResultsFormat: 'VSTest' - testRunTitle: WindowsCompressedMetadata testDesktop Batch3 + testRunTitle: WindowsCompressedMetadata testDesktop Batch$(batchNumber) mergeTestResults: true testResultsFiles: '*.trx' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' @@ -692,7 +556,7 @@ stages: continueOnError: true inputs: PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows testDesktop Batch3 binlogs' + ArtifactName: 'Windows testDesktop Batch$(batchNumber) binlogs' ArtifactType: Container parallel: true - task: PublishBuildArtifacts@1 @@ -701,14 +565,14 @@ stages: continueOnError: true inputs: PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' - ArtifactName: 'Windows testDesktop Batch3 process dumps' + ArtifactName: 'Windows testDesktop Batch$(batchNumber) process dumps' ArtifactType: Container parallel: true - task: PublishBuildArtifacts@1 displayName: Publish Test Logs inputs: PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release' - ArtifactName: 'Windows testDesktop Batch3 test logs' + ArtifactName: 'Windows testDesktop Batch$(batchNumber) test logs' publishLocation: Container continueOnError: true condition: always() @@ -719,7 +583,7 @@ stages: displayName: Publish NuGet cache contents inputs: PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' - ArtifactName: 'NuGetPackageContents Windows testDesktop Batch3' + ArtifactName: 'NuGetPackageContents Windows testDesktop Batch$(batchNumber)' publishLocation: Container continueOnError: true condition: failed() From d80ef7b86551d876a588f91a8652a2843c696427 Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Thu, 26 Feb 2026 14:45:03 +0100 Subject: [PATCH 3/6] Use spekt for test results --- eng/Build.ps1 | 12 ++-- eng/Versions.props | 1 + eng/build.sh | 10 ++-- eng/templates/batched-test-job.yml | 93 ++++++++++++++++++++++++++++++ eng/tests/TestSplit.fsx | 1 + tests/Directory.Build.props | 2 + 6 files changed, 106 insertions(+), 13 deletions(-) create mode 100644 eng/templates/batched-test-job.yml diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 8a8c4afdab4..8f858b21876 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -380,14 +380,10 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str # MTP requires --solution flag for .sln files $testTarget = if ($testProject.EndsWith('.sln')) { "--solution ""$testProject""" } else { "--project ""$testProject""" } - # For solutions, omit --report-xunit-trx-filename so each test assembly generates a unique .trx file. - # With a static filename, all assemblies overwrite the same file and only the last one's results survive. - if ($testProject.EndsWith('.sln')) { - $reportArgs = "--report-xunit-trx" - } else { - $testLogFileName = "${projectName}_${targetFramework}.trx" - $reportArgs = "--report-xunit-trx --report-xunit-trx-filename ""$testLogFileName""" - } + # Xunit XML report via XunitXml.TestLogger with CI-friendly filenames + $jobName = if ($env:SYSTEM_JOBNAME) { $env:SYSTEM_JOBNAME } else { "local" } + $xunitLogFileName = "{assembly}.{framework}.${jobName}.xml" + $reportArgs = "--report-spekt-xunit --report-spekt-xunit-filename ""$xunitLogFileName""" $test_args = "test $testTarget -c $configuration -f $targetFramework $reportArgs --results-directory ""$testResultsDir"" /bl:$testBinLogPath" # MTP HangDump extension replaces VSTest --blame-hang-timeout diff --git a/eng/Versions.props b/eng/Versions.props index 0032205fef2..8c9852c7620 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -170,6 +170,7 @@ 13.0.3 3.2.2 3.2.2 + 8.0.0 diff --git a/eng/build.sh b/eng/build.sh index 06df1423012..913be4b1690 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -233,17 +233,17 @@ function Test() { testresultsdir="$artifacts_dir/TestResults/$configuration" # MTP requires --solution flag for .sln files - # For solutions, omit --report-xunit-trx-filename so each test assembly generates a unique .trx file. - # With a static filename, all assemblies overwrite the same file and only the last one's results survive. if [[ "$testproject" == *.sln ]]; then testtarget="--solution" - reportargs="--report-xunit-trx" else testtarget="--project" - testlogfilename="${projectname}_${targetframework}.trx" - reportargs="--report-xunit-trx --report-xunit-trx-filename $testlogfilename" fi + # Xunit XML report via XunitXml.TestLogger with CI-friendly filenames + jobname="${SYSTEM_JOBNAME:-local}" + xunitlogfilename="{assembly}.{framework}.${jobname}.xml" + reportargs="--report-spekt-xunit --report-spekt-xunit-filename $xunitlogfilename" + args=(test $testtarget "$testproject" --no-build -c "$configuration" -f "$targetframework" $reportargs --results-directory "$testresultsdir" --hangdump --hangdump-timeout 5m --hangdump-type Full) "$DOTNET_INSTALL_DIR/dotnet" "${args[@]}" || exit $? diff --git a/eng/templates/batched-test-job.yml b/eng/templates/batched-test-job.yml new file mode 100644 index 00000000000..acc2a842624 --- /dev/null +++ b/eng/templates/batched-test-job.yml @@ -0,0 +1,93 @@ +# Template for running test jobs split into parallel batches via strategy:matrix. +# Place this alongside (not inside) the Arcade jobs.yml template reference, +# since Arcade's ${{ each job }} loop cannot expand nested template references. + +parameters: + jobName: '' + pool: {} + timeoutInMinutes: 120 + buildCommand: '' # Full build/test command including $(batchNumber) + buildEnv: {} + batches: [1, 2, 3] + variables: [] + testRunTitlePrefix: '' # e.g. 'WindowsCompressedMetadata testDesktop' + artifactNamePrefix: '' # e.g. 'Windows testDesktop' + configuration: 'Release' + publishBinLog: false + binLogPath: '' + publishDumps: false + +jobs: +- job: ${{ parameters.jobName }} + strategy: + matrix: + ${{ each batch in parameters.batches }}: + Batch${{ batch }}: + batchNumber: ${{ batch }} + pool: ${{ parameters.pool }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + variables: + - ${{ each var in parameters.variables }}: + - name: ${{ var.name }} + value: ${{ var.value }} + steps: + - checkout: self + clean: true + + - script: ${{ parameters.buildCommand }} + env: ${{ parameters.buildEnv }} + displayName: Build / Test + + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'VSTest' + testRunTitle: ${{ parameters.testRunTitlePrefix }} Batch$(batchNumber) + mergeTestResults: true + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.configuration }}' + continueOnError: true + condition: succeededOrFailed() + + - ${{ if parameters.publishBinLog }}: + - task: PublishBuildArtifacts@1 + displayName: Publish BinLog + continueOnError: true + inputs: + PathToPublish: ${{ parameters.binLogPath }} + ArtifactName: '${{ parameters.artifactNamePrefix }} Batch$(batchNumber) binlogs' + ArtifactType: Container + parallel: true + + - ${{ if parameters.publishDumps }}: + - task: PublishBuildArtifacts@1 + displayName: Publish Dumps + condition: failed() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/${{ parameters.configuration }}' + ArtifactName: '${{ parameters.artifactNamePrefix }} Batch$(batchNumber) process dumps' + ArtifactType: Container + parallel: true + + - task: PublishBuildArtifacts@1 + displayName: Publish Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.configuration }}' + ArtifactName: '${{ parameters.artifactNamePrefix }} Batch$(batchNumber) test logs' + publishLocation: Container + continueOnError: true + condition: always() + + - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj + displayName: Dump NuGet cache contents + condition: failed() + + - task: PublishBuildArtifacts@1 + displayName: Publish NuGet cache contents + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/NugetPackageRootContents' + ArtifactName: 'NuGetPackageContents ${{ parameters.artifactNamePrefix }} Batch$(batchNumber)' + publishLocation: Container + continueOnError: true + condition: failed() diff --git a/eng/tests/TestSplit.fsx b/eng/tests/TestSplit.fsx index 42980e1f162..b5389c658fd 100644 --- a/eng/tests/TestSplit.fsx +++ b/eng/tests/TestSplit.fsx @@ -42,6 +42,7 @@ let otherProjects = "tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj", 1 "tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj", 3 "tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj", 3 + "tests/fsharp/FSharpSuite.Tests.fsproj", 3 ] // ── filter generation ── diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props index bf1b81f2bef..dc7ed710a3a 100644 --- a/tests/Directory.Build.props +++ b/tests/Directory.Build.props @@ -13,6 +13,8 @@ + + From 969a0991bd624a8808090aacf71c25068aceb8b8 Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Thu, 26 Feb 2026 16:07:56 +0100 Subject: [PATCH 4/6] Add ObjectModel nuget --- tests/Directory.Build.props | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props index dc7ed710a3a..ccc7e44ffa3 100644 --- a/tests/Directory.Build.props +++ b/tests/Directory.Build.props @@ -15,6 +15,8 @@ + + From c1b3b00de4fdbf4176e9bcfea354503c9aea41c4 Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Thu, 26 Feb 2026 17:46:40 +0100 Subject: [PATCH 5/6] Fix missing references --- tests/EndToEndBuildTests/Directory.Build.props | 2 ++ .../tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj | 2 ++ vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj | 2 ++ 3 files changed, 6 insertions(+) diff --git a/tests/EndToEndBuildTests/Directory.Build.props b/tests/EndToEndBuildTests/Directory.Build.props index 496de8645f5..f97db4e1684 100644 --- a/tests/EndToEndBuildTests/Directory.Build.props +++ b/tests/EndToEndBuildTests/Directory.Build.props @@ -7,6 +7,8 @@ 3.2.2 3.2.2 2.0.2 + 8.0.0 + 17.14.1 diff --git a/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj b/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj index 284ec6eeb1d..52ba8624a30 100644 --- a/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj +++ b/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj @@ -108,6 +108,8 @@ + + diff --git a/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj b/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj index 009d4f0a8fa..cf8cc25e837 100644 --- a/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj +++ b/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj @@ -128,6 +128,8 @@ + + From b36d8a4afa505b5d9f1d9fbf4e71aa91b649436c Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:21:19 +0100 Subject: [PATCH 6/6] Change test result format --- azure-pipelines-PR.yml | 24 ++++++++++++------------ azure-pipelines.yml | 4 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index b3abd12e12e..57dc682614a 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -267,10 +267,10 @@ stages: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' + testResultsFormat: 'XUnit' testRunTitle: WindowsNoRealsig_testCoreclr mergeTestResults: true - testResultsFiles: '*.trx' + testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' condition: succeededOrFailed() @@ -314,10 +314,10 @@ stages: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' + testResultsFormat: 'XUnit' testRunTitle: WindowsNoRealsig_testDesktop mergeTestResults: true - testResultsFiles: '*.trx' + testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' condition: succeededOrFailed() continueOnError: true @@ -462,10 +462,10 @@ stages: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' + testResultsFormat: 'XUnit' testRunTitle: WindowsCompressedMetadata $(_testKind) $(transparentCompiler) mergeTestResults: true - testResultsFiles: '*.trx' + testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_configuration)' continueOnError: true condition: succeededOrFailed() @@ -543,10 +543,10 @@ stages: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' + testResultsFormat: 'XUnit' testRunTitle: WindowsCompressedMetadata testDesktop Batch$(batchNumber) mergeTestResults: true - testResultsFiles: '*.trx' + testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' continueOnError: true condition: succeededOrFailed() @@ -616,9 +616,9 @@ stages: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' + testResultsFormat: 'XUnit' testRunTitle: Linux - testResultsFiles: '*.trx' + testResultsFiles: '*.xml' mergeTestResults: true searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' continueOnError: true @@ -661,8 +661,8 @@ stages: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' + testResultsFormat: 'XUnit' + testResultsFiles: '*.xml' testRunTitle: MacOS mergeTestResults: true searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 170d26397e3..e4f0c294362 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -152,8 +152,8 @@ extends: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' + testResultsFormat: 'XUnit' + testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' continueOnError: true condition: ne(variables['SkipTests'], 'true')