diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index d775d9ed907..57dc682614a 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -267,10 +267,10 @@ stages: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' + testResultsFormat: 'XUnit' testRunTitle: WindowsNoRealsig_testCoreclr mergeTestResults: true - testResultsFiles: '*.trx' + testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' condition: succeededOrFailed() @@ -314,10 +314,10 @@ stages: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' + testResultsFormat: 'XUnit' testRunTitle: WindowsNoRealsig_testDesktop mergeTestResults: true - testResultsFiles: '*.trx' + testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' condition: succeededOrFailed() continueOnError: true @@ -462,10 +462,10 @@ stages: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' + testResultsFormat: 'XUnit' testRunTitle: WindowsCompressedMetadata $(_testKind) $(transparentCompiler) mergeTestResults: true - testResultsFiles: '*.trx' + testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_configuration)' continueOnError: true condition: succeededOrFailed() @@ -507,8 +507,16 @@ stages: continueOnError: true condition: failed() - # Windows With Compressed Metadata Desktop + # Windows With Compressed Metadata Desktop (split into 3 batches) - job: WindowsCompressedMetadata_Desktop + strategy: + matrix: + Batch1: + batchNumber: 1 + Batch2: + batchNumber: 2 + Batch3: + batchNumber: 3 variables: - name: XUNIT_LOGS value: $(Build.SourcesDirectory)\artifacts\TestResults\Release @@ -523,7 +531,7 @@ stages: - checkout: self clean: true - - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktop + - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch $(batchNumber) env: FSharp_CacheEvictionImmediate: true DOTNET_DbgEnableMiniDump: 1 @@ -535,10 +543,10 @@ stages: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' - testRunTitle: WindowsCompressedMetadata testDesktop + testResultsFormat: 'XUnit' + testRunTitle: WindowsCompressedMetadata testDesktop Batch$(batchNumber) mergeTestResults: true - testResultsFiles: '*.trx' + testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' continueOnError: true condition: succeededOrFailed() @@ -548,7 +556,7 @@ stages: continueOnError: true inputs: PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows testDesktop binlogs' + ArtifactName: 'Windows testDesktop Batch$(batchNumber) binlogs' ArtifactType: Container parallel: true - task: PublishBuildArtifacts@1 @@ -557,14 +565,14 @@ stages: continueOnError: true inputs: PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' - ArtifactName: 'Windows testDesktop process dumps' + ArtifactName: 'Windows testDesktop Batch$(batchNumber) process dumps' ArtifactType: Container parallel: true - task: PublishBuildArtifacts@1 displayName: Publish Test Logs inputs: PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release' - ArtifactName: 'Windows testDesktop test logs' + ArtifactName: 'Windows testDesktop Batch$(batchNumber) test logs' publishLocation: Container continueOnError: true condition: always() @@ -575,7 +583,7 @@ stages: displayName: Publish NuGet cache contents inputs: PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' - ArtifactName: 'NuGetPackageContents Windows testDesktop' + ArtifactName: 'NuGetPackageContents Windows testDesktop Batch$(batchNumber)' publishLocation: Container continueOnError: true condition: failed() @@ -608,9 +616,9 @@ stages: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' + testResultsFormat: 'XUnit' testRunTitle: Linux - testResultsFiles: '*.trx' + testResultsFiles: '*.xml' mergeTestResults: true searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' continueOnError: true @@ -653,8 +661,8 @@ stages: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' + testResultsFormat: 'XUnit' + testResultsFiles: '*.xml' testRunTitle: MacOS mergeTestResults: true searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 170d26397e3..e4f0c294362 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -152,8 +152,8 @@ extends: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' + testResultsFormat: 'XUnit' + testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' continueOnError: true condition: ne(variables['SkipTests'], 'true') diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 770e9e88c8a..8f858b21876 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -49,6 +49,7 @@ param ( [switch]$dontUseGlobalNuGetCache = $false, [switch]$warnAsError = $true, [switch][Alias('test')]$testDesktop, + [string]$testDesktopBatch = "", [switch]$testCoreClr, [switch]$testCambridge, [switch]$testCompiler, @@ -121,6 +122,7 @@ function Print-Usage() { Write-Host " -testCompilerService Run FSharpCompilerService unit tests" Write-Host " -testCompilerComponentTests Run FSharpCompilerService component tests" Write-Host " -testDesktop Run tests against full .NET Framework" + Write-Host " -testDesktopBatch <1|2|3> Run a specific batch of the desktop test split (implies -testDesktop)" Write-Host " -testCoreClr Run tests against CoreCLR" Write-Host " -testFSharpCore Run FSharpCore unit tests" Write-Host " -testIntegration Run F# integration tests" @@ -193,6 +195,10 @@ function Process-Arguments() { $script:testEditor = $True } + if ($script:testDesktopBatch -ne "") { + $script:testDesktop = $True + } + if ([System.Boolean]::Parse($script:officialSkipTests)) { $script:testAll = $False $script:testAllButIntegration = $False @@ -374,14 +380,10 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str # MTP requires --solution flag for .sln files $testTarget = if ($testProject.EndsWith('.sln')) { "--solution ""$testProject""" } else { "--project ""$testProject""" } - # For solutions, omit --report-xunit-trx-filename so each test assembly generates a unique .trx file. - # With a static filename, all assemblies overwrite the same file and only the last one's results survive. - if ($testProject.EndsWith('.sln')) { - $reportArgs = "--report-xunit-trx" - } else { - $testLogFileName = "${projectName}_${targetFramework}.trx" - $reportArgs = "--report-xunit-trx --report-xunit-trx-filename ""$testLogFileName""" - } + # Xunit XML report via XunitXml.TestLogger with CI-friendly filenames + $jobName = if ($env:SYSTEM_JOBNAME) { $env:SYSTEM_JOBNAME } else { "local" } + $xunitLogFileName = "{assembly}.{framework}.${jobName}.xml" + $reportArgs = "--report-spekt-xunit --report-spekt-xunit-filename ""$xunitLogFileName""" $test_args = "test $testTarget -c $configuration -f $targetFramework $reportArgs --results-directory ""$testResultsDir"" /bl:$testBinLogPath" # MTP HangDump extension replaces VSTest --blame-hang-timeout @@ -605,7 +607,23 @@ try { } if ($testDesktop) { - TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework + if ($testDesktopBatch -ne "") { + $dotnetPath = InitializeDotNetCli + $dotnetExe = Join-Path $dotnetPath "dotnet.exe" + $splitScript = Join-Path $RepoRoot "eng\tests\TestSplit.fsx" + $splitOutput = & $dotnetExe fsi $splitScript $testDesktopBatch + if ($LASTEXITCODE -ne 0) { throw "TestSplit.fsx failed with exit code $LASTEXITCODE" } + foreach ($line in $splitOutput) { + if ($line -match '^dotnet test (\S+) --no-build -c Release\s*(.*)$') { + $proj = $Matches[1] -replace '/', '\' + $projPath = Join-Path $RepoRoot $proj + $settings = $Matches[2].Trim() + TestUsingMSBuild -testProject $projPath -targetFramework $script:desktopTargetFramework -settings $settings + } + } + } else { + TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework + } } if ($testFSharpCore) { diff --git a/eng/Versions.props b/eng/Versions.props index 0032205fef2..8c9852c7620 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -170,6 +170,7 @@ 13.0.3 3.2.2 3.2.2 + 8.0.0 diff --git a/eng/build.sh b/eng/build.sh index 06df1423012..913be4b1690 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -233,17 +233,17 @@ function Test() { testresultsdir="$artifacts_dir/TestResults/$configuration" # MTP requires --solution flag for .sln files - # For solutions, omit --report-xunit-trx-filename so each test assembly generates a unique .trx file. - # With a static filename, all assemblies overwrite the same file and only the last one's results survive. if [[ "$testproject" == *.sln ]]; then testtarget="--solution" - reportargs="--report-xunit-trx" else testtarget="--project" - testlogfilename="${projectname}_${targetframework}.trx" - reportargs="--report-xunit-trx --report-xunit-trx-filename $testlogfilename" fi + # Xunit XML report via XunitXml.TestLogger with CI-friendly filenames + jobname="${SYSTEM_JOBNAME:-local}" + xunitlogfilename="{assembly}.{framework}.${jobname}.xml" + reportargs="--report-spekt-xunit --report-spekt-xunit-filename $xunitlogfilename" + args=(test $testtarget "$testproject" --no-build -c "$configuration" -f "$targetframework" $reportargs --results-directory "$testresultsdir" --hangdump --hangdump-timeout 5m --hangdump-type Full) "$DOTNET_INSTALL_DIR/dotnet" "${args[@]}" || exit $? diff --git a/eng/templates/batched-test-job.yml b/eng/templates/batched-test-job.yml new file mode 100644 index 00000000000..acc2a842624 --- /dev/null +++ b/eng/templates/batched-test-job.yml @@ -0,0 +1,93 @@ +# Template for running test jobs split into parallel batches via strategy:matrix. +# Place this alongside (not inside) the Arcade jobs.yml template reference, +# since Arcade's ${{ each job }} loop cannot expand nested template references. + +parameters: + jobName: '' + pool: {} + timeoutInMinutes: 120 + buildCommand: '' # Full build/test command including $(batchNumber) + buildEnv: {} + batches: [1, 2, 3] + variables: [] + testRunTitlePrefix: '' # e.g. 'WindowsCompressedMetadata testDesktop' + artifactNamePrefix: '' # e.g. 'Windows testDesktop' + configuration: 'Release' + publishBinLog: false + binLogPath: '' + publishDumps: false + +jobs: +- job: ${{ parameters.jobName }} + strategy: + matrix: + ${{ each batch in parameters.batches }}: + Batch${{ batch }}: + batchNumber: ${{ batch }} + pool: ${{ parameters.pool }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + variables: + - ${{ each var in parameters.variables }}: + - name: ${{ var.name }} + value: ${{ var.value }} + steps: + - checkout: self + clean: true + + - script: ${{ parameters.buildCommand }} + env: ${{ parameters.buildEnv }} + displayName: Build / Test + + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'VSTest' + testRunTitle: ${{ parameters.testRunTitlePrefix }} Batch$(batchNumber) + mergeTestResults: true + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.configuration }}' + continueOnError: true + condition: succeededOrFailed() + + - ${{ if parameters.publishBinLog }}: + - task: PublishBuildArtifacts@1 + displayName: Publish BinLog + continueOnError: true + inputs: + PathToPublish: ${{ parameters.binLogPath }} + ArtifactName: '${{ parameters.artifactNamePrefix }} Batch$(batchNumber) binlogs' + ArtifactType: Container + parallel: true + + - ${{ if parameters.publishDumps }}: + - task: PublishBuildArtifacts@1 + displayName: Publish Dumps + condition: failed() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/${{ parameters.configuration }}' + ArtifactName: '${{ parameters.artifactNamePrefix }} Batch$(batchNumber) process dumps' + ArtifactType: Container + parallel: true + + - task: PublishBuildArtifacts@1 + displayName: Publish Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.configuration }}' + ArtifactName: '${{ parameters.artifactNamePrefix }} Batch$(batchNumber) test logs' + publishLocation: Container + continueOnError: true + condition: always() + + - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj + displayName: Dump NuGet cache contents + condition: failed() + + - task: PublishBuildArtifacts@1 + displayName: Publish NuGet cache contents + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/NugetPackageRootContents' + ArtifactName: 'NuGetPackageContents ${{ parameters.artifactNamePrefix }} Batch$(batchNumber)' + publishLocation: Container + continueOnError: true + condition: failed() diff --git a/eng/tests/TestSplit.fsx b/eng/tests/TestSplit.fsx new file mode 100644 index 00000000000..b5389c658fd --- /dev/null +++ b/eng/tests/TestSplit.fsx @@ -0,0 +1,74 @@ +/// Test split table for parallel CI. +/// Edit the batch assignments below, then run: +/// dotnet fsi eng/tests/TestSplit.fsx +/// to get the dotnet test commands for that batch. + +let totalBatches = 3 +let residualBatch = 2 // uses negation filter; catches unlisted atoms + future namespaces + +// MTP --filter-namespace uses starts-with matching on the test namespace. +// Unlisted atoms go to the residual batch automatically via --filter-not-namespace. + +let componentTestsAtoms = + [// atom batch + "CompilerDirectives", 1 + "CompilerService", 1 + "ErrorMessages", 1 + "FSharpChecker", 1 + "Import", 1 + "Language", 1 + "Miscellaneous", 1 + "XmlComments", 1 + + "Libraries", 2 + "Globalization", 2 + + "EmittedIL", 3 + "Interop", 3 + "InteractiveSession", 3 + "CompilerOptions", 3 + "Conformance", 3 + "Diagnostics", 3 + "Signatures", 3 + "ConstraintSolver", 3 + "Debugger", 3 + "Scripting", 3 + "TypeChecks", 3 + ] + +let otherProjects = + [// project path batch + "tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj", 1 + "tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj", 1 + "tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj", 3 + "tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj", 3 + "tests/fsharp/FSharpSuite.Tests.fsproj", 3 + ] + +// ── filter generation ── + +let batch = + match fsi.CommandLineArgs with + | [| _; n |] -> + let v = int n + if v < 1 || v > totalBatches then failwith $"Batch number must be between 1 and {totalBatches}, got {v}" + v + | _ -> failwith "Usage: dotnet fsi eng/tests/TestSplit.fsx " + +let atomsForBatch b = componentTestsAtoms |> List.filter (fun (_, ba) -> ba = b) |> List.map fst |> List.sort +let otherBatchesAtoms = componentTestsAtoms |> List.filter (fun (_, b) -> b <> batch) |> List.map fst |> List.sort + +let filterArgs = + if batch = residualBatch then + let atoms = otherBatchesAtoms |> String.concat " " + $"--filter-not-namespace {atoms}" + else + let atoms = atomsForBatch batch |> String.concat " " + $"--filter-namespace {atoms}" + +let componentTests = "tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj" + +printfn $"dotnet test {componentTests} --no-build -c Release {filterArgs}" + +for (proj, _) in otherProjects |> List.filter (fun (_, b) -> b = batch) do + printfn $"dotnet test {proj} --no-build -c Release" diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props index bf1b81f2bef..ccc7e44ffa3 100644 --- a/tests/Directory.Build.props +++ b/tests/Directory.Build.props @@ -13,6 +13,10 @@ + + + + diff --git a/tests/EndToEndBuildTests/Directory.Build.props b/tests/EndToEndBuildTests/Directory.Build.props index 496de8645f5..f97db4e1684 100644 --- a/tests/EndToEndBuildTests/Directory.Build.props +++ b/tests/EndToEndBuildTests/Directory.Build.props @@ -7,6 +7,8 @@ 3.2.2 3.2.2 2.0.2 + 8.0.0 + 17.14.1 diff --git a/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj b/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj index 284ec6eeb1d..52ba8624a30 100644 --- a/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj +++ b/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj @@ -108,6 +108,8 @@ + + diff --git a/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj b/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj index 009d4f0a8fa..cf8cc25e837 100644 --- a/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj +++ b/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj @@ -128,6 +128,8 @@ + +