diff --git a/Directory.Build.props b/Directory.Build.props index 60757b50b4..45e417c62d 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -66,12 +66,26 @@ --> 2.0.0 + alpha + + + 1.0.0 + alpha 1.0.0 + + alpha + + + 1.0.0 diff --git a/Microsoft.Testing.Platform.slnf b/Microsoft.Testing.Platform.slnf index afd4071b11..ef379078e4 100644 --- a/Microsoft.Testing.Platform.slnf +++ b/Microsoft.Testing.Platform.slnf @@ -9,6 +9,7 @@ "src\\Platform\\Microsoft.Testing.Extensions.AzureDevOpsReport\\Microsoft.Testing.Extensions.AzureDevOpsReport.csproj", "src\\Platform\\Microsoft.Testing.Extensions.AzureFoundry\\Microsoft.Testing.Extensions.AzureFoundry.csproj", "src\\Platform\\Microsoft.Testing.Extensions.CrashDump\\Microsoft.Testing.Extensions.CrashDump.csproj", + "src\\Platform\\Microsoft.Testing.Extensions.CtrfReport\\Microsoft.Testing.Extensions.CtrfReport.csproj", "src\\Platform\\Microsoft.Testing.Extensions.HangDump\\Microsoft.Testing.Extensions.HangDump.csproj", "src\\Platform\\Microsoft.Testing.Extensions.HotReload\\Microsoft.Testing.Extensions.HotReload.csproj", "src\\Platform\\Microsoft.Testing.Extensions.HtmlReport\\Microsoft.Testing.Extensions.HtmlReport.csproj", diff --git a/NonWindowsTests.slnf b/NonWindowsTests.slnf index 02ffd01ded..7b9c4bf70e 100644 --- a/NonWindowsTests.slnf +++ b/NonWindowsTests.slnf @@ -14,6 +14,7 @@ "src\\Platform\\Microsoft.Testing.Extensions.AzureDevOpsReport\\Microsoft.Testing.Extensions.AzureDevOpsReport.csproj", "src\\Platform\\Microsoft.Testing.Extensions.AzureFoundry\\Microsoft.Testing.Extensions.AzureFoundry.csproj", "src\\Platform\\Microsoft.Testing.Extensions.CrashDump\\Microsoft.Testing.Extensions.CrashDump.csproj", + "src\\Platform\\Microsoft.Testing.Extensions.CtrfReport\\Microsoft.Testing.Extensions.CtrfReport.csproj", "src\\Platform\\Microsoft.Testing.Extensions.HangDump\\Microsoft.Testing.Extensions.HangDump.csproj", "src\\Platform\\Microsoft.Testing.Extensions.HotReload\\Microsoft.Testing.Extensions.HotReload.csproj", "src\\Platform\\Microsoft.Testing.Extensions.HtmlReport\\Microsoft.Testing.Extensions.HtmlReport.csproj", diff --git a/TestFx.slnx b/TestFx.slnx index 991e9dc8db..f5a5085d29 100644 --- a/TestFx.slnx +++ b/TestFx.slnx @@ -1,5 +1,7 @@ + + @@ -39,6 +41,7 @@ + diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 37f9367e1f..080023d264 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -110,6 +110,9 @@ stages: # options that makes sense in Release: # --report-trx + --report-trx-filename "{asm}_{tfm}.trx" — TRX per (assembly x TFM), # resolved by Microsoft.Testing.Platform.Services.ArtifactNamingHelper. + # --report-ctrf + --report-ctrf-filename "{asm}_{tfm}.ctrf.json" — CTRF report per + # (assembly x TFM). Same placeholder resolver as TRX. Dogfoods the new CTRF extension + # so any regression in the report shape surfaces on every CI run. # --report-azdo — surfaces failed tests in the AzDO UI. # --report-azdo-progress — emits ##vso[task.logdetail ...;progress=...] timeline records so # long-running test sessions show live progress on the AzDO timeline. No-op outside AzDO @@ -132,7 +135,7 @@ stages: # and this glob mixes net4x and net8+/net9+ hosts. # --coverage — Codecov upload is Debug-only (see "Upload coverage to codecov.io" step # below), so collecting coverage in Release would just write unused .coverage files. - - script: dotnet test --no-build --test-modules "artifacts/bin/*UnitTests*/$(_BuildConfig)/**/*UnitTests.exe" --results-directory $(BUILD.SOURCESDIRECTORY)\artifacts\TestResults\$(_BuildConfig) --no-progress --output detailed --report-trx --report-trx-filename "{asm}_{tfm}.trx" --report-azdo --report-azdo-progress --report-azdo-summary --report-junit --report-junit-filename "{asm}_{tfm}.xml" --hangdump --hangdump-timeout 15m --diagnostic --diagnostic-output-directory $(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig) --diagnostic-verbosity trace + - script: dotnet test --no-build --test-modules "artifacts/bin/*UnitTests*/$(_BuildConfig)/**/*UnitTests.exe" --results-directory $(BUILD.SOURCESDIRECTORY)\artifacts\TestResults\$(_BuildConfig) --no-progress --output detailed --report-trx --report-trx-filename "{asm}_{tfm}.trx" --report-ctrf --report-ctrf-filename "{asm}_{tfm}.ctrf.json" --report-azdo --report-azdo-progress --report-azdo-summary --report-junit --report-junit-filename "{asm}_{tfm}.xml" --hangdump --hangdump-timeout 15m --diagnostic --diagnostic-output-directory $(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig) --diagnostic-verbosity trace name: TestRelease displayName: Test (unit tests only) condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) diff --git a/samples/CtrfPlayground/Mtp/Mtp.csproj b/samples/CtrfPlayground/Mtp/Mtp.csproj new file mode 100644 index 0000000000..1f58392bc7 --- /dev/null +++ b/samples/CtrfPlayground/Mtp/Mtp.csproj @@ -0,0 +1,36 @@ + + + + Exe + net9.0 + enable + $(NoWarn);NETSDK1023;SA0001;EnableGenerateDocumentationFile;TPEXP + + + true + false + false + + + + + + + + + + + + + + + + + diff --git a/samples/CtrfPlayground/Mtp/Program.cs b/samples/CtrfPlayground/Mtp/Program.cs new file mode 100644 index 0000000000..6259c50a5c --- /dev/null +++ b/samples/CtrfPlayground/Mtp/Program.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Reflection; + +using Microsoft.Testing.Extensions; +using Microsoft.Testing.Platform.Builder; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace CtrfPlayground.Mtp; + +public static class Program +{ + public static async Task Main(string[] args) + { + ITestApplicationBuilder testApplicationBuilder = await TestApplication.CreateBuilderAsync(args); + testApplicationBuilder.AddMSTest(() => [Assembly.GetEntryAssembly()!]); + testApplicationBuilder.AddCtrfReportProvider(); + using ITestApplication app = await testApplicationBuilder.BuildAsync(); + return await app.RunAsync(); + } +} diff --git a/samples/CtrfPlayground/Mtp/Tests.cs b/samples/CtrfPlayground/Mtp/Tests.cs new file mode 100644 index 0000000000..173d084b8d --- /dev/null +++ b/samples/CtrfPlayground/Mtp/Tests.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace CtrfPlayground.Mtp; + +[TestClass] +public class SampleTests +{ + [TestMethod] + public void PassingTest() + => Assert.AreEqual(2, 1 + 1); + + [TestMethod] + public void FailingTest() + => Assert.AreEqual(3, 1 + 1, "intentional failure to exercise CTRF status mapping"); + + [TestMethod] + [Ignore("intentionally skipped to exercise CTRF status mapping")] + public void SkippedTest() + { + } + + [TestMethod] + public void ThrowingTest() + => throw new InvalidOperationException("intentional exception to exercise CTRF error fields"); + + [DataTestMethod] + [DataRow(1, 1, 2)] + [DataRow(2, 3, 5)] + [DataRow(2, 2, 5)] // intentional failure + public void DataDrivenTest(int a, int b, int expected) + => Assert.AreEqual(expected, a + b); +} diff --git a/samples/CtrfPlayground/README.md b/samples/CtrfPlayground/README.md new file mode 100644 index 0000000000..c83bec86ed --- /dev/null +++ b/samples/CtrfPlayground/README.md @@ -0,0 +1,38 @@ +# CtrfPlayground + +This sample produces side-by-side [CTRF (Common Test Report Format)](https://ctrf.io/) reports +from the same set of tests using two different CTRF generators that both target +[Microsoft.Testing.Platform](https://aka.ms/testingplatform): + +| Project | Test framework | CTRF generator | +| ------- | -------------- | -------------- | +| [`Mtp`](./Mtp) | MSTest | `Microsoft.Testing.Extensions.CtrfReport` (this repository, **experimental**) | +| [`XunitMtp`](./XunitMtp) | [`xunit.v3`](https://www.nuget.org/packages/xunit.v3) | `xunit.v3`'s built-in `-ctrf ` reporter | + +The tests in each project are intentionally equivalent (pass / fail / skip / theory / throw) +so that the two CTRF reports can be diffed to validate the parity of the new extension. + +## Running + +From the repository root: + +```pwsh +# MSTest + Microsoft.Testing.Extensions.CtrfReport +dotnet run --project samples/CtrfPlayground/Mtp -- --report-ctrf --results-directory ./out/mtp + +# xunit.v3 with its built-in CTRF reporter +dotnet run --project samples/CtrfPlayground/XunitMtp -- -ctrf ./out/xunit/ctrf-report.json +``` + +Both runs write a CTRF JSON file. Open them side by side (e.g. +`code -d ./out/mtp/.ctrf.json ./out/xunit/ctrf-report.json`) to compare the two outputs. + +> [!NOTE] +> The CTRF extension shipped by this repository is currently marked **experimental** +> (`[Experimental("TPEXP")]`). API shape and report content may change. + + +> [!TIP] +> The two projects are intentionally **kept separate**: combining a CTRF-producing +> extension with another test framework that already exposes its own CTRF reporter would +> conflict on command-line options inside a single test host. diff --git a/samples/CtrfPlayground/XunitMtp/Tests.cs b/samples/CtrfPlayground/XunitMtp/Tests.cs new file mode 100644 index 0000000000..db7e614893 --- /dev/null +++ b/samples/CtrfPlayground/XunitMtp/Tests.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Xunit; + +namespace CtrfPlayground.XunitMtp; + +public class SampleTests +{ + [Fact] + public void PassingTest() + => Assert.Equal(2, 1 + 1); + + [Fact] + public void FailingTest() + => Assert.Fail("intentional failure to exercise CTRF status mapping"); + + [Fact(Skip = "intentionally skipped to exercise CTRF status mapping")] + public void SkippedTest() + { + } + + [Fact] + public void ThrowingTest() + => throw new InvalidOperationException("intentional exception to exercise CTRF error fields"); + + [Theory] + [InlineData(1, 1, 2)] + [InlineData(2, 3, 5)] + [InlineData(2, 2, 5)] // intentional failure + public void DataDrivenTest(int a, int b, int expected) + => Assert.Equal(expected, a + b); +} diff --git a/samples/CtrfPlayground/XunitMtp/XunitMtp.csproj b/samples/CtrfPlayground/XunitMtp/XunitMtp.csproj new file mode 100644 index 0000000000..f88ff7c0dc --- /dev/null +++ b/samples/CtrfPlayground/XunitMtp/XunitMtp.csproj @@ -0,0 +1,24 @@ + + + + Exe + net9.0 + enable + false + $(NoWarn);NETSDK1023;SA0001;EnableGenerateDocumentationFile + + + true + false + + + + + + + diff --git a/src/Package/MSTest.Sdk/MSTest.Sdk.csproj b/src/Package/MSTest.Sdk/MSTest.Sdk.csproj index 7132d2b545..ff417bd69c 100644 --- a/src/Package/MSTest.Sdk/MSTest.Sdk.csproj +++ b/src/Package/MSTest.Sdk/MSTest.Sdk.csproj @@ -43,6 +43,13 @@ <_MSTestSourceGenerationVersion>$(MSTestSourceGenerationVersionPrefix) <_MSTestSourceGenerationVersion Condition="'$(_MSTestSourceGenerationVersionSuffix)' != ''">$(_MSTestSourceGenerationVersion)-$(_MSTestSourceGenerationVersionSuffix) + <_MicrosoftTestingExtensionsCtrfReportPreReleaseVersionLabel>$(MicrosoftTestingExtensionsCtrfReportPreReleaseVersionLabel) + <_MicrosoftTestingExtensionsCtrfReportPreReleaseVersionLabel Condition="'$(ContinuousIntegrationBuild)' == 'true' and '$(OfficialBuild)' != 'true'">ci + <_MicrosoftTestingExtensionsCtrfReportPreReleaseVersionLabel Condition="'$(ContinuousIntegrationBuild)' != 'true' and '$(OfficialBuild)' != 'true'">dev + <_MicrosoftTestingExtensionsCtrfReportVersionSuffix>$(_MicrosoftTestingExtensionsCtrfReportPreReleaseVersionLabel)$(_BuildNumberLabels) + <_MicrosoftTestingExtensionsCtrfReportVersion>$(MicrosoftTestingExtensionsCtrfReportVersionPrefix) + <_MicrosoftTestingExtensionsCtrfReportVersion Condition="'$(_MicrosoftTestingExtensionsCtrfReportVersionSuffix)' != ''">$(_MicrosoftTestingExtensionsCtrfReportVersion)-$(_MicrosoftTestingExtensionsCtrfReportVersionSuffix) + <_MicrosoftTestingExtensionsJUnitReportPreReleaseVersionLabel>$(MicrosoftTestingExtensionsJUnitReportPreReleaseVersionLabel) <_MicrosoftTestingExtensionsJUnitReportPreReleaseVersionLabel Condition="'$(ContinuousIntegrationBuild)' == 'true' and '$(OfficialBuild)' != 'true'">ci <_MicrosoftTestingExtensionsJUnitReportPreReleaseVersionLabel Condition="'$(ContinuousIntegrationBuild)' != 'true' and '$(OfficialBuild)' != 'true'">dev @@ -50,7 +57,14 @@ <_MicrosoftTestingExtensionsJUnitReportVersion>$(MicrosoftTestingExtensionsJUnitReportVersionPrefix) <_MicrosoftTestingExtensionsJUnitReportVersion Condition="'$(_MicrosoftTestingExtensionsJUnitReportVersionSuffix)' != ''">$(_MicrosoftTestingExtensionsJUnitReportVersion)-$(_MicrosoftTestingExtensionsJUnitReportVersionSuffix) - <_TemplateProperties>MSTestSourceGenerationVersion=$(_MSTestSourceGenerationVersion);MSTestVersion=$(Version);MicrosoftTestingPlatformVersion=$(Version.Replace('$(VersionPrefix)', '$(TestingPlatformVersionPrefix)'));MicrosoftNETTestSdkVersion=$(MicrosoftNETTestSdkVersion);MicrosoftTestingExtensionsCodeCoverageVersion=$(MicrosoftTestingExtensionsCodeCoverageVersion);MicrosoftPlaywrightVersion=$(MicrosoftPlaywrightVersion);AspireHostingTestingVersion=$(AspireHostingTestingVersion);MicrosoftTestingExtensionsFakesVersion=$(MicrosoftTestingExtensionsFakesVersion);MicrosoftTestingExtensionsJUnitReportVersion=$(_MicrosoftTestingExtensionsJUnitReportVersion) + <_MicrosoftTestingExtensionsOpenTelemetryPreReleaseVersionLabel>$(MicrosoftTestingExtensionsOpenTelemetryPreReleaseVersionLabel) + <_MicrosoftTestingExtensionsOpenTelemetryPreReleaseVersionLabel Condition="'$(ContinuousIntegrationBuild)' == 'true' and '$(OfficialBuild)' != 'true'">ci + <_MicrosoftTestingExtensionsOpenTelemetryPreReleaseVersionLabel Condition="'$(ContinuousIntegrationBuild)' != 'true' and '$(OfficialBuild)' != 'true'">dev + <_MicrosoftTestingExtensionsOpenTelemetryVersionSuffix>$(_MicrosoftTestingExtensionsOpenTelemetryPreReleaseVersionLabel)$(_BuildNumberLabels) + <_MicrosoftTestingExtensionsOpenTelemetryVersion>$(MicrosoftTestingExtensionsOpenTelemetryVersionPrefix) + <_MicrosoftTestingExtensionsOpenTelemetryVersion Condition="'$(_MicrosoftTestingExtensionsOpenTelemetryVersionSuffix)' != ''">$(_MicrosoftTestingExtensionsOpenTelemetryVersion)-$(_MicrosoftTestingExtensionsOpenTelemetryVersionSuffix) + + <_TemplateProperties>MSTestSourceGenerationVersion=$(_MSTestSourceGenerationVersion);MSTestVersion=$(Version);MicrosoftTestingPlatformVersion=$(Version.Replace('$(VersionPrefix)', '$(TestingPlatformVersionPrefix)'));MicrosoftNETTestSdkVersion=$(MicrosoftNETTestSdkVersion);MicrosoftTestingExtensionsCodeCoverageVersion=$(MicrosoftTestingExtensionsCodeCoverageVersion);MicrosoftPlaywrightVersion=$(MicrosoftPlaywrightVersion);AspireHostingTestingVersion=$(AspireHostingTestingVersion);MicrosoftTestingExtensionsFakesVersion=$(MicrosoftTestingExtensionsFakesVersion);MicrosoftTestingExtensionsCtrfReportVersion=$(_MicrosoftTestingExtensionsCtrfReportVersion);MicrosoftTestingExtensionsJUnitReportVersion=$(_MicrosoftTestingExtensionsJUnitReportVersion);MicrosoftTestingExtensionsOpenTelemetryVersion=$(_MicrosoftTestingExtensionsOpenTelemetryVersion) + true + $(MicrosoftTestingExtensionsCommonVersion) + + + + + true $(MicrosoftTestingExtensionsFakesVersion) @@ -132,12 +152,33 @@ + + $(MicrosoftTestingExtensionsCtrfReportVersion) + + + + + $(MicrosoftTestingExtensionsHtmlReportVersion) + + + $(MicrosoftTestingExtensionsJUnitReportVersion) + + + $(MicrosoftTestingExtensionsOpenTelemetryVersion) + + diff --git a/src/Package/MSTest.Sdk/Sdk/Sdk.props.template b/src/Package/MSTest.Sdk/Sdk/Sdk.props.template index 396ee6d9a9..004df17238 100644 --- a/src/Package/MSTest.Sdk/Sdk/Sdk.props.template +++ b/src/Package/MSTest.Sdk/Sdk/Sdk.props.template @@ -20,8 +20,10 @@ ${MicrosoftNETTestSdkVersion} ${MicrosoftPlaywrightVersion} ${MicrosoftTestingExtensionsCodeCoverageVersion} + ${MicrosoftTestingExtensionsCtrfReportVersion} ${MicrosoftTestingExtensionsFakesVersion} ${MicrosoftTestingExtensionsJUnitReportVersion} + ${MicrosoftTestingExtensionsOpenTelemetryVersion} ${MicrosoftTestingPlatformVersion} ${MSTestSourceGenerationVersion} ${MSTestVersion} diff --git a/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets b/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets index 4eb6773f36..dcc6680431 100644 --- a/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets +++ b/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets @@ -2,11 +2,15 @@ + + + + diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/BannedSymbols.txt b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/BannedSymbols.txt new file mode 100644 index 0000000000..64ef236c50 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/BannedSymbols.txt @@ -0,0 +1,9 @@ +P:System.DateTime.Now; Use 'IClock' instead +P:System.DateTime.UtcNow; Use 'IClock' instead +M:System.Threading.Tasks.Task.Run(System.Action); Use 'ITask' instead +M:System.Threading.Tasks.Task.WhenAll(System.Threading.Tasks.Task[]); Use 'ITask' instead +M:System.Threading.Tasks.Task.WhenAll(System.Collections.Generic.IEnumerable{System.Threading.Tasks.Task}); Use 'ITask' instead +M:System.String.IsNullOrEmpty(System.String); Use 'RoslynString.IsNullOrEmpty' instead +M:System.String.IsNullOrWhiteSpace(System.String); Use 'RoslynString.IsNullOrWhiteSpace' instead +M:System.Diagnostics.Debug.Assert(System.Boolean); Use 'RoslynDebug.Assert' instead +M:System.Diagnostics.Debug.Assert(System.Boolean,System.String); Use 'RoslynDebug.Assert' instead diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CapturedTestResult.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CapturedTestResult.cs new file mode 100644 index 0000000000..f35dbf32a3 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CapturedTestResult.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Extensions.CtrfReport; + +// Minimal capped-size projection of a TestNodeUpdateMessage. The consumer projects +// each message into this DTO immediately so that we don't retain entire test nodes +// (and their potentially huge stdout/stderr/stack trace strings) in memory for the +// whole session. All variable-length text fields are already truncated at this point +// so the engine doesn't need to truncate again. +internal sealed class CapturedTestResult +{ + public required string Uid { get; init; } + + public required string DisplayName { get; init; } + + // CTRF status (passed/failed/skipped/pending/other) — already normalized. + public required string Status { get; init; } + + // Preserves the original MTP outcome when it doesn't map 1:1 to CTRF + // (e.g. "timedOut", "errored", "cancelled"). Surfaced via CTRF `rawStatus`. + public string? RawStatus { get; init; } + + public required TimeSpan Duration { get; init; } + + public DateTimeOffset? StartTime { get; init; } + + public DateTimeOffset? EndTime { get; init; } + + public string? Namespace { get; init; } + + public string? ClassName { get; init; } + + public string? MethodName { get; init; } + + public string? ErrorMessage { get; init; } + + public string? ExceptionType { get; init; } + + public string? StackTrace { get; init; } + + public string? StandardOutput { get; init; } + + public string? StandardError { get; init; } + + public string? FilePath { get; init; } + + public int? Line { get; init; } + + public IReadOnlyList>? Traits { get; init; } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs new file mode 100644 index 0000000000..49e1af7fb2 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs @@ -0,0 +1,778 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Text.Json; + +using Microsoft.Testing.Extensions.CtrfReport.Resources; +using Microsoft.Testing.Platform; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Configurations; +using Microsoft.Testing.Platform.Extensions.TestFramework; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.OutputDevice; +using Microsoft.Testing.Platform.Services; + +namespace Microsoft.Testing.Extensions.CtrfReport; + +internal sealed class CtrfReportEngine +{ + // CTRF spec: https://github.com/ctrf-io/ctrf + private const string CtrfReportFormat = "CTRF"; + private const string CtrfSpecVersion = "0.0.0"; + + private readonly IFileSystem _fileSystem; + private readonly ITestApplicationModuleInfo _testApplicationModuleInfo; + private readonly IEnvironment _environment; + private readonly ICommandLineOptions _commandLineOptions; + private readonly IConfiguration _configuration; + private readonly IClock _clock; + private readonly ITestFramework _testFramework; + private readonly DateTimeOffset _testStartTime; + private readonly int _exitCode; + private readonly CancellationToken _cancellationToken; + + public CtrfReportEngine( + IFileSystem fileSystem, + ITestApplicationModuleInfo testApplicationModuleInfo, + IEnvironment environment, + ICommandLineOptions commandLineOptions, + IConfiguration configuration, + IClock clock, + ITestFramework testFramework, + DateTimeOffset testStartTime, + int exitCode, + CancellationToken cancellationToken) + { + _fileSystem = fileSystem; + _testApplicationModuleInfo = testApplicationModuleInfo; + _environment = environment; + _commandLineOptions = commandLineOptions; + _configuration = configuration; + _clock = clock; + _testFramework = testFramework; + _testStartTime = testStartTime; + _exitCode = exitCode; + _cancellationToken = cancellationToken; + } + + public Task<(string FileName, string? Warning)> GenerateReportAsync(CapturedTestResult[] results) + => GenerateReportCoreAsync(results, _clock.UtcNow); + + private async Task<(string FileName, string? Warning)> GenerateReportCoreAsync(CapturedTestResult[] results, DateTimeOffset finishTime) + { + _cancellationToken.ThrowIfCancellationRequested(); + + bool fileNameExplicitlyProvided = _commandLineOptions.TryGetOptionArgumentList( + CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName, + out string[]? providedFileName); + + string fileName = fileNameExplicitlyProvided + ? ResolveJsonFileName(GetProvidedFileName(providedFileName)) + : BuildDefaultFileName(finishTime); + + string outputDirectory = _configuration.GetTestResultDirectory(); + // Path.Combine short-circuits when the second argument is rooted, so an absolute + // user-provided file name overrides the test results directory while validated + // relative paths stay nested under it. + string finalPath = Path.Combine(outputDirectory, fileName); + string? finalDirectory = Path.GetDirectoryName(finalPath); + if (!RoslynString.IsNullOrEmpty(finalDirectory)) + { + _fileSystem.CreateDirectory(finalDirectory); + } + + byte[] bytes = BuildCtrfJson(results, finishTime); + + return await WriteWithRetryAsync(finalPath, bytes, fileNameExplicitlyProvided).ConfigureAwait(false); + } + + private static string GetProvidedFileName(string[]? providedFileName) + => providedFileName is { Length: > 0 } + ? providedFileName[0] + : throw ApplicationStateGuard.Unreachable(); + + private async Task<(string FileName, string? Warning)> WriteWithRetryAsync(string finalPath, byte[] bytes, bool fileNameExplicitlyProvided) + { + // Explicit file names: use FileMode.Create (overwrite). Default-generated file + // names: use FileMode.CreateNew but retry with disambiguating suffixes when the + // file already exists, so concurrent runs (or two runs within the same second + // sharing the result directory) don't fail with IOException. + if (fileNameExplicitlyProvided) + { + bool willOverwrite = _fileSystem.ExistFile(finalPath); + await WriteAsync(finalPath, FileMode.Create, bytes).ConfigureAwait(false); + return ( + finalPath, + willOverwrite + ? string.Format(CultureInfo.InvariantCulture, ExtensionResources.CtrfReportFileExistsAndWillBeOverwritten, finalPath) + : null); + } + + DateTimeOffset firstTry = _clock.UtcNow; + string directory = Path.GetDirectoryName(finalPath) ?? string.Empty; + string fileName = Path.GetFileName(finalPath); + SplitCtrfExtension(fileName, out string baseName, out string extension); + string candidate = finalPath; + int attempt = 0; + + while (true) + { + _cancellationToken.ThrowIfCancellationRequested(); + + try + { + await WriteAsync(candidate, FileMode.CreateNew, bytes).ConfigureAwait(false); + return (candidate, null); + } + catch (IOException) when (_fileSystem.ExistFile(candidate)) + { + // The IOException was caused by the file already existing. Try a + // suffixed name. Any other IOException (disk full, permission, path + // too long, etc.) is not caught here and will propagate to the caller. + // Bound by both wall-clock (5s) and attempt count (1000) so we never + // spin forever in pathological cases like a clock that doesn't advance. + if (_clock.UtcNow - firstTry > TimeSpan.FromSeconds(5) || attempt >= 1_000) + { + throw; + } + + attempt++; + candidate = Path.Combine(directory, $"{baseName}_{attempt}{extension}"); + } + } + } + + // Split a file name into base + extension while preserving the CTRF + // double-extension convention (`*.ctrf.json`). The disambiguation suffix + // must land before `.ctrf.json` so that downstream CTRF readers continue to + // recognize the file by its conventional extension. + private static void SplitCtrfExtension(string fileName, out string baseName, out string extension) + { + const string ctrfJsonSuffix = ".ctrf.json"; + if (fileName.EndsWith(ctrfJsonSuffix, StringComparison.OrdinalIgnoreCase) && fileName.Length > ctrfJsonSuffix.Length) + { + baseName = fileName.Substring(0, fileName.Length - ctrfJsonSuffix.Length); + extension = fileName.Substring(fileName.Length - ctrfJsonSuffix.Length); + return; + } + + baseName = Path.GetFileNameWithoutExtension(fileName); + extension = Path.GetExtension(fileName); + } + + private async Task WriteAsync(string path, FileMode mode, byte[] bytes) + { + // Note that we need to dispose the IFileStream, not the inner stream. + // IFileStream implementations will be responsible to dispose their inner stream. + using IFileStream stream = _fileSystem.NewFileStream(path, mode); +#if NETCOREAPP + await stream.Stream.WriteAsync(bytes.AsMemory(), _cancellationToken).ConfigureAwait(false); +#else + await stream.Stream.WriteAsync(bytes, 0, bytes.Length, _cancellationToken).ConfigureAwait(false); +#endif + } + + private string BuildDefaultFileName(DateTimeOffset finishTime) + { + string user = _environment.GetEnvironmentVariable("UserName") + ?? _environment.GetEnvironmentVariable("USER") + ?? "user"; + string moduleName = Path.GetFileNameWithoutExtension(_testApplicationModuleInfo.GetCurrentTestApplicationFullPath()); + string targetFrameworkMoniker = GetTargetFrameworkMoniker(); + string raw = $"{user}_{_environment.MachineName}_{moduleName}_{targetFrameworkMoniker}_{finishTime:yyyy-MM-dd_HH_mm_ss}.ctrf.json"; + return ReplaceInvalidFileNameChars(raw); + } + + private string ResolveJsonFileName(string template) + { + string processName = Path.GetFileNameWithoutExtension(_testApplicationModuleInfo.GetCurrentTestApplicationFullPath()); + string processId = _environment.ProcessId.ToString(CultureInfo.InvariantCulture); + Dictionary replacements = ArtifactNamingHelper.GetStandardReplacements(processName, processId, _clock.UtcNow); + string resolved = ArtifactNamingHelper.ResolveTemplate(template, replacements); + string directoryPart = Path.GetDirectoryName(resolved) ?? string.Empty; + string sanitizedFileName = ReplaceInvalidFileNameChars(Path.GetFileName(resolved)); + return directoryPart.Length == 0 + ? sanitizedFileName + : Path.Combine(directoryPart, sanitizedFileName); + } + + private static string GetTargetFrameworkMoniker() + => TargetFrameworkParser.GetShortTargetFramework( + Assembly.GetEntryAssembly()?.GetCustomAttribute()?.FrameworkDisplayName) + ?? TargetFrameworkParser.GetShortTargetFramework(RuntimeInformation.FrameworkDescription) + ?? "unknown"; + + // CTRF `osPlatform` is the short Node-style platform identifier (e.g. "win32", + // "linux", "darwin"). The full descriptive OS string belongs in `osVersion`. + private static string GetCtrfOsPlatform() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return "win32"; + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + return "linux"; + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return "darwin"; + } + +#if NETCOREAPP + if (RuntimeInformation.IsOSPlatform(OSPlatform.FreeBSD)) + { + return "freebsd"; + } +#endif + + return "unknown"; + } + + private static string ReplaceInvalidFileNameChars(string fileName) + { + var sb = new StringBuilder(fileName.Length); + foreach (char c in fileName) + { + sb.Append(IsInvalidFileNameChar(c) ? '_' : c); + } + + string replaced = sb.ToString().TrimEnd(); + if (IsReservedFileName(replaced)) + { + replaced = '_' + replaced; + } + + return replaced; + } + + private static bool IsInvalidFileNameChar(char c) + // Keep the explicit file-name sanitization aligned with TRX report naming so + // placeholders and cross-platform reserved characters produce compatible names. + => c is < ' ' or '"' or '<' or '>' or '|' or ':' or '*' or '?' or '\\' or '/' or '@' or '(' or ')' or '^' or ' '; + + private static bool IsReservedFileName(string fileName) + { + string bareName = fileName; + int dot = bareName.IndexOf('.'); + if (dot >= 0) + { + bareName = bareName.Substring(0, dot); + } + + return bareName.Equals("CON", StringComparison.OrdinalIgnoreCase) + || bareName.Equals("PRN", StringComparison.OrdinalIgnoreCase) + || bareName.Equals("AUX", StringComparison.OrdinalIgnoreCase) + || bareName.Equals("NUL", StringComparison.OrdinalIgnoreCase) + || bareName.Equals("CLOCK$", StringComparison.OrdinalIgnoreCase) + || IsReservedNameWithNumber(bareName, "COM") + || IsReservedNameWithNumber(bareName, "LPT"); + + static bool IsReservedNameWithNumber(string bareName, string prefix) + => bareName.Length == 4 + && bareName.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) + && bareName[3] is >= '1' and <= '9'; + } + + private byte[] BuildCtrfJson(CapturedTestResult[] results, DateTimeOffset finishTime) + { + // Collapse multiple captures sharing the same UID into a single CTRF test + // entry. The CTRF spec models retries as nested `retryAttempts[]` records + // and exposes `flaky: true` on the final passing row; emitting separate + // top-level rows for retries would inflate `summary.tests` and double-count + // outcomes. + List collapsed = CollapseAttempts(results); + + // Aggregate summary counts from the collapsed (final) outcomes only. + int passed = 0; + int failed = 0; + int skipped = 0; + int pending = 0; + int other = 0; + int flaky = 0; + foreach (CollapsedTestResult c in collapsed) + { + switch (c.Final.Status) + { + case "passed": passed++; break; + case "failed": failed++; break; + case "skipped": skipped++; break; + case "pending": pending++; break; + default: other++; break; + } + + if (c.IsFlaky) + { + flaky++; + } + } + + long startMs = _testStartTime.ToUnixTimeMilliseconds(); + long stopMs = finishTime.ToUnixTimeMilliseconds(); + long durationMs = Math.Max(0, stopMs - startMs); + + using var ms = new MemoryStream(capacity: 8 * 1024); + // We deliberately use the default Default encoder rather than + // UnsafeRelaxedJsonEscaping: CTRF documents routinely flow into web + // dashboards that embed JSON into HTML/JS, and test names/messages are + // attacker-controllable. The default safe encoder keeps `<`, `>`, `&` + // escaped so a test display name like `` can't + // become an XSS vector in downstream consumers. + var writerOptions = new JsonWriterOptions + { + Indented = true, + }; + + using (var writer = new Utf8JsonWriter(ms, writerOptions)) + { + writer.WriteStartObject(); + + writer.WriteString("reportFormat", CtrfReportFormat); + // CTRF is still in pre-1.0; the upstream spec is at "0.0.0" today + // (see https://github.com/ctrf-io/ctrf/blob/main/spec/ctrf.md). + // Bump this constant whenever we update against a newer schema revision. + writer.WriteString("specVersion", CtrfSpecVersion); + writer.WriteString("reportId", Guid.NewGuid().ToString("D")); + writer.WriteString("timestamp", finishTime.ToString("O", CultureInfo.InvariantCulture)); + writer.WriteString( + "generatedBy", + $"Microsoft.Testing.Extensions.CtrfReport@{ExtensionVersion.DefaultSemVer}"); + + writer.WritePropertyName("results"); + writer.WriteStartObject(); + + // results.tool + writer.WritePropertyName("tool"); + writer.WriteStartObject(); + // CTRF spec requires `tool.name` to be a non-empty string. Fall back to + // a sentinel rather than emitting an empty string (which would fail + // strict schema validation by downstream CTRF consumers). + string toolName = RoslynString.IsNullOrEmpty(_testFramework.DisplayName) + ? "unknown" + : _testFramework.DisplayName; + writer.WriteString("name", toolName); + if (!RoslynString.IsNullOrEmpty(_testFramework.Version)) + { + writer.WriteString("version", _testFramework.Version); + } + + writer.WritePropertyName("extra"); + writer.WriteStartObject(); + writer.WriteString("uid", _testFramework.Uid); + writer.WriteEndObject(); + writer.WriteEndObject(); + + // results.summary + writer.WritePropertyName("summary"); + writer.WriteStartObject(); + writer.WriteNumber("tests", collapsed.Count); + writer.WriteNumber("passed", passed); + writer.WriteNumber("failed", failed); + writer.WriteNumber("skipped", skipped); + writer.WriteNumber("pending", pending); + writer.WriteNumber("other", other); + writer.WriteNumber("flaky", flaky); + writer.WriteNumber("start", startMs); + writer.WriteNumber("stop", stopMs); + writer.WriteNumber("duration", durationMs); + writer.WriteEndObject(); + + // results.environment + writer.WritePropertyName("environment"); + writer.WriteStartObject(); + string user = _environment.GetEnvironmentVariable("UserName") + ?? _environment.GetEnvironmentVariable("USER") + ?? string.Empty; + // CTRF `osPlatform` expects a short identifier such as "win32", "linux" or + // "darwin"; the full descriptive string belongs in `osVersion`. + writer.WriteString("osPlatform", GetCtrfOsPlatform()); + writer.WriteString("osVersion", RuntimeInformation.OSDescription); + // CTRF `extra` MUST be an object (schema enforces additionalProperties: false + // on environment, with `extra` typed as object). We surface the test module + // path and process exit code here rather than as top-level environment fields + // because there is no first-class CTRF slot for them. + writer.WritePropertyName("extra"); + writer.WriteStartObject(); + writer.WriteString("user", user); + writer.WriteString("machine", _environment.MachineName); + writer.WriteNumber("exitCode", _exitCode); + writer.WriteString("testApplication", _testApplicationModuleInfo.GetCurrentTestApplicationFullPath()); + writer.WriteEndObject(); + writer.WriteEndObject(); + + // results.tests + writer.WritePropertyName("tests"); + writer.WriteStartArray(); + + foreach (CollapsedTestResult c in collapsed) + { + WriteTest(writer, c); + } + + writer.WriteEndArray(); + writer.WriteEndObject(); + + writer.WriteEndObject(); + } + + return ms.ToArray(); + } + + private static void WriteTest(Utf8JsonWriter writer, CollapsedTestResult c) + { + CapturedTestResult r = c.Final; + writer.WriteStartObject(); + + // CTRF spec: tests[i].name MUST be a non-empty string. Fall back to UID + // (also non-empty) when the framework didn't supply a display name. + string name = RoslynString.IsNullOrEmpty(r.DisplayName) ? r.Uid : r.DisplayName; + writer.WriteString("name", name); + writer.WriteString("status", r.Status); + writer.WriteNumber("duration", (long)Math.Max(0, r.Duration.TotalMilliseconds)); + + if (r.StartTime is { } start) + { + writer.WriteNumber("start", start.ToUnixTimeMilliseconds()); + } + + if (r.EndTime is { } end) + { + writer.WriteNumber("stop", end.ToUnixTimeMilliseconds()); + } + + if (r.RawStatus is not null) + { + writer.WriteString("rawStatus", r.RawStatus); + } + + // CTRF `suite` is an array of strings (minItems: 1) representing the test + // hierarchy (e.g. ["MyNamespace", "MyClass"]). + if (r.Namespace is not null || r.ClassName is not null) + { + writer.WritePropertyName("suite"); + writer.WriteStartArray(); + if (r.Namespace is not null) + { + writer.WriteStringValue(r.Namespace); + } + + if (r.ClassName is not null) + { + writer.WriteStringValue(r.ClassName); + } + + writer.WriteEndArray(); + } + + if (r.ErrorMessage is not null) + { + writer.WriteString("message", r.ErrorMessage); + } + + if (r.StackTrace is not null) + { + writer.WriteString("trace", r.StackTrace); + } + + if (r.FilePath is not null) + { + writer.WriteString("filePath", r.FilePath); + } + + if (r.Line is { } lineNumber) + { + writer.WriteNumber("line", lineNumber); + } + + if (c.PriorAttempts.Count > 0) + { + writer.WriteNumber("retries", c.PriorAttempts.Count); + writer.WritePropertyName("retryAttempts"); + writer.WriteStartArray(); + for (int i = 0; i < c.PriorAttempts.Count; i++) + { + WriteRetryAttempt(writer, c.PriorAttempts[i], attemptNumber: i + 1); + } + + writer.WriteEndArray(); + } + + if (c.IsFlaky) + { + writer.WriteBoolean("flaky", true); + } + + WriteOutputLines(writer, "stdout", r.StandardOutput); + WriteOutputLines(writer, "stderr", r.StandardError); + + // CTRF spec 9.14 (`tags`): top-level string array on the Test object used + // for keyless classification. We promote MSTest `[TestCategory("…")]` trait + // values here so CTRF consumers can filter/group by category without having + // to walk the structured `labels` object. The full set of traits (including + // TestCategory) is also emitted under `labels` below. + if (r.Traits is { Count: > 0 }) + { + bool tagsArrayStarted = false; + foreach (KeyValuePair trait in r.Traits) + { + if (string.Equals(trait.Key, "TestCategory", StringComparison.Ordinal)) + { + if (!tagsArrayStarted) + { + writer.WritePropertyName("tags"); + writer.WriteStartArray(); + tagsArrayStarted = true; + } + + writer.WriteStringValue(trait.Value); + } + } + + if (tagsArrayStarted) + { + writer.WriteEndArray(); + } + } + + // CTRF spec 9.15 (`labels`): top-level object on the Test object for + // structured key/value test metadata. Per ctrf-io/ctrf#53, this is the + // intended home for arbitrary framework-defined traits (Option A confirmed + // by the spec maintainer), and array values are an accepted extension for + // multi-valued keys (e.g. multiple `[TestCategory]` attributes on the same + // MSTest method). We emit a scalar string for single-valued keys and an + // array of strings when the same key appears more than once. Keys appear + // in first-seen order; values keep their original declaration order. + if (r.Traits is { Count: > 0 }) + { + writer.WritePropertyName("labels"); + writer.WriteStartObject(); + for (int i = 0; i < r.Traits.Count; i++) + { + string key = r.Traits[i].Key; + if (HasSameKeyEarlier(r.Traits, i)) + { + continue; + } + + writer.WritePropertyName(key); + + int duplicateCount = 0; + for (int j = i + 1; j < r.Traits.Count; j++) + { + if (string.Equals(r.Traits[j].Key, key, StringComparison.Ordinal)) + { + duplicateCount++; + } + } + + if (duplicateCount == 0) + { + writer.WriteStringValue(r.Traits[i].Value); + } + else + { + writer.WriteStartArray(); + writer.WriteStringValue(r.Traits[i].Value); + for (int j = i + 1; j < r.Traits.Count; j++) + { + if (string.Equals(r.Traits[j].Key, key, StringComparison.Ordinal)) + { + writer.WriteStringValue(r.Traits[j].Value); + } + } + + writer.WriteEndArray(); + } + } + + writer.WriteEndObject(); + } + + // CTRF `extra` (free-form object) — the CTRF spec doesn't define a + // dedicated stable identifier so we surface the MTP UID here for + // cross-tool correlation, alongside other framework metadata. + writer.WritePropertyName("extra"); + writer.WriteStartObject(); + writer.WriteString("uid", r.Uid); + if (r.MethodName is not null) + { + writer.WriteString("method", r.MethodName); + } + + if (r.ExceptionType is not null) + { + writer.WriteString("exceptionType", r.ExceptionType); + } + + writer.WriteEndObject(); + + writer.WriteEndObject(); + } + + private static bool HasSameKeyEarlier(IReadOnlyList> traits, int index) + { + string key = traits[index].Key; + for (int k = 0; k < index; k++) + { + if (string.Equals(traits[k].Key, key, StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } + + private static void WriteRetryAttempt(Utf8JsonWriter writer, CapturedTestResult attempt, int attemptNumber) + { + writer.WriteStartObject(); + writer.WriteNumber("attempt", attemptNumber); + writer.WriteString("status", attempt.Status); + writer.WriteNumber("duration", (long)Math.Max(0, attempt.Duration.TotalMilliseconds)); + if (attempt.StartTime is { } start) + { + writer.WriteNumber("start", start.ToUnixTimeMilliseconds()); + } + + if (attempt.EndTime is { } end) + { + writer.WriteNumber("stop", end.ToUnixTimeMilliseconds()); + } + + if (attempt.ErrorMessage is not null) + { + writer.WriteString("message", attempt.ErrorMessage); + } + + if (attempt.StackTrace is not null) + { + writer.WriteString("trace", attempt.StackTrace); + } + + if (attempt.Line is { } line) + { + writer.WriteNumber("line", line); + } + + WriteOutputLines(writer, "stdout", attempt.StandardOutput); + WriteOutputLines(writer, "stderr", attempt.StandardError); + + if (attempt.RawStatus is not null || attempt.ExceptionType is not null) + { + writer.WritePropertyName("extra"); + writer.WriteStartObject(); + if (attempt.RawStatus is not null) + { + writer.WriteString("rawStatus", attempt.RawStatus); + } + + if (attempt.ExceptionType is not null) + { + writer.WriteString("exceptionType", attempt.ExceptionType); + } + + writer.WriteEndObject(); + } + + writer.WriteEndObject(); + } + + // CTRF `stdout`/`stderr` are typed as an array of lines (each item is one line + // of captured output). Splitting on LF (handling optional CR) preserves the + // original line structure for consumers that present output per-line. + private static void WriteOutputLines(Utf8JsonWriter writer, string propertyName, string? output) + { + if (output is null) + { + return; + } + + writer.WritePropertyName(propertyName); + writer.WriteStartArray(); + int start = 0; + for (int i = 0; i < output.Length; i++) + { + if (output[i] == '\n') + { + int end = i; + if (end > start && output[end - 1] == '\r') + { + end--; + } + + writer.WriteStringValue(output.AsSpan(start, end - start)); + start = i + 1; + } + } + + if (start < output.Length) + { + // Emit the trailing segment after the last LF (no trailing entry when + // the input ends with LF — a trailing newline isn't an additional line). + int end = output.Length; + if (end > start && output[end - 1] == '\r') + { + end--; + } + + writer.WriteStringValue(output.AsSpan(start, end - start)); + } + + writer.WriteEndArray(); + } + + private static List CollapseAttempts(CapturedTestResult[] results) + { + // For each UID, group all captures in arrival order: the latest entry becomes the + // final test record, earlier entries become `retryAttempts[]`. Preserves the + // insertion order of first-seen UIDs in the output (stable across runs). + var byUid = new Dictionary(StringComparer.Ordinal); + var collapsed = new List(results.Length); + foreach (CapturedTestResult r in results) + { + if (byUid.TryGetValue(r.Uid, out int existingIndex)) + { + CollapsedTestResult existing = collapsed[existingIndex]; + existing.PriorAttempts.Add(existing.Final); + collapsed[existingIndex] = existing with { Final = r }; + } + else + { + byUid.Add(r.Uid, collapsed.Count); + collapsed.Add(new CollapsedTestResult(r)); + } + } + + return collapsed; + } + + private readonly record struct CollapsedTestResult(CapturedTestResult Final) + { + public List PriorAttempts { get; } = []; + + // CTRF "flaky" is true iff the final status is "passed" AND at least one + // previous attempt failed. + public bool IsFlaky + { + get + { + if (Final.Status != "passed" || PriorAttempts.Count == 0) + { + return false; + } + + foreach (CapturedTestResult attempt in PriorAttempts) + { + if (attempt.Status == "failed") + { + return true; + } + } + + return false; + } + } + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportExtensions.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportExtensions.cs new file mode 100644 index 0000000000..49eecdc5a0 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportExtensions.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.CtrfReport; +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.Services; + +namespace Microsoft.Testing.Extensions; + +/// +/// Provides extension methods for adding CTRF (Common Test Report Format) report generation to a test application. +/// +[Experimental("TPEXP", UrlFormat = "https://aka.ms/testingplatform/diagnostics#{0}")] +public static class CtrfReportExtensions +{ + /// + /// Adds CTRF (Common Test Report Format) report generation to a test application. + /// + /// The test application builder. + public static void AddCtrfReportProvider(this ITestApplicationBuilder builder) + { + var commandLine = new CtrfReportGeneratorCommandLine(); + + var compositeCtrfReportGenerator = + new CompositeExtensionFactory(serviceProvider => + new CtrfReportGenerator( + serviceProvider.GetConfiguration(), + serviceProvider.GetCommandLineOptions(), + serviceProvider.GetRequiredService(), + serviceProvider.GetTestApplicationModuleInfo(), + serviceProvider.GetMessageBus(), + serviceProvider.GetSystemClock(), + serviceProvider.GetEnvironment(), + serviceProvider.GetOutputDevice(), + serviceProvider.GetTestFramework(), + serviceProvider.GetTestApplicationProcessExitCode(), + serviceProvider.GetLoggerFactory().CreateLogger())); + + builder.TestHost.AddDataConsumer(compositeCtrfReportGenerator); + builder.TestHost.AddTestSessionLifetimeHandler(compositeCtrfReportGenerator); + + builder.CommandLine.AddProvider(() => commandLine); + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportGenerator.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportGenerator.cs new file mode 100644 index 0000000000..9a1d937bc2 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportGenerator.cs @@ -0,0 +1,158 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.CtrfReport.Resources; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Configurations; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.OutputDevice; +using Microsoft.Testing.Platform.Extensions.TestFramework; +using Microsoft.Testing.Platform.Extensions.TestHost; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.Messages; +using Microsoft.Testing.Platform.OutputDevice; +using Microsoft.Testing.Platform.Services; + +namespace Microsoft.Testing.Extensions.CtrfReport; + +internal sealed class CtrfReportGenerator : + IDataConsumer, + ITestSessionLifetimeHandler, + IDataProducer, + IOutputDeviceDataProducer +{ + private readonly IConfiguration _configuration; + private readonly ICommandLineOptions _commandLineOptions; + private readonly IFileSystem _fileSystem; + private readonly ITestApplicationModuleInfo _testApplicationModuleInfo; + private readonly IMessageBus _messageBus; + private readonly IClock _clock; + private readonly IEnvironment _environment; + private readonly IOutputDevice _outputDevice; + private readonly ITestFramework _testFramework; + private readonly ITestApplicationProcessExitCode _testApplicationProcessExitCode; + private readonly ILogger _logger; + private readonly List _tests = []; + private readonly bool _isEnabled; + + private DateTimeOffset? _testStartTime; + + public CtrfReportGenerator( + IConfiguration configuration, + ICommandLineOptions commandLineOptions, + IFileSystem fileSystem, + ITestApplicationModuleInfo testApplicationModuleInfo, + IMessageBus messageBus, + IClock clock, + IEnvironment environment, + IOutputDevice outputDevice, + ITestFramework testFramework, + ITestApplicationProcessExitCode testApplicationProcessExitCode, + ILogger logger) + { + _configuration = configuration; + _commandLineOptions = commandLineOptions; + _fileSystem = fileSystem; + _testApplicationModuleInfo = testApplicationModuleInfo; + _messageBus = messageBus; + _clock = clock; + _environment = environment; + _outputDevice = outputDevice; + _testFramework = testFramework; + _testApplicationProcessExitCode = testApplicationProcessExitCode; + _logger = logger; + _isEnabled = commandLineOptions.IsOptionSet(CtrfReportGeneratorCommandLine.CtrfReportOptionName); + } + + public Type[] DataTypesConsumed { get; } = + [ + typeof(TestNodeUpdateMessage), + ]; + + public Type[] DataTypesProduced { get; } = [typeof(SessionFileArtifact)]; + + /// + public string Uid => nameof(CtrfReportGenerator); + + /// + public string Version => ExtensionVersion.DefaultSemVer; + + /// + public string DisplayName { get; } = ExtensionResources.CtrfReportGeneratorDisplayName; + + /// + public string Description { get; } = ExtensionResources.CtrfReportGeneratorDescription; + + /// + public Task IsEnabledAsync() => Task.FromResult(_isEnabled); + + public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (value is TestNodeUpdateMessage update) + { + // Project to a capped DTO immediately so we don't retain the original + // TestNode (and its potentially huge stdout/stderr/stack trace strings) + // for the whole session. We capture every TestNode update as-is here; the + // engine later collapses captures that share the same UID into a single + // CTRF test entry (earlier captures become `retryAttempts[]`, marking the + // test as `flaky` when an earlier attempt failed). See + // CtrfReportEngine.CollapseAttempts for the deduplication logic. + CapturedTestResult? captured = TestResultCapture.TryCapture(update.TestNode); + if (captured is not null) + { + _tests.Add(captured); + } + } + + return Task.CompletedTask; + } + + public Task OnTestSessionStartingAsync(ITestSessionContext testSessionContext) + { + testSessionContext.CancellationToken.ThrowIfCancellationRequested(); + _testStartTime = _clock.UtcNow; + return Task.CompletedTask; + } + + public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionContext) + { + CancellationToken cancellationToken = testSessionContext.CancellationToken; + cancellationToken.ThrowIfCancellationRequested(); + + ApplicationStateGuard.Ensure(_testStartTime is not null); + + await _logger.LogTraceAsync($"Generating CTRF report for {_tests.Count} test result(s).").ConfigureAwait(false); + + int exitCode = _testApplicationProcessExitCode.GetProcessExitCode(); + var engine = new CtrfReportEngine( + _fileSystem, + _testApplicationModuleInfo, + _environment, + _commandLineOptions, + _configuration, + _clock, + _testFramework, + _testStartTime.Value, + exitCode, + cancellationToken); + + (string reportFileName, string? warning) = await engine.GenerateReportAsync([.. _tests]).ConfigureAwait(false); + + if (warning is not null) + { + await _outputDevice.DisplayAsync(this, new WarningMessageOutputDeviceData(warning), cancellationToken).ConfigureAwait(false); + } + + await _messageBus.PublishAsync( + this, + new SessionFileArtifact( + testSessionContext.SessionUid, + new FileInfo(reportFileName), + ExtensionResources.CtrfReportArtifactDisplayName, + ExtensionResources.CtrfReportArtifactDescription)).ConfigureAwait(false); + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportGeneratorCommandLine.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportGeneratorCommandLine.cs new file mode 100644 index 0000000000..6877d575fb --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportGeneratorCommandLine.cs @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.CtrfReport.Resources; +using Microsoft.Testing.Platform; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.CommandLine; + +namespace Microsoft.Testing.Extensions.CtrfReport; + +internal sealed class CtrfReportGeneratorCommandLine : CommandLineOptionsProviderBase +{ + public const string CtrfReportOptionName = "report-ctrf"; + public const string CtrfReportFileNameOptionName = "report-ctrf-filename"; + + private static readonly char[] DirectorySeparators = [Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar]; + + public CtrfReportGeneratorCommandLine() + : base( + nameof(CtrfReportGeneratorCommandLine), + ExtensionVersion.DefaultSemVer, + ExtensionResources.CtrfReportGeneratorDisplayName, + ExtensionResources.CtrfReportGeneratorDescription, + [ + new(CtrfReportOptionName, ExtensionResources.CtrfReportOptionDescription, ArgumentArity.Zero, false), + new(CtrfReportFileNameOptionName, ExtensionResources.CtrfReportFileNameOptionDescription, ArgumentArity.ExactlyOne, false), + ]) + { + } + + public override Task ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments) + { + if (commandOption.Name == CtrfReportFileNameOptionName) + { + if (arguments.Length is 0) + { + return ValidationResult.InvalidTask(ExtensionResources.CtrfReportFileNameMustNotBeEmpty); + } + + string argument = arguments[0]; + + string fileNamePart = Path.GetFileName(argument); + if (RoslynString.IsNullOrWhiteSpace(fileNamePart)) + { + return ValidationResult.InvalidTask(ExtensionResources.CtrfReportFileNameMustNotBeEmpty); + } + + if (!fileNamePart.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) + { + return ValidationResult.InvalidTask(ExtensionResources.CtrfReportFileNameExtensionIsNotJson); + } + + if (EscapesResultsDirectory(argument)) + { + return ValidationResult.InvalidTask(ExtensionResources.CtrfReportFileNameRelativePathMustStayUnderResultsDirectory); + } + } + + return ValidationResult.ValidTask; + } + + public override Task ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions) + => commandLineOptions.IsOptionSet(CtrfReportFileNameOptionName) && !commandLineOptions.IsOptionSet(CtrfReportOptionName) + ? ValidationResult.InvalidTask(ExtensionResources.CtrfReportFileNameRequiresCtrfReport) + : commandLineOptions.IsOptionSet(CtrfReportOptionName) && commandLineOptions.IsOptionSet(PlatformCommandLineProvider.DiscoverTestsOptionKey) + ? ValidationResult.InvalidTask(ExtensionResources.CtrfReportIsNotValidForDiscovery) + : ValidationResult.ValidTask; + + private static bool EscapesResultsDirectory(string path) + { + // Fully-qualified paths (e.g. "C:\foo.json", "\\server\share\foo.json" or "/foo.json") are + // accepted as-is and validated by the OS when we open the file - the user explicitly opted + // out of writing under the test results directory. + if (IsPathFullyQualified(path)) + { + return false; + } + + // Drive-relative paths on Windows such as "C:foo.json" are "rooted" but not fully qualified - + // they resolve against the current directory of the drive, which is unpredictable and would + // silently escape the test results directory. Reject them. On non-Windows OSes + // Path.IsPathRooted only returns true for paths starting with "/", which are already handled + // above, so this check is effectively Windows-only and matches the TRX option behavior. + if (Path.IsPathRooted(path)) + { + return true; + } + + // Any remaining ".." segment in a relative path would escape the test results directory. + return path.Split(DirectorySeparators, StringSplitOptions.RemoveEmptyEntries).Any(segment => segment == ".."); + } + + private static bool IsPathFullyQualified(string path) + { +#if NETCOREAPP + return Path.IsPathFullyQualified(path); +#else + // Mirrors the runtime implementation that is missing on .NET Framework and netstandard2.0. + if (path.Length < 2) + { + return false; + } + + // UNC paths like "\\server\share" (or with forward slashes). + if (IsDirectorySeparator(path[0]) && IsDirectorySeparator(path[1])) + { + return true; + } + + // On Unix, only paths starting with "/" are fully qualified. + if (Path.DirectorySeparatorChar == '/') + { + return path[0] == '/'; + } + + // On Windows, fully qualified drive paths must be "X:\" or "X:/". + return path.Length >= 3 + && IsValidDriveLetter(path[0]) + && path[1] == ':' + && IsDirectorySeparator(path[2]); + + static bool IsDirectorySeparator(char c) + => c == Path.DirectorySeparatorChar || c == Path.AltDirectorySeparatorChar; + + static bool IsValidDriveLetter(char c) + => c is (>= 'A' and <= 'Z') or (>= 'a' and <= 'z'); +#endif + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj new file mode 100644 index 0000000000..b0e7ba2722 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj @@ -0,0 +1,78 @@ + + + + netstandard2.0;$(SupportedNetFrameworks) + $(MicrosoftTestingExtensionsCtrfReportVersionPrefix) + $(MicrosoftTestingExtensionsCtrfReportPreReleaseVersionLabel) + true + true + $(RepoRoot)src\Platform\SharedExtensionHelpers\BuildInfo.cs.template + + + + + + + + + + + + + + true + buildMultiTargeting + + + buildTransitive/$(TargetFramework) + + + build/$(TargetFramework) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PACKAGE.md b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PACKAGE.md new file mode 100644 index 0000000000..a7348d5731 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PACKAGE.md @@ -0,0 +1,32 @@ +# Microsoft.Testing.Extensions.CtrfReport + +Microsoft.Testing.Extensions.CtrfReport is an extension for [Microsoft.Testing.Platform](https://www.nuget.org/packages/Microsoft.Testing.Platform) that generates a test report in the [Common Test Report Format (CTRF)](https://ctrf.io) at the end of a test session. + +Microsoft.Testing.Platform is open source. You can find `Microsoft.Testing.Extensions.CtrfReport` code in the [microsoft/testfx](https://github.com/microsoft/testfx) GitHub repository. + +## Install the package + +```dotnetcli +dotnet add package Microsoft.Testing.Extensions.CtrfReport +``` + +## About + +> **⚠️ Experimental:** This extension is currently experimental. The API, CLI options and on-disk format may change in future releases without notice. The CTRF specification itself is also pre-1.0 and may evolve. + +This package extends Microsoft.Testing.Platform with: + +- **CTRF (Common Test Report Format) report**: a single JSON file conforming to the [CTRF schema](https://github.com/ctrf-io/ctrf/blob/main/schema/ctrf.schema.json) that can be consumed by any tool that understands CTRF (dashboards, CI integrations, AI agents, etc.) without requiring a TRX or JUnit XML parser. +- **Cross-tool interoperability**: same shape as outputs produced by other testing frameworks that adopt CTRF, so results from multiple test runs can be aggregated by a single consumer. + +Enable the report via the `--report-ctrf` command line option. The report file name can be overridden with `--report-ctrf-filename .json`. + +## Documentation + +For comprehensive documentation, see . + +For the CTRF specification, see . + +## Feedback & contributing + +Microsoft.Testing.Platform is an open source project. Provide feedback or report issues in the [microsoft/testfx](https://github.com/microsoft/testfx/issues) GitHub repository. diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PublicAPI/PublicAPI.Shipped.txt b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PublicAPI/PublicAPI.Shipped.txt new file mode 100644 index 0000000000..7dc5c58110 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PublicAPI/PublicAPI.Shipped.txt @@ -0,0 +1 @@ +#nullable enable diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PublicAPI/PublicAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PublicAPI/PublicAPI.Unshipped.txt new file mode 100644 index 0000000000..7221ec44cd --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PublicAPI/PublicAPI.Unshipped.txt @@ -0,0 +1,5 @@ +#nullable enable +[TPEXP]Microsoft.Testing.Extensions.CtrfReport.TestingPlatformBuilderHook +[TPEXP]Microsoft.Testing.Extensions.CtrfReportExtensions +[TPEXP]static Microsoft.Testing.Extensions.CtrfReport.TestingPlatformBuilderHook.AddExtensions(Microsoft.Testing.Platform.Builder.ITestApplicationBuilder! testApplicationBuilder, string![]! _) -> void +[TPEXP]static Microsoft.Testing.Extensions.CtrfReportExtensions.AddCtrfReportProvider(this Microsoft.Testing.Platform.Builder.ITestApplicationBuilder! builder) -> void diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/ExtensionResources.resx b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/ExtensionResources.resx new file mode 100644 index 0000000000..83f65d6b09 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/ExtensionResources.resx @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + CTRF (Common Test Report Format) JSON test report + + + CTRF Report + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + CTRF report generator + + + '--report-ctrf' cannot be enabled when using '--list-tests' + + + Enable generating a CTRF (Common Test Report Format) JSON report + + diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.cs.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.cs.xlf new file mode 100644 index 0000000000..bfe0165a83 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.cs.xlf @@ -0,0 +1,71 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.de.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.de.xlf new file mode 100644 index 0000000000..15bbc8f994 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.de.xlf @@ -0,0 +1,71 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.es.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.es.xlf new file mode 100644 index 0000000000..6caade8705 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.es.xlf @@ -0,0 +1,71 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.fr.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.fr.xlf new file mode 100644 index 0000000000..f58b40f646 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.fr.xlf @@ -0,0 +1,71 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.it.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.it.xlf new file mode 100644 index 0000000000..20c821c636 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.it.xlf @@ -0,0 +1,71 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ja.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ja.xlf new file mode 100644 index 0000000000..a6445f9550 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ja.xlf @@ -0,0 +1,71 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ko.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ko.xlf new file mode 100644 index 0000000000..0fa0072875 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ko.xlf @@ -0,0 +1,71 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pl.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pl.xlf new file mode 100644 index 0000000000..1d376ab6e3 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pl.xlf @@ -0,0 +1,71 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pt-BR.xlf new file mode 100644 index 0000000000..53e9e5f64b --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pt-BR.xlf @@ -0,0 +1,71 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ru.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ru.xlf new file mode 100644 index 0000000000..87f95c747d --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ru.xlf @@ -0,0 +1,71 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.tr.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.tr.xlf new file mode 100644 index 0000000000..c64a501627 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.tr.xlf @@ -0,0 +1,71 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hans.xlf new file mode 100644 index 0000000000..d6145343b2 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hans.xlf @@ -0,0 +1,71 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hant.xlf new file mode 100644 index 0000000000..ea01d029b7 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hant.xlf @@ -0,0 +1,71 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/TestResultCapture.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/TestResultCapture.cs new file mode 100644 index 0000000000..55d357250d --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/TestResultCapture.cs @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Messages; + +namespace Microsoft.Testing.Extensions.CtrfReport; + +// Projects a TestNodeUpdateMessage into a capped CapturedTestResult so that the +// generator does not retain entire test node payloads (and their potentially huge +// stdout/stderr/stack traces) in memory for the whole session. +internal static class TestResultCapture +{ + internal const int MaxStandardStreamLength = 32 * 1024; + internal const int MaxStackTraceLength = 32 * 1024; + internal const int MaxMessageLength = 16 * 1024; + internal const int MaxIdentityFieldLength = 4 * 1024; + internal const int MaxTraitFieldLength = 1024; + + public static CapturedTestResult? TryCapture(TestNode node) + { + TestNodeStateProperty? state = node.Properties.SingleOrDefault(); + if (state is null or DiscoveredTestNodeStateProperty or InProgressTestNodeStateProperty) + { + return null; + } + + (string status, string? rawStatus) = ClassifyStatus(state); + + TimingProperty? timing = node.Properties.SingleOrDefault(); + TimeSpan duration = timing?.GlobalTiming.Duration ?? TimeSpan.Zero; + + (string? ns, string? className, string? methodName) = GetClassAndMethodName(node); + + string? errorMessage = state.Explanation; + string? stackTrace = null; + string? exceptionType = null; + Exception? exception = state switch + { + FailedTestNodeStateProperty f => f.Exception, + ErrorTestNodeStateProperty e => e.Exception, + TimeoutTestNodeStateProperty t => t.Exception, +#pragma warning disable CS0618, MTP0001 // CancelledTestNodeStateProperty is obsolete + CancelledTestNodeStateProperty c => c.Exception, +#pragma warning restore CS0618, MTP0001 + _ => null, + }; + + if (exception is not null) + { + errorMessage ??= exception.Message; + stackTrace = exception.StackTrace; + exceptionType = exception.GetType().FullName; + } + + string? stdout = node.Properties.SingleOrDefault()?.StandardOutput; + string? stderr = node.Properties.SingleOrDefault()?.StandardError; + + TestFileLocationProperty? location = node.Properties.SingleOrDefault(); + string? filePath = location?.FilePath; + int? line = location?.LineSpan.Start.Line; + + // Collect traits without using LINQ to avoid an enumerator allocation per node. + // Trait keys and values are also test-controlled so we truncate them as well to + // bound the size of the in-memory result list and generated report. + List>? traits = null; + foreach (IProperty p in node.Properties) + { + if (p is TestMetadataProperty meta) + { + traits ??= []; + traits.Add(new KeyValuePair( + Truncate(meta.Key, MaxTraitFieldLength)!, + Truncate(meta.Value, MaxTraitFieldLength)!)); + } + } + + return new CapturedTestResult + { + // Identity fields are test-controlled and can be unbounded (e.g. very long + // UIDs/display names from generated data), so we also cap them to keep the + // session-wide result list and generated report within a predictable budget. + Uid = Truncate(node.Uid.Value, MaxIdentityFieldLength)!, + DisplayName = Truncate(node.DisplayName, MaxIdentityFieldLength)!, + Status = status, + RawStatus = rawStatus, + Duration = duration, + StartTime = timing?.GlobalTiming.StartTime, + EndTime = timing?.GlobalTiming.EndTime, + Namespace = Truncate(ns, MaxIdentityFieldLength), + ClassName = Truncate(className, MaxIdentityFieldLength), + MethodName = Truncate(methodName, MaxIdentityFieldLength), + ErrorMessage = Truncate(errorMessage, MaxMessageLength), + ExceptionType = exceptionType, + StackTrace = Truncate(stackTrace, MaxStackTraceLength), + StandardOutput = Truncate(stdout, MaxStandardStreamLength), + StandardError = Truncate(stderr, MaxStandardStreamLength), + FilePath = Truncate(filePath, MaxIdentityFieldLength), + Line = line, + Traits = traits, + }; + } + + // CTRF status enum: passed, failed, skipped, pending, other. + // Returns the CTRF status plus an optional `rawStatus` preserving the original + // MTP outcome when it doesn't map 1:1 (timedOut, errored, cancelled). + private static (string Status, string? RawStatus) ClassifyStatus(TestNodeStateProperty state) + => state switch + { + PassedTestNodeStateProperty => ("passed", null), + SkippedTestNodeStateProperty => ("skipped", null), + TimeoutTestNodeStateProperty => ("failed", "timedOut"), + ErrorTestNodeStateProperty => ("failed", "errored"), + FailedTestNodeStateProperty => ("failed", null), +#pragma warning disable CS0618, MTP0001 // CancelledTestNodeStateProperty is obsolete + CancelledTestNodeStateProperty => ("other", "cancelled"), +#pragma warning restore CS0618, MTP0001 + _ when Array.IndexOf(TestNodePropertiesCategories.WellKnownTestNodeTestRunOutcomeFailedProperties, state.GetType()) >= 0 + => ("failed", null), + _ => throw ApplicationStateGuard.Unreachable(), + }; + + private static (string? Namespace, string? ClassName, string? MethodName) GetClassAndMethodName(TestNode node) + { + TestMethodIdentifierProperty? identifier = node.Properties.SingleOrDefault(); + if (identifier is null) + { + return (null, null, null); + } + + string? ns = RoslynString.IsNullOrEmpty(identifier.Namespace) ? null : identifier.Namespace; + return (ns, identifier.TypeName, identifier.MethodName); + } + + internal static string? Truncate(string? value, int maxLength) + => value is null || value.Length <= maxLength + ? value + : value.Substring(0, maxLength) + + $"\n…[truncated, original length: {value.Length.ToString(CultureInfo.InvariantCulture)}]"; +} diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/TestingPlatformBuilderHook.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/TestingPlatformBuilderHook.cs new file mode 100644 index 0000000000..e7ba703b99 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/TestingPlatformBuilderHook.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Builder; + +namespace Microsoft.Testing.Extensions.CtrfReport; + +/// +/// This class is used by Microsoft.Testing.Platform.MSBuild to hook into the Testing Platform Builder +/// to add CTRF report support. +/// +[Experimental("TPEXP", UrlFormat = "https://aka.ms/testingplatform/diagnostics#{0}")] +public static class TestingPlatformBuilderHook +{ + /// + /// Adds CTRF report support to the Testing Platform Builder. + /// + /// The test application builder. + /// The command line arguments. + public static void AddExtensions(ITestApplicationBuilder testApplicationBuilder, string[] _) + => testApplicationBuilder.AddCtrfReportProvider(); +} diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/build/Microsoft.Testing.Extensions.CtrfReport.props b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/build/Microsoft.Testing.Extensions.CtrfReport.props new file mode 100644 index 0000000000..a3820670bd --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/build/Microsoft.Testing.Extensions.CtrfReport.props @@ -0,0 +1,3 @@ + + + diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/buildMultiTargeting/Microsoft.Testing.Extensions.CtrfReport.props b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/buildMultiTargeting/Microsoft.Testing.Extensions.CtrfReport.props new file mode 100644 index 0000000000..f7de696d4a --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/buildMultiTargeting/Microsoft.Testing.Extensions.CtrfReport.props @@ -0,0 +1,13 @@ + + + + + + Microsoft.Testing.Extensions.CtrfReport + Microsoft.Testing.Extensions.CtrfReport.TestingPlatformBuilderHook + + + diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/buildTransitive/Microsoft.Testing.Extensions.CtrfReport.props b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/buildTransitive/Microsoft.Testing.Extensions.CtrfReport.props new file mode 100644 index 0000000000..a3820670bd --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/buildTransitive/Microsoft.Testing.Extensions.CtrfReport.props @@ -0,0 +1,3 @@ + + + diff --git a/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/Microsoft.Testing.Extensions.OpenTelemetry.csproj b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/Microsoft.Testing.Extensions.OpenTelemetry.csproj index 281970d978..bf8362f4d7 100644 --- a/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/Microsoft.Testing.Extensions.OpenTelemetry.csproj +++ b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/Microsoft.Testing.Extensions.OpenTelemetry.csproj @@ -2,8 +2,8 @@ netstandard2.0;$(SupportedNetFrameworks) - 1.0.0 - alpha + $(MicrosoftTestingExtensionsOpenTelemetryVersionPrefix) + $(MicrosoftTestingExtensionsOpenTelemetryPreReleaseVersionLabel) true true $(RepoRoot)src\Platform\SharedExtensionHelpers\BuildInfo.cs.template diff --git a/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj b/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj index 86e38622e8..9c400d8455 100644 --- a/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj +++ b/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj @@ -42,6 +42,7 @@ This package provides the core platform and the .NET implementation of the proto + diff --git a/test/Directory.Build.targets b/test/Directory.Build.targets index f7729861a4..5f2c7283bf 100644 --- a/test/Directory.Build.targets +++ b/test/Directory.Build.targets @@ -16,6 +16,7 @@ $(TestingPlatformCommandLineArguments) --crashdump $(TestingPlatformCommandLineArguments) --hangdump --hangdump-timeout 15m $(TestingPlatformCommandLineArguments) --report-azdo + $(TestingPlatformCommandLineArguments) --report-ctrf --report-ctrf-filename $(MSBuildProjectName)_$(TargetFramework).ctrf.json @@ -44,6 +45,7 @@ + diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/CtrfReportTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/CtrfReportTests.cs new file mode 100644 index 0000000000..e6259ba5e0 --- /dev/null +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/CtrfReportTests.cs @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Acceptance.IntegrationTests; +using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; +using Microsoft.Testing.Platform.Helpers; + +namespace MSTest.Acceptance.IntegrationTests; + +[TestClass] +public sealed class CtrfReportTests : AcceptanceTestBase +{ + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CtrfReport_WhenTestsArePassingAndFailing_CtrfFileIsGeneratedWithBothOutcomes(string tfm) + { + string fileName = Guid.NewGuid().ToString("N") + ".ctrf.json"; + var testHost = TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + $"--report-ctrf --report-ctrf-filename {fileName}", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.AtLeastOneTestFailed); + + string ctrfFile = Directory.GetFiles(testHost.DirectoryName, fileName, SearchOption.AllDirectories).Single(); + string ctrfContent = File.ReadAllText(ctrfFile); + + // Top-level CTRF document shape + Assert.Contains(@"""reportFormat"": ""CTRF""", ctrfContent, ctrfContent); + Assert.Contains(@"""specVersion""", ctrfContent, ctrfContent); + Assert.Contains(@"""generatedBy"": ""Microsoft.Testing.Extensions.CtrfReport@", ctrfContent, ctrfContent); + + // Summary counts include both outcomes + Assert.Contains(@"""tests"": 2", ctrfContent, ctrfContent); + Assert.Contains(@"""passed"": 1", ctrfContent, ctrfContent); + Assert.Contains(@"""failed"": 1", ctrfContent, ctrfContent); + + // Both per-test entries are present + Assert.Contains(@"""name"": ""PassingTest""", ctrfContent, ctrfContent); + Assert.Contains(@"""name"": ""FailingTest""", ctrfContent, ctrfContent); + Assert.Contains(@"""status"": ""passed""", ctrfContent, ctrfContent); + Assert.Contains(@"""status"": ""failed""", ctrfContent, ctrfContent); + + // The failed test exposes the assertion message in CTRF's `message` field. + Assert.Contains(@"""message"":", ctrfContent, ctrfContent); + Assert.Contains("Assert.AreEqual", ctrfContent, ctrfContent); + } + + public sealed class TestAssetFixture() : TestAssetFixtureBase() + { + public const string ProjectName = "MSTestCtrfReport"; + + public string TargetAssetPath => GetAssetPath(ProjectName); + + public override (string ID, string Name, string Code) GetAssetsToGenerate() => (ProjectName, ProjectName, + SourceCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) + .PatchCodeWithReplace("$MicrosoftTestingExtensionsCtrfReportVersion$", MicrosoftTestingExtensionsCtrfReportVersion) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + private const string SourceCode = """ +#file MSTestCtrfReport.csproj + + + + Exe + true + $TargetFrameworks$ + latest + + + + + + + + + + +#file UnitTest1.cs +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace MSTestCtrfReport; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + public void PassingTest() + { + } + + [TestMethod] + public void FailingTest() + { + Assert.AreEqual(1, 2); + } +} +"""; + } + + public TestContext TestContext { get; set; } = null!; +} diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/Program.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/Program.cs index 972fb170d5..40fda96bff 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/Program.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -24,6 +24,7 @@ builder.AddTrxReportProvider(); builder.AddJUnitReportProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); // Dogfood the OpenTelemetry extension: subscribe to the Microsoft.Testing.Platform activity source // and meter so the OpenTelemetryResultHandler pipeline is exercised end-to-end in CI. No exporter diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs index 82f6afe166..b2db816def 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs @@ -201,10 +201,28 @@ public async Task RunTests_With_CentralPackageManagement_Standalone(string multi "--report-azdo", "--crashdump")); + yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, + "true", + "--report-ctrf", + "--crashdump")); + + yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, + "true", + "--report-html", + "--crashdump")); + yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, "true", "--report-junit", "--crashdump")); + + // OpenTelemetry is API-only (no CLI flag); the enable property only opts the package into the build. + // We pass an empty enable arg to validate the package restores and the test host runs cleanly, and we + // assert that an unrelated extension's CLI arg ('--crashdump') is rejected to prove no other extensions leaked in. + yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, + "true", + string.Empty, + "--crashdump")); } } @@ -251,7 +269,7 @@ public async Task RunTests_With_MSTestRunner_Standalone_EnableAll_Extensions(str foreach (string tfm in multiTfm.Split(";")) { var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, tfm, buildConfiguration: buildConfiguration); - TestHostResult testHostResult = await testHost.ExecuteAsync(command: "--coverage --retry-failed-tests 3 --report-trx --crashdump --hangdump --report-azdo", cancellationToken: TestContext.CancellationToken); + TestHostResult testHostResult = await testHost.ExecuteAsync(command: "--coverage --retry-failed-tests 3 --report-trx --crashdump --hangdump --report-azdo --report-html", cancellationToken: TestContext.CancellationToken); testHostResult.AssertOutputContainsSummary(0, 1, 0); } } diff --git a/test/IntegrationTests/MSTest.IntegrationTests/Program.cs b/test/IntegrationTests/MSTest.IntegrationTests/Program.cs index 8d8c9cce07..024e949e4a 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/Program.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -19,6 +19,7 @@ builder.AddCrashDumpProvider(ignoreIfNotSupported: true); builder.AddRetryProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); // Dogfood the OpenTelemetry extension: subscribe to the Microsoft.Testing.Platform activity source // and meter so the OpenTelemetryResultHandler pipeline is exercised end-to-end in CI. No exporter diff --git a/test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/Program.cs b/test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/Program.cs index eece9b9207..5da8b309d7 100644 --- a/test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/Program.cs +++ b/test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -19,6 +19,7 @@ builder.AddCrashDumpProvider(ignoreIfNotSupported: true); builder.AddRetryProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); // Dogfood the OpenTelemetry extension: subscribe to the Microsoft.Testing.Platform activity source // and meter so the OpenTelemetryResultHandler pipeline is exercised end-to-end in CI. No exporter diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs new file mode 100644 index 0000000000..48891a2b24 --- /dev/null +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs @@ -0,0 +1,363 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; + +[TestClass] +public class CtrfReportTests : AcceptanceTestBase +{ + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task Ctrf_WhenReportCtrfIsNotSpecified_CtrfReportIsNotGenerated(string tfm) + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); + + // The dummy framework emits at least one failing test so the host exits with + // AtLeastOneTestFailed regardless of whether the CTRF report is enabled. + testHostResult.AssertExitCodeIs(ExitCode.AtLeastOneTestFailed); + + // The CTRF report is published as an in-process artifact; check the correct block. + string outputPattern = """ + In process file artifacts produced: + - .+?\.ctrf\.json +"""; + testHostResult.AssertOutputDoesNotMatchRegex(outputPattern); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task Ctrf_WhenReportCtrfIsSpecified_CtrfReportIsGeneratedInDefaultLocation(string tfm) + { + string testResultsPath = Path.Combine(AssetFixture.TargetAssetPath, "bin", "Release", tfm, "TestResults"); + string ctrfPathPattern = Regex.Escape(testResultsPath + Path.DirectorySeparatorChar) + @".+?\.ctrf\.json"; + + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--report-ctrf", cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.AtLeastOneTestFailed); + + string outputPattern = $""" + In process file artifacts produced: + - {ctrfPathPattern} +"""; + testHostResult.AssertOutputMatchesRegex(outputPattern); + + Match match = Regex.Match(testHostResult.StandardOutput, ctrfPathPattern); + Assert.IsTrue(match.Success, $"CTRF report path not found in output:\n{testHostResult.StandardOutput}"); + + AssertCtrfReportShape(match.Value); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task Ctrf_WhenReportCtrfFilenameIsSpecified_CtrfReportIsGeneratedWithThatName(string tfm) + { + const string customFileName = "my-custom-report.ctrf.json"; + string testResultsPath = Path.Combine(AssetFixture.TargetAssetPath, "bin", "Release", tfm, "TestResults"); + string customFilePath = Path.Combine(testResultsPath, customFileName); + string expectedFilePath = Regex.Escape(customFilePath); + + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + $"--report-ctrf --report-ctrf-filename {customFileName}", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.AtLeastOneTestFailed); + + string outputPattern = $""" + In process file artifacts produced: + - {expectedFilePath} +"""; + testHostResult.AssertOutputMatchesRegex(outputPattern); + + Assert.IsTrue( + File.Exists(customFilePath), + $"Expected custom CTRF report file '{customFileName}' was not found in '{testResultsPath}'."); + + AssertCtrfReportShape(customFilePath); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task Ctrf_WhenReportCtrfFilenameContainsPath_CtrfReportIsGeneratedInThatPath(string tfm) + { + string customFileName = Path.Combine("subdir", "report.ctrf.json"); + string testResultsPath = Path.Combine(AssetFixture.TargetAssetPath, "bin", "Release", tfm, "TestResults"); + string customFilePath = Path.Combine(testResultsPath, customFileName); + string expectedFilePath = Regex.Escape(customFilePath); + + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + $"--report-ctrf --report-ctrf-filename {customFileName}", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.AtLeastOneTestFailed); + + string outputPattern = $""" + In process file artifacts produced: + - {expectedFilePath} +"""; + testHostResult.AssertOutputMatchesRegex(outputPattern); + + Assert.IsTrue( + File.Exists(customFilePath), + $"Expected custom CTRF report file '{customFileName}' was not found in '{testResultsPath}'."); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task Ctrf_WhenReportCtrfFilenameIsSpecifiedWithoutReportCtrf_ErrorIsDisplayed(string tfm) + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--report-ctrf-filename report.ctrf.json", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.InvalidCommandLine); + testHostResult.AssertOutputContains("'--report-ctrf-filename' requires '--report-ctrf' to be enabled"); + } + + private static void AssertCtrfReportShape(string filePath) + { + // Snapshot the full CTRF JSON against an exact expected document. Runtime-variable + // fields (GUID report id, ISO timestamp, epoch-ms times, machine name, user name, + // OS info, the extension's own version, the absolute path of the test application, + // and the test host exit code) are masked with deterministic tokens so the + // comparison is hermetic across machines and runs. Anything else — including key + // order, indentation, escaping, conditional-emission shape, and the actual values + // baked from the dummy test framework — must match byte-for-byte. + // + // The dummy framework emits three tests that exercise the CTRF status and retry + // model end to end: + // - PassingTest → status: "passed" + // - FailingTest → status: "failed" + `message` + // - FlakyTest (retried) → first attempt failed, retried successfully: + // final status: "passed", retries: 1, + // retryAttempts[].status: "failed", flaky: true + string actual = File.ReadAllText(filePath); + string normalized = NormalizeCtrfReport(actual); + + const string expected = """ +{ + "reportFormat": "CTRF", + "specVersion": "0.0.0", + "reportId": "", + "timestamp": "", + "generatedBy": "Microsoft.Testing.Extensions.CtrfReport@", + "results": { + "tool": { + "name": "DummyTestFramework", + "version": "2.0.0", + "extra": { + "uid": "DummyTestFramework" + } + }, + "summary": { + "tests": 3, + "passed": 2, + "failed": 1, + "skipped": 0, + "pending": 0, + "other": 0, + "flaky": 1, + "start": , + "stop": , + "duration": + }, + "environment": { + "osPlatform": "", + "osVersion": "", + "extra": { + "user": "", + "machine": "", + "exitCode": , + "testApplication": "" + } + }, + "tests": [ + { + "name": "PassingTest", + "status": "passed", + "duration": , + "extra": { + "uid": "test-1" + } + }, + { + "name": "FailingTest", + "status": "failed", + "duration": , + "message": "Expected 1 but got 2", + "extra": { + "uid": "test-2" + } + }, + { + "name": "FlakyTest", + "status": "passed", + "duration": , + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "duration": , + "message": "Transient failure" + } + ], + "flaky": true, + "extra": { + "uid": "test-3" + } + } + ] + } +} +"""; + + Assert.AreEqual( + NormalizeLineEndings(expected), + NormalizeLineEndings(normalized), + $"Generated CTRF JSON does not match the expected snapshot.\n\nNormalized actual:\n{normalized}\n\nRaw actual:\n{actual}"); + } + + private static string NormalizeCtrfReport(string actual) + { + // Field-scoped regexes so per-test attribute order and JSON shape are still + // anchored, but runtime-variable values are folded into stable tokens. + string normalized = actual; + normalized = Regex.Replace(normalized, @"""reportId"": ""[^""]+""", @"""reportId"": """""); + normalized = Regex.Replace(normalized, @"""timestamp"": ""[^""]+""", @"""timestamp"": """""); + normalized = Regex.Replace(normalized, @"""generatedBy"": ""Microsoft\.Testing\.Extensions\.CtrfReport@[^""]+""", @"""generatedBy"": ""Microsoft.Testing.Extensions.CtrfReport@"""); + normalized = Regex.Replace(normalized, @"""start"": \d+", @"""start"": "); + normalized = Regex.Replace(normalized, @"""stop"": \d+", @"""stop"": "); + normalized = Regex.Replace(normalized, @"""duration"": \d+", @"""duration"": "); + normalized = Regex.Replace(normalized, @"""osPlatform"": ""[^""]*""", @"""osPlatform"": """""); + normalized = Regex.Replace(normalized, @"""osVersion"": ""[^""]*""", @"""osVersion"": """""); + normalized = Regex.Replace(normalized, @"""user"": ""[^""]*""", @"""user"": """""); + normalized = Regex.Replace(normalized, @"""machine"": ""[^""]*""", @"""machine"": """""); + normalized = Regex.Replace(normalized, @"""exitCode"": -?\d+", @"""exitCode"": "); + normalized = Regex.Replace(normalized, @"""testApplication"": ""[^""]*""", @"""testApplication"": """""); + return normalized; + } + + private static string NormalizeLineEndings(string s) => s.Replace("\r\n", "\n").Trim('\n'); + + public sealed class TestAssetFixture() : TestAssetFixtureBase() + { + public const string AssetName = "CtrfReportTest"; + + private const string TestCode = """ +#file CtrfReportTest.csproj + + + $TargetFrameworks$ + enable + enable + Exe + preview + + + + + + +#file Program.cs +using Microsoft.Testing.Extensions; +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestFramework; + +public class Program +{ + public static async Task Main(string[] args) + { + ITestApplicationBuilder builder = await TestApplication.CreateBuilderAsync(args); + builder.RegisterTestFramework( + sp => new TestFrameworkCapabilities(), + (_, __) => new DummyTestFramework()); +#pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + builder.AddCtrfReportProvider(); +#pragma warning restore TPEXP + using ITestApplication app = await builder.BuildAsync(); + return await app.RunAsync(); + } +} + +public class DummyTestFramework : ITestFramework, IDataProducer +{ + public string Uid => nameof(DummyTestFramework); + public string Version => "2.0.0"; + public string DisplayName => nameof(DummyTestFramework); + public string Description => nameof(DummyTestFramework); + public Type[] DataTypesProduced => [typeof(TestNodeUpdateMessage)]; + + public Task IsEnabledAsync() => Task.FromResult(true); + + public Task CreateTestSessionAsync(CreateTestSessionContext context) + => Task.FromResult(new CreateTestSessionResult() { IsSuccess = true }); + + public Task CloseTestSessionAsync(CloseTestSessionContext context) + => Task.FromResult(new CloseTestSessionResult() { IsSuccess = true }); + + public async Task ExecuteRequestAsync(ExecuteRequestContext context) + { + // 1) A plain passing test. + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage( + context.Request.Session.SessionUid, + new TestNode() + { + Uid = "test-1", + DisplayName = "PassingTest", + Properties = new PropertyBag(PassedTestNodeStateProperty.CachedInstance), + })); + + // 2) A plain failing test (no Exception object so no stack trace / exception type + // are emitted; only the explanation propagates as CTRF `message`). + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage( + context.Request.Session.SessionUid, + new TestNode() + { + Uid = "test-2", + DisplayName = "FailingTest", + Properties = new PropertyBag(new FailedTestNodeStateProperty("Expected 1 but got 2")), + })); + + // 3) A flaky test: same Uid published twice — first failing, then passing. + // The CTRF engine collapses these into a single test entry with retries=1, + // retryAttempts[0].status=failed, and flaky=true on the final passing record. + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage( + context.Request.Session.SessionUid, + new TestNode() + { + Uid = "test-3", + DisplayName = "FlakyTest", + Properties = new PropertyBag(new FailedTestNodeStateProperty("Transient failure")), + })); + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage( + context.Request.Session.SessionUid, + new TestNode() + { + Uid = "test-3", + DisplayName = "FlakyTest", + Properties = new PropertyBag(PassedTestNodeStateProperty.CachedInstance), + })); + + context.Complete(); + } +} +"""; + + public string TargetAssetPath => GetAssetPath(AssetName); + + public override (string ID, string Name, string Code) GetAssetsToGenerate() => (AssetName, AssetName, + TestCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) + .PatchCodeWithReplace("$MicrosoftTestingExtensionsCtrfReportVersion$", MicrosoftTestingExtensionsCtrfReportVersion)); + } + + public TestContext TestContext { get; set; } +} diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs index d0b71aabf1..2b29974eed 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs @@ -146,6 +146,12 @@ Path to a text file that lists quarantined test fully qualified names or glob pa Override the Azure DevOps artifact container name. Defaults to 'TestResults_{assemblyName}_{tfm}'. --report-azdo-upload-artifacts Upload test result files and/or add build tags to Azure DevOps. Options are: off (default), tags-only, files, and all. + --report-ctrf + Enable generating a CTRF (Common Test Report Format) JSON report + --report-ctrf-filename + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. + Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). + Example: MyReport_{tfm}.ctrf.json --report-html Enable generating an HTML report --report-html-filename @@ -455,6 +461,21 @@ The file makes it possible to identify the tests that were running at the time o Description: Specify the type of the dump. Valid values are 'Mini', 'Heap', 'Triage' or 'Full'. Default type is 'Full'. For more information visit https://learn.microsoft.com/dotnet/core/diagnostics/collect-dumps-crash#types-of-mini-dumps + CtrfReportGeneratorCommandLine + Name: CTRF report generator + Version: * + Description: Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Options: + --report-ctrf + Arity: 0 + Hidden: False + Description: Enable generating a CTRF (Common Test Report Format) JSON report + --report-ctrf-filename + Arity: 1 + Hidden: False + Description: The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. + Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). + Example: MyReport_{tfm}.ctrf.json HangDumpCommandLineProvider Name: Hang dump Version: * @@ -619,6 +640,7 @@ public sealed class TestAssetFixture() : TestAssetFixtureBase() + @@ -679,6 +701,7 @@ public override (string ID, string Name, string Code) GetAssetsToGenerate() => ( AllExtensionsTestCode .PatchTargetFrameworks(TargetFrameworks.All) .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) + .PatchCodeWithReplace("$MicrosoftTestingExtensionsCtrfReportVersion$", MicrosoftTestingExtensionsCtrfReportVersion) .PatchCodeWithReplace("$MicrosoftTestingExtensionsJUnitReportVersion$", MicrosoftTestingExtensionsJUnitReportVersion)); } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs index 2952733bf8..e737ca827f 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs @@ -16,6 +16,7 @@ static AcceptanceTestBase() MicrosoftTestingPlatformVersion = ExtractVersionFromPackage(Constants.ArtifactsPackagesShipping, "Microsoft.Testing.Platform."); MSTestSourceGenerationVersion = ExtractVersionFromPackage(Constants.ArtifactsPackagesShipping, "MSTest.SourceGeneration."); MicrosoftTestingExtensionsLoggingVersion = ExtractVersionFromPackage(Constants.ArtifactsPackagesShipping, "Microsoft.Testing.Extensions.Logging."); + MicrosoftTestingExtensionsCtrfReportVersion = ExtractVersionFromPackage(Constants.ArtifactsPackagesShipping, "Microsoft.Testing.Extensions.CtrfReport."); MicrosoftTestingExtensionsJUnitReportVersion = ExtractVersionFromPackage(Constants.ArtifactsPackagesShipping, "Microsoft.Testing.Extensions.JUnitReport."); } @@ -38,6 +39,8 @@ static AcceptanceTestBase() public static string MicrosoftTestingExtensionsLoggingVersion { get; private set; } + public static string MicrosoftTestingExtensionsCtrfReportVersion { get; private set; } + public static string MicrosoftTestingExtensionsJUnitReportVersion { get; private set; } private static string ExtractVersionFromPackage(string rootFolder, string packagePrefixName) diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs index 87d1b74917..b415802881 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs @@ -19,6 +19,7 @@ public async Task Microsoft_Testing_Platform_Extensions_ShouldBe_Correctly_Regis SourceCode .PatchCodeWithReplace("$TargetFrameworks$", tfm) .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) + .PatchCodeWithReplace("$MicrosoftTestingExtensionsCtrfReportVersion$", MicrosoftTestingExtensionsCtrfReportVersion) .PatchCodeWithReplace("$MicrosoftTestingExtensionsJUnitReportVersion$", MicrosoftTestingExtensionsJUnitReportVersion)); DotnetMuxerResult result = await DotnetCli.RunAsync($"{(verb == Verb.publish ? $"publish -f {tfm}" : "build")} -c {compilationMode} -r {RID} {testAsset.TargetAssetPath} -v:n", cancellationToken: TestContext.CancellationToken); string binlogFile = result.BinlogPath!; @@ -29,6 +30,7 @@ public async Task Microsoft_Testing_Platform_Extensions_ShouldBe_Correctly_Regis testHostResult.AssertOutputContains("--hangdump"); testHostResult.AssertOutputContains("--publish-azdo-run-name"); testHostResult.AssertOutputContains("--publish-azdo-test-results"); + testHostResult.AssertOutputContains("--report-ctrf"); testHostResult.AssertOutputContains("--report-html"); testHostResult.AssertOutputContains("--report-junit"); testHostResult.AssertOutputContains("--report-trx"); @@ -41,6 +43,7 @@ public async Task Microsoft_Testing_Platform_Extensions_ShouldBe_Correctly_Regis Assert.Contains("Microsoft.Testing.Extensions.AzureDevOpsReport.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); Assert.Contains("Microsoft.Testing.Extensions.CrashDump.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); + Assert.Contains("Microsoft.Testing.Extensions.CtrfReport.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); Assert.Contains("Microsoft.Testing.Extensions.HangDump.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); Assert.Contains("Microsoft.Testing.Extensions.HotReload.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); Assert.Contains("Microsoft.Testing.Extensions.HtmlReport.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); @@ -103,6 +106,7 @@ public async Task TestingPlatformBuilderHook_With_Conflicting_Metadata_Fails_Bui + diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Program.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Program.cs index 38321157e8..6ab5309331 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Program.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -25,6 +25,7 @@ builder.AddJUnitReportProvider(); builder.AddRetryProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); // Dogfood the OpenTelemetry extension: subscribe to the Microsoft.Testing.Platform activity source // and meter so the OpenTelemetryResultHandler pipeline is exercised end-to-end in CI. No exporter diff --git a/test/IntegrationTests/PlatformServices.Desktop.IntegrationTests/Program.cs b/test/IntegrationTests/PlatformServices.Desktop.IntegrationTests/Program.cs index 62891b6492..474a775f52 100644 --- a/test/IntegrationTests/PlatformServices.Desktop.IntegrationTests/Program.cs +++ b/test/IntegrationTests/PlatformServices.Desktop.IntegrationTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -19,6 +19,7 @@ builder.AddCrashDumpProvider(ignoreIfNotSupported: true); builder.AddRetryProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); // Dogfood the OpenTelemetry extension: subscribe to the Microsoft.Testing.Platform activity source // and meter so the OpenTelemetryResultHandler pipeline is exercised end-to-end in CI. No exporter diff --git a/test/UnitTests/MSTest.Analyzers.UnitTests/Program.cs b/test/UnitTests/MSTest.Analyzers.UnitTests/Program.cs index 5b2790e3e5..01c0e36bb7 100644 --- a/test/UnitTests/MSTest.Analyzers.UnitTests/Program.cs +++ b/test/UnitTests/MSTest.Analyzers.UnitTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -22,6 +22,7 @@ builder.AddTrxReportProvider(); builder.AddJUnitReportProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); // Dogfood the OpenTelemetry extension: subscribe to the Microsoft.Testing.Platform activity source // and meter so the OpenTelemetryResultHandler pipeline is exercised end-to-end in CI. No exporter diff --git a/test/UnitTests/MSTest.SelfRealExamples.UnitTests/Program.cs b/test/UnitTests/MSTest.SelfRealExamples.UnitTests/Program.cs index 039c9dfd1c..daddb48f25 100644 --- a/test/UnitTests/MSTest.SelfRealExamples.UnitTests/Program.cs +++ b/test/UnitTests/MSTest.SelfRealExamples.UnitTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -21,6 +21,7 @@ #endif testApplicationBuilder.AddAzureDevOpsProvider(); +testApplicationBuilder.AddCtrfReportProvider(); // Dogfood the OpenTelemetry extension: subscribe to the Microsoft.Testing.Platform activity source // and meter so the OpenTelemetryResultHandler pipeline is exercised end-to-end in CI. No exporter diff --git a/test/UnitTests/MSTest.SourceGeneration.UnitTests/Program.cs b/test/UnitTests/MSTest.SourceGeneration.UnitTests/Program.cs index e606d22b6d..c1b8076a92 100644 --- a/test/UnitTests/MSTest.SourceGeneration.UnitTests/Program.cs +++ b/test/UnitTests/MSTest.SourceGeneration.UnitTests/Program.cs @@ -18,6 +18,7 @@ builder.AddJUnitReportProvider(); builder.AddAppInsightsTelemetryProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); // Dogfood the OpenTelemetry extension: subscribe to the Microsoft.Testing.Platform activity source // and meter so the OpenTelemetryResultHandler pipeline is exercised end-to-end in CI. No exporter diff --git a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Program.cs b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Program.cs index 62891b6492..474a775f52 100644 --- a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Program.cs +++ b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -19,6 +19,7 @@ builder.AddCrashDumpProvider(ignoreIfNotSupported: true); builder.AddRetryProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); // Dogfood the OpenTelemetry extension: subscribe to the Microsoft.Testing.Platform activity source // and meter so the OpenTelemetryResultHandler pipeline is exercised end-to-end in CI. No exporter diff --git a/test/UnitTests/MSTestAdapter.UnitTests/Program.cs b/test/UnitTests/MSTestAdapter.UnitTests/Program.cs index 62891b6492..474a775f52 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/Program.cs +++ b/test/UnitTests/MSTestAdapter.UnitTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -19,6 +19,7 @@ builder.AddCrashDumpProvider(ignoreIfNotSupported: true); builder.AddRetryProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); // Dogfood the OpenTelemetry extension: subscribe to the Microsoft.Testing.Platform activity source // and meter so the OpenTelemetryResultHandler pipeline is exercised end-to-end in CI. No exporter diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs new file mode 100644 index 0000000000..1c0299ad7d --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs @@ -0,0 +1,869 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Text.Json; + +using Microsoft.Testing.Extensions.CtrfReport; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Configurations; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestFramework; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Services; + +using Moq; + +namespace Microsoft.Testing.Extensions.UnitTests; + +[TestClass] +public class CtrfReportEngineTests +{ + private readonly Mock _environmentMock = new(); + private readonly Mock _commandLineOptionsMock = new(); + private readonly Mock _configurationMock = new(); + private readonly Mock _clockMock = new(); + private readonly Mock _testFrameworkMock = new(); + private readonly Mock _testApplicationModuleInfoMock = new(); + private readonly Mock _fileSystem = new(); + + [TestMethod] + public async Task GenerateReportAsync_WritesValidCtrfJson_WithRequiredTopLevelFields() + { + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + Captured("p1", "Passing test", "passed"), + Captured("f1", "Failing test", "failed", errorMessage: "expected 1, got 2"), + Captured("s1", "Skipped test", "skipped", errorMessage: "not relevant"), + ]; + + (string fileName, string? warning) = await engine.GenerateReportAsync(tests); + + Assert.IsNotNull(fileName); + Assert.IsNull(warning); + + // Parse the produced JSON to validate the CTRF document structure (this is the + // schema contract for the consumers at https://github.com/ctrf-io/ctrf). + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement root = document.RootElement; + + Assert.AreEqual("CTRF", root.GetProperty("reportFormat").GetString()); + Assert.AreEqual("0.0.0", root.GetProperty("specVersion").GetString()); + Assert.IsGreaterThan(0, root.GetProperty("reportId").GetString()!.Length); + Assert.IsGreaterThan(0, root.GetProperty("timestamp").GetString()!.Length); + Assert.IsTrue(root.GetProperty("generatedBy").GetString()!.StartsWith("Microsoft.Testing.Extensions.CtrfReport", StringComparison.Ordinal)); + + JsonElement results = root.GetProperty("results"); + + JsonElement tool = results.GetProperty("tool"); + Assert.IsNotNull(tool.GetProperty("name").GetString()); + Assert.IsNotNull(tool.GetProperty("version").GetString()); + + JsonElement summary = results.GetProperty("summary"); + Assert.AreEqual(3, summary.GetProperty("tests").GetInt32()); + Assert.AreEqual(1, summary.GetProperty("passed").GetInt32()); + Assert.AreEqual(1, summary.GetProperty("failed").GetInt32()); + Assert.AreEqual(1, summary.GetProperty("skipped").GetInt32()); + + JsonElement testArray = results.GetProperty("tests"); + Assert.AreEqual(3, testArray.GetArrayLength()); + } + + [TestMethod] + [DataRow("passed", typeof(PassedTestNodeStateProperty))] + [DataRow("skipped", typeof(SkippedTestNodeStateProperty))] + [DataRow("failed", typeof(FailedTestNodeStateProperty))] + public void TestResultCapture_ClassifiesTerminalOutcomes_ToCtrfStatus(string expectedStatus, Type stateType) + { + TestNodeStateProperty state = stateType switch + { + Type t when t == typeof(PassedTestNodeStateProperty) => PassedTestNodeStateProperty.CachedInstance, + Type t when t == typeof(SkippedTestNodeStateProperty) => SkippedTestNodeStateProperty.CachedInstance, + Type t when t == typeof(FailedTestNodeStateProperty) => new FailedTestNodeStateProperty("x"), + _ => throw new InvalidOperationException(), + }; + + TestNode node = new() { Uid = "id", DisplayName = "T", Properties = new(state) }; + + CapturedTestResult result = TestResultCapture.TryCapture(node)!; + + Assert.AreEqual(expectedStatus, result.Status); + Assert.IsNull(result.RawStatus, "Pure CTRF outcomes should not carry rawStatus."); + } + + [TestMethod] + public void TestResultCapture_ErrorState_MapsToFailed_With_RawStatus_Errored() + { + TestNode node = new() { Uid = "id", DisplayName = "T", Properties = new(new ErrorTestNodeStateProperty("boom")) }; + + CapturedTestResult result = TestResultCapture.TryCapture(node)!; + + Assert.AreEqual("failed", result.Status); + Assert.AreEqual("errored", result.RawStatus); + } + + [TestMethod] + public void TestResultCapture_TimeoutState_MapsToFailed_With_RawStatus_TimedOut() + { + TestNode node = new() { Uid = "id", DisplayName = "T", Properties = new(new TimeoutTestNodeStateProperty("slow")) }; + + CapturedTestResult result = TestResultCapture.TryCapture(node)!; + + Assert.AreEqual("failed", result.Status); + Assert.AreEqual("timedOut", result.RawStatus); + } + + [TestMethod] + public void TestResultCapture_Truncates_OverLength_StandardOutput_AtBoundary() + { + string huge = new('a', TestResultCapture.MaxStandardStreamLength + 7); + + var bag = new PropertyBag(PassedTestNodeStateProperty.CachedInstance); + bag.Add(new StandardOutputProperty(huge)); + TestNode node = new() { Uid = "id", DisplayName = "T", Properties = bag }; + + CapturedTestResult result = TestResultCapture.TryCapture(node)!; + + Assert.IsNotNull(result); + Assert.IsNotNull(result.StandardOutput); + Assert.StartsWith(new string('a', TestResultCapture.MaxStandardStreamLength), result.StandardOutput!); + Assert.Contains("[truncated, original length:", result.StandardOutput); + Assert.Contains((TestResultCapture.MaxStandardStreamLength + 7).ToString(CultureInfo.InvariantCulture), result.StandardOutput); + } + + [TestMethod] + public void TestResultCapture_Returns_Null_For_NonTerminalStates() + { + TestNode discovered = new() { Uid = "a", DisplayName = "x", Properties = new(DiscoveredTestNodeStateProperty.CachedInstance) }; + TestNode inProgress = new() { Uid = "b", DisplayName = "y", Properties = new(InProgressTestNodeStateProperty.CachedInstance) }; + + Assert.IsNull(TestResultCapture.TryCapture(discovered)); + Assert.IsNull(TestResultCapture.TryCapture(inProgress)); + } + + [TestMethod] + public async Task GenerateReportAsync_CountsAllOutcomeKindsSeparately() + { + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + Captured("p1", "Passed", "passed"), + Captured("f1", "Failed", "failed"), + Captured("s1", "Skipped", "skipped"), + CapturedRaw("e1", "Errored", "failed", "errored"), + CapturedRaw("t1", "Timed out", "failed", "timedOut"), + ]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement summary = document.RootElement.GetProperty("results").GetProperty("summary"); + Assert.AreEqual(5, summary.GetProperty("tests").GetInt32()); + Assert.AreEqual(1, summary.GetProperty("passed").GetInt32()); + Assert.AreEqual(3, summary.GetProperty("failed").GetInt32(), "errored + timedOut + failed all map to CTRF 'failed'."); + Assert.AreEqual(1, summary.GetProperty("skipped").GetInt32()); + } + + [TestMethod] + public async Task GenerateReportAsync_CollapsesDuplicateUidsIntoRetryAttempts_AndFlagsFlaky() + { + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + Captured("dup", "Row A", "failed", errorMessage: "first failure"), + Captured("dup", "Row B", "failed", errorMessage: "second failure"), + Captured("dup", "Row C", "passed"), + Captured("unique", "Solo", "passed"), + ]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement results = document.RootElement.GetProperty("results"); + JsonElement testArray = results.GetProperty("tests"); + + // Duplicate-UID captures must collapse into a single CTRF test entry; the + // earlier attempts are recorded as nested retryAttempts[]. Top-level + // entries should therefore equal the number of unique UIDs. + Assert.AreEqual(2, testArray.GetArrayLength(), "Duplicate UIDs must collapse to one tests[] row."); + + JsonElement summary = results.GetProperty("summary"); + Assert.AreEqual(2, summary.GetProperty("tests").GetInt32(), "summary.tests must count unique UIDs only."); + Assert.AreEqual(2, summary.GetProperty("passed").GetInt32()); + Assert.AreEqual(0, summary.GetProperty("failed").GetInt32()); + Assert.AreEqual(1, summary.GetProperty("flaky").GetInt32()); + + JsonElement dupRow = default; + JsonElement soloRow = default; + foreach (JsonElement t in testArray.EnumerateArray()) + { + if (t.GetProperty("name").GetString() == "Row C") + { + dupRow = t; + } + else if (t.GetProperty("name").GetString() == "Solo") + { + soloRow = t; + } + } + + Assert.AreNotEqual(JsonValueKind.Undefined, dupRow.ValueKind, "Final attempt name must surface as the collapsed test name."); + Assert.AreEqual("passed", dupRow.GetProperty("status").GetString()); + Assert.AreEqual(2, dupRow.GetProperty("retries").GetInt32(), "retries must equal the number of prior attempts."); + Assert.IsTrue(dupRow.GetProperty("flaky").GetBoolean(), "passed-after-failed runs must be marked flaky."); + + JsonElement retryAttempts = dupRow.GetProperty("retryAttempts"); + Assert.AreEqual(2, retryAttempts.GetArrayLength(), "retryAttempts must record every prior attempt."); + + var attemptNumbers = new List(); + var attemptStatuses = new List(); + foreach (JsonElement a in retryAttempts.EnumerateArray()) + { + attemptNumbers.Add(a.GetProperty("attempt").GetInt32()); + attemptStatuses.Add(a.GetProperty("status").GetString()!); + } + + Assert.AreSequenceEqual(new[] { 1, 2 }, attemptNumbers); + Assert.AreSequenceEqual(new[] { "failed", "failed" }, attemptStatuses); + + // Single-attempt entries must not surface retry metadata. + Assert.IsFalse(soloRow.TryGetProperty("retries", out _)); + Assert.IsFalse(soloRow.TryGetProperty("retryAttempts", out _)); + Assert.IsFalse(soloRow.TryGetProperty("flaky", out _)); + } + + [TestMethod] + public async Task GenerateReportAsync_PerTest_ContainsRequiredFields() + { + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + new CapturedTestResult + { + Uid = "id-1", + DisplayName = "MyTest", + Status = "passed", + Duration = TimeSpan.FromMilliseconds(42), + Namespace = "MyNs", + ClassName = "MyClass", + MethodName = "MyMethod", + }, + ]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement test = document.RootElement.GetProperty("results").GetProperty("tests")[0]; + + // CTRF spec required fields per test: name, status, duration. + Assert.AreEqual("MyTest", test.GetProperty("name").GetString()); + Assert.AreEqual("passed", test.GetProperty("status").GetString()); + Assert.AreEqual(42, test.GetProperty("duration").GetInt64()); + + // CTRF `suite` (array of strings) when class/namespace are known. + JsonElement suite = test.GetProperty("suite"); + Assert.AreEqual(2, suite.GetArrayLength()); + Assert.AreEqual("MyNs", suite[0].GetString()); + Assert.AreEqual("MyClass", suite[1].GetString()); + + // UID must be surfaced under `extra` for cross-tool correlation. + Assert.AreEqual("id-1", test.GetProperty("extra").GetProperty("uid").GetString()); + } + + [TestMethod] + public async Task GenerateReportAsync_OmitsSuite_WhenNoClassOrNamespace() + { + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = [Captured("u", "T", "passed")]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement test = document.RootElement.GetProperty("results").GetProperty("tests")[0]; + Assert.IsFalse(test.TryGetProperty("suite", out _), "suite must be omitted when className/namespace are unknown (CTRF requires minItems:1)."); + } + + [TestMethod] + public async Task GenerateReportAsync_RoundTripsErrorMessageAndStackTrace() + { + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + new CapturedTestResult + { + Uid = "u", + DisplayName = "T", + Status = "failed", + Duration = TimeSpan.Zero, + ErrorMessage = "expected 1 got 2", + StackTrace = "at MyAssembly.MyType.MyMethod()", + }, + ]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement test = document.RootElement.GetProperty("results").GetProperty("tests")[0]; + Assert.AreEqual("expected 1 got 2", test.GetProperty("message").GetString()); + Assert.AreEqual("at MyAssembly.MyType.MyMethod()", test.GetProperty("trace").GetString()); + } + + [TestMethod] + public async Task GenerateReportAsync_PromotesTraitsToLabelsAndTestCategoryToTags() + { + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + new CapturedTestResult + { + Uid = "id", + DisplayName = "T", + Status = "passed", + Duration = TimeSpan.Zero, + Traits = + [ + // Multiple [TestCategory] attributes on the same MSTest method produce + // repeated trait entries with the same key. Per ctrf-io/ctrf#53 the + // CTRF maintainer confirmed array values for top-level `labels` + // (spec 9.15), so multi-valued keys serialize as JSON arrays and + // single-valued keys serialize as scalar strings. + new KeyValuePair("TestCategory", "Fast"), + new KeyValuePair("TestCategory", "Smoke"), + new KeyValuePair("Owner", "alice"), + ], + }, + ]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement test = document.RootElement.GetProperty("results").GetProperty("tests")[0]; + + // TestCategory values are promoted to the CTRF top-level `tags` array (spec 9.14) + // so consumers can filter/group by category without walking the labels object. + // The values are preserved in declaration order. + JsonElement tags = test.GetProperty("tags"); + Assert.AreEqual(JsonValueKind.Array, tags.ValueKind); + Assert.AreEqual(2, tags.GetArrayLength()); + Assert.AreEqual("Fast", tags[0].GetString()); + Assert.AreEqual("Smoke", tags[1].GetString()); + + // All traits — including TestCategory — round-trip under the CTRF top-level + // `labels` object (spec 9.15). Single-valued keys are emitted as scalar + // strings; multi-valued keys are emitted as arrays of strings. + JsonElement labels = test.GetProperty("labels"); + Assert.AreEqual(JsonValueKind.Object, labels.ValueKind); + + JsonElement testCategory = labels.GetProperty("TestCategory"); + Assert.AreEqual(JsonValueKind.Array, testCategory.ValueKind); + Assert.AreEqual(2, testCategory.GetArrayLength()); + Assert.AreEqual("Fast", testCategory[0].GetString()); + Assert.AreEqual("Smoke", testCategory[1].GetString()); + + JsonElement owner = labels.GetProperty("Owner"); + Assert.AreEqual(JsonValueKind.String, owner.ValueKind); + Assert.AreEqual("alice", owner.GetString()); + + // `extra.traits` was the previous home for these values; it is no longer + // emitted now that the spec-defined `labels` field is populated. + Assert.IsFalse( + test.GetProperty("extra").TryGetProperty("traits", out _), + "Traits are now emitted under top-level labels, not extra.traits."); + } + + [TestMethod] + public async Task GenerateReportAsync_DefaultFileName_IncludesModuleNameAndTargetFramework() + { + string? pathSeen = null; + _ = _fileSystem.Setup(x => x.ExistFile(It.IsAny())).Returns(false); + _ = _fileSystem.Setup(x => x.NewFileStream(It.IsAny(), FileMode.CreateNew)) + .Returns((path, _) => + { + pathSeen = path; + return new MemoryFileStream(); + }); + + _ = _configurationMock.SetupGet(_ => _[It.IsAny()]).Returns("out"); + _ = _environmentMock.SetupGet(_ => _.MachineName).Returns("M"); + _ = _environmentMock.Setup(_ => _.GetEnvironmentVariable(It.IsAny())).Returns("u"); + _ = _testApplicationModuleInfoMock.Setup(_ => _.GetCurrentTestApplicationFullPath()).Returns(Path.Combine("tmp", "My.Test.Module.dll")); + _ = _testFrameworkMock.SetupGet(_ => _.Uid).Returns("uid"); + _ = _testFrameworkMock.SetupGet(_ => _.Version).Returns("0.0"); + _ = _testFrameworkMock.SetupGet(_ => _.DisplayName).Returns("F"); + _ = _clockMock.SetupGet(_ => _.UtcNow).Returns(new DateTimeOffset(2026, 2, 3, 4, 5, 6, TimeSpan.Zero)); + + var engine = new CtrfReportEngine( + _fileSystem.Object, + _testApplicationModuleInfoMock.Object, + _environmentMock.Object, + _commandLineOptionsMock.Object, + _configurationMock.Object, + _clockMock.Object, + _testFrameworkMock.Object, + DateTimeOffset.UtcNow, + 0, + CancellationToken.None); + + (string finalPath, _) = await engine.GenerateReportAsync([Captured("a", "A", "passed")]); + + const string ExpectedFileNamePattern = "^u_M_My\\.Test\\.Module_net[0-9]+(\\.[0-9]+)?_2026-02-03_04_05_06\\.ctrf\\.json$"; + Assert.AreEqual(pathSeen, finalPath); + Assert.IsTrue(Regex.IsMatch(Path.GetFileName(finalPath), ExpectedFileNamePattern)); + } + + [TestMethod] + public async Task GenerateReportAsync_ExplicitRelativePath_IsResolvedUnderResultsDirectory() + { + string[]? jsonFileName = [Path.Combine("nested", "custom.json")]; + _ = _commandLineOptionsMock.Setup(_ => _.TryGetOptionArgumentList(CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName, out jsonFileName)).Returns(true); + + string? pathSeen = null; + var directories = new List(); + _ = _fileSystem.Setup(x => x.ExistFile(It.IsAny())).Returns(false); + _ = _fileSystem.Setup(x => x.CreateDirectory(It.IsAny())) + .Callback(directories.Add) + .Returns(path => path); + _ = _fileSystem.Setup(x => x.NewFileStream(It.IsAny(), FileMode.Create)) + .Returns((path, _) => + { + pathSeen = path; + return new MemoryFileStream(); + }); + + CtrfReportEngine engine = CreateEngine(); + _ = _configurationMock.SetupGet(_ => _[It.IsAny()]).Returns("out"); + + (string finalPath, _) = await engine.GenerateReportAsync([Captured("a", "A", "passed")]); + + string expectedPath = Path.Combine("out", "nested", "custom.json"); + Assert.AreEqual(expectedPath, finalPath); + Assert.AreEqual(expectedPath, pathSeen); + Assert.Contains(Path.Combine("out", "nested"), directories); + } + + [TestMethod] + public async Task GenerateReportAsync_ExplicitAbsolutePath_OverridesResultsDirectory() + { + string absolutePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".json"); + string[]? jsonFileName = [absolutePath]; + _ = _commandLineOptionsMock.Setup(_ => _.TryGetOptionArgumentList(CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName, out jsonFileName)).Returns(true); + + string? pathSeen = null; + _ = _fileSystem.Setup(x => x.ExistFile(It.IsAny())).Returns(false); + _ = _fileSystem.Setup(x => x.CreateDirectory(It.IsAny())).Returns(path => path); + _ = _fileSystem.Setup(x => x.NewFileStream(It.IsAny(), FileMode.Create)) + .Returns((path, _) => + { + pathSeen = path; + return new MemoryFileStream(); + }); + + CtrfReportEngine engine = CreateEngine(); + _ = _configurationMock.SetupGet(_ => _[It.IsAny()]).Returns("out"); + + (string finalPath, _) = await engine.GenerateReportAsync([Captured("a", "A", "passed")]); + + Assert.AreEqual(absolutePath, finalPath); + Assert.AreEqual(absolutePath, pathSeen); + } + + [TestMethod] + public async Task GenerateReportAsync_AppendsDisambiguatingSuffix_When_DefaultFileExists() + { + var bytesSeen = new List(); + int callCount = 0; + _ = _fileSystem.Setup(x => x.NewFileStream(It.IsAny(), FileMode.CreateNew)) + .Returns((path, _) => + { + callCount++; + bytesSeen.Add(path); + return callCount == 1 + ? throw new IOException("file exists") + : new MemoryFileStream(); + }); + + _ = _fileSystem.Setup(x => x.ExistFile(It.IsAny())).Returns(true); + + _ = _configurationMock.SetupGet(_ => _[It.IsAny()]).Returns(string.Empty); + _ = _environmentMock.SetupGet(_ => _.MachineName).Returns("M"); + _ = _environmentMock.Setup(_ => _.GetEnvironmentVariable(It.IsAny())).Returns("u"); + _ = _testApplicationModuleInfoMock.Setup(_ => _.GetCurrentTestApplicationFullPath()).Returns("app"); + _ = _testFrameworkMock.SetupGet(_ => _.Uid).Returns("uid"); + _ = _testFrameworkMock.SetupGet(_ => _.Version).Returns("0.0"); + _ = _testFrameworkMock.SetupGet(_ => _.DisplayName).Returns("F"); + _ = _clockMock.SetupGet(_ => _.UtcNow).Returns(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + + var engine = new CtrfReportEngine( + _fileSystem.Object, + _testApplicationModuleInfoMock.Object, + _environmentMock.Object, + _commandLineOptionsMock.Object, + _configurationMock.Object, + _clockMock.Object, + _testFrameworkMock.Object, + DateTimeOffset.UtcNow, + 0, + CancellationToken.None); + + (string finalPath, _) = await engine.GenerateReportAsync([Captured("a", "A", "passed")]); + + Assert.AreEqual(2, callCount); + Assert.AreEqual(bytesSeen[1], finalPath); + Assert.Contains("_1.ctrf.json", finalPath); + } + + [TestMethod] + public async Task GenerateReportAsync_PropagatesIOException_When_FileDoesNotExist() + { + int callCount = 0; + _ = _fileSystem.Setup(x => x.NewFileStream(It.IsAny(), FileMode.CreateNew)) + .Returns((path, _) => + { + callCount++; + throw new IOException("disk full"); + }); + + _ = _fileSystem.Setup(x => x.ExistFile(It.IsAny())).Returns(false); + + _ = _configurationMock.SetupGet(_ => _[It.IsAny()]).Returns(string.Empty); + _ = _environmentMock.SetupGet(_ => _.MachineName).Returns("M"); + _ = _environmentMock.Setup(_ => _.GetEnvironmentVariable(It.IsAny())).Returns("u"); + _ = _testApplicationModuleInfoMock.Setup(_ => _.GetCurrentTestApplicationFullPath()).Returns("app"); + _ = _testFrameworkMock.SetupGet(_ => _.Uid).Returns("uid"); + _ = _testFrameworkMock.SetupGet(_ => _.Version).Returns("0.0"); + _ = _testFrameworkMock.SetupGet(_ => _.DisplayName).Returns("F"); + _ = _clockMock.SetupGet(_ => _.UtcNow).Returns(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + + var engine = new CtrfReportEngine( + _fileSystem.Object, + _testApplicationModuleInfoMock.Object, + _environmentMock.Object, + _commandLineOptionsMock.Object, + _configurationMock.Object, + _clockMock.Object, + _testFrameworkMock.Object, + DateTimeOffset.UtcNow, + 0, + CancellationToken.None); + + await Assert.ThrowsExactlyAsync(() => engine.GenerateReportAsync([Captured("a", "A", "passed")])); + Assert.AreEqual(1, callCount); + } + + private static CapturedTestResult Captured(string uid, string name, string status, + TimeSpan? duration = null, string? errorMessage = null) + => new() + { + Uid = uid, + DisplayName = name, + Status = status, + Duration = duration ?? TimeSpan.Zero, + ErrorMessage = errorMessage, + }; + + private static CapturedTestResult CapturedRaw(string uid, string name, string status, string rawStatus) + => new() + { + Uid = uid, + DisplayName = name, + Status = status, + RawStatus = rawStatus, + Duration = TimeSpan.Zero, + }; + + [TestMethod] + public async Task GenerateReportAsync_Environment_HasSchemaCompliantShape() + { + // CTRF schema (additionalProperties: false on environment): + // * `extra` MUST be an object — emitting a string here breaks strict validators. + // * `osPlatform` is the short identifier (win32/linux/darwin/...); the full + // descriptive string belongs in `osVersion`. + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = [Captured("u", "T", "passed")]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement env = document.RootElement.GetProperty("results").GetProperty("environment"); + + // `extra` must be an object (the most common spec violation). + JsonElement extra = env.GetProperty("extra"); + Assert.AreEqual(JsonValueKind.Object, extra.ValueKind, "environment.extra MUST be a JSON object per CTRF schema."); + Assert.AreEqual("user", extra.GetProperty("user").GetString()); + Assert.AreEqual("MachineName", extra.GetProperty("machine").GetString()); + Assert.AreEqual(0, extra.GetProperty("exitCode").GetInt32()); + Assert.AreEqual("TestAppPath", extra.GetProperty("testApplication").GetString()); + + // `osPlatform` is one of the short identifiers, not the descriptive name. + string osPlatform = env.GetProperty("osPlatform").GetString()!; + Assert.Contains(osPlatform, new[] { "win32", "linux", "darwin", "freebsd", "unknown" }); + + // Descriptive OS string goes into `osVersion`. + Assert.IsGreaterThan(0, env.GetProperty("osVersion").GetString()!.Length); + } + + [TestMethod] + public async Task GenerateReportAsync_TestExtra_CarriesMethodNameAndExceptionType() + { + // method, exceptionType, and uid all live under `extra` so the per-test object + // surfaces framework-defined metadata next to the CTRF-defined `labels` + // (spec 9.15) and `tags` (spec 9.14) fields. A test with no traits should + // emit none of those optional fields. + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + new CapturedTestResult + { + Uid = "u-1", + DisplayName = "T", + Status = "failed", + Duration = TimeSpan.Zero, + MethodName = "MyMethod", + ExceptionType = "System.InvalidOperationException", + }, + ]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement test = document.RootElement.GetProperty("results").GetProperty("tests")[0]; + + JsonElement extra = test.GetProperty("extra"); + Assert.AreEqual("u-1", extra.GetProperty("uid").GetString()); + Assert.AreEqual("MyMethod", extra.GetProperty("method").GetString()); + Assert.AreEqual("System.InvalidOperationException", extra.GetProperty("exceptionType").GetString()); + + // No `labels` is emitted when there are no traits at all (spec 9.15). + Assert.IsFalse(test.TryGetProperty("labels", out _), "labels is only emitted when traits exist."); + + // No `tags` is emitted when there is no TestCategory trait, and no `traits` is + // emitted under `extra` (we no longer write `extra.traits` — see labels above). + Assert.IsFalse(test.TryGetProperty("tags", out _), "tags is only emitted when TestCategory traits exist."); + Assert.IsFalse(extra.TryGetProperty("traits", out _), "extra.traits is no longer emitted; traits go to top-level labels."); + } + + [TestMethod] + public async Task GenerateReportAsync_ToolName_FallsBackToUnknown_WhenFrameworkDisplayNameEmpty() + { + // CTRF spec: results.tool.name MUST be a non-empty string. + using var memoryStream = new MemoryFileStream(); + _ = _testFrameworkMock.SetupGet(_ => _.DisplayName).Returns(string.Empty); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = [Captured("u", "T", "passed")]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement tool = document.RootElement.GetProperty("results").GetProperty("tool"); + string name = tool.GetProperty("name").GetString()!; + Assert.IsGreaterThan(0, name.Length, "CTRF requires tool.name to be a non-empty string."); + } + + [TestMethod] + public async Task GenerateReportAsync_EmptyResults_ProducesValidDocument() + { + // The summary block must still be present with zeroed counts when no tests + // ran (the schema requires summary fields to exist, and `tests[]` should be + // an empty array rather than absent). + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + + await engine.GenerateReportAsync([]); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement results = document.RootElement.GetProperty("results"); + JsonElement summary = results.GetProperty("summary"); + Assert.AreEqual(0, summary.GetProperty("tests").GetInt32()); + Assert.AreEqual(0, summary.GetProperty("passed").GetInt32()); + Assert.AreEqual(0, summary.GetProperty("failed").GetInt32()); + Assert.AreEqual(0, summary.GetProperty("skipped").GetInt32()); + Assert.AreEqual(0, summary.GetProperty("pending").GetInt32()); + Assert.AreEqual(0, summary.GetProperty("other").GetInt32()); + Assert.AreEqual(0, summary.GetProperty("flaky").GetInt32()); + + JsonElement testsArray = results.GetProperty("tests"); + Assert.AreEqual(JsonValueKind.Array, testsArray.ValueKind); + Assert.AreEqual(0, testsArray.GetArrayLength()); + } + + [TestMethod] + public async Task GenerateReportAsync_TestName_IsNeverEmpty() + { + // CTRF schema: tests[i].name has minLength: 1. We must surface a non-empty + // value even if the test framework forwarded an empty DisplayName. + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + new CapturedTestResult + { + Uid = "uid-with-no-display-name", + DisplayName = string.Empty, + Status = "passed", + Duration = TimeSpan.Zero, + }, + ]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement test = document.RootElement.GetProperty("results").GetProperty("tests")[0]; + string name = test.GetProperty("name").GetString()!; + Assert.IsGreaterThan(0, name.Length, "CTRF requires tests[].name to be non-empty (minLength: 1)."); + Assert.AreEqual("uid-with-no-display-name", name, "Empty DisplayName must fall back to UID."); + } + + [TestMethod] + public async Task GenerateReportAsync_SpecialCharactersInName_AreSafelyEscaped() + { + // Test that names with HTML/JS metacharacters are escaped (no + // UnsafeRelaxedJsonEscaping). The bytes must contain a unicode-escaped + // form rather than the raw `", "passed"), + ]; + + await engine.GenerateReportAsync(tests); + + byte[] raw = System.Text.Encoding.UTF8.GetBytes(memoryStream.GetUtf8Content()); + string rawString = System.Text.Encoding.UTF8.GetString(raw); + + // The raw `<` must not appear in the encoded JSON — it must be \u003C. + Assert.IsFalse(rawString.Contains("", test.GetProperty("name").GetString()); + } + + [TestMethod] + public async Task GenerateReportAsync_SplitsStandardOutputAndError_OnNewlines() + { + // CTRF schema types stdout/stderr as an array of "lines of output"; + // we must split on LF (handling CRLF) and not include a trailing empty + // entry for inputs that end with a newline. + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + new CapturedTestResult + { + Uid = "id-multiline", + DisplayName = "MultiLine", + Status = "passed", + Duration = TimeSpan.Zero, + StandardOutput = "line1\nline2\r\nline3\n", + StandardError = "errA\nerrB", + }, + ]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement test = document.RootElement.GetProperty("results").GetProperty("tests")[0]; + + JsonElement stdout = test.GetProperty("stdout"); + Assert.AreEqual(JsonValueKind.Array, stdout.ValueKind); + Assert.AreEqual(3, stdout.GetArrayLength(), "Trailing newline must not produce an extra empty entry."); + Assert.AreEqual("line1", stdout[0].GetString()); + Assert.AreEqual("line2", stdout[1].GetString(), "CR before LF must be stripped (CRLF normalization)."); + Assert.AreEqual("line3", stdout[2].GetString()); + + JsonElement stderr = test.GetProperty("stderr"); + Assert.AreEqual(2, stderr.GetArrayLength()); + Assert.AreEqual("errA", stderr[0].GetString()); + Assert.AreEqual("errB", stderr[1].GetString(), "Final segment without trailing newline must still be emitted."); + } + + [TestMethod] + public async Task GenerateReportAsync_SingleLineOutput_EmitsOneArrayEntry() + { + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + new CapturedTestResult + { + Uid = "id-single", + DisplayName = "SingleLine", + Status = "passed", + Duration = TimeSpan.Zero, + StandardOutput = "only-line", + }, + ]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement test = document.RootElement.GetProperty("results").GetProperty("tests")[0]; + + JsonElement stdout = test.GetProperty("stdout"); + Assert.AreEqual(1, stdout.GetArrayLength()); + Assert.AreEqual("only-line", stdout[0].GetString()); + } + + private CtrfReportEngine CreateEngine(MemoryFileStream stream) + { + _ = _fileSystem.Setup(x => x.ExistFile(It.IsAny())).Returns(false); + _ = _fileSystem.Setup(x => x.NewFileStream(It.IsAny(), It.IsAny())).Returns(stream); + + return CreateEngine(); + } + + private CtrfReportEngine CreateEngine() + { + _ = _configurationMock.SetupGet(_ => _[It.IsAny()]).Returns(string.Empty); + _ = _environmentMock.SetupGet(_ => _.MachineName).Returns("MachineName"); + _ = _environmentMock.Setup(_ => _.GetEnvironmentVariable(It.IsAny())).Returns("user"); + _ = _testApplicationModuleInfoMock.Setup(_ => _.GetCurrentTestApplicationFullPath()).Returns("TestAppPath"); + _ = _testFrameworkMock.SetupGet(_ => _.Uid).Returns("fake-uid"); + _ = _testFrameworkMock.SetupGet(_ => _.Version).Returns("0.0.0"); + _ = _testFrameworkMock.SetupGet(_ => _.DisplayName).Returns("Fake"); + + return new CtrfReportEngine( + _fileSystem.Object, + _testApplicationModuleInfoMock.Object, + _environmentMock.Object, + _commandLineOptionsMock.Object, + _configurationMock.Object, + _clockMock.Object, + _testFrameworkMock.Object, + DateTimeOffset.UtcNow, + 0, + CancellationToken.None); + } + + internal sealed class MemoryFileStream : IFileStream + { + public MemoryFileStream() => Stream = new MemoryStream(); + + public MemoryStream Stream { get; } + + Stream IFileStream.Stream => Stream; + + string IFileStream.Name => string.Empty; + + public string GetUtf8Content() => Encoding.UTF8.GetString(Stream.ToArray()); + + void IDisposable.Dispose() => Stream.Dispose(); + +#if NETCOREAPP + ValueTask IAsyncDisposable.DisposeAsync() => Stream.DisposeAsync(); +#endif + } +} diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportGeneratorCommandLineTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportGeneratorCommandLineTests.cs new file mode 100644 index 0000000000..8e437e8f63 --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportGeneratorCommandLineTests.cs @@ -0,0 +1,194 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.CtrfReport; +using Microsoft.Testing.Extensions.UnitTests.Helpers; +using Microsoft.Testing.Platform.CommandLine; + +namespace Microsoft.Testing.Extensions.UnitTests; + +[TestClass] +public sealed class CtrfReportGeneratorCommandLineTests +{ + [TestMethod] + [DataRow("report.json")] + [DataRow("report.ctrf.json")] + [DataRow("sub/report.json")] + public async Task IsValid_If_JsonFileNameOrNestedRelativePath_Is_Provided(string fileName) + { + var provider = new CtrfReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, [fileName]).ConfigureAwait(false); + + Assert.IsTrue(result.IsValid); + Assert.IsTrue(string.IsNullOrEmpty(result.ErrorMessage)); + } + + [TestMethod] + [OSCondition(OperatingSystems.Windows)] + public async Task IsValid_If_JsonFileNameUsesBackslashSeparator_OnWindows() + { + // The '\' character is only a directory separator on Windows; on Unix it would be treated + // as part of the leaf file name (and later sanitized at write time). + var provider = new CtrfReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, ["sub\\report.json"]).ConfigureAwait(false); + + Assert.IsTrue(result.IsValid); + Assert.IsTrue(string.IsNullOrEmpty(result.ErrorMessage)); + } + + [TestMethod] + public async Task IsValid_If_JsonFile_Has_Absolute_Path() + { + var provider = new CtrfReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName); + string fileName = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".json"); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, [fileName]).ConfigureAwait(false); + + Assert.IsTrue(result.IsValid); + Assert.IsTrue(string.IsNullOrEmpty(result.ErrorMessage)); + } + + [TestMethod] + [DataRow("report.txt")] // wrong extension + [DataRow("report")] // no extension + [DataRow("REPORT.JSO")] // wrong extension (truncated) + [DataRow("report.html")] // html is not json + public async Task IsInvalid_If_FileName_Does_Not_End_With_Json(string fileName) + { + var provider = new CtrfReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, [fileName]).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(CtrfReport.Resources.ExtensionResources.CtrfReportFileNameExtensionIsNotJson, result.ErrorMessage); + } + + [TestMethod] + [DataRow("../report.json")] + [DataRow("nested/../report.json")] + public async Task IsInvalid_If_RelativePath_Contains_ParentDirectorySegment(string fileName) + { + var provider = new CtrfReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, [fileName]).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(CtrfReport.Resources.ExtensionResources.CtrfReportFileNameRelativePathMustStayUnderResultsDirectory, result.ErrorMessage); + } + + [TestMethod] + [OSCondition(OperatingSystems.Windows)] + public async Task IsInvalid_If_JsonFile_Uses_DriveRelativePath_OnWindows() + { + // Drive-relative paths such as "C:report.json" are "rooted" but not fully qualified, so they + // would silently escape the test results directory. Validate that they are rejected on Windows. + var provider = new CtrfReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, ["C:report.json"]).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(CtrfReport.Resources.ExtensionResources.CtrfReportFileNameRelativePathMustStayUnderResultsDirectory, result.ErrorMessage); + } + + [TestMethod] + [DataRow(" ")] + [DataRow("sub/")] + [DataRow("sub/ ")] + public async Task IsInvalid_If_FileNamePart_Is_Empty_Or_Whitespace(string fileName) + { + var provider = new CtrfReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, [fileName]).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(CtrfReport.Resources.ExtensionResources.CtrfReportFileNameMustNotBeEmpty, result.ErrorMessage); + } + + [TestMethod] + public async Task IsInvalid_If_No_Argument_Provided() + { + var provider = new CtrfReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, []).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(CtrfReport.Resources.ExtensionResources.CtrfReportFileNameMustNotBeEmpty, result.ErrorMessage); + } + + [TestMethod] + public async Task IsInvalid_If_FileName_Provided_Without_CtrfReport_Flag() + { + var provider = new CtrfReportGeneratorCommandLine(); + var options = new Dictionary + { + [CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName] = ["report.json"], + }; + + ValidationResult result = await provider.ValidateCommandLineOptionsAsync(new TestCommandLineOptions(options)).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(CtrfReport.Resources.ExtensionResources.CtrfReportFileNameRequiresCtrfReport, result.ErrorMessage); + } + + [TestMethod] + public async Task IsInvalid_If_CtrfReport_Used_With_DiscoverTests() + { + var provider = new CtrfReportGeneratorCommandLine(); + var options = new Dictionary + { + [CtrfReportGeneratorCommandLine.CtrfReportOptionName] = [], + [PlatformCommandLineProvider.DiscoverTestsOptionKey] = [], + }; + + ValidationResult result = await provider.ValidateCommandLineOptionsAsync(new TestCommandLineOptions(options)).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(CtrfReport.Resources.ExtensionResources.CtrfReportIsNotValidForDiscovery, result.ErrorMessage); + } + + [TestMethod] + public async Task IsValid_When_CtrfReport_Used_Alone() + { + var provider = new CtrfReportGeneratorCommandLine(); + var options = new Dictionary + { + [CtrfReportGeneratorCommandLine.CtrfReportOptionName] = [], + }; + + ValidationResult result = await provider.ValidateCommandLineOptionsAsync(new TestCommandLineOptions(options)).ConfigureAwait(false); + + Assert.IsTrue(result.IsValid); + } + + [TestMethod] + [DataRow("report*.json")] + [DataRow("CON.json")] + public async Task IsValid_When_FileName_WillBeSanitized_AtWriteTime(string fileName) + { + var provider = new CtrfReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, [fileName]).ConfigureAwait(false); + + Assert.IsTrue(result.IsValid); + } +} diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj index e6390cfbe6..52c551cfa3 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj @@ -39,6 +39,9 @@ TargetFramework=netstandard2.0 + + TargetFramework=netstandard2.0 + TargetFramework=netstandard2.0 diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Program.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Program.cs index ac8d470cd1..26c0e5d9bf 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Program.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -22,6 +22,7 @@ builder.AddJUnitReportProvider(); builder.AddAppInsightsTelemetryProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); // Dogfood the OpenTelemetry extension: subscribe to the Microsoft.Testing.Platform activity source // and meter so the OpenTelemetryResultHandler pipeline is exercised end-to-end in CI. No exporter diff --git a/test/UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests/Program.cs b/test/UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests/Program.cs index 05d4e55015..281a69986d 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests/Program.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -43,6 +43,7 @@ builder.AddTrxReportProvider(); builder.AddJUnitReportProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); using ITestApplication app = await builder.BuildAsync(); return await app.RunAsync(); diff --git a/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/Program.cs b/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/Program.cs index e97e20f20b..242506a101 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/Program.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -19,6 +19,7 @@ builder.AddTrxReportProvider(); builder.AddJUnitReportProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); // Dogfood the OpenTelemetry extension: subscribe to the Microsoft.Testing.Platform activity source // and meter so the OpenTelemetryResultHandler pipeline is exercised end-to-end in CI. No exporter diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Program.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Program.cs index 70e94b7751..f46c9fc5cf 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Program.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -33,6 +33,7 @@ builder.AddTrxReportProvider(); builder.AddJUnitReportProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); // Dogfood the OpenTelemetry extension: subscribe to the Microsoft.Testing.Platform activity source // and meter so the OpenTelemetryResultHandler pipeline is exercised end-to-end in CI. No exporter diff --git a/test/UnitTests/TestFramework.UnitTests/Program.cs b/test/UnitTests/TestFramework.UnitTests/Program.cs index 62891b6492..474a775f52 100644 --- a/test/UnitTests/TestFramework.UnitTests/Program.cs +++ b/test/UnitTests/TestFramework.UnitTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -19,6 +19,7 @@ builder.AddCrashDumpProvider(ignoreIfNotSupported: true); builder.AddRetryProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); // Dogfood the OpenTelemetry extension: subscribe to the Microsoft.Testing.Platform activity source // and meter so the OpenTelemetryResultHandler pipeline is exercised end-to-end in CI. No exporter