From 02ef3f8f7bcac6b55bed7157e4d79d2f6db31b59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Fri, 5 Jun 2026 18:27:49 +0200 Subject: [PATCH 01/17] Add Microsoft.Testing.Extensions.CtrfReport extension Adds a new MTP extension that emits a CTRF (Common Test Report Format, https://ctrf.io) JSON file at the end of a test session, mirroring the layout of Microsoft.Testing.Extensions.HtmlReport. CLI options: - --report-ctrf: enables generation - --report-ctrf-filename : optional file name (default: {user}_{machine}_{module}_{tfm}_{timestamp}.ctrf.json) Refs #8858. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Microsoft.Testing.Platform.slnf | 1 + NonWindowsTests.slnf | 1 + TestFx.slnx | 1 + .../BannedSymbols.txt | 9 + .../CapturedTestResult.cs | 51 ++ .../CtrfReportEngine.cs | 487 +++++++++++++++ .../CtrfReportExtensions.cs | 53 ++ .../CtrfReportGenerator.cs | 157 +++++ .../CtrfReportGeneratorCommandLine.cs | 130 ++++ ...osoft.Testing.Extensions.CtrfReport.csproj | 75 +++ .../PACKAGE.md | 32 + .../PublicAPI/PublicAPI.Shipped.txt | 1 + .../PublicAPI/PublicAPI.Unshipped.txt | 5 + .../Resources/ExtensionResources.resx | 102 +++ .../Resources/xlf/ExtensionResources.cs.xlf | 76 +++ .../Resources/xlf/ExtensionResources.de.xlf | 76 +++ .../Resources/xlf/ExtensionResources.es.xlf | 76 +++ .../Resources/xlf/ExtensionResources.fr.xlf | 76 +++ .../Resources/xlf/ExtensionResources.it.xlf | 76 +++ .../Resources/xlf/ExtensionResources.ja.xlf | 76 +++ .../Resources/xlf/ExtensionResources.ko.xlf | 76 +++ .../Resources/xlf/ExtensionResources.pl.xlf | 76 +++ .../xlf/ExtensionResources.pt-BR.xlf | 76 +++ .../Resources/xlf/ExtensionResources.ru.xlf | 76 +++ .../Resources/xlf/ExtensionResources.tr.xlf | 76 +++ .../xlf/ExtensionResources.zh-Hans.xlf | 76 +++ .../xlf/ExtensionResources.zh-Hant.xlf | 76 +++ .../TestResultCapture.cs | 142 +++++ .../TestingPlatformBuilderHook.cs | 21 + ...rosoft.Testing.Extensions.CtrfReport.props | 3 + ...rosoft.Testing.Extensions.CtrfReport.props | 13 + ...rosoft.Testing.Extensions.CtrfReport.props | 3 + .../Microsoft.Testing.Platform.csproj | 1 + .../CtrfReportTests.cs | 228 +++++++ .../HelpInfoAllExtensionsTests.cs | 22 + .../MSBuild.KnownExtensionRegistration.cs | 3 + .../CtrfReportEngineTests.cs | 591 ++++++++++++++++++ .../CtrfReportGeneratorCommandLineTests.cs | 194 ++++++ ...rosoft.Testing.Extensions.UnitTests.csproj | 3 + 39 files changed, 3317 insertions(+) create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/BannedSymbols.txt create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/CapturedTestResult.cs create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportExtensions.cs create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportGenerator.cs create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportGeneratorCommandLine.cs create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/PACKAGE.md create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/PublicAPI/PublicAPI.Shipped.txt create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/PublicAPI/PublicAPI.Unshipped.txt create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/ExtensionResources.resx create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.cs.xlf create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.de.xlf create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.es.xlf create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.fr.xlf create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.it.xlf create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ja.xlf create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ko.xlf create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pl.xlf create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pt-BR.xlf create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ru.xlf create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.tr.xlf create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hans.xlf create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hant.xlf create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/TestResultCapture.cs create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/TestingPlatformBuilderHook.cs create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/build/Microsoft.Testing.Extensions.CtrfReport.props create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/buildMultiTargeting/Microsoft.Testing.Extensions.CtrfReport.props create mode 100644 src/Platform/Microsoft.Testing.Extensions.CtrfReport/buildTransitive/Microsoft.Testing.Extensions.CtrfReport.props create mode 100644 test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs create mode 100644 test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs create mode 100644 test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportGeneratorCommandLineTests.cs diff --git a/Microsoft.Testing.Platform.slnf b/Microsoft.Testing.Platform.slnf index f738a37771..4c3b6714bb 100644 --- a/Microsoft.Testing.Platform.slnf +++ b/Microsoft.Testing.Platform.slnf @@ -9,6 +9,7 @@ "src\\Platform\\Microsoft.Testing.Extensions.AzureDevOpsReport\\Microsoft.Testing.Extensions.AzureDevOpsReport.csproj", "src\\Platform\\Microsoft.Testing.Extensions.AzureFoundry\\Microsoft.Testing.Extensions.AzureFoundry.csproj", "src\\Platform\\Microsoft.Testing.Extensions.CrashDump\\Microsoft.Testing.Extensions.CrashDump.csproj", + "src\\Platform\\Microsoft.Testing.Extensions.CtrfReport\\Microsoft.Testing.Extensions.CtrfReport.csproj", "src\\Platform\\Microsoft.Testing.Extensions.HangDump\\Microsoft.Testing.Extensions.HangDump.csproj", "src\\Platform\\Microsoft.Testing.Extensions.HotReload\\Microsoft.Testing.Extensions.HotReload.csproj", "src\\Platform\\Microsoft.Testing.Extensions.HtmlReport\\Microsoft.Testing.Extensions.HtmlReport.csproj", diff --git a/NonWindowsTests.slnf b/NonWindowsTests.slnf index babc740760..12176d97cc 100644 --- a/NonWindowsTests.slnf +++ b/NonWindowsTests.slnf @@ -14,6 +14,7 @@ "src\\Platform\\Microsoft.Testing.Extensions.AzureDevOpsReport\\Microsoft.Testing.Extensions.AzureDevOpsReport.csproj", "src\\Platform\\Microsoft.Testing.Extensions.AzureFoundry\\Microsoft.Testing.Extensions.AzureFoundry.csproj", "src\\Platform\\Microsoft.Testing.Extensions.CrashDump\\Microsoft.Testing.Extensions.CrashDump.csproj", + "src\\Platform\\Microsoft.Testing.Extensions.CtrfReport\\Microsoft.Testing.Extensions.CtrfReport.csproj", "src\\Platform\\Microsoft.Testing.Extensions.HangDump\\Microsoft.Testing.Extensions.HangDump.csproj", "src\\Platform\\Microsoft.Testing.Extensions.HotReload\\Microsoft.Testing.Extensions.HotReload.csproj", "src\\Platform\\Microsoft.Testing.Extensions.HtmlReport\\Microsoft.Testing.Extensions.HtmlReport.csproj", diff --git a/TestFx.slnx b/TestFx.slnx index 68f70e5c5f..6470bc7c3f 100644 --- a/TestFx.slnx +++ b/TestFx.slnx @@ -39,6 +39,7 @@ + diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/BannedSymbols.txt b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/BannedSymbols.txt new file mode 100644 index 0000000000..64ef236c50 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/BannedSymbols.txt @@ -0,0 +1,9 @@ +P:System.DateTime.Now; Use 'IClock' instead +P:System.DateTime.UtcNow; Use 'IClock' instead +M:System.Threading.Tasks.Task.Run(System.Action); Use 'ITask' instead +M:System.Threading.Tasks.Task.WhenAll(System.Threading.Tasks.Task[]); Use 'ITask' instead +M:System.Threading.Tasks.Task.WhenAll(System.Collections.Generic.IEnumerable{System.Threading.Tasks.Task}); Use 'ITask' instead +M:System.String.IsNullOrEmpty(System.String); Use 'RoslynString.IsNullOrEmpty' instead +M:System.String.IsNullOrWhiteSpace(System.String); Use 'RoslynString.IsNullOrWhiteSpace' instead +M:System.Diagnostics.Debug.Assert(System.Boolean); Use 'RoslynDebug.Assert' instead +M:System.Diagnostics.Debug.Assert(System.Boolean,System.String); Use 'RoslynDebug.Assert' instead diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CapturedTestResult.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CapturedTestResult.cs new file mode 100644 index 0000000000..f35dbf32a3 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CapturedTestResult.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Extensions.CtrfReport; + +// Minimal capped-size projection of a TestNodeUpdateMessage. The consumer projects +// each message into this DTO immediately so that we don't retain entire test nodes +// (and their potentially huge stdout/stderr/stack trace strings) in memory for the +// whole session. All variable-length text fields are already truncated at this point +// so the engine doesn't need to truncate again. +internal sealed class CapturedTestResult +{ + public required string Uid { get; init; } + + public required string DisplayName { get; init; } + + // CTRF status (passed/failed/skipped/pending/other) — already normalized. + public required string Status { get; init; } + + // Preserves the original MTP outcome when it doesn't map 1:1 to CTRF + // (e.g. "timedOut", "errored", "cancelled"). Surfaced via CTRF `rawStatus`. + public string? RawStatus { get; init; } + + public required TimeSpan Duration { get; init; } + + public DateTimeOffset? StartTime { get; init; } + + public DateTimeOffset? EndTime { get; init; } + + public string? Namespace { get; init; } + + public string? ClassName { get; init; } + + public string? MethodName { get; init; } + + public string? ErrorMessage { get; init; } + + public string? ExceptionType { get; init; } + + public string? StackTrace { get; init; } + + public string? StandardOutput { get; init; } + + public string? StandardError { get; init; } + + public string? FilePath { get; init; } + + public int? Line { get; init; } + + public IReadOnlyList>? Traits { get; init; } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs new file mode 100644 index 0000000000..ed03487faa --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs @@ -0,0 +1,487 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Text.Json; + +using Microsoft.Testing.Extensions.CtrfReport.Resources; +using Microsoft.Testing.Platform; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Configurations; +using Microsoft.Testing.Platform.Extensions.TestFramework; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.OutputDevice; +using Microsoft.Testing.Platform.Services; + +namespace Microsoft.Testing.Extensions.CtrfReport; + +internal sealed class CtrfReportEngine +{ + // CTRF spec: https://github.com/ctrf-io/ctrf + private const string CtrfReportFormat = "CTRF"; + private const string CtrfSpecVersion = "0.0.0"; + + private readonly IFileSystem _fileSystem; + private readonly ITestApplicationModuleInfo _testApplicationModuleInfo; + private readonly IEnvironment _environment; + private readonly ICommandLineOptions _commandLineOptions; + private readonly IConfiguration _configuration; + private readonly IClock _clock; + private readonly ITestFramework _testFramework; + private readonly DateTimeOffset _testStartTime; + private readonly int _exitCode; + private readonly CancellationToken _cancellationToken; + + public CtrfReportEngine( + IFileSystem fileSystem, + ITestApplicationModuleInfo testApplicationModuleInfo, + IEnvironment environment, + ICommandLineOptions commandLineOptions, + IConfiguration configuration, + IClock clock, + ITestFramework testFramework, + DateTimeOffset testStartTime, + int exitCode, + CancellationToken cancellationToken) + { + _fileSystem = fileSystem; + _testApplicationModuleInfo = testApplicationModuleInfo; + _environment = environment; + _commandLineOptions = commandLineOptions; + _configuration = configuration; + _clock = clock; + _testFramework = testFramework; + _testStartTime = testStartTime; + _exitCode = exitCode; + _cancellationToken = cancellationToken; + } + + public Task<(string FileName, string? Warning)> GenerateReportAsync(CapturedTestResult[] results) + => GenerateReportCoreAsync(results, _clock.UtcNow); + + private async Task<(string FileName, string? Warning)> GenerateReportCoreAsync(CapturedTestResult[] results, DateTimeOffset finishTime) + { + _cancellationToken.ThrowIfCancellationRequested(); + + bool fileNameExplicitlyProvided = _commandLineOptions.TryGetOptionArgumentList( + CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName, + out string[]? providedFileName); + + string fileName = fileNameExplicitlyProvided + ? ResolveJsonFileName(GetProvidedFileName(providedFileName)) + : BuildDefaultFileName(finishTime); + + string outputDirectory = _configuration.GetTestResultDirectory(); + // Path.Combine short-circuits when the second argument is rooted, so an absolute + // user-provided file name overrides the test results directory while validated + // relative paths stay nested under it. + string finalPath = Path.Combine(outputDirectory, fileName); + string? finalDirectory = Path.GetDirectoryName(finalPath); + if (!RoslynString.IsNullOrEmpty(finalDirectory)) + { + _fileSystem.CreateDirectory(finalDirectory); + } + + byte[] bytes = BuildCtrfJson(results, finishTime); + + return await WriteWithRetryAsync(finalPath, bytes, fileNameExplicitlyProvided).ConfigureAwait(false); + } + + private static string GetProvidedFileName(string[]? providedFileName) + => providedFileName is { Length: > 0 } + ? providedFileName[0] + : throw ApplicationStateGuard.Unreachable(); + + private async Task<(string FileName, string? Warning)> WriteWithRetryAsync(string finalPath, byte[] bytes, bool fileNameExplicitlyProvided) + { + // Explicit file names: use FileMode.Create (overwrite). Default-generated file + // names: use FileMode.CreateNew but retry with disambiguating suffixes when the + // file already exists, so concurrent runs (or two runs within the same second + // sharing the result directory) don't fail with IOException. + if (fileNameExplicitlyProvided) + { + bool willOverwrite = _fileSystem.ExistFile(finalPath); + await WriteAsync(finalPath, FileMode.Create, bytes).ConfigureAwait(false); + return ( + finalPath, + willOverwrite + ? string.Format(CultureInfo.InvariantCulture, ExtensionResources.CtrfReportFileExistsAndWillBeOverwritten, finalPath) + : null); + } + + DateTimeOffset firstTry = _clock.UtcNow; + string directory = Path.GetDirectoryName(finalPath) ?? string.Empty; + string fileName = Path.GetFileName(finalPath); + SplitCtrfExtension(fileName, out string baseName, out string extension); + string candidate = finalPath; + int attempt = 0; + + while (true) + { + _cancellationToken.ThrowIfCancellationRequested(); + + try + { + await WriteAsync(candidate, FileMode.CreateNew, bytes).ConfigureAwait(false); + return (candidate, null); + } + catch (IOException) when (_fileSystem.ExistFile(candidate)) + { + // The IOException was caused by the file already existing. Try a + // suffixed name. Any other IOException (disk full, permission, path + // too long, etc.) is not caught here and will propagate to the caller. + if (_clock.UtcNow - firstTry > TimeSpan.FromSeconds(5)) + { + throw; + } + + attempt++; + candidate = Path.Combine(directory, $"{baseName}_{attempt}{extension}"); + } + } + } + + // Split a file name into base + extension while preserving the CTRF + // double-extension convention (`*.ctrf.json`). The disambiguation suffix + // must land before `.ctrf.json` so that downstream CTRF readers continue to + // recognize the file by its conventional extension. + private static void SplitCtrfExtension(string fileName, out string baseName, out string extension) + { + const string ctrfJsonSuffix = ".ctrf.json"; + if (fileName.EndsWith(ctrfJsonSuffix, StringComparison.OrdinalIgnoreCase) && fileName.Length > ctrfJsonSuffix.Length) + { + baseName = fileName.Substring(0, fileName.Length - ctrfJsonSuffix.Length); + extension = fileName.Substring(fileName.Length - ctrfJsonSuffix.Length); + return; + } + + baseName = Path.GetFileNameWithoutExtension(fileName); + extension = Path.GetExtension(fileName); + } + + private async Task WriteAsync(string path, FileMode mode, byte[] bytes) + { + // Note that we need to dispose the IFileStream, not the inner stream. + // IFileStream implementations will be responsible to dispose their inner stream. + using IFileStream stream = _fileSystem.NewFileStream(path, mode); +#if NETCOREAPP + await stream.Stream.WriteAsync(bytes.AsMemory(), _cancellationToken).ConfigureAwait(false); +#else + await stream.Stream.WriteAsync(bytes, 0, bytes.Length, _cancellationToken).ConfigureAwait(false); +#endif + } + + private string BuildDefaultFileName(DateTimeOffset finishTime) + { + string user = _environment.GetEnvironmentVariable("UserName") + ?? _environment.GetEnvironmentVariable("USER") + ?? "user"; + string moduleName = Path.GetFileNameWithoutExtension(_testApplicationModuleInfo.GetCurrentTestApplicationFullPath()); + string targetFrameworkMoniker = GetTargetFrameworkMoniker(); + string raw = $"{user}_{_environment.MachineName}_{moduleName}_{targetFrameworkMoniker}_{finishTime:yyyy-MM-dd_HH_mm_ss}.ctrf.json"; + return ReplaceInvalidFileNameChars(raw); + } + + private string ResolveJsonFileName(string template) + { + string processName = Path.GetFileNameWithoutExtension(_testApplicationModuleInfo.GetCurrentTestApplicationFullPath()); + string processId = _environment.ProcessId.ToString(CultureInfo.InvariantCulture); + Dictionary replacements = ArtifactNamingHelper.GetStandardReplacements(processName, processId, _clock.UtcNow); + string resolved = ArtifactNamingHelper.ResolveTemplate(template, replacements); + string directoryPart = Path.GetDirectoryName(resolved) ?? string.Empty; + string sanitizedFileName = ReplaceInvalidFileNameChars(Path.GetFileName(resolved)); + return directoryPart.Length == 0 + ? sanitizedFileName + : Path.Combine(directoryPart, sanitizedFileName); + } + + private static string GetTargetFrameworkMoniker() + => TargetFrameworkParser.GetShortTargetFramework( + Assembly.GetEntryAssembly()?.GetCustomAttribute()?.FrameworkDisplayName) + ?? TargetFrameworkParser.GetShortTargetFramework(RuntimeInformation.FrameworkDescription) + ?? "unknown"; + + private static string ReplaceInvalidFileNameChars(string fileName) + { + var sb = new StringBuilder(fileName.Length); + foreach (char c in fileName) + { + sb.Append(IsInvalidFileNameChar(c) ? '_' : c); + } + + string replaced = sb.ToString().TrimEnd(); + if (IsReservedFileName(replaced)) + { + replaced = '_' + replaced; + } + + return replaced; + } + + private static bool IsInvalidFileNameChar(char c) + // Keep the explicit file-name sanitization aligned with TRX report naming so + // placeholders and cross-platform reserved characters produce compatible names. + => c is < ' ' or '"' or '<' or '>' or '|' or ':' or '*' or '?' or '\\' or '/' or '@' or '(' or ')' or '^' or ' '; + + private static bool IsReservedFileName(string fileName) + { + string bareName = fileName; + int dot = bareName.IndexOf('.'); + if (dot >= 0) + { + bareName = bareName.Substring(0, dot); + } + + return bareName.Equals("CON", StringComparison.OrdinalIgnoreCase) + || bareName.Equals("PRN", StringComparison.OrdinalIgnoreCase) + || bareName.Equals("AUX", StringComparison.OrdinalIgnoreCase) + || bareName.Equals("NUL", StringComparison.OrdinalIgnoreCase) + || bareName.Equals("CLOCK$", StringComparison.OrdinalIgnoreCase) + || IsReservedNameWithNumber(bareName, "COM") + || IsReservedNameWithNumber(bareName, "LPT"); + + static bool IsReservedNameWithNumber(string bareName, string prefix) + => bareName.Length == 4 + && bareName.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) + && bareName[3] is >= '1' and <= '9'; + } + + private byte[] BuildCtrfJson(CapturedTestResult[] results, DateTimeOffset finishTime) + { + // Aggregate summary counts. + int passed = 0; + int failed = 0; + int skipped = 0; + int pending = 0; + int other = 0; + foreach (CapturedTestResult r in results) + { + switch (r.Status) + { + case "passed": passed++; break; + case "failed": failed++; break; + case "skipped": skipped++; break; + case "pending": pending++; break; + default: other++; break; + } + } + + long startMs = _testStartTime.ToUnixTimeMilliseconds(); + long stopMs = finishTime.ToUnixTimeMilliseconds(); + long durationMs = Math.Max(0, stopMs - startMs); + + using var ms = new MemoryStream(capacity: 8 * 1024); + var writerOptions = new JsonWriterOptions + { + Indented = true, + // CTRF documents are intended to be embedded in JSON/JS contexts. + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + using (var writer = new Utf8JsonWriter(ms, writerOptions)) + { + writer.WriteStartObject(); + + writer.WriteString("reportFormat", CtrfReportFormat); + writer.WriteString("specVersion", CtrfSpecVersion); + writer.WriteString("reportId", Guid.NewGuid().ToString("D")); + writer.WriteString("timestamp", finishTime.ToString("O", CultureInfo.InvariantCulture)); + writer.WriteString( + "generatedBy", + $"Microsoft.Testing.Extensions.CtrfReport@{ExtensionVersion.DefaultSemVer}"); + + writer.WritePropertyName("results"); + writer.WriteStartObject(); + + // results.tool + writer.WritePropertyName("tool"); + writer.WriteStartObject(); + writer.WriteString("name", _testFramework.DisplayName); + writer.WriteString("version", _testFramework.Version); + writer.WritePropertyName("extra"); + writer.WriteStartObject(); + writer.WriteString("uid", _testFramework.Uid); + writer.WriteEndObject(); + writer.WriteEndObject(); + + // results.summary + writer.WritePropertyName("summary"); + writer.WriteStartObject(); + writer.WriteNumber("tests", results.Length); + writer.WriteNumber("passed", passed); + writer.WriteNumber("failed", failed); + writer.WriteNumber("skipped", skipped); + writer.WriteNumber("pending", pending); + writer.WriteNumber("other", other); + writer.WriteNumber("start", startMs); + writer.WriteNumber("stop", stopMs); + writer.WriteNumber("duration", durationMs); + writer.WriteEndObject(); + + // results.environment + writer.WritePropertyName("environment"); + writer.WriteStartObject(); + string user = _environment.GetEnvironmentVariable("UserName") + ?? _environment.GetEnvironmentVariable("USER") + ?? string.Empty; + writer.WriteString( + "extra", + string.Format(CultureInfo.InvariantCulture, "user={0};machine={1};exitCode={2}", user, _environment.MachineName, _exitCode)); + writer.WriteString("osPlatform", RuntimeInformation.OSDescription); + writer.WriteString("testEnvironment", _testApplicationModuleInfo.GetCurrentTestApplicationFullPath()); + writer.WriteEndObject(); + + // results.tests + writer.WritePropertyName("tests"); + writer.WriteStartArray(); + + // First pass: count attempts per UID so we can annotate rows that share a UID + // with retry/attempt metadata in `extra`. Multiple terminal results per UID + // are intentional (parameterized rows, in-process retries) and must not be + // dropped. + Dictionary countByUid = []; + foreach (CapturedTestResult r in results) + { + countByUid[r.Uid] = countByUid.TryGetValue(r.Uid, out int existing) ? existing + 1 : 1; + } + + Dictionary emittedByUid = []; + + foreach (CapturedTestResult r in results) + { + int attemptOf = countByUid[r.Uid]; + int attemptIndex = emittedByUid.TryGetValue(r.Uid, out int alreadyEmitted) ? alreadyEmitted + 1 : 1; + emittedByUid[r.Uid] = attemptIndex; + + writer.WriteStartObject(); + + writer.WriteString("name", r.DisplayName); + writer.WriteString("status", r.Status); + writer.WriteNumber("duration", (long)Math.Max(0, r.Duration.TotalMilliseconds)); + + if (r.StartTime is { } start) + { + writer.WriteNumber("start", start.ToUnixTimeMilliseconds()); + } + + if (r.EndTime is { } end) + { + writer.WriteNumber("stop", end.ToUnixTimeMilliseconds()); + } + + if (r.RawStatus is not null) + { + writer.WriteString("rawStatus", r.RawStatus); + } + + // CTRF `suite` is an array of strings (minItems: 1) representing the test + // hierarchy (e.g. ["MyNamespace", "MyClass"]). + if (r.Namespace is not null || r.ClassName is not null) + { + writer.WritePropertyName("suite"); + writer.WriteStartArray(); + if (r.Namespace is not null) + { + writer.WriteStringValue(r.Namespace); + } + + if (r.ClassName is not null) + { + writer.WriteStringValue(r.ClassName); + } + + writer.WriteEndArray(); + } + + if (r.ErrorMessage is not null) + { + writer.WriteString("message", r.ErrorMessage); + } + + if (r.StackTrace is not null) + { + writer.WriteString("trace", r.StackTrace); + } + + if (r.FilePath is not null) + { + writer.WriteString("filePath", r.FilePath); + } + + if (r.Line is { } lineNumber) + { + writer.WriteNumber("line", lineNumber); + } + + if (attemptOf > 1) + { + writer.WriteNumber("retries", attemptOf - 1); + } + + if (r.StandardOutput is not null) + { + writer.WritePropertyName("stdout"); + writer.WriteStartArray(); + writer.WriteStringValue(r.StandardOutput); + writer.WriteEndArray(); + } + + if (r.StandardError is not null) + { + writer.WritePropertyName("stderr"); + writer.WriteStartArray(); + writer.WriteStringValue(r.StandardError); + writer.WriteEndArray(); + } + + if (r.Traits is { Count: > 0 } || r.MethodName is not null + || r.ExceptionType is not null || attemptOf > 1) + { + writer.WritePropertyName("labels"); + writer.WriteStartObject(); + if (r.Traits is { Count: > 0 }) + { + foreach (KeyValuePair trait in r.Traits) + { + writer.WriteString(trait.Key, trait.Value); + } + } + + if (r.MethodName is not null) + { + writer.WriteString("method", r.MethodName); + } + + if (r.ExceptionType is not null) + { + writer.WriteString("exceptionType", r.ExceptionType); + } + + if (attemptOf > 1) + { + writer.WriteString("attemptIndex", attemptIndex.ToString(CultureInfo.InvariantCulture)); + writer.WriteString("attemptOf", attemptOf.ToString(CultureInfo.InvariantCulture)); + } + + writer.WriteEndObject(); + } + + // CTRF `extra` (free-form) for the test UID — the CTRF spec doesn't + // define a dedicated stable identifier so we surface the MTP UID here + // for cross-tool correlation. + writer.WritePropertyName("extra"); + writer.WriteStartObject(); + writer.WriteString("uid", r.Uid); + writer.WriteEndObject(); + + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + writer.WriteEndObject(); + + writer.WriteEndObject(); + } + + return ms.ToArray(); + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportExtensions.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportExtensions.cs new file mode 100644 index 0000000000..5d8a03502c --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportExtensions.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.CtrfReport; +using Microsoft.Testing.Extensions.CtrfReport.Resources; +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.Services; + +namespace Microsoft.Testing.Extensions; + +/// +/// Provides extension methods for adding CTRF (Common Test Report Format) report generation to a test application. +/// +[Experimental("TPEXP", UrlFormat = "https://aka.ms/testingplatform/diagnostics#{0}")] +public static class CtrfReportExtensions +{ + /// + /// Adds CTRF (Common Test Report Format) report generation to a test application. + /// + /// The test application builder. + public static void AddCtrfReportProvider(this ITestApplicationBuilder builder) + { + if (builder is not TestApplicationBuilder) + { + throw new InvalidOperationException(ExtensionResources.InvalidTestApplicationBuilderType); + } + + var commandLine = new CtrfReportGeneratorCommandLine(); + + var compositeCtrfReportGenerator = + new CompositeExtensionFactory(serviceProvider => + new CtrfReportGenerator( + serviceProvider.GetConfiguration(), + serviceProvider.GetCommandLineOptions(), + serviceProvider.GetRequiredService(), + serviceProvider.GetTestApplicationModuleInfo(), + serviceProvider.GetMessageBus(), + serviceProvider.GetSystemClock(), + serviceProvider.GetEnvironment(), + serviceProvider.GetOutputDevice(), + serviceProvider.GetTestFramework(), + serviceProvider.GetTestApplicationProcessExitCode(), + serviceProvider.GetLoggerFactory().CreateLogger())); + + builder.TestHost.AddDataConsumer(compositeCtrfReportGenerator); + builder.TestHost.AddTestSessionLifetimeHandler(compositeCtrfReportGenerator); + + builder.CommandLine.AddProvider(() => commandLine); + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportGenerator.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportGenerator.cs new file mode 100644 index 0000000000..a1e7c3979d --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportGenerator.cs @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.CtrfReport.Resources; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Configurations; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.OutputDevice; +using Microsoft.Testing.Platform.Extensions.TestFramework; +using Microsoft.Testing.Platform.Extensions.TestHost; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.Messages; +using Microsoft.Testing.Platform.OutputDevice; +using Microsoft.Testing.Platform.Services; + +namespace Microsoft.Testing.Extensions.CtrfReport; + +internal sealed class CtrfReportGenerator : + IDataConsumer, + ITestSessionLifetimeHandler, + IDataProducer, + IOutputDeviceDataProducer +{ + private readonly IConfiguration _configuration; + private readonly ICommandLineOptions _commandLineOptions; + private readonly IFileSystem _fileSystem; + private readonly ITestApplicationModuleInfo _testApplicationModuleInfo; + private readonly IMessageBus _messageBus; + private readonly IClock _clock; + private readonly IEnvironment _environment; + private readonly IOutputDevice _outputDevice; + private readonly ITestFramework _testFramework; + private readonly ITestApplicationProcessExitCode _testApplicationProcessExitCode; + private readonly ILogger _logger; + private readonly List _tests = []; + private readonly bool _isEnabled; + + private DateTimeOffset? _testStartTime; + + public CtrfReportGenerator( + IConfiguration configuration, + ICommandLineOptions commandLineOptions, + IFileSystem fileSystem, + ITestApplicationModuleInfo testApplicationModuleInfo, + IMessageBus messageBus, + IClock clock, + IEnvironment environment, + IOutputDevice outputDevice, + ITestFramework testFramework, + ITestApplicationProcessExitCode testApplicationProcessExitCode, + ILogger logger) + { + _configuration = configuration; + _commandLineOptions = commandLineOptions; + _fileSystem = fileSystem; + _testApplicationModuleInfo = testApplicationModuleInfo; + _messageBus = messageBus; + _clock = clock; + _environment = environment; + _outputDevice = outputDevice; + _testFramework = testFramework; + _testApplicationProcessExitCode = testApplicationProcessExitCode; + _logger = logger; + _isEnabled = commandLineOptions.IsOptionSet(CtrfReportGeneratorCommandLine.CtrfReportOptionName); + } + + public Type[] DataTypesConsumed { get; } = + [ + typeof(TestNodeUpdateMessage), + ]; + + public Type[] DataTypesProduced { get; } = [typeof(SessionFileArtifact)]; + + /// + public string Uid => nameof(CtrfReportGenerator); + + /// + public string Version => ExtensionVersion.DefaultSemVer; + + /// + public string DisplayName { get; } = ExtensionResources.CtrfReportGeneratorDisplayName; + + /// + public string Description { get; } = ExtensionResources.CtrfReportGeneratorDescription; + + /// + public Task IsEnabledAsync() => Task.FromResult(_isEnabled); + + public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (value is TestNodeUpdateMessage update) + { + // Project to a capped DTO immediately so we don't retain the original + // TestNode (and its potentially huge stdout/stderr/stack trace strings) + // for the whole session. We never deduplicate on TestNode.Uid: some + // frameworks emit several distinct results sharing the same UID + // (parameterized rows, theory data, in-process retries). The engine + // surfaces all of them so no data is dropped. + CapturedTestResult? captured = TestResultCapture.TryCapture(update.TestNode); + if (captured is not null) + { + _tests.Add(captured); + } + } + + return Task.CompletedTask; + } + + public Task OnTestSessionStartingAsync(ITestSessionContext testSessionContext) + { + testSessionContext.CancellationToken.ThrowIfCancellationRequested(); + _testStartTime = _clock.UtcNow; + return Task.CompletedTask; + } + + public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionContext) + { + CancellationToken cancellationToken = testSessionContext.CancellationToken; + cancellationToken.ThrowIfCancellationRequested(); + + ApplicationStateGuard.Ensure(_testStartTime is not null); + + await _logger.LogTraceAsync($"Generating CTRF report for {_tests.Count} test result(s).").ConfigureAwait(false); + + int exitCode = _testApplicationProcessExitCode.GetProcessExitCode(); + var engine = new CtrfReportEngine( + _fileSystem, + _testApplicationModuleInfo, + _environment, + _commandLineOptions, + _configuration, + _clock, + _testFramework, + _testStartTime.Value, + exitCode, + cancellationToken); + + (string reportFileName, string? warning) = await engine.GenerateReportAsync([.. _tests]).ConfigureAwait(false); + + if (warning is not null) + { + await _outputDevice.DisplayAsync(this, new WarningMessageOutputDeviceData(warning), cancellationToken).ConfigureAwait(false); + } + + await _messageBus.PublishAsync( + this, + new SessionFileArtifact( + testSessionContext.SessionUid, + new FileInfo(reportFileName), + ExtensionResources.CtrfReportArtifactDisplayName, + ExtensionResources.CtrfReportArtifactDescription)).ConfigureAwait(false); + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportGeneratorCommandLine.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportGeneratorCommandLine.cs new file mode 100644 index 0000000000..6877d575fb --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportGeneratorCommandLine.cs @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.CtrfReport.Resources; +using Microsoft.Testing.Platform; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.CommandLine; + +namespace Microsoft.Testing.Extensions.CtrfReport; + +internal sealed class CtrfReportGeneratorCommandLine : CommandLineOptionsProviderBase +{ + public const string CtrfReportOptionName = "report-ctrf"; + public const string CtrfReportFileNameOptionName = "report-ctrf-filename"; + + private static readonly char[] DirectorySeparators = [Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar]; + + public CtrfReportGeneratorCommandLine() + : base( + nameof(CtrfReportGeneratorCommandLine), + ExtensionVersion.DefaultSemVer, + ExtensionResources.CtrfReportGeneratorDisplayName, + ExtensionResources.CtrfReportGeneratorDescription, + [ + new(CtrfReportOptionName, ExtensionResources.CtrfReportOptionDescription, ArgumentArity.Zero, false), + new(CtrfReportFileNameOptionName, ExtensionResources.CtrfReportFileNameOptionDescription, ArgumentArity.ExactlyOne, false), + ]) + { + } + + public override Task ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments) + { + if (commandOption.Name == CtrfReportFileNameOptionName) + { + if (arguments.Length is 0) + { + return ValidationResult.InvalidTask(ExtensionResources.CtrfReportFileNameMustNotBeEmpty); + } + + string argument = arguments[0]; + + string fileNamePart = Path.GetFileName(argument); + if (RoslynString.IsNullOrWhiteSpace(fileNamePart)) + { + return ValidationResult.InvalidTask(ExtensionResources.CtrfReportFileNameMustNotBeEmpty); + } + + if (!fileNamePart.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) + { + return ValidationResult.InvalidTask(ExtensionResources.CtrfReportFileNameExtensionIsNotJson); + } + + if (EscapesResultsDirectory(argument)) + { + return ValidationResult.InvalidTask(ExtensionResources.CtrfReportFileNameRelativePathMustStayUnderResultsDirectory); + } + } + + return ValidationResult.ValidTask; + } + + public override Task ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions) + => commandLineOptions.IsOptionSet(CtrfReportFileNameOptionName) && !commandLineOptions.IsOptionSet(CtrfReportOptionName) + ? ValidationResult.InvalidTask(ExtensionResources.CtrfReportFileNameRequiresCtrfReport) + : commandLineOptions.IsOptionSet(CtrfReportOptionName) && commandLineOptions.IsOptionSet(PlatformCommandLineProvider.DiscoverTestsOptionKey) + ? ValidationResult.InvalidTask(ExtensionResources.CtrfReportIsNotValidForDiscovery) + : ValidationResult.ValidTask; + + private static bool EscapesResultsDirectory(string path) + { + // Fully-qualified paths (e.g. "C:\foo.json", "\\server\share\foo.json" or "/foo.json") are + // accepted as-is and validated by the OS when we open the file - the user explicitly opted + // out of writing under the test results directory. + if (IsPathFullyQualified(path)) + { + return false; + } + + // Drive-relative paths on Windows such as "C:foo.json" are "rooted" but not fully qualified - + // they resolve against the current directory of the drive, which is unpredictable and would + // silently escape the test results directory. Reject them. On non-Windows OSes + // Path.IsPathRooted only returns true for paths starting with "/", which are already handled + // above, so this check is effectively Windows-only and matches the TRX option behavior. + if (Path.IsPathRooted(path)) + { + return true; + } + + // Any remaining ".." segment in a relative path would escape the test results directory. + return path.Split(DirectorySeparators, StringSplitOptions.RemoveEmptyEntries).Any(segment => segment == ".."); + } + + private static bool IsPathFullyQualified(string path) + { +#if NETCOREAPP + return Path.IsPathFullyQualified(path); +#else + // Mirrors the runtime implementation that is missing on .NET Framework and netstandard2.0. + if (path.Length < 2) + { + return false; + } + + // UNC paths like "\\server\share" (or with forward slashes). + if (IsDirectorySeparator(path[0]) && IsDirectorySeparator(path[1])) + { + return true; + } + + // On Unix, only paths starting with "/" are fully qualified. + if (Path.DirectorySeparatorChar == '/') + { + return path[0] == '/'; + } + + // On Windows, fully qualified drive paths must be "X:\" or "X:/". + return path.Length >= 3 + && IsValidDriveLetter(path[0]) + && path[1] == ':' + && IsDirectorySeparator(path[2]); + + static bool IsDirectorySeparator(char c) + => c == Path.DirectorySeparatorChar || c == Path.AltDirectorySeparatorChar; + + static bool IsValidDriveLetter(char c) + => c is (>= 'A' and <= 'Z') or (>= 'a' and <= 'z'); +#endif + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj new file mode 100644 index 0000000000..52798c8e65 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj @@ -0,0 +1,75 @@ + + + + netstandard2.0;$(SupportedNetFrameworks) + true + $(RepoRoot)src\Platform\SharedExtensionHelpers\BuildInfo.cs.template + + + + + + + + + + + + + + true + buildMultiTargeting + + + buildTransitive/$(TargetFramework) + + + build/$(TargetFramework) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PACKAGE.md b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PACKAGE.md new file mode 100644 index 0000000000..a7348d5731 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PACKAGE.md @@ -0,0 +1,32 @@ +# Microsoft.Testing.Extensions.CtrfReport + +Microsoft.Testing.Extensions.CtrfReport is an extension for [Microsoft.Testing.Platform](https://www.nuget.org/packages/Microsoft.Testing.Platform) that generates a test report in the [Common Test Report Format (CTRF)](https://ctrf.io) at the end of a test session. + +Microsoft.Testing.Platform is open source. You can find `Microsoft.Testing.Extensions.CtrfReport` code in the [microsoft/testfx](https://github.com/microsoft/testfx) GitHub repository. + +## Install the package + +```dotnetcli +dotnet add package Microsoft.Testing.Extensions.CtrfReport +``` + +## About + +> **⚠️ Experimental:** This extension is currently experimental. The API, CLI options and on-disk format may change in future releases without notice. The CTRF specification itself is also pre-1.0 and may evolve. + +This package extends Microsoft.Testing.Platform with: + +- **CTRF (Common Test Report Format) report**: a single JSON file conforming to the [CTRF schema](https://github.com/ctrf-io/ctrf/blob/main/schema/ctrf.schema.json) that can be consumed by any tool that understands CTRF (dashboards, CI integrations, AI agents, etc.) without requiring a TRX or JUnit XML parser. +- **Cross-tool interoperability**: same shape as outputs produced by other testing frameworks that adopt CTRF, so results from multiple test runs can be aggregated by a single consumer. + +Enable the report via the `--report-ctrf` command line option. The report file name can be overridden with `--report-ctrf-filename .json`. + +## Documentation + +For comprehensive documentation, see . + +For the CTRF specification, see . + +## Feedback & contributing + +Microsoft.Testing.Platform is an open source project. Provide feedback or report issues in the [microsoft/testfx](https://github.com/microsoft/testfx/issues) GitHub repository. diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PublicAPI/PublicAPI.Shipped.txt b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PublicAPI/PublicAPI.Shipped.txt new file mode 100644 index 0000000000..7dc5c58110 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PublicAPI/PublicAPI.Shipped.txt @@ -0,0 +1 @@ +#nullable enable diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PublicAPI/PublicAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PublicAPI/PublicAPI.Unshipped.txt new file mode 100644 index 0000000000..b3862b10ff --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PublicAPI/PublicAPI.Unshipped.txt @@ -0,0 +1,5 @@ +#nullable enable +Microsoft.Testing.Extensions.CtrfReport.TestingPlatformBuilderHook +[TPEXP]Microsoft.Testing.Extensions.CtrfReportExtensions +static Microsoft.Testing.Extensions.CtrfReport.TestingPlatformBuilderHook.AddExtensions(Microsoft.Testing.Platform.Builder.ITestApplicationBuilder! testApplicationBuilder, string![]! _) -> void +[TPEXP]static Microsoft.Testing.Extensions.CtrfReportExtensions.AddCtrfReportProvider(this Microsoft.Testing.Platform.Builder.ITestApplicationBuilder! builder) -> void diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/ExtensionResources.resx b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/ExtensionResources.resx new file mode 100644 index 0000000000..10b13c193b --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/ExtensionResources.resx @@ -0,0 +1,102 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + CTRF (Common Test Report Format) JSON test report + + + CTRF Report + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + CTRF report generator + + + '--report-ctrf' cannot be enabled when using '--list-tests' + + + Enable generating a CTRF (Common Test Report Format) JSON report + + + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + + diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.cs.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.cs.xlf new file mode 100644 index 0000000000..665835d295 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.cs.xlf @@ -0,0 +1,76 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.de.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.de.xlf new file mode 100644 index 0000000000..3b2fc606ff --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.de.xlf @@ -0,0 +1,76 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.es.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.es.xlf new file mode 100644 index 0000000000..7768fa8a2c --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.es.xlf @@ -0,0 +1,76 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.fr.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.fr.xlf new file mode 100644 index 0000000000..4edf2ed2dd --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.fr.xlf @@ -0,0 +1,76 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.it.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.it.xlf new file mode 100644 index 0000000000..a56086c445 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.it.xlf @@ -0,0 +1,76 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ja.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ja.xlf new file mode 100644 index 0000000000..76e08daa3b --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ja.xlf @@ -0,0 +1,76 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ko.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ko.xlf new file mode 100644 index 0000000000..c0d69d1593 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ko.xlf @@ -0,0 +1,76 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pl.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pl.xlf new file mode 100644 index 0000000000..1c148b3b63 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pl.xlf @@ -0,0 +1,76 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pt-BR.xlf new file mode 100644 index 0000000000..95f41add47 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pt-BR.xlf @@ -0,0 +1,76 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ru.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ru.xlf new file mode 100644 index 0000000000..4b2e3b3118 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ru.xlf @@ -0,0 +1,76 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.tr.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.tr.xlf new file mode 100644 index 0000000000..5896bb5b25 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.tr.xlf @@ -0,0 +1,76 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hans.xlf new file mode 100644 index 0000000000..144ee1136a --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hans.xlf @@ -0,0 +1,76 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hant.xlf new file mode 100644 index 0000000000..c70140b47e --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hant.xlf @@ -0,0 +1,76 @@ + + + + + + CTRF (Common Test Report Format) JSON test report + CTRF (Common Test Report Format) JSON test report + + + + CTRF Report + CTRF Report + + + + Warning: CTRF report file '{0}' already exists and will be overwritten. + Warning: CTRF report file '{0}' already exists and will be overwritten. + + + + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name argument must end with '.json' (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + '--report-ctrf-filename' file name part must not be empty (e.g. --report-ctrf-filename myreport.ctrf.json) + + + + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. +Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). +Example: MyReport_{tfm}.ctrf.json + + + + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + '--report-ctrf-filename' relative paths must stay under the test results directory (e.g. --report-ctrf-filename nested/myreport.ctrf.json) + + + + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + '--report-ctrf-filename' requires '--report-ctrf' to be enabled + + + + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + + + + CTRF report generator + CTRF report generator + + + + '--report-ctrf' cannot be enabled when using '--list-tests' + '--report-ctrf' cannot be enabled when using '--list-tests' + + + + Enable generating a CTRF (Common Test Report Format) JSON report + Enable generating a CTRF (Common Test Report Format) JSON report + + + + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' + + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/TestResultCapture.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/TestResultCapture.cs new file mode 100644 index 0000000000..55d357250d --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/TestResultCapture.cs @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Messages; + +namespace Microsoft.Testing.Extensions.CtrfReport; + +// Projects a TestNodeUpdateMessage into a capped CapturedTestResult so that the +// generator does not retain entire test node payloads (and their potentially huge +// stdout/stderr/stack traces) in memory for the whole session. +internal static class TestResultCapture +{ + internal const int MaxStandardStreamLength = 32 * 1024; + internal const int MaxStackTraceLength = 32 * 1024; + internal const int MaxMessageLength = 16 * 1024; + internal const int MaxIdentityFieldLength = 4 * 1024; + internal const int MaxTraitFieldLength = 1024; + + public static CapturedTestResult? TryCapture(TestNode node) + { + TestNodeStateProperty? state = node.Properties.SingleOrDefault(); + if (state is null or DiscoveredTestNodeStateProperty or InProgressTestNodeStateProperty) + { + return null; + } + + (string status, string? rawStatus) = ClassifyStatus(state); + + TimingProperty? timing = node.Properties.SingleOrDefault(); + TimeSpan duration = timing?.GlobalTiming.Duration ?? TimeSpan.Zero; + + (string? ns, string? className, string? methodName) = GetClassAndMethodName(node); + + string? errorMessage = state.Explanation; + string? stackTrace = null; + string? exceptionType = null; + Exception? exception = state switch + { + FailedTestNodeStateProperty f => f.Exception, + ErrorTestNodeStateProperty e => e.Exception, + TimeoutTestNodeStateProperty t => t.Exception, +#pragma warning disable CS0618, MTP0001 // CancelledTestNodeStateProperty is obsolete + CancelledTestNodeStateProperty c => c.Exception, +#pragma warning restore CS0618, MTP0001 + _ => null, + }; + + if (exception is not null) + { + errorMessage ??= exception.Message; + stackTrace = exception.StackTrace; + exceptionType = exception.GetType().FullName; + } + + string? stdout = node.Properties.SingleOrDefault()?.StandardOutput; + string? stderr = node.Properties.SingleOrDefault()?.StandardError; + + TestFileLocationProperty? location = node.Properties.SingleOrDefault(); + string? filePath = location?.FilePath; + int? line = location?.LineSpan.Start.Line; + + // Collect traits without using LINQ to avoid an enumerator allocation per node. + // Trait keys and values are also test-controlled so we truncate them as well to + // bound the size of the in-memory result list and generated report. + List>? traits = null; + foreach (IProperty p in node.Properties) + { + if (p is TestMetadataProperty meta) + { + traits ??= []; + traits.Add(new KeyValuePair( + Truncate(meta.Key, MaxTraitFieldLength)!, + Truncate(meta.Value, MaxTraitFieldLength)!)); + } + } + + return new CapturedTestResult + { + // Identity fields are test-controlled and can be unbounded (e.g. very long + // UIDs/display names from generated data), so we also cap them to keep the + // session-wide result list and generated report within a predictable budget. + Uid = Truncate(node.Uid.Value, MaxIdentityFieldLength)!, + DisplayName = Truncate(node.DisplayName, MaxIdentityFieldLength)!, + Status = status, + RawStatus = rawStatus, + Duration = duration, + StartTime = timing?.GlobalTiming.StartTime, + EndTime = timing?.GlobalTiming.EndTime, + Namespace = Truncate(ns, MaxIdentityFieldLength), + ClassName = Truncate(className, MaxIdentityFieldLength), + MethodName = Truncate(methodName, MaxIdentityFieldLength), + ErrorMessage = Truncate(errorMessage, MaxMessageLength), + ExceptionType = exceptionType, + StackTrace = Truncate(stackTrace, MaxStackTraceLength), + StandardOutput = Truncate(stdout, MaxStandardStreamLength), + StandardError = Truncate(stderr, MaxStandardStreamLength), + FilePath = Truncate(filePath, MaxIdentityFieldLength), + Line = line, + Traits = traits, + }; + } + + // CTRF status enum: passed, failed, skipped, pending, other. + // Returns the CTRF status plus an optional `rawStatus` preserving the original + // MTP outcome when it doesn't map 1:1 (timedOut, errored, cancelled). + private static (string Status, string? RawStatus) ClassifyStatus(TestNodeStateProperty state) + => state switch + { + PassedTestNodeStateProperty => ("passed", null), + SkippedTestNodeStateProperty => ("skipped", null), + TimeoutTestNodeStateProperty => ("failed", "timedOut"), + ErrorTestNodeStateProperty => ("failed", "errored"), + FailedTestNodeStateProperty => ("failed", null), +#pragma warning disable CS0618, MTP0001 // CancelledTestNodeStateProperty is obsolete + CancelledTestNodeStateProperty => ("other", "cancelled"), +#pragma warning restore CS0618, MTP0001 + _ when Array.IndexOf(TestNodePropertiesCategories.WellKnownTestNodeTestRunOutcomeFailedProperties, state.GetType()) >= 0 + => ("failed", null), + _ => throw ApplicationStateGuard.Unreachable(), + }; + + private static (string? Namespace, string? ClassName, string? MethodName) GetClassAndMethodName(TestNode node) + { + TestMethodIdentifierProperty? identifier = node.Properties.SingleOrDefault(); + if (identifier is null) + { + return (null, null, null); + } + + string? ns = RoslynString.IsNullOrEmpty(identifier.Namespace) ? null : identifier.Namespace; + return (ns, identifier.TypeName, identifier.MethodName); + } + + internal static string? Truncate(string? value, int maxLength) + => value is null || value.Length <= maxLength + ? value + : value.Substring(0, maxLength) + + $"\n…[truncated, original length: {value.Length.ToString(CultureInfo.InvariantCulture)}]"; +} diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/TestingPlatformBuilderHook.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/TestingPlatformBuilderHook.cs new file mode 100644 index 0000000000..a69da97b23 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/TestingPlatformBuilderHook.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Builder; + +namespace Microsoft.Testing.Extensions.CtrfReport; + +/// +/// This class is used by Microsoft.Testing.Platform.MSBuild to hook into the Testing Platform Builder +/// to add CTRF report support. +/// +public static class TestingPlatformBuilderHook +{ + /// + /// Adds CTRF report support to the Testing Platform Builder. + /// + /// The test application builder. + /// The command line arguments. + public static void AddExtensions(ITestApplicationBuilder testApplicationBuilder, string[] _) + => testApplicationBuilder.AddCtrfReportProvider(); +} diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/build/Microsoft.Testing.Extensions.CtrfReport.props b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/build/Microsoft.Testing.Extensions.CtrfReport.props new file mode 100644 index 0000000000..a3820670bd --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/build/Microsoft.Testing.Extensions.CtrfReport.props @@ -0,0 +1,3 @@ + + + diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/buildMultiTargeting/Microsoft.Testing.Extensions.CtrfReport.props b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/buildMultiTargeting/Microsoft.Testing.Extensions.CtrfReport.props new file mode 100644 index 0000000000..f7de696d4a --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/buildMultiTargeting/Microsoft.Testing.Extensions.CtrfReport.props @@ -0,0 +1,13 @@ + + + + + + Microsoft.Testing.Extensions.CtrfReport + Microsoft.Testing.Extensions.CtrfReport.TestingPlatformBuilderHook + + + diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/buildTransitive/Microsoft.Testing.Extensions.CtrfReport.props b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/buildTransitive/Microsoft.Testing.Extensions.CtrfReport.props new file mode 100644 index 0000000000..a3820670bd --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/buildTransitive/Microsoft.Testing.Extensions.CtrfReport.props @@ -0,0 +1,3 @@ + + + diff --git a/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj b/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj index 9aae122e9a..7e9e55b261 100644 --- a/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj +++ b/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj @@ -42,6 +42,7 @@ This package provides the core platform and the .NET implementation of the proto + diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs new file mode 100644 index 0000000000..09f2e72445 --- /dev/null +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs @@ -0,0 +1,228 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Text.Json; + +namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; + +[TestClass] +public class CtrfReportTests : AcceptanceTestBase +{ + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task Ctrf_WhenReportCtrfIsNotSpecified_CtrfReportIsNotGenerated(string tfm) + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + + // The CTRF report is published as an in-process artifact; check the correct block. + string outputPattern = """ + In process file artifacts produced: + - .+?\.ctrf\.json +"""; + testHostResult.AssertOutputDoesNotMatchRegex(outputPattern); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task Ctrf_WhenReportCtrfIsSpecified_CtrfReportIsGeneratedInDefaultLocation(string tfm) + { + string testResultsPath = Path.Combine(AssetFixture.TargetAssetPath, "bin", "Release", tfm, "TestResults"); + string ctrfPathPattern = Regex.Escape(testResultsPath + Path.DirectorySeparatorChar) + @".+?\.ctrf\.json"; + + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--report-ctrf", cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + + string outputPattern = $""" + In process file artifacts produced: + - {ctrfPathPattern} +"""; + testHostResult.AssertOutputMatchesRegex(outputPattern); + + Match match = Regex.Match(testHostResult.StandardOutput, ctrfPathPattern); + Assert.IsTrue(match.Success, $"CTRF report path not found in output:\n{testHostResult.StandardOutput}"); + + AssertCtrfReportShape(match.Value); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task Ctrf_WhenReportCtrfFilenameIsSpecified_CtrfReportIsGeneratedWithThatName(string tfm) + { + const string customFileName = "my-custom-report.ctrf.json"; + string testResultsPath = Path.Combine(AssetFixture.TargetAssetPath, "bin", "Release", tfm, "TestResults"); + string customFilePath = Path.Combine(testResultsPath, customFileName); + string expectedFilePath = Regex.Escape(customFilePath); + + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + $"--report-ctrf --report-ctrf-filename {customFileName}", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + + string outputPattern = $""" + In process file artifacts produced: + - {expectedFilePath} +"""; + testHostResult.AssertOutputMatchesRegex(outputPattern); + + Assert.IsTrue( + File.Exists(customFilePath), + $"Expected custom CTRF report file '{customFileName}' was not found in '{testResultsPath}'."); + + AssertCtrfReportShape(customFilePath); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task Ctrf_WhenReportCtrfFilenameContainsPath_CtrfReportIsGeneratedInThatPath(string tfm) + { + string customFileName = Path.Combine("subdir", "report.ctrf.json"); + string testResultsPath = Path.Combine(AssetFixture.TargetAssetPath, "bin", "Release", tfm, "TestResults"); + string customFilePath = Path.Combine(testResultsPath, customFileName); + string expectedFilePath = Regex.Escape(customFilePath); + + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + $"--report-ctrf --report-ctrf-filename {customFileName}", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + + string outputPattern = $""" + In process file artifacts produced: + - {expectedFilePath} +"""; + testHostResult.AssertOutputMatchesRegex(outputPattern); + + Assert.IsTrue( + File.Exists(customFilePath), + $"Expected custom CTRF report file '{customFileName}' was not found in '{testResultsPath}'."); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task Ctrf_WhenReportCtrfFilenameIsSpecifiedWithoutReportCtrf_ErrorIsDisplayed(string tfm) + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--report-ctrf-filename report.ctrf.json", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.InvalidCommandLine); + testHostResult.AssertOutputContains("'--report-ctrf-filename' requires '--report-ctrf' to be enabled"); + } + + private static void AssertCtrfReportShape(string filePath) + { + string jsonContent = File.ReadAllText(filePath); + using var document = JsonDocument.Parse(jsonContent); + JsonElement root = document.RootElement; + + Assert.AreEqual("CTRF", root.GetProperty("reportFormat").GetString()); + Assert.AreEqual("0.0.0", root.GetProperty("specVersion").GetString()); + + JsonElement results = root.GetProperty("results"); + Assert.IsTrue(results.TryGetProperty("tool", out _), "CTRF report is missing 'results.tool'."); + Assert.IsTrue(results.TryGetProperty("summary", out JsonElement summary), "CTRF report is missing 'results.summary'."); + Assert.IsTrue(results.TryGetProperty("tests", out JsonElement tests), "CTRF report is missing 'results.tests'."); + Assert.AreEqual(JsonValueKind.Array, tests.ValueKind); + Assert.AreEqual(1, tests.GetArrayLength()); + + JsonElement firstTest = tests[0]; + Assert.AreEqual("PassingTest", firstTest.GetProperty("name").GetString()); + Assert.AreEqual("passed", firstTest.GetProperty("status").GetString()); + + Assert.AreEqual(1, summary.GetProperty("tests").GetInt32()); + Assert.AreEqual(1, summary.GetProperty("passed").GetInt32()); + Assert.AreEqual(0, summary.GetProperty("failed").GetInt32()); + } + + public sealed class TestAssetFixture() : TestAssetFixtureBase() + { + public const string AssetName = "CtrfReportTest"; + + private const string TestCode = """ +#file CtrfReportTest.csproj + + + $TargetFrameworks$ + enable + enable + Exe + preview + + + + + + +#file Program.cs +using Microsoft.Testing.Extensions; +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestFramework; + +public class Program +{ + public static async Task Main(string[] args) + { + ITestApplicationBuilder builder = await TestApplication.CreateBuilderAsync(args); + builder.RegisterTestFramework( + sp => new TestFrameworkCapabilities(), + (_, __) => new DummyTestFramework()); +#pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + builder.AddCtrfReportProvider(); +#pragma warning restore TPEXP + using ITestApplication app = await builder.BuildAsync(); + return await app.RunAsync(); + } +} + +public class DummyTestFramework : ITestFramework, IDataProducer +{ + public string Uid => nameof(DummyTestFramework); + public string Version => "2.0.0"; + public string DisplayName => nameof(DummyTestFramework); + public string Description => nameof(DummyTestFramework); + public Type[] DataTypesProduced => [typeof(TestNodeUpdateMessage)]; + + public Task IsEnabledAsync() => Task.FromResult(true); + + public Task CreateTestSessionAsync(CreateTestSessionContext context) + => Task.FromResult(new CreateTestSessionResult() { IsSuccess = true }); + + public Task CloseTestSessionAsync(CloseTestSessionContext context) + => Task.FromResult(new CloseTestSessionResult() { IsSuccess = true }); + + public async Task ExecuteRequestAsync(ExecuteRequestContext context) + { + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage( + context.Request.Session.SessionUid, + new TestNode() + { + Uid = "test-1", + DisplayName = "PassingTest", + Properties = new PropertyBag(PassedTestNodeStateProperty.CachedInstance), + })); + context.Complete(); + } +} +"""; + + public string TargetAssetPath => GetAssetPath(AssetName); + + public override (string ID, string Name, string Code) GetAssetsToGenerate() => (AssetName, AssetName, + TestCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion)); + } + + public TestContext TestContext { get; set; } +} diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs index fce3b92cc0..7666204ecd 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs @@ -146,6 +146,12 @@ Path to a text file that lists quarantined test fully qualified names or glob pa Override the Azure DevOps artifact container name. Defaults to 'TestResults_{assemblyName}_{tfm}'. --report-azdo-upload-artifacts Upload test result files and/or add build tags to Azure DevOps. Options are: off (default), tags-only, files, and all. + --report-ctrf + Enable generating a CTRF (Common Test Report Format) JSON report + --report-ctrf-filename + The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. + Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). + Example: MyReport_{tfm}.ctrf.json --report-html Enable generating an HTML report --report-html-filename @@ -500,6 +506,21 @@ Default type is 'Full' Description: The name of the generated HTML report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). Example: MyReport_{tfm}.html + CtrfReportGeneratorCommandLine + Name: CTRF report generator + Version: * + Description: Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Options: + --report-ctrf + Arity: 0 + Hidden: False + Description: Enable generating a CTRF (Common Test Report Format) JSON report + --report-ctrf-filename + Arity: 1 + Hidden: False + Description: The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. + Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). + Example: MyReport_{tfm}.ctrf.json MSBuildCommandLineProvider Name: MSBuildCommandLineProvider Version: * @@ -598,6 +619,7 @@ public sealed class TestAssetFixture() : TestAssetFixtureBase() + diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs index d4c31a958f..d514a1db0a 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs @@ -28,6 +28,7 @@ public async Task Microsoft_Testing_Platform_Extensions_ShouldBe_Correctly_Regis testHostResult.AssertOutputContains("--hangdump"); testHostResult.AssertOutputContains("--publish-azdo-run-name"); testHostResult.AssertOutputContains("--publish-azdo-test-results"); + testHostResult.AssertOutputContains("--report-ctrf"); testHostResult.AssertOutputContains("--report-html"); testHostResult.AssertOutputContains("--report-trx"); testHostResult.AssertOutputContains("--retry-failed-tests"); @@ -39,6 +40,7 @@ public async Task Microsoft_Testing_Platform_Extensions_ShouldBe_Correctly_Regis Assert.Contains("Microsoft.Testing.Extensions.AzureDevOpsReport.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); Assert.Contains("Microsoft.Testing.Extensions.CrashDump.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); + Assert.Contains("Microsoft.Testing.Extensions.CtrfReport.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); Assert.Contains("Microsoft.Testing.Extensions.HangDump.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); Assert.Contains("Microsoft.Testing.Extensions.HotReload.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); Assert.Contains("Microsoft.Testing.Extensions.HtmlReport.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); @@ -100,6 +102,7 @@ public async Task TestingPlatformBuilderHook_With_Conflicting_Metadata_Fails_Bui + diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs new file mode 100644 index 0000000000..6d44777132 --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs @@ -0,0 +1,591 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Text.Json; + +using Microsoft.Testing.Extensions.CtrfReport; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Configurations; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestFramework; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Services; + +using Moq; + +namespace Microsoft.Testing.Extensions.UnitTests; + +[TestClass] +public class CtrfReportEngineTests +{ + private readonly Mock _environmentMock = new(); + private readonly Mock _commandLineOptionsMock = new(); + private readonly Mock _configurationMock = new(); + private readonly Mock _clockMock = new(); + private readonly Mock _testFrameworkMock = new(); + private readonly Mock _testApplicationModuleInfoMock = new(); + private readonly Mock _fileSystem = new(); + + [TestMethod] + public async Task GenerateReportAsync_WritesValidCtrfJson_WithRequiredTopLevelFields() + { + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + Captured("p1", "Passing test", "passed"), + Captured("f1", "Failing test", "failed", errorMessage: "expected 1, got 2"), + Captured("s1", "Skipped test", "skipped", errorMessage: "not relevant"), + ]; + + (string fileName, string? warning) = await engine.GenerateReportAsync(tests); + + Assert.IsNotNull(fileName); + Assert.IsNull(warning); + + // Parse the produced JSON to validate the CTRF document structure (this is the + // schema contract for the consumers at https://github.com/ctrf-io/ctrf). + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement root = document.RootElement; + + Assert.AreEqual("CTRF", root.GetProperty("reportFormat").GetString()); + Assert.AreEqual("0.0.0", root.GetProperty("specVersion").GetString()); + Assert.IsGreaterThan(0, root.GetProperty("reportId").GetString()!.Length); + Assert.IsGreaterThan(0, root.GetProperty("timestamp").GetString()!.Length); + Assert.IsTrue(root.GetProperty("generatedBy").GetString()!.StartsWith("Microsoft.Testing.Extensions.CtrfReport", StringComparison.Ordinal)); + + JsonElement results = root.GetProperty("results"); + + JsonElement tool = results.GetProperty("tool"); + Assert.IsNotNull(tool.GetProperty("name").GetString()); + Assert.IsNotNull(tool.GetProperty("version").GetString()); + + JsonElement summary = results.GetProperty("summary"); + Assert.AreEqual(3, summary.GetProperty("tests").GetInt32()); + Assert.AreEqual(1, summary.GetProperty("passed").GetInt32()); + Assert.AreEqual(1, summary.GetProperty("failed").GetInt32()); + Assert.AreEqual(1, summary.GetProperty("skipped").GetInt32()); + + JsonElement testArray = results.GetProperty("tests"); + Assert.AreEqual(3, testArray.GetArrayLength()); + } + + [TestMethod] + [DataRow("passed", typeof(PassedTestNodeStateProperty))] + [DataRow("skipped", typeof(SkippedTestNodeStateProperty))] + [DataRow("failed", typeof(FailedTestNodeStateProperty))] + public void TestResultCapture_ClassifiesTerminalOutcomes_ToCtrfStatus(string expectedStatus, Type stateType) + { + TestNodeStateProperty state = stateType switch + { + Type t when t == typeof(PassedTestNodeStateProperty) => PassedTestNodeStateProperty.CachedInstance, + Type t when t == typeof(SkippedTestNodeStateProperty) => SkippedTestNodeStateProperty.CachedInstance, + Type t when t == typeof(FailedTestNodeStateProperty) => new FailedTestNodeStateProperty("x"), + _ => throw new InvalidOperationException(), + }; + + TestNode node = new() { Uid = "id", DisplayName = "T", Properties = new(state) }; + + CapturedTestResult result = TestResultCapture.TryCapture(node)!; + + Assert.AreEqual(expectedStatus, result.Status); + Assert.IsNull(result.RawStatus, "Pure CTRF outcomes should not carry rawStatus."); + } + + [TestMethod] + public void TestResultCapture_ErrorState_MapsToFailed_With_RawStatus_Errored() + { + TestNode node = new() { Uid = "id", DisplayName = "T", Properties = new(new ErrorTestNodeStateProperty("boom")) }; + + CapturedTestResult result = TestResultCapture.TryCapture(node)!; + + Assert.AreEqual("failed", result.Status); + Assert.AreEqual("errored", result.RawStatus); + } + + [TestMethod] + public void TestResultCapture_TimeoutState_MapsToFailed_With_RawStatus_TimedOut() + { + TestNode node = new() { Uid = "id", DisplayName = "T", Properties = new(new TimeoutTestNodeStateProperty("slow")) }; + + CapturedTestResult result = TestResultCapture.TryCapture(node)!; + + Assert.AreEqual("failed", result.Status); + Assert.AreEqual("timedOut", result.RawStatus); + } + + [TestMethod] + public void TestResultCapture_Truncates_OverLength_StandardOutput_AtBoundary() + { + string huge = new('a', TestResultCapture.MaxStandardStreamLength + 7); + + var bag = new PropertyBag(PassedTestNodeStateProperty.CachedInstance); + bag.Add(new StandardOutputProperty(huge)); + TestNode node = new() { Uid = "id", DisplayName = "T", Properties = bag }; + + CapturedTestResult result = TestResultCapture.TryCapture(node)!; + + Assert.IsNotNull(result); + Assert.IsNotNull(result.StandardOutput); + Assert.StartsWith(new string('a', TestResultCapture.MaxStandardStreamLength), result.StandardOutput!); + Assert.Contains("[truncated, original length:", result.StandardOutput); + Assert.Contains((TestResultCapture.MaxStandardStreamLength + 7).ToString(CultureInfo.InvariantCulture), result.StandardOutput); + } + + [TestMethod] + public void TestResultCapture_Returns_Null_For_NonTerminalStates() + { + TestNode discovered = new() { Uid = "a", DisplayName = "x", Properties = new(DiscoveredTestNodeStateProperty.CachedInstance) }; + TestNode inProgress = new() { Uid = "b", DisplayName = "y", Properties = new(InProgressTestNodeStateProperty.CachedInstance) }; + + Assert.IsNull(TestResultCapture.TryCapture(discovered)); + Assert.IsNull(TestResultCapture.TryCapture(inProgress)); + } + + [TestMethod] + public async Task GenerateReportAsync_CountsAllOutcomeKindsSeparately() + { + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + Captured("p1", "Passed", "passed"), + Captured("f1", "Failed", "failed"), + Captured("s1", "Skipped", "skipped"), + CapturedRaw("e1", "Errored", "failed", "errored"), + CapturedRaw("t1", "Timed out", "failed", "timedOut"), + ]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement summary = document.RootElement.GetProperty("results").GetProperty("summary"); + Assert.AreEqual(5, summary.GetProperty("tests").GetInt32()); + Assert.AreEqual(1, summary.GetProperty("passed").GetInt32()); + Assert.AreEqual(3, summary.GetProperty("failed").GetInt32(), "errored + timedOut + failed all map to CTRF 'failed'."); + Assert.AreEqual(1, summary.GetProperty("skipped").GetInt32()); + } + + [TestMethod] + public async Task GenerateReportAsync_PreservesAllResultsForDuplicateUids_AndAnnotatesAttemptOf() + { + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + Captured("dup", "Row A", "failed", errorMessage: "first failure"), + Captured("dup", "Row B", "failed", errorMessage: "second failure"), + Captured("dup", "Row C", "passed"), + Captured("unique", "Solo", "passed"), + ]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement results = document.RootElement.GetProperty("results"); + JsonElement testArray = results.GetProperty("tests"); + Assert.AreEqual(4, testArray.GetArrayLength()); + + // All four rows must appear (no de-duplication on UID). + var names = new List(); + foreach (JsonElement t in testArray.EnumerateArray()) + { + names.Add(t.GetProperty("name").GetString()!); + } + + CollectionAssert.AreEquivalent(new[] { "Row A", "Row B", "Row C", "Solo" }, names); + + // Each duplicate row carries retries=2 + attemptIndex/attemptOf in labels. + var retries = new List(); + var attemptIndices = new List(); + foreach (JsonElement t in testArray.EnumerateArray()) + { + string name = t.GetProperty("name").GetString()!; + if (name.StartsWith("Row ", StringComparison.Ordinal)) + { + Assert.AreEqual(2, t.GetProperty("retries").GetInt32(), $"Row {name} should report 2 retries (3 total attempts)."); + Assert.IsTrue(t.GetProperty("labels").TryGetProperty("attemptIndex", out JsonElement idx)); + attemptIndices.Add(idx.GetString()); + Assert.AreEqual("3", t.GetProperty("labels").GetProperty("attemptOf").GetString()); + } + } + + CollectionAssert.AreEquivalent(new[] { "1", "2", "3" }, attemptIndices); + + // The unique UID row does not get a retries field or attempts annotation. + JsonElement solo = default; + foreach (JsonElement t in testArray.EnumerateArray()) + { + if (t.GetProperty("name").GetString() == "Solo") + { + solo = t; + break; + } + } + + Assert.IsFalse(solo.TryGetProperty("retries", out _), "Unique UIDs should not carry retries."); + if (solo.TryGetProperty("labels", out JsonElement labels)) + { + Assert.IsFalse(labels.TryGetProperty("attemptIndex", out _), "Unique UIDs should not carry attemptIndex."); + Assert.IsFalse(labels.TryGetProperty("attemptOf", out _), "Unique UIDs should not carry attemptOf."); + } + } + + [TestMethod] + public async Task GenerateReportAsync_PerTest_ContainsRequiredFields() + { + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + new CapturedTestResult + { + Uid = "id-1", + DisplayName = "MyTest", + Status = "passed", + Duration = TimeSpan.FromMilliseconds(42), + Namespace = "MyNs", + ClassName = "MyClass", + MethodName = "MyMethod", + }, + ]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement test = document.RootElement.GetProperty("results").GetProperty("tests")[0]; + + // CTRF spec required fields per test: name, status, duration. + Assert.AreEqual("MyTest", test.GetProperty("name").GetString()); + Assert.AreEqual("passed", test.GetProperty("status").GetString()); + Assert.AreEqual(42, test.GetProperty("duration").GetInt64()); + + // CTRF `suite` (array of strings) when class/namespace are known. + JsonElement suite = test.GetProperty("suite"); + Assert.AreEqual(2, suite.GetArrayLength()); + Assert.AreEqual("MyNs", suite[0].GetString()); + Assert.AreEqual("MyClass", suite[1].GetString()); + + // UID must be surfaced under `extra` for cross-tool correlation. + Assert.AreEqual("id-1", test.GetProperty("extra").GetProperty("uid").GetString()); + } + + [TestMethod] + public async Task GenerateReportAsync_OmitsSuite_WhenNoClassOrNamespace() + { + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = [Captured("u", "T", "passed")]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement test = document.RootElement.GetProperty("results").GetProperty("tests")[0]; + Assert.IsFalse(test.TryGetProperty("suite", out _), "suite must be omitted when className/namespace are unknown (CTRF requires minItems:1)."); + } + + [TestMethod] + public async Task GenerateReportAsync_RoundTripsErrorMessageAndStackTrace() + { + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + new CapturedTestResult + { + Uid = "u", + DisplayName = "T", + Status = "failed", + Duration = TimeSpan.Zero, + ErrorMessage = "expected 1 got 2", + StackTrace = "at MyAssembly.MyType.MyMethod()", + }, + ]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement test = document.RootElement.GetProperty("results").GetProperty("tests")[0]; + Assert.AreEqual("expected 1 got 2", test.GetProperty("message").GetString()); + Assert.AreEqual("at MyAssembly.MyType.MyMethod()", test.GetProperty("trace").GetString()); + } + + [TestMethod] + public async Task GenerateReportAsync_IncludesTraitsUnderLabels() + { + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + new CapturedTestResult + { + Uid = "id", + DisplayName = "T", + Status = "passed", + Duration = TimeSpan.Zero, + Traits = + [ + new KeyValuePair("Category", "FastTest"), + new KeyValuePair("Owner", "alice"), + ], + }, + ]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement labels = document.RootElement.GetProperty("results").GetProperty("tests")[0].GetProperty("labels"); + Assert.AreEqual("FastTest", labels.GetProperty("Category").GetString()); + Assert.AreEqual("alice", labels.GetProperty("Owner").GetString()); + } + + [TestMethod] + public async Task GenerateReportAsync_DefaultFileName_IncludesModuleNameAndTargetFramework() + { + string? pathSeen = null; + _ = _fileSystem.Setup(x => x.ExistFile(It.IsAny())).Returns(false); + _ = _fileSystem.Setup(x => x.NewFileStream(It.IsAny(), FileMode.CreateNew)) + .Returns((path, _) => + { + pathSeen = path; + return new MemoryFileStream(); + }); + + _ = _configurationMock.SetupGet(_ => _[It.IsAny()]).Returns("out"); + _ = _environmentMock.SetupGet(_ => _.MachineName).Returns("M"); + _ = _environmentMock.Setup(_ => _.GetEnvironmentVariable(It.IsAny())).Returns("u"); + _ = _testApplicationModuleInfoMock.Setup(_ => _.GetCurrentTestApplicationFullPath()).Returns(Path.Combine("tmp", "My.Test.Module.dll")); + _ = _testFrameworkMock.SetupGet(_ => _.Uid).Returns("uid"); + _ = _testFrameworkMock.SetupGet(_ => _.Version).Returns("0.0"); + _ = _testFrameworkMock.SetupGet(_ => _.DisplayName).Returns("F"); + _ = _clockMock.SetupGet(_ => _.UtcNow).Returns(new DateTimeOffset(2026, 2, 3, 4, 5, 6, TimeSpan.Zero)); + + var engine = new CtrfReportEngine( + _fileSystem.Object, + _testApplicationModuleInfoMock.Object, + _environmentMock.Object, + _commandLineOptionsMock.Object, + _configurationMock.Object, + _clockMock.Object, + _testFrameworkMock.Object, + DateTimeOffset.UtcNow, + 0, + CancellationToken.None); + + (string finalPath, _) = await engine.GenerateReportAsync([Captured("a", "A", "passed")]); + + const string ExpectedFileNamePattern = "^u_M_My\\.Test\\.Module_net[0-9]+(\\.[0-9]+)?_2026-02-03_04_05_06\\.ctrf\\.json$"; + Assert.AreEqual(pathSeen, finalPath); + Assert.IsTrue(Regex.IsMatch(Path.GetFileName(finalPath), ExpectedFileNamePattern)); + } + + [TestMethod] + public async Task GenerateReportAsync_ExplicitRelativePath_IsResolvedUnderResultsDirectory() + { + string[]? jsonFileName = [Path.Combine("nested", "custom.json")]; + _ = _commandLineOptionsMock.Setup(_ => _.TryGetOptionArgumentList(CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName, out jsonFileName)).Returns(true); + + string? pathSeen = null; + var directories = new List(); + _ = _fileSystem.Setup(x => x.ExistFile(It.IsAny())).Returns(false); + _ = _fileSystem.Setup(x => x.CreateDirectory(It.IsAny())) + .Callback(directories.Add) + .Returns(path => path); + _ = _fileSystem.Setup(x => x.NewFileStream(It.IsAny(), FileMode.Create)) + .Returns((path, _) => + { + pathSeen = path; + return new MemoryFileStream(); + }); + + CtrfReportEngine engine = CreateEngine(); + _ = _configurationMock.SetupGet(_ => _[It.IsAny()]).Returns("out"); + + (string finalPath, _) = await engine.GenerateReportAsync([Captured("a", "A", "passed")]); + + string expectedPath = Path.Combine("out", "nested", "custom.json"); + Assert.AreEqual(expectedPath, finalPath); + Assert.AreEqual(expectedPath, pathSeen); + Assert.Contains(Path.Combine("out", "nested"), directories); + } + + [TestMethod] + public async Task GenerateReportAsync_ExplicitAbsolutePath_OverridesResultsDirectory() + { + string absolutePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".json"); + string[]? jsonFileName = [absolutePath]; + _ = _commandLineOptionsMock.Setup(_ => _.TryGetOptionArgumentList(CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName, out jsonFileName)).Returns(true); + + string? pathSeen = null; + _ = _fileSystem.Setup(x => x.ExistFile(It.IsAny())).Returns(false); + _ = _fileSystem.Setup(x => x.CreateDirectory(It.IsAny())).Returns(path => path); + _ = _fileSystem.Setup(x => x.NewFileStream(It.IsAny(), FileMode.Create)) + .Returns((path, _) => + { + pathSeen = path; + return new MemoryFileStream(); + }); + + CtrfReportEngine engine = CreateEngine(); + _ = _configurationMock.SetupGet(_ => _[It.IsAny()]).Returns("out"); + + (string finalPath, _) = await engine.GenerateReportAsync([Captured("a", "A", "passed")]); + + Assert.AreEqual(absolutePath, finalPath); + Assert.AreEqual(absolutePath, pathSeen); + } + + [TestMethod] + public async Task GenerateReportAsync_AppendsDisambiguatingSuffix_When_DefaultFileExists() + { + var bytesSeen = new List(); + int callCount = 0; + _ = _fileSystem.Setup(x => x.NewFileStream(It.IsAny(), FileMode.CreateNew)) + .Returns((path, _) => + { + callCount++; + bytesSeen.Add(path); + return callCount == 1 + ? throw new IOException("file exists") + : new MemoryFileStream(); + }); + + _ = _fileSystem.Setup(x => x.ExistFile(It.IsAny())).Returns(true); + + _ = _configurationMock.SetupGet(_ => _[It.IsAny()]).Returns(string.Empty); + _ = _environmentMock.SetupGet(_ => _.MachineName).Returns("M"); + _ = _environmentMock.Setup(_ => _.GetEnvironmentVariable(It.IsAny())).Returns("u"); + _ = _testApplicationModuleInfoMock.Setup(_ => _.GetCurrentTestApplicationFullPath()).Returns("app"); + _ = _testFrameworkMock.SetupGet(_ => _.Uid).Returns("uid"); + _ = _testFrameworkMock.SetupGet(_ => _.Version).Returns("0.0"); + _ = _testFrameworkMock.SetupGet(_ => _.DisplayName).Returns("F"); + _ = _clockMock.SetupGet(_ => _.UtcNow).Returns(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + + var engine = new CtrfReportEngine( + _fileSystem.Object, + _testApplicationModuleInfoMock.Object, + _environmentMock.Object, + _commandLineOptionsMock.Object, + _configurationMock.Object, + _clockMock.Object, + _testFrameworkMock.Object, + DateTimeOffset.UtcNow, + 0, + CancellationToken.None); + + (string finalPath, _) = await engine.GenerateReportAsync([Captured("a", "A", "passed")]); + + Assert.AreEqual(2, callCount); + Assert.AreEqual(bytesSeen[1], finalPath); + Assert.Contains("_1.ctrf.json", finalPath); + } + + [TestMethod] + public async Task GenerateReportAsync_PropagatesIOException_When_FileDoesNotExist() + { + int callCount = 0; + _ = _fileSystem.Setup(x => x.NewFileStream(It.IsAny(), FileMode.CreateNew)) + .Returns((path, _) => + { + callCount++; + throw new IOException("disk full"); + }); + + _ = _fileSystem.Setup(x => x.ExistFile(It.IsAny())).Returns(false); + + _ = _configurationMock.SetupGet(_ => _[It.IsAny()]).Returns(string.Empty); + _ = _environmentMock.SetupGet(_ => _.MachineName).Returns("M"); + _ = _environmentMock.Setup(_ => _.GetEnvironmentVariable(It.IsAny())).Returns("u"); + _ = _testApplicationModuleInfoMock.Setup(_ => _.GetCurrentTestApplicationFullPath()).Returns("app"); + _ = _testFrameworkMock.SetupGet(_ => _.Uid).Returns("uid"); + _ = _testFrameworkMock.SetupGet(_ => _.Version).Returns("0.0"); + _ = _testFrameworkMock.SetupGet(_ => _.DisplayName).Returns("F"); + _ = _clockMock.SetupGet(_ => _.UtcNow).Returns(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + + var engine = new CtrfReportEngine( + _fileSystem.Object, + _testApplicationModuleInfoMock.Object, + _environmentMock.Object, + _commandLineOptionsMock.Object, + _configurationMock.Object, + _clockMock.Object, + _testFrameworkMock.Object, + DateTimeOffset.UtcNow, + 0, + CancellationToken.None); + + await Assert.ThrowsExactlyAsync(() => engine.GenerateReportAsync([Captured("a", "A", "passed")])); + Assert.AreEqual(1, callCount); + } + + private static CapturedTestResult Captured(string uid, string name, string status, + TimeSpan? duration = null, string? errorMessage = null) + => new() + { + Uid = uid, + DisplayName = name, + Status = status, + Duration = duration ?? TimeSpan.Zero, + ErrorMessage = errorMessage, + }; + + private static CapturedTestResult CapturedRaw(string uid, string name, string status, string rawStatus) + => new() + { + Uid = uid, + DisplayName = name, + Status = status, + RawStatus = rawStatus, + Duration = TimeSpan.Zero, + }; + + private CtrfReportEngine CreateEngine(MemoryFileStream stream) + { + _ = _fileSystem.Setup(x => x.ExistFile(It.IsAny())).Returns(false); + _ = _fileSystem.Setup(x => x.NewFileStream(It.IsAny(), It.IsAny())).Returns(stream); + + return CreateEngine(); + } + + private CtrfReportEngine CreateEngine() + { + _ = _configurationMock.SetupGet(_ => _[It.IsAny()]).Returns(string.Empty); + _ = _environmentMock.SetupGet(_ => _.MachineName).Returns("MachineName"); + _ = _environmentMock.Setup(_ => _.GetEnvironmentVariable(It.IsAny())).Returns("user"); + _ = _testApplicationModuleInfoMock.Setup(_ => _.GetCurrentTestApplicationFullPath()).Returns("TestAppPath"); + _ = _testFrameworkMock.SetupGet(_ => _.Uid).Returns("fake-uid"); + _ = _testFrameworkMock.SetupGet(_ => _.Version).Returns("0.0.0"); + _ = _testFrameworkMock.SetupGet(_ => _.DisplayName).Returns("Fake"); + + return new CtrfReportEngine( + _fileSystem.Object, + _testApplicationModuleInfoMock.Object, + _environmentMock.Object, + _commandLineOptionsMock.Object, + _configurationMock.Object, + _clockMock.Object, + _testFrameworkMock.Object, + DateTimeOffset.UtcNow, + 0, + CancellationToken.None); + } + + internal sealed class MemoryFileStream : IFileStream + { + public MemoryFileStream() => Stream = new MemoryStream(); + + public MemoryStream Stream { get; } + + Stream IFileStream.Stream => Stream; + + string IFileStream.Name => string.Empty; + + public string GetUtf8Content() => Encoding.UTF8.GetString(Stream.ToArray()); + + void IDisposable.Dispose() => Stream.Dispose(); + +#if NETCOREAPP + ValueTask IAsyncDisposable.DisposeAsync() => Stream.DisposeAsync(); +#endif + } +} diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportGeneratorCommandLineTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportGeneratorCommandLineTests.cs new file mode 100644 index 0000000000..8e437e8f63 --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportGeneratorCommandLineTests.cs @@ -0,0 +1,194 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.CtrfReport; +using Microsoft.Testing.Extensions.UnitTests.Helpers; +using Microsoft.Testing.Platform.CommandLine; + +namespace Microsoft.Testing.Extensions.UnitTests; + +[TestClass] +public sealed class CtrfReportGeneratorCommandLineTests +{ + [TestMethod] + [DataRow("report.json")] + [DataRow("report.ctrf.json")] + [DataRow("sub/report.json")] + public async Task IsValid_If_JsonFileNameOrNestedRelativePath_Is_Provided(string fileName) + { + var provider = new CtrfReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, [fileName]).ConfigureAwait(false); + + Assert.IsTrue(result.IsValid); + Assert.IsTrue(string.IsNullOrEmpty(result.ErrorMessage)); + } + + [TestMethod] + [OSCondition(OperatingSystems.Windows)] + public async Task IsValid_If_JsonFileNameUsesBackslashSeparator_OnWindows() + { + // The '\' character is only a directory separator on Windows; on Unix it would be treated + // as part of the leaf file name (and later sanitized at write time). + var provider = new CtrfReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, ["sub\\report.json"]).ConfigureAwait(false); + + Assert.IsTrue(result.IsValid); + Assert.IsTrue(string.IsNullOrEmpty(result.ErrorMessage)); + } + + [TestMethod] + public async Task IsValid_If_JsonFile_Has_Absolute_Path() + { + var provider = new CtrfReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName); + string fileName = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".json"); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, [fileName]).ConfigureAwait(false); + + Assert.IsTrue(result.IsValid); + Assert.IsTrue(string.IsNullOrEmpty(result.ErrorMessage)); + } + + [TestMethod] + [DataRow("report.txt")] // wrong extension + [DataRow("report")] // no extension + [DataRow("REPORT.JSO")] // wrong extension (truncated) + [DataRow("report.html")] // html is not json + public async Task IsInvalid_If_FileName_Does_Not_End_With_Json(string fileName) + { + var provider = new CtrfReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, [fileName]).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(CtrfReport.Resources.ExtensionResources.CtrfReportFileNameExtensionIsNotJson, result.ErrorMessage); + } + + [TestMethod] + [DataRow("../report.json")] + [DataRow("nested/../report.json")] + public async Task IsInvalid_If_RelativePath_Contains_ParentDirectorySegment(string fileName) + { + var provider = new CtrfReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, [fileName]).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(CtrfReport.Resources.ExtensionResources.CtrfReportFileNameRelativePathMustStayUnderResultsDirectory, result.ErrorMessage); + } + + [TestMethod] + [OSCondition(OperatingSystems.Windows)] + public async Task IsInvalid_If_JsonFile_Uses_DriveRelativePath_OnWindows() + { + // Drive-relative paths such as "C:report.json" are "rooted" but not fully qualified, so they + // would silently escape the test results directory. Validate that they are rejected on Windows. + var provider = new CtrfReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, ["C:report.json"]).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(CtrfReport.Resources.ExtensionResources.CtrfReportFileNameRelativePathMustStayUnderResultsDirectory, result.ErrorMessage); + } + + [TestMethod] + [DataRow(" ")] + [DataRow("sub/")] + [DataRow("sub/ ")] + public async Task IsInvalid_If_FileNamePart_Is_Empty_Or_Whitespace(string fileName) + { + var provider = new CtrfReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, [fileName]).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(CtrfReport.Resources.ExtensionResources.CtrfReportFileNameMustNotBeEmpty, result.ErrorMessage); + } + + [TestMethod] + public async Task IsInvalid_If_No_Argument_Provided() + { + var provider = new CtrfReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, []).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(CtrfReport.Resources.ExtensionResources.CtrfReportFileNameMustNotBeEmpty, result.ErrorMessage); + } + + [TestMethod] + public async Task IsInvalid_If_FileName_Provided_Without_CtrfReport_Flag() + { + var provider = new CtrfReportGeneratorCommandLine(); + var options = new Dictionary + { + [CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName] = ["report.json"], + }; + + ValidationResult result = await provider.ValidateCommandLineOptionsAsync(new TestCommandLineOptions(options)).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(CtrfReport.Resources.ExtensionResources.CtrfReportFileNameRequiresCtrfReport, result.ErrorMessage); + } + + [TestMethod] + public async Task IsInvalid_If_CtrfReport_Used_With_DiscoverTests() + { + var provider = new CtrfReportGeneratorCommandLine(); + var options = new Dictionary + { + [CtrfReportGeneratorCommandLine.CtrfReportOptionName] = [], + [PlatformCommandLineProvider.DiscoverTestsOptionKey] = [], + }; + + ValidationResult result = await provider.ValidateCommandLineOptionsAsync(new TestCommandLineOptions(options)).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(CtrfReport.Resources.ExtensionResources.CtrfReportIsNotValidForDiscovery, result.ErrorMessage); + } + + [TestMethod] + public async Task IsValid_When_CtrfReport_Used_Alone() + { + var provider = new CtrfReportGeneratorCommandLine(); + var options = new Dictionary + { + [CtrfReportGeneratorCommandLine.CtrfReportOptionName] = [], + }; + + ValidationResult result = await provider.ValidateCommandLineOptionsAsync(new TestCommandLineOptions(options)).ConfigureAwait(false); + + Assert.IsTrue(result.IsValid); + } + + [TestMethod] + [DataRow("report*.json")] + [DataRow("CON.json")] + public async Task IsValid_When_FileName_WillBeSanitized_AtWriteTime(string fileName) + { + var provider = new CtrfReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == CtrfReportGeneratorCommandLine.CtrfReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, [fileName]).ConfigureAwait(false); + + Assert.IsTrue(result.IsValid); + } +} diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj index 401e5457c9..b89432bfa8 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj @@ -36,6 +36,9 @@ TargetFramework=netstandard2.0 + + TargetFramework=netstandard2.0 + TargetFramework=netstandard2.0 From a2bd24fef24500b241f013b5ffe6212509f49b6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Sun, 7 Jun 2026 10:12:47 +0200 Subject: [PATCH 02/17] Address CTRF expert review: collapse retries, safe JSON encoder, name fallback Apply spec/expert-review fixes to the CTRF report extension: * Collapse same-UID captures into a single tests[] entry with retryAttempts[]; set retries=N-1 and flaky=true when the final attempt passed after earlier failures (CTRF retry idiom). * Update summary.tests to count unique UIDs (and add summary.flaky) so CTRF aggregators no longer double-count retries. * Drop UnsafeRelaxedJsonEscaping: HTML/JS metachars in test names could become XSS vectors in downstream CTRF dashboards. * Cap WriteWithRetryAsync attempts at 1000 in addition to the existing 5s wall-clock bound. * Fall back to UID when DisplayName is empty so tests[].name keeps the schema-required minLength: 1. * Add unit tests for the collapsed retry shape, empty result set, HTML-escaping behavior, and DisplayName -> UID fallback. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../CtrfReportEngine.cs | 456 +++++++++++++----- .../CtrfReportEngineTests.cs | 237 +++++++-- 2 files changed, 529 insertions(+), 164 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs index ed03487faa..71a5b3ed14 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Text.Json; @@ -129,7 +129,9 @@ private static string GetProvidedFileName(string[]? providedFileName) // The IOException was caused by the file already existing. Try a // suffixed name. Any other IOException (disk full, permission, path // too long, etc.) is not caught here and will propagate to the caller. - if (_clock.UtcNow - firstTry > TimeSpan.FromSeconds(5)) + // Bound by both wall-clock (5s) and attempt count (1000) so we never + // spin forever in pathological cases like a clock that doesn't advance. + if (_clock.UtcNow - firstTry > TimeSpan.FromSeconds(5) || attempt >= 1_000) { throw; } @@ -200,6 +202,35 @@ private static string GetTargetFrameworkMoniker() ?? TargetFrameworkParser.GetShortTargetFramework(RuntimeInformation.FrameworkDescription) ?? "unknown"; + // CTRF `osPlatform` is the short Node-style platform identifier (e.g. "win32", + // "linux", "darwin"). The full descriptive OS string belongs in `osVersion`. + private static string GetCtrfOsPlatform() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return "win32"; + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + return "linux"; + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return "darwin"; + } + +#if NETCOREAPP + if (RuntimeInformation.IsOSPlatform(OSPlatform.FreeBSD)) + { + return "freebsd"; + } +#endif + + return "unknown"; + } + private static string ReplaceInvalidFileNameChars(string fileName) { var sb = new StringBuilder(fileName.Length); @@ -247,15 +278,23 @@ static bool IsReservedNameWithNumber(string bareName, string prefix) private byte[] BuildCtrfJson(CapturedTestResult[] results, DateTimeOffset finishTime) { - // Aggregate summary counts. + // Collapse multiple captures sharing the same UID into a single CTRF test + // entry. The CTRF spec models retries as nested `retryAttempts[]` records + // and exposes `flaky: true` on the final passing row; emitting separate + // top-level rows for retries would inflate `summary.tests` and double-count + // outcomes. + List collapsed = CollapseAttempts(results); + + // Aggregate summary counts from the collapsed (final) outcomes only. int passed = 0; int failed = 0; int skipped = 0; int pending = 0; int other = 0; - foreach (CapturedTestResult r in results) + int flaky = 0; + foreach (CollapsedTestResult c in collapsed) { - switch (r.Status) + switch (c.Final.Status) { case "passed": passed++; break; case "failed": failed++; break; @@ -263,6 +302,11 @@ private byte[] BuildCtrfJson(CapturedTestResult[] results, DateTimeOffset finish case "pending": pending++; break; default: other++; break; } + + if (c.IsFlaky) + { + flaky++; + } } long startMs = _testStartTime.ToUnixTimeMilliseconds(); @@ -270,11 +314,15 @@ private byte[] BuildCtrfJson(CapturedTestResult[] results, DateTimeOffset finish long durationMs = Math.Max(0, stopMs - startMs); using var ms = new MemoryStream(capacity: 8 * 1024); + // We deliberately use the default Default encoder rather than + // UnsafeRelaxedJsonEscaping: CTRF documents routinely flow into web + // dashboards that embed JSON into HTML/JS, and test names/messages are + // attacker-controllable. The default safe encoder keeps `<`, `>`, `&` + // escaped so a test display name like `` can't + // become an XSS vector in downstream consumers. var writerOptions = new JsonWriterOptions { Indented = true, - // CTRF documents are intended to be embedded in JSON/JS contexts. - Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, }; using (var writer = new Utf8JsonWriter(ms, writerOptions)) @@ -282,6 +330,9 @@ private byte[] BuildCtrfJson(CapturedTestResult[] results, DateTimeOffset finish writer.WriteStartObject(); writer.WriteString("reportFormat", CtrfReportFormat); + // CTRF is still in pre-1.0; the upstream spec is at "0.0.0" today + // (see https://github.com/ctrf-io/ctrf/blob/main/spec/ctrf.md). + // Bump this constant whenever we update against a newer schema revision. writer.WriteString("specVersion", CtrfSpecVersion); writer.WriteString("reportId", Guid.NewGuid().ToString("D")); writer.WriteString("timestamp", finishTime.ToString("O", CultureInfo.InvariantCulture)); @@ -295,8 +346,18 @@ private byte[] BuildCtrfJson(CapturedTestResult[] results, DateTimeOffset finish // results.tool writer.WritePropertyName("tool"); writer.WriteStartObject(); - writer.WriteString("name", _testFramework.DisplayName); - writer.WriteString("version", _testFramework.Version); + // CTRF spec requires `tool.name` to be a non-empty string. Fall back to + // a sentinel rather than emitting an empty string (which would fail + // strict schema validation by downstream CTRF consumers). + string toolName = RoslynString.IsNullOrEmpty(_testFramework.DisplayName) + ? "unknown" + : _testFramework.DisplayName; + writer.WriteString("name", toolName); + if (!RoslynString.IsNullOrEmpty(_testFramework.Version)) + { + writer.WriteString("version", _testFramework.Version); + } + writer.WritePropertyName("extra"); writer.WriteStartObject(); writer.WriteString("uid", _testFramework.Uid); @@ -306,12 +367,13 @@ private byte[] BuildCtrfJson(CapturedTestResult[] results, DateTimeOffset finish // results.summary writer.WritePropertyName("summary"); writer.WriteStartObject(); - writer.WriteNumber("tests", results.Length); + writer.WriteNumber("tests", collapsed.Count); writer.WriteNumber("passed", passed); writer.WriteNumber("failed", failed); writer.WriteNumber("skipped", skipped); writer.WriteNumber("pending", pending); writer.WriteNumber("other", other); + writer.WriteNumber("flaky", flaky); writer.WriteNumber("start", startMs); writer.WriteNumber("stop", stopMs); writer.WriteNumber("duration", durationMs); @@ -323,165 +385,297 @@ private byte[] BuildCtrfJson(CapturedTestResult[] results, DateTimeOffset finish string user = _environment.GetEnvironmentVariable("UserName") ?? _environment.GetEnvironmentVariable("USER") ?? string.Empty; - writer.WriteString( - "extra", - string.Format(CultureInfo.InvariantCulture, "user={0};machine={1};exitCode={2}", user, _environment.MachineName, _exitCode)); - writer.WriteString("osPlatform", RuntimeInformation.OSDescription); - writer.WriteString("testEnvironment", _testApplicationModuleInfo.GetCurrentTestApplicationFullPath()); + // CTRF `osPlatform` expects a short identifier such as "win32", "linux" or + // "darwin"; the full descriptive string belongs in `osVersion`. + writer.WriteString("osPlatform", GetCtrfOsPlatform()); + writer.WriteString("osVersion", RuntimeInformation.OSDescription); + // CTRF `extra` MUST be an object (schema enforces additionalProperties: false + // on environment, with `extra` typed as object). We surface the test module + // path and process exit code here rather than as top-level environment fields + // because there is no first-class CTRF slot for them. + writer.WritePropertyName("extra"); + writer.WriteStartObject(); + writer.WriteString("user", user); + writer.WriteString("machine", _environment.MachineName); + writer.WriteNumber("exitCode", _exitCode); + writer.WriteString("testApplication", _testApplicationModuleInfo.GetCurrentTestApplicationFullPath()); + writer.WriteEndObject(); writer.WriteEndObject(); // results.tests writer.WritePropertyName("tests"); writer.WriteStartArray(); - // First pass: count attempts per UID so we can annotate rows that share a UID - // with retry/attempt metadata in `extra`. Multiple terminal results per UID - // are intentional (parameterized rows, in-process retries) and must not be - // dropped. - Dictionary countByUid = []; - foreach (CapturedTestResult r in results) + foreach (CollapsedTestResult c in collapsed) { - countByUid[r.Uid] = countByUid.TryGetValue(r.Uid, out int existing) ? existing + 1 : 1; + WriteTest(writer, c); } - Dictionary emittedByUid = []; + writer.WriteEndArray(); + writer.WriteEndObject(); + + writer.WriteEndObject(); + } + + return ms.ToArray(); + } + + private static void WriteTest(Utf8JsonWriter writer, CollapsedTestResult c) + { + CapturedTestResult r = c.Final; + writer.WriteStartObject(); + + // CTRF spec: tests[i].name MUST be a non-empty string. Fall back to UID + // (also non-empty) when the framework didn't supply a display name. + string name = RoslynString.IsNullOrEmpty(r.DisplayName) ? r.Uid : r.DisplayName; + writer.WriteString("name", name); + writer.WriteString("status", r.Status); + writer.WriteNumber("duration", (long)Math.Max(0, r.Duration.TotalMilliseconds)); + + if (r.StartTime is { } start) + { + writer.WriteNumber("start", start.ToUnixTimeMilliseconds()); + } + + if (r.EndTime is { } end) + { + writer.WriteNumber("stop", end.ToUnixTimeMilliseconds()); + } + + if (r.RawStatus is not null) + { + writer.WriteString("rawStatus", r.RawStatus); + } - foreach (CapturedTestResult r in results) + // CTRF `suite` is an array of strings (minItems: 1) representing the test + // hierarchy (e.g. ["MyNamespace", "MyClass"]). + if (r.Namespace is not null || r.ClassName is not null) + { + writer.WritePropertyName("suite"); + writer.WriteStartArray(); + if (r.Namespace is not null) { - int attemptOf = countByUid[r.Uid]; - int attemptIndex = emittedByUid.TryGetValue(r.Uid, out int alreadyEmitted) ? alreadyEmitted + 1 : 1; - emittedByUid[r.Uid] = attemptIndex; + writer.WriteStringValue(r.Namespace); + } - writer.WriteStartObject(); + if (r.ClassName is not null) + { + writer.WriteStringValue(r.ClassName); + } - writer.WriteString("name", r.DisplayName); - writer.WriteString("status", r.Status); - writer.WriteNumber("duration", (long)Math.Max(0, r.Duration.TotalMilliseconds)); + writer.WriteEndArray(); + } - if (r.StartTime is { } start) - { - writer.WriteNumber("start", start.ToUnixTimeMilliseconds()); - } + if (r.ErrorMessage is not null) + { + writer.WriteString("message", r.ErrorMessage); + } - if (r.EndTime is { } end) - { - writer.WriteNumber("stop", end.ToUnixTimeMilliseconds()); - } + if (r.StackTrace is not null) + { + writer.WriteString("trace", r.StackTrace); + } - if (r.RawStatus is not null) - { - writer.WriteString("rawStatus", r.RawStatus); - } + if (r.FilePath is not null) + { + writer.WriteString("filePath", r.FilePath); + } - // CTRF `suite` is an array of strings (minItems: 1) representing the test - // hierarchy (e.g. ["MyNamespace", "MyClass"]). - if (r.Namespace is not null || r.ClassName is not null) - { - writer.WritePropertyName("suite"); - writer.WriteStartArray(); - if (r.Namespace is not null) - { - writer.WriteStringValue(r.Namespace); - } + if (r.Line is { } lineNumber) + { + writer.WriteNumber("line", lineNumber); + } - if (r.ClassName is not null) - { - writer.WriteStringValue(r.ClassName); - } + if (c.PriorAttempts.Count > 0) + { + writer.WriteNumber("retries", c.PriorAttempts.Count); + writer.WritePropertyName("retryAttempts"); + writer.WriteStartArray(); + for (int i = 0; i < c.PriorAttempts.Count; i++) + { + WriteRetryAttempt(writer, c.PriorAttempts[i], attemptNumber: i + 1); + } - writer.WriteEndArray(); - } + writer.WriteEndArray(); + } - if (r.ErrorMessage is not null) - { - writer.WriteString("message", r.ErrorMessage); - } + if (c.IsFlaky) + { + writer.WriteBoolean("flaky", true); + } - if (r.StackTrace is not null) - { - writer.WriteString("trace", r.StackTrace); - } + if (r.StandardOutput is not null) + { + writer.WritePropertyName("stdout"); + writer.WriteStartArray(); + writer.WriteStringValue(r.StandardOutput); + writer.WriteEndArray(); + } - if (r.FilePath is not null) - { - writer.WriteString("filePath", r.FilePath); - } + if (r.StandardError is not null) + { + writer.WritePropertyName("stderr"); + writer.WriteStartArray(); + writer.WriteStringValue(r.StandardError); + writer.WriteEndArray(); + } - if (r.Line is { } lineNumber) - { - writer.WriteNumber("line", lineNumber); - } + // CTRF `labels` is reserved for user-controlled, classification-style + // metadata (priority, severity, external IDs, etc.). We only emit the + // traits collected from MTP TestMetadataProperty here. Synthetic + // framework-generated metadata (method name, exception type, MTP UID) + // lives in the per-test `extra` object instead so CTRF consumers can + // filter/group by labels without seeing our internals. + if (r.Traits is { Count: > 0 }) + { + writer.WritePropertyName("labels"); + writer.WriteStartObject(); + foreach (KeyValuePair trait in r.Traits) + { + writer.WriteString(trait.Key, trait.Value); + } - if (attemptOf > 1) - { - writer.WriteNumber("retries", attemptOf - 1); - } + writer.WriteEndObject(); + } - if (r.StandardOutput is not null) - { - writer.WritePropertyName("stdout"); - writer.WriteStartArray(); - writer.WriteStringValue(r.StandardOutput); - writer.WriteEndArray(); - } + // CTRF `extra` (free-form object) — the CTRF spec doesn't define a + // dedicated stable identifier so we surface the MTP UID here for + // cross-tool correlation, alongside other framework metadata. + writer.WritePropertyName("extra"); + writer.WriteStartObject(); + writer.WriteString("uid", r.Uid); + if (r.MethodName is not null) + { + writer.WriteString("method", r.MethodName); + } - if (r.StandardError is not null) - { - writer.WritePropertyName("stderr"); - writer.WriteStartArray(); - writer.WriteStringValue(r.StandardError); - writer.WriteEndArray(); - } + if (r.ExceptionType is not null) + { + writer.WriteString("exceptionType", r.ExceptionType); + } - if (r.Traits is { Count: > 0 } || r.MethodName is not null - || r.ExceptionType is not null || attemptOf > 1) - { - writer.WritePropertyName("labels"); - writer.WriteStartObject(); - if (r.Traits is { Count: > 0 }) - { - foreach (KeyValuePair trait in r.Traits) - { - writer.WriteString(trait.Key, trait.Value); - } - } + writer.WriteEndObject(); - if (r.MethodName is not null) - { - writer.WriteString("method", r.MethodName); - } + writer.WriteEndObject(); + } - if (r.ExceptionType is not null) - { - writer.WriteString("exceptionType", r.ExceptionType); - } + private static void WriteRetryAttempt(Utf8JsonWriter writer, CapturedTestResult attempt, int attemptNumber) + { + writer.WriteStartObject(); + writer.WriteNumber("attempt", attemptNumber); + writer.WriteString("status", attempt.Status); + writer.WriteNumber("duration", (long)Math.Max(0, attempt.Duration.TotalMilliseconds)); + if (attempt.StartTime is { } start) + { + writer.WriteNumber("start", start.ToUnixTimeMilliseconds()); + } - if (attemptOf > 1) - { - writer.WriteString("attemptIndex", attemptIndex.ToString(CultureInfo.InvariantCulture)); - writer.WriteString("attemptOf", attemptOf.ToString(CultureInfo.InvariantCulture)); - } + if (attempt.EndTime is { } end) + { + writer.WriteNumber("stop", end.ToUnixTimeMilliseconds()); + } - writer.WriteEndObject(); - } + if (attempt.ErrorMessage is not null) + { + writer.WriteString("message", attempt.ErrorMessage); + } - // CTRF `extra` (free-form) for the test UID — the CTRF spec doesn't - // define a dedicated stable identifier so we surface the MTP UID here - // for cross-tool correlation. - writer.WritePropertyName("extra"); - writer.WriteStartObject(); - writer.WriteString("uid", r.Uid); - writer.WriteEndObject(); + if (attempt.StackTrace is not null) + { + writer.WriteString("trace", attempt.StackTrace); + } - writer.WriteEndObject(); - } + if (attempt.Line is { } line) + { + writer.WriteNumber("line", line); + } + if (attempt.StandardOutput is not null) + { + writer.WritePropertyName("stdout"); + writer.WriteStartArray(); + writer.WriteStringValue(attempt.StandardOutput); writer.WriteEndArray(); - writer.WriteEndObject(); + } + + if (attempt.StandardError is not null) + { + writer.WritePropertyName("stderr"); + writer.WriteStartArray(); + writer.WriteStringValue(attempt.StandardError); + writer.WriteEndArray(); + } + + if (attempt.RawStatus is not null || attempt.ExceptionType is not null) + { + writer.WritePropertyName("extra"); + writer.WriteStartObject(); + if (attempt.RawStatus is not null) + { + writer.WriteString("rawStatus", attempt.RawStatus); + } + + if (attempt.ExceptionType is not null) + { + writer.WriteString("exceptionType", attempt.ExceptionType); + } writer.WriteEndObject(); } - return ms.ToArray(); + writer.WriteEndObject(); + } + + private static List CollapseAttempts(CapturedTestResult[] results) + { + // For each UID, group consecutive captures: the latest entry becomes the + // final test record, earlier entries become `retryAttempts[]`. Preserves + // insertion order of unique UIDs in the output (stable across runs). + var byUid = new Dictionary(StringComparer.Ordinal); + var collapsed = new List(results.Length); + foreach (CapturedTestResult r in results) + { + if (byUid.TryGetValue(r.Uid, out int existingIndex)) + { + CollapsedTestResult existing = collapsed[existingIndex]; + existing.PriorAttempts.Add(existing.Final); + collapsed[existingIndex] = existing with { Final = r }; + } + else + { + byUid.Add(r.Uid, collapsed.Count); + collapsed.Add(new CollapsedTestResult(r)); + } + } + + return collapsed; + } + + private readonly record struct CollapsedTestResult(CapturedTestResult Final) + { + public List PriorAttempts { get; } = []; + + // CTRF "flaky" is true iff the final status is "passed" AND at least one + // previous attempt failed. + public bool IsFlaky + { + get + { + if (Final.Status != "passed" || PriorAttempts.Count == 0) + { + return false; + } + + foreach (CapturedTestResult attempt in PriorAttempts) + { + if (attempt.Status == "failed") + { + return true; + } + } + + return false; + } + } } } diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs index 6d44777132..8724861217 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs @@ -167,7 +167,7 @@ public async Task GenerateReportAsync_CountsAllOutcomeKindsSeparately() } [TestMethod] - public async Task GenerateReportAsync_PreservesAllResultsForDuplicateUids_AndAnnotatesAttemptOf() + public async Task GenerateReportAsync_CollapsesDuplicateUidsIntoRetryAttempts_AndFlagsFlaky() { using var memoryStream = new MemoryFileStream(); CtrfReportEngine engine = CreateEngine(memoryStream); @@ -184,51 +184,55 @@ public async Task GenerateReportAsync_PreservesAllResultsForDuplicateUids_AndAnn using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); JsonElement results = document.RootElement.GetProperty("results"); JsonElement testArray = results.GetProperty("tests"); - Assert.AreEqual(4, testArray.GetArrayLength()); - // All four rows must appear (no de-duplication on UID). - var names = new List(); - foreach (JsonElement t in testArray.EnumerateArray()) - { - names.Add(t.GetProperty("name").GetString()!); - } + // Duplicate-UID captures must collapse into a single CTRF test entry; the + // earlier attempts are recorded as nested retryAttempts[]. Top-level + // entries should therefore equal the number of unique UIDs. + Assert.AreEqual(2, testArray.GetArrayLength(), "Duplicate UIDs must collapse to one tests[] row."); - CollectionAssert.AreEquivalent(new[] { "Row A", "Row B", "Row C", "Solo" }, names); + JsonElement summary = results.GetProperty("summary"); + Assert.AreEqual(2, summary.GetProperty("tests").GetInt32(), "summary.tests must count unique UIDs only."); + Assert.AreEqual(2, summary.GetProperty("passed").GetInt32()); + Assert.AreEqual(0, summary.GetProperty("failed").GetInt32()); + Assert.AreEqual(1, summary.GetProperty("flaky").GetInt32()); - // Each duplicate row carries retries=2 + attemptIndex/attemptOf in labels. - var retries = new List(); - var attemptIndices = new List(); + JsonElement dupRow = default; + JsonElement soloRow = default; foreach (JsonElement t in testArray.EnumerateArray()) { - string name = t.GetProperty("name").GetString()!; - if (name.StartsWith("Row ", StringComparison.Ordinal)) + if (t.GetProperty("name").GetString() == "Row C") { - Assert.AreEqual(2, t.GetProperty("retries").GetInt32(), $"Row {name} should report 2 retries (3 total attempts)."); - Assert.IsTrue(t.GetProperty("labels").TryGetProperty("attemptIndex", out JsonElement idx)); - attemptIndices.Add(idx.GetString()); - Assert.AreEqual("3", t.GetProperty("labels").GetProperty("attemptOf").GetString()); + dupRow = t; } - } - - CollectionAssert.AreEquivalent(new[] { "1", "2", "3" }, attemptIndices); - - // The unique UID row does not get a retries field or attempts annotation. - JsonElement solo = default; - foreach (JsonElement t in testArray.EnumerateArray()) - { - if (t.GetProperty("name").GetString() == "Solo") + else if (t.GetProperty("name").GetString() == "Solo") { - solo = t; - break; + soloRow = t; } } - Assert.IsFalse(solo.TryGetProperty("retries", out _), "Unique UIDs should not carry retries."); - if (solo.TryGetProperty("labels", out JsonElement labels)) + Assert.AreNotEqual(JsonValueKind.Undefined, dupRow.ValueKind, "Final attempt name must surface as the collapsed test name."); + Assert.AreEqual("passed", dupRow.GetProperty("status").GetString()); + Assert.AreEqual(2, dupRow.GetProperty("retries").GetInt32(), "retries must equal the number of prior attempts."); + Assert.IsTrue(dupRow.GetProperty("flaky").GetBoolean(), "passed-after-failed runs must be marked flaky."); + + JsonElement retryAttempts = dupRow.GetProperty("retryAttempts"); + Assert.AreEqual(2, retryAttempts.GetArrayLength(), "retryAttempts must record every prior attempt."); + + var attemptNumbers = new List(); + var attemptStatuses = new List(); + foreach (JsonElement a in retryAttempts.EnumerateArray()) { - Assert.IsFalse(labels.TryGetProperty("attemptIndex", out _), "Unique UIDs should not carry attemptIndex."); - Assert.IsFalse(labels.TryGetProperty("attemptOf", out _), "Unique UIDs should not carry attemptOf."); + attemptNumbers.Add(a.GetProperty("attempt").GetInt32()); + attemptStatuses.Add(a.GetProperty("status").GetString()!); } + + CollectionAssert.AreEqual(new[] { 1, 2 }, attemptNumbers); + CollectionAssert.AreEqual(new[] { "failed", "failed" }, attemptStatuses); + + // Single-attempt entries must not surface retry metadata. + Assert.IsFalse(soloRow.TryGetProperty("retries", out _)); + Assert.IsFalse(soloRow.TryGetProperty("retryAttempts", out _)); + Assert.IsFalse(soloRow.TryGetProperty("flaky", out _)); } [TestMethod] @@ -539,6 +543,173 @@ private static CapturedTestResult CapturedRaw(string uid, string name, string st Duration = TimeSpan.Zero, }; + [TestMethod] + public async Task GenerateReportAsync_Environment_HasSchemaCompliantShape() + { + // CTRF schema (additionalProperties: false on environment): + // * `extra` MUST be an object — emitting a string here breaks strict validators. + // * `osPlatform` is the short identifier (win32/linux/darwin/...); the full + // descriptive string belongs in `osVersion`. + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = [Captured("u", "T", "passed")]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement env = document.RootElement.GetProperty("results").GetProperty("environment"); + + // `extra` must be an object (the most common spec violation). + JsonElement extra = env.GetProperty("extra"); + Assert.AreEqual(JsonValueKind.Object, extra.ValueKind, "environment.extra MUST be a JSON object per CTRF schema."); + Assert.AreEqual("user", extra.GetProperty("user").GetString()); + Assert.AreEqual("MachineName", extra.GetProperty("machine").GetString()); + Assert.AreEqual(0, extra.GetProperty("exitCode").GetInt32()); + Assert.AreEqual("TestAppPath", extra.GetProperty("testApplication").GetString()); + + // `osPlatform` is one of the short identifiers, not the descriptive name. + string osPlatform = env.GetProperty("osPlatform").GetString()!; + CollectionAssert.Contains(new[] { "win32", "linux", "darwin", "freebsd", "unknown" }, osPlatform); + + // Descriptive OS string goes into `osVersion`. + Assert.IsGreaterThan(0, env.GetProperty("osVersion").GetString()!.Length); + } + + [TestMethod] + public async Task GenerateReportAsync_TestExtra_CarriesMethodNameAndExceptionType() + { + // method, exceptionType, and uid all live under `extra` (not `labels`), + // so `labels` is reserved for user-controlled trait metadata only. + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + new CapturedTestResult + { + Uid = "u-1", + DisplayName = "T", + Status = "failed", + Duration = TimeSpan.Zero, + MethodName = "MyMethod", + ExceptionType = "System.InvalidOperationException", + }, + ]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement test = document.RootElement.GetProperty("results").GetProperty("tests")[0]; + + JsonElement extra = test.GetProperty("extra"); + Assert.AreEqual("u-1", extra.GetProperty("uid").GetString()); + Assert.AreEqual("MyMethod", extra.GetProperty("method").GetString()); + Assert.AreEqual("System.InvalidOperationException", extra.GetProperty("exceptionType").GetString()); + + // No `labels` is emitted when there are no user traits — `labels` is for + // classification (priority/severity/owner), not framework internals. + Assert.IsFalse(test.TryGetProperty("labels", out _), "Synthetic framework metadata must not appear in labels."); + } + + [TestMethod] + public async Task GenerateReportAsync_ToolName_FallsBackToUnknown_WhenFrameworkDisplayNameEmpty() + { + // CTRF spec: results.tool.name MUST be a non-empty string. + using var memoryStream = new MemoryFileStream(); + _ = _testFrameworkMock.SetupGet(_ => _.DisplayName).Returns(string.Empty); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = [Captured("u", "T", "passed")]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement tool = document.RootElement.GetProperty("results").GetProperty("tool"); + string name = tool.GetProperty("name").GetString()!; + Assert.IsGreaterThan(0, name.Length, "CTRF requires tool.name to be a non-empty string."); + } + + [TestMethod] + public async Task GenerateReportAsync_EmptyResults_ProducesValidDocument() + { + // The summary block must still be present with zeroed counts when no tests + // ran (the schema requires summary fields to exist, and `tests[]` should be + // an empty array rather than absent). + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + + await engine.GenerateReportAsync([]); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement results = document.RootElement.GetProperty("results"); + JsonElement summary = results.GetProperty("summary"); + Assert.AreEqual(0, summary.GetProperty("tests").GetInt32()); + Assert.AreEqual(0, summary.GetProperty("passed").GetInt32()); + Assert.AreEqual(0, summary.GetProperty("failed").GetInt32()); + Assert.AreEqual(0, summary.GetProperty("skipped").GetInt32()); + Assert.AreEqual(0, summary.GetProperty("pending").GetInt32()); + Assert.AreEqual(0, summary.GetProperty("other").GetInt32()); + Assert.AreEqual(0, summary.GetProperty("flaky").GetInt32()); + + JsonElement testsArray = results.GetProperty("tests"); + Assert.AreEqual(JsonValueKind.Array, testsArray.ValueKind); + Assert.AreEqual(0, testsArray.GetArrayLength()); + } + + [TestMethod] + public async Task GenerateReportAsync_TestName_IsNeverEmpty() + { + // CTRF schema: tests[i].name has minLength: 1. We must surface a non-empty + // value even if the test framework forwarded an empty DisplayName. + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + new CapturedTestResult + { + Uid = "uid-with-no-display-name", + DisplayName = string.Empty, + Status = "passed", + Duration = TimeSpan.Zero, + }, + ]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement test = document.RootElement.GetProperty("results").GetProperty("tests")[0]; + string name = test.GetProperty("name").GetString()!; + Assert.IsGreaterThan(0, name.Length, "CTRF requires tests[].name to be non-empty (minLength: 1)."); + Assert.AreEqual("uid-with-no-display-name", name, "Empty DisplayName must fall back to UID."); + } + + [TestMethod] + public async Task GenerateReportAsync_SpecialCharactersInName_AreSafelyEscaped() + { + // Test that names with HTML/JS metacharacters are escaped (no + // UnsafeRelaxedJsonEscaping). The bytes must contain a unicode-escaped + // form rather than the raw `", "passed"), + ]; + + await engine.GenerateReportAsync(tests); + + byte[] raw = System.Text.Encoding.UTF8.GetBytes(memoryStream.GetUtf8Content()); + string rawString = System.Text.Encoding.UTF8.GetString(raw); + + // The raw `<` must not appear in the encoded JSON — it must be \u003C. + Assert.IsFalse(rawString.Contains("", test.GetProperty("name").GetString()); + } + private CtrfReportEngine CreateEngine(MemoryFileStream stream) { _ = _fileSystem.Setup(x => x.ExistFile(It.IsAny())).Returns(false); From 288620d5c08ed37e106e8201f1d0857a9e5ebcb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Mon, 8 Jun 2026 10:29:27 +0200 Subject: [PATCH 03/17] Add CtrfPlayground sample (MSTest + xunit.v3) for side-by-side CTRF comparison Provides two MTP test apps that generate CTRF reports for the same set of tests (pass / fail / skip / theory / throw) using two different generators: - samples/CtrfPlayground/Mtp: MSTest + Microsoft.Testing.Extensions.CtrfReport (the experimental extension shipped by this repository) - samples/CtrfPlayground/XunitMtp: xunit.v3 with its built-in '-ctrf ' reporter This allows reviewers to diff the CTRF JSON produced by both stacks against the same test fixtures. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- TestFx.slnx | 2 + samples/CtrfPlayground/Mtp/Mtp.csproj | 28 ++++++++++++++ samples/CtrfPlayground/Mtp/Program.cs | 22 +++++++++++ samples/CtrfPlayground/Mtp/Tests.cs | 35 ++++++++++++++++++ samples/CtrfPlayground/README.md | 37 +++++++++++++++++++ samples/CtrfPlayground/XunitMtp/Tests.cs | 33 +++++++++++++++++ .../CtrfPlayground/XunitMtp/XunitMtp.csproj | 17 +++++++++ 7 files changed, 174 insertions(+) create mode 100644 samples/CtrfPlayground/Mtp/Mtp.csproj create mode 100644 samples/CtrfPlayground/Mtp/Program.cs create mode 100644 samples/CtrfPlayground/Mtp/Tests.cs create mode 100644 samples/CtrfPlayground/README.md create mode 100644 samples/CtrfPlayground/XunitMtp/Tests.cs create mode 100644 samples/CtrfPlayground/XunitMtp/XunitMtp.csproj diff --git a/TestFx.slnx b/TestFx.slnx index 6470bc7c3f..b2562ecb26 100644 --- a/TestFx.slnx +++ b/TestFx.slnx @@ -1,5 +1,7 @@ + + diff --git a/samples/CtrfPlayground/Mtp/Mtp.csproj b/samples/CtrfPlayground/Mtp/Mtp.csproj new file mode 100644 index 0000000000..8c2ca53541 --- /dev/null +++ b/samples/CtrfPlayground/Mtp/Mtp.csproj @@ -0,0 +1,28 @@ + + + + Exe + net9.0 + enable + $(NoWarn);NETSDK1023;SA0001;EnableGenerateDocumentationFile;TPEXP + + true + false + false + + + + + + + + + + + + + + + + + diff --git a/samples/CtrfPlayground/Mtp/Program.cs b/samples/CtrfPlayground/Mtp/Program.cs new file mode 100644 index 0000000000..6259c50a5c --- /dev/null +++ b/samples/CtrfPlayground/Mtp/Program.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Reflection; + +using Microsoft.Testing.Extensions; +using Microsoft.Testing.Platform.Builder; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace CtrfPlayground.Mtp; + +public static class Program +{ + public static async Task Main(string[] args) + { + ITestApplicationBuilder testApplicationBuilder = await TestApplication.CreateBuilderAsync(args); + testApplicationBuilder.AddMSTest(() => [Assembly.GetEntryAssembly()!]); + testApplicationBuilder.AddCtrfReportProvider(); + using ITestApplication app = await testApplicationBuilder.BuildAsync(); + return await app.RunAsync(); + } +} diff --git a/samples/CtrfPlayground/Mtp/Tests.cs b/samples/CtrfPlayground/Mtp/Tests.cs new file mode 100644 index 0000000000..173d084b8d --- /dev/null +++ b/samples/CtrfPlayground/Mtp/Tests.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace CtrfPlayground.Mtp; + +[TestClass] +public class SampleTests +{ + [TestMethod] + public void PassingTest() + => Assert.AreEqual(2, 1 + 1); + + [TestMethod] + public void FailingTest() + => Assert.AreEqual(3, 1 + 1, "intentional failure to exercise CTRF status mapping"); + + [TestMethod] + [Ignore("intentionally skipped to exercise CTRF status mapping")] + public void SkippedTest() + { + } + + [TestMethod] + public void ThrowingTest() + => throw new InvalidOperationException("intentional exception to exercise CTRF error fields"); + + [DataTestMethod] + [DataRow(1, 1, 2)] + [DataRow(2, 3, 5)] + [DataRow(2, 2, 5)] // intentional failure + public void DataDrivenTest(int a, int b, int expected) + => Assert.AreEqual(expected, a + b); +} diff --git a/samples/CtrfPlayground/README.md b/samples/CtrfPlayground/README.md new file mode 100644 index 0000000000..ca7427db5c --- /dev/null +++ b/samples/CtrfPlayground/README.md @@ -0,0 +1,37 @@ +# CtrfPlayground + +This sample produces side-by-side [CTRF (Common Test Report Format)](https://ctrf.io/) reports +from the same set of tests using two different CTRF generators that both target +[Microsoft.Testing.Platform](https://aka.ms/testingplatform): + +| Project | Test framework | CTRF generator | +| ------- | -------------- | -------------- | +| [`Mtp`](./Mtp) | MSTest | `Microsoft.Testing.Extensions.CtrfReport` (this repository, **experimental**) | +| [`XunitMtp`](./XunitMtp) | [`xunit.v3`](https://www.nuget.org/packages/xunit.v3) | `xunit.v3`'s built-in `-ctrf ` reporter | + +The tests in each project are intentionally equivalent (pass / fail / skip / theory / throw) +so that the two CTRF reports can be diffed to validate the parity of the new extension. + +## Running + +From the repository root: + +```pwsh +# MSTest + Microsoft.Testing.Extensions.CtrfReport +dotnet run --project samples/CtrfPlayground/Mtp -- --report-ctrf --results-directory ./out/mtp + +# xunit.v3 with its built-in CTRF reporter +dotnet run --project samples/CtrfPlayground/XunitMtp -- -ctrf ./out/xunit/ctrf-report.json +``` + +Both runs write a CTRF JSON file. Open them side by side (e.g. +`code -d ./out/mtp/.ctrf.json ./out/xunit/ctrf-report.json`) to compare the two outputs. + +> [!NOTE] +> The CTRF extension shipped by this repository is currently marked **experimental** +> (`[Experimental("TPEXP")]`). API shape and report content may change. + +> [!TIP] +> The two projects are intentionally **kept separate**: combining a CTRF-producing +> extension with another test framework that already exposes its own CTRF reporter would +> conflict on command-line options inside a single test host. diff --git a/samples/CtrfPlayground/XunitMtp/Tests.cs b/samples/CtrfPlayground/XunitMtp/Tests.cs new file mode 100644 index 0000000000..db7e614893 --- /dev/null +++ b/samples/CtrfPlayground/XunitMtp/Tests.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Xunit; + +namespace CtrfPlayground.XunitMtp; + +public class SampleTests +{ + [Fact] + public void PassingTest() + => Assert.Equal(2, 1 + 1); + + [Fact] + public void FailingTest() + => Assert.Fail("intentional failure to exercise CTRF status mapping"); + + [Fact(Skip = "intentionally skipped to exercise CTRF status mapping")] + public void SkippedTest() + { + } + + [Fact] + public void ThrowingTest() + => throw new InvalidOperationException("intentional exception to exercise CTRF error fields"); + + [Theory] + [InlineData(1, 1, 2)] + [InlineData(2, 3, 5)] + [InlineData(2, 2, 5)] // intentional failure + public void DataDrivenTest(int a, int b, int expected) + => Assert.Equal(expected, a + b); +} diff --git a/samples/CtrfPlayground/XunitMtp/XunitMtp.csproj b/samples/CtrfPlayground/XunitMtp/XunitMtp.csproj new file mode 100644 index 0000000000..89c5c7b345 --- /dev/null +++ b/samples/CtrfPlayground/XunitMtp/XunitMtp.csproj @@ -0,0 +1,17 @@ + + + + Exe + net9.0 + enable + false + $(NoWarn);NETSDK1023;SA0001;EnableGenerateDocumentationFile + + true + + + + + + + From f7aeac3ecd9a26cef2599a6a954c43a8d48af554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Mon, 8 Jun 2026 10:45:18 +0200 Subject: [PATCH 04/17] CtrfReport: use alpha pre-release label for experimental packages Match the versioning scheme used by other experimental Microsoft.Testing.Platform extensions (Logging, OpenTelemetry, AI). Packages will now ship as 1.0.0-alpha.* instead of the repo-default *-preview.*. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Microsoft.Testing.Extensions.CtrfReport.csproj | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj index 52798c8e65..8f74a9894d 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj @@ -2,6 +2,9 @@ netstandard2.0;$(SupportedNetFrameworks) + 1.0.0 + alpha + true true $(RepoRoot)src\Platform\SharedExtensionHelpers\BuildInfo.cs.template From 2a0fd39de41dae31cbfa8d1d82df4077d80bf86a Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Mon, 8 Jun 2026 11:43:17 +0200 Subject: [PATCH 05/17] Convert CTRF acceptance test to full-content snapshot Replace structural contains-style assertions with a verbatim snapshot of the entire JSON document. Runtime-variable fields (GUID report id, ISO timestamp, epoch-ms times, extension version, OS info, user/machine, absolute test app path) are normalized via field-scoped regexes to deterministic tokens, while everything else (key order, indentation, conditional emission, dummy framework values) must match byte-for-byte. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../CtrfReportTests.cs | 109 ++++++++++++++---- 1 file changed, 86 insertions(+), 23 deletions(-) diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs index 09f2e72445..660f4e0092 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Text.Json; - namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; [TestClass] @@ -120,29 +118,94 @@ public async Task Ctrf_WhenReportCtrfFilenameIsSpecifiedWithoutReportCtrf_ErrorI private static void AssertCtrfReportShape(string filePath) { - string jsonContent = File.ReadAllText(filePath); - using var document = JsonDocument.Parse(jsonContent); - JsonElement root = document.RootElement; - - Assert.AreEqual("CTRF", root.GetProperty("reportFormat").GetString()); - Assert.AreEqual("0.0.0", root.GetProperty("specVersion").GetString()); - - JsonElement results = root.GetProperty("results"); - Assert.IsTrue(results.TryGetProperty("tool", out _), "CTRF report is missing 'results.tool'."); - Assert.IsTrue(results.TryGetProperty("summary", out JsonElement summary), "CTRF report is missing 'results.summary'."); - Assert.IsTrue(results.TryGetProperty("tests", out JsonElement tests), "CTRF report is missing 'results.tests'."); - Assert.AreEqual(JsonValueKind.Array, tests.ValueKind); - Assert.AreEqual(1, tests.GetArrayLength()); - - JsonElement firstTest = tests[0]; - Assert.AreEqual("PassingTest", firstTest.GetProperty("name").GetString()); - Assert.AreEqual("passed", firstTest.GetProperty("status").GetString()); - - Assert.AreEqual(1, summary.GetProperty("tests").GetInt32()); - Assert.AreEqual(1, summary.GetProperty("passed").GetInt32()); - Assert.AreEqual(0, summary.GetProperty("failed").GetInt32()); + // Snapshot the full CTRF JSON against an exact expected document. Runtime-variable + // fields (GUID report id, ISO timestamp, epoch-ms times, machine name, user name, + // OS info, the extension's own version, and the absolute path of the test + // application) are masked with deterministic tokens so the comparison is + // hermetic across machines and runs. Anything else — including key order, + // indentation, escaping, conditional-emission shape, and the actual values + // baked from the dummy test framework — must match byte-for-byte. + string actual = File.ReadAllText(filePath); + string normalized = NormalizeCtrfReport(actual); + + const string expected = """ +{ + "reportFormat": "CTRF", + "specVersion": "0.0.0", + "reportId": "", + "timestamp": "", + "generatedBy": "Microsoft.Testing.Extensions.CtrfReport@", + "results": { + "tool": { + "name": "DummyTestFramework", + "version": "2.0.0", + "extra": { + "uid": "DummyTestFramework" + } + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "flaky": 0, + "start": , + "stop": , + "duration": + }, + "environment": { + "osPlatform": "", + "osVersion": "", + "extra": { + "user": "", + "machine": "", + "exitCode": 0, + "testApplication": "" + } + }, + "tests": [ + { + "name": "PassingTest", + "status": "passed", + "duration": , + "extra": { + "uid": "test-1" + } + } + ] + } +} +"""; + + Assert.AreEqual( + NormalizeLineEndings(expected), + NormalizeLineEndings(normalized), + $"Generated CTRF JSON does not match the expected snapshot.\n\nNormalized actual:\n{normalized}\n\nRaw actual:\n{actual}"); } + private static string NormalizeCtrfReport(string actual) + { + // Field-scoped regexes so per-test attribute order and JSON shape are still + // anchored, but runtime-variable values are folded into stable tokens. + string normalized = actual; + normalized = Regex.Replace(normalized, @"""reportId"": ""[^""]+""", @"""reportId"": """""); + normalized = Regex.Replace(normalized, @"""timestamp"": ""[^""]+""", @"""timestamp"": """""); + normalized = Regex.Replace(normalized, @"""generatedBy"": ""Microsoft\.Testing\.Extensions\.CtrfReport@[^""]+""", @"""generatedBy"": ""Microsoft.Testing.Extensions.CtrfReport@"""); + normalized = Regex.Replace(normalized, @"""start"": \d+", @"""start"": "); + normalized = Regex.Replace(normalized, @"""stop"": \d+", @"""stop"": "); + normalized = Regex.Replace(normalized, @"""duration"": \d+", @"""duration"": "); + normalized = Regex.Replace(normalized, @"""osPlatform"": ""[^""]*""", @"""osPlatform"": """""); + normalized = Regex.Replace(normalized, @"""osVersion"": ""[^""]*""", @"""osVersion"": """""); + normalized = Regex.Replace(normalized, @"""user"": ""[^""]*""", @"""user"": """""); + normalized = Regex.Replace(normalized, @"""machine"": ""[^""]*""", @"""machine"": """""); + normalized = Regex.Replace(normalized, @"""testApplication"": ""[^""]*""", @"""testApplication"": """""); + return normalized; + } + + private static string NormalizeLineEndings(string s) => s.Replace("\r\n", "\n").Trim('\n'); + public sealed class TestAssetFixture() : TestAssetFixtureBase() { public const string AssetName = "CtrfReportTest"; From 4704b5421f54d075790339857bf71f88ce07c6b6 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Mon, 8 Jun 2026 12:12:12 +0200 Subject: [PATCH 06/17] Plumb experimental Ctrf package version through acceptance test asset After the CtrfReport package was moved to the experimental versioning scheme (1.0.0-alpha) it no longer matches the platform version requested by the acceptance test asset csproj. Introduce a dedicated MicrosoftTestingExtensionsCtrfReportVersion property on AcceptanceTestBase and a \\\$\ placeholder so the dummy app references the package at the version actually produced by the local pack. Also fix two pre-existing MSTEST0068 analyzer errors in CtrfReportEngineTests.cs that block packing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../CtrfReportTests.cs | 5 +++-- .../Helpers/AcceptanceTestBase.cs | 3 +++ .../CtrfReportEngineTests.cs | 6 +++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs index 660f4e0092..3494ca7056 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs @@ -221,7 +221,7 @@ public sealed class TestAssetFixture() : TestAssetFixtureBase() preview - + @@ -284,7 +284,8 @@ public async Task ExecuteRequestAsync(ExecuteRequestContext context) public override (string ID, string Name, string Code) GetAssetsToGenerate() => (AssetName, AssetName, TestCode .PatchTargetFrameworks(TargetFrameworks.All) - .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion)); + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) + .PatchCodeWithReplace("$MicrosoftTestingExtensionsCtrfReportVersion$", MicrosoftTestingExtensionsCtrfReportVersion)); } public TestContext TestContext { get; set; } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs index 7b5996a6d3..ff5fd5dce1 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs @@ -16,6 +16,7 @@ static AcceptanceTestBase() MicrosoftTestingPlatformVersion = ExtractVersionFromPackage(Constants.ArtifactsPackagesShipping, "Microsoft.Testing.Platform."); MSTestSourceGenerationVersion = ExtractVersionFromPackage(Constants.ArtifactsPackagesShipping, "MSTest.SourceGeneration."); MicrosoftTestingExtensionsLoggingVersion = ExtractVersionFromPackage(Constants.ArtifactsPackagesShipping, "Microsoft.Testing.Extensions.Logging."); + MicrosoftTestingExtensionsCtrfReportVersion = ExtractVersionFromPackage(Constants.ArtifactsPackagesShipping, "Microsoft.Testing.Extensions.CtrfReport."); } internal static string RID { get; } @@ -37,6 +38,8 @@ static AcceptanceTestBase() public static string MicrosoftTestingExtensionsLoggingVersion { get; private set; } + public static string MicrosoftTestingExtensionsCtrfReportVersion { get; private set; } + private static string ExtractVersionFromPackage(string rootFolder, string packagePrefixName) { string[] matches = Directory.GetFiles(rootFolder, packagePrefixName + "*" + NuGetPackageExtensionName, SearchOption.TopDirectoryOnly); diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs index 8724861217..321b03644a 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs @@ -226,8 +226,8 @@ public async Task GenerateReportAsync_CollapsesDuplicateUidsIntoRetryAttempts_An attemptStatuses.Add(a.GetProperty("status").GetString()!); } - CollectionAssert.AreEqual(new[] { 1, 2 }, attemptNumbers); - CollectionAssert.AreEqual(new[] { "failed", "failed" }, attemptStatuses); + Assert.AreSequenceEqual(new[] { 1, 2 }, attemptNumbers); + Assert.AreSequenceEqual(new[] { "failed", "failed" }, attemptStatuses); // Single-attempt entries must not surface retry metadata. Assert.IsFalse(soloRow.TryGetProperty("retries", out _)); @@ -569,7 +569,7 @@ public async Task GenerateReportAsync_Environment_HasSchemaCompliantShape() // `osPlatform` is one of the short identifiers, not the descriptive name. string osPlatform = env.GetProperty("osPlatform").GetString()!; - CollectionAssert.Contains(new[] { "win32", "linux", "darwin", "freebsd", "unknown" }, osPlatform); + Assert.Contains(osPlatform, new[] { "win32", "linux", "darwin", "freebsd", "unknown" }); // Descriptive OS string goes into `osVersion`. Assert.IsGreaterThan(0, env.GetProperty("osVersion").GetString()!.Length); From 22d2f1d2b15befd574abc44fb93e80fee56558b1 Mon Sep 17 00:00:00 2001 From: Evangelink Date: Mon, 8 Jun 2026 13:44:58 +0200 Subject: [PATCH 07/17] Fix XunitMtp comment and HelpInfoAllExtensions package version - Correct the misleading comment in samples/CtrfPlayground/XunitMtp/XunitMtp.csproj: xunit.v3 exposes its built-in CTRF reporter via the `-ctrf ` switch, not a generic `report-ctrf` option. - Switch the HelpInfoAllExtensionsTests asset csproj to the dedicated \\$ placeholder (the CTRF extension now follows the experimental versioning scheme and no longer matches \\$). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- samples/CtrfPlayground/XunitMtp/XunitMtp.csproj | 2 +- .../HelpInfoAllExtensionsTests.cs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/samples/CtrfPlayground/XunitMtp/XunitMtp.csproj b/samples/CtrfPlayground/XunitMtp/XunitMtp.csproj index 89c5c7b345..75b4b63807 100644 --- a/samples/CtrfPlayground/XunitMtp/XunitMtp.csproj +++ b/samples/CtrfPlayground/XunitMtp/XunitMtp.csproj @@ -11,7 +11,7 @@ - + diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs index 7666204ecd..31a07859fa 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs @@ -619,7 +619,7 @@ public sealed class TestAssetFixture() : TestAssetFixtureBase() - + @@ -678,7 +678,8 @@ public Task ExecuteRequestAsync(ExecuteRequestContext context) public override (string ID, string Name, string Code) GetAssetsToGenerate() => (AllExtensionsAssetName, AllExtensionsAssetName, AllExtensionsTestCode .PatchTargetFrameworks(TargetFrameworks.All) - .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion)); + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) + .PatchCodeWithReplace("$MicrosoftTestingExtensionsCtrfReportVersion$", MicrosoftTestingExtensionsCtrfReportVersion)); } public TestContext TestContext { get; set; } From 818f2011a1b040c95468bfc530b0275e956d7227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Mon, 8 Jun 2026 14:37:22 +0200 Subject: [PATCH 08/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../CtrfReportEngine.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs index 71a5b3ed14..612ebc34f3 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs @@ -628,9 +628,9 @@ private static void WriteRetryAttempt(Utf8JsonWriter writer, CapturedTestResult private static List CollapseAttempts(CapturedTestResult[] results) { - // For each UID, group consecutive captures: the latest entry becomes the - // final test record, earlier entries become `retryAttempts[]`. Preserves - // insertion order of unique UIDs in the output (stable across runs). + // For each UID, group all captures in arrival order: the latest entry becomes the + // final test record, earlier entries become `retryAttempts[]`. Preserves the + // insertion order of first-seen UIDs in the output (stable across runs). var byUid = new Dictionary(StringComparer.Ordinal); var collapsed = new List(results.Length); foreach (CapturedTestResult r in results) From dcd8e1c707723498f77900cd47350417df9d74bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Mon, 8 Jun 2026 14:39:12 +0200 Subject: [PATCH 09/17] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Amaury Levé --- samples/CtrfPlayground/XunitMtp/XunitMtp.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/CtrfPlayground/XunitMtp/XunitMtp.csproj b/samples/CtrfPlayground/XunitMtp/XunitMtp.csproj index 75b4b63807..2ec72e77db 100644 --- a/samples/CtrfPlayground/XunitMtp/XunitMtp.csproj +++ b/samples/CtrfPlayground/XunitMtp/XunitMtp.csproj @@ -7,11 +7,11 @@ false $(NoWarn);NETSDK1023;SA0001;EnableGenerateDocumentationFile - true + true - + From 5329d64bb64289266ed0b3d8891142342892edb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Mon, 8 Jun 2026 15:10:04 +0200 Subject: [PATCH 10/17] Address PR #8903 review: fix CtrfReport version, add MSTest acceptance test, expand snapshot, fix markdown lint Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- samples/CtrfPlayground/README.md | 1 + .../CtrfReportTests.cs | 104 ++++++++++++++++++ .../CtrfReportTests.cs | 97 +++++++++++++--- .../HelpInfoAllExtensionsTests.cs | 30 ++--- .../MSBuild.KnownExtensionRegistration.cs | 5 +- 5 files changed, 207 insertions(+), 30 deletions(-) create mode 100644 test/IntegrationTests/MSTest.Acceptance.IntegrationTests/CtrfReportTests.cs diff --git a/samples/CtrfPlayground/README.md b/samples/CtrfPlayground/README.md index ca7427db5c..c83bec86ed 100644 --- a/samples/CtrfPlayground/README.md +++ b/samples/CtrfPlayground/README.md @@ -31,6 +31,7 @@ Both runs write a CTRF JSON file. Open them side by side (e.g. > The CTRF extension shipped by this repository is currently marked **experimental** > (`[Experimental("TPEXP")]`). API shape and report content may change. + > [!TIP] > The two projects are intentionally **kept separate**: combining a CTRF-producing > extension with another test framework that already exposes its own CTRF reporter would diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/CtrfReportTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/CtrfReportTests.cs new file mode 100644 index 0000000000..e6259ba5e0 --- /dev/null +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/CtrfReportTests.cs @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Acceptance.IntegrationTests; +using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; +using Microsoft.Testing.Platform.Helpers; + +namespace MSTest.Acceptance.IntegrationTests; + +[TestClass] +public sealed class CtrfReportTests : AcceptanceTestBase +{ + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task CtrfReport_WhenTestsArePassingAndFailing_CtrfFileIsGeneratedWithBothOutcomes(string tfm) + { + string fileName = Guid.NewGuid().ToString("N") + ".ctrf.json"; + var testHost = TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + $"--report-ctrf --report-ctrf-filename {fileName}", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.AtLeastOneTestFailed); + + string ctrfFile = Directory.GetFiles(testHost.DirectoryName, fileName, SearchOption.AllDirectories).Single(); + string ctrfContent = File.ReadAllText(ctrfFile); + + // Top-level CTRF document shape + Assert.Contains(@"""reportFormat"": ""CTRF""", ctrfContent, ctrfContent); + Assert.Contains(@"""specVersion""", ctrfContent, ctrfContent); + Assert.Contains(@"""generatedBy"": ""Microsoft.Testing.Extensions.CtrfReport@", ctrfContent, ctrfContent); + + // Summary counts include both outcomes + Assert.Contains(@"""tests"": 2", ctrfContent, ctrfContent); + Assert.Contains(@"""passed"": 1", ctrfContent, ctrfContent); + Assert.Contains(@"""failed"": 1", ctrfContent, ctrfContent); + + // Both per-test entries are present + Assert.Contains(@"""name"": ""PassingTest""", ctrfContent, ctrfContent); + Assert.Contains(@"""name"": ""FailingTest""", ctrfContent, ctrfContent); + Assert.Contains(@"""status"": ""passed""", ctrfContent, ctrfContent); + Assert.Contains(@"""status"": ""failed""", ctrfContent, ctrfContent); + + // The failed test exposes the assertion message in CTRF's `message` field. + Assert.Contains(@"""message"":", ctrfContent, ctrfContent); + Assert.Contains("Assert.AreEqual", ctrfContent, ctrfContent); + } + + public sealed class TestAssetFixture() : TestAssetFixtureBase() + { + public const string ProjectName = "MSTestCtrfReport"; + + public string TargetAssetPath => GetAssetPath(ProjectName); + + public override (string ID, string Name, string Code) GetAssetsToGenerate() => (ProjectName, ProjectName, + SourceCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) + .PatchCodeWithReplace("$MicrosoftTestingExtensionsCtrfReportVersion$", MicrosoftTestingExtensionsCtrfReportVersion) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + private const string SourceCode = """ +#file MSTestCtrfReport.csproj + + + + Exe + true + $TargetFrameworks$ + latest + + + + + + + + + + +#file UnitTest1.cs +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace MSTestCtrfReport; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + public void PassingTest() + { + } + + [TestMethod] + public void FailingTest() + { + Assert.AreEqual(1, 2); + } +} +"""; + } + + public TestContext TestContext { get; set; } = null!; +} diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs index 3494ca7056..48891a2b24 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs @@ -13,7 +13,9 @@ public async Task Ctrf_WhenReportCtrfIsNotSpecified_CtrfReportIsNotGenerated(str var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); - testHostResult.AssertExitCodeIs(ExitCode.Success); + // The dummy framework emits at least one failing test so the host exits with + // AtLeastOneTestFailed regardless of whether the CTRF report is enabled. + testHostResult.AssertExitCodeIs(ExitCode.AtLeastOneTestFailed); // The CTRF report is published as an in-process artifact; check the correct block. string outputPattern = """ @@ -33,7 +35,7 @@ public async Task Ctrf_WhenReportCtrfIsSpecified_CtrfReportIsGeneratedInDefaultL var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync("--report-ctrf", cancellationToken: TestContext.CancellationToken); - testHostResult.AssertExitCodeIs(ExitCode.Success); + testHostResult.AssertExitCodeIs(ExitCode.AtLeastOneTestFailed); string outputPattern = $""" In process file artifacts produced: @@ -61,7 +63,7 @@ public async Task Ctrf_WhenReportCtrfFilenameIsSpecified_CtrfReportIsGeneratedWi $"--report-ctrf --report-ctrf-filename {customFileName}", cancellationToken: TestContext.CancellationToken); - testHostResult.AssertExitCodeIs(ExitCode.Success); + testHostResult.AssertExitCodeIs(ExitCode.AtLeastOneTestFailed); string outputPattern = $""" In process file artifacts produced: @@ -90,7 +92,7 @@ public async Task Ctrf_WhenReportCtrfFilenameContainsPath_CtrfReportIsGeneratedI $"--report-ctrf --report-ctrf-filename {customFileName}", cancellationToken: TestContext.CancellationToken); - testHostResult.AssertExitCodeIs(ExitCode.Success); + testHostResult.AssertExitCodeIs(ExitCode.AtLeastOneTestFailed); string outputPattern = $""" In process file artifacts produced: @@ -120,11 +122,19 @@ private static void AssertCtrfReportShape(string filePath) { // Snapshot the full CTRF JSON against an exact expected document. Runtime-variable // fields (GUID report id, ISO timestamp, epoch-ms times, machine name, user name, - // OS info, the extension's own version, and the absolute path of the test - // application) are masked with deterministic tokens so the comparison is - // hermetic across machines and runs. Anything else — including key order, - // indentation, escaping, conditional-emission shape, and the actual values + // OS info, the extension's own version, the absolute path of the test application, + // and the test host exit code) are masked with deterministic tokens so the + // comparison is hermetic across machines and runs. Anything else — including key + // order, indentation, escaping, conditional-emission shape, and the actual values // baked from the dummy test framework — must match byte-for-byte. + // + // The dummy framework emits three tests that exercise the CTRF status and retry + // model end to end: + // - PassingTest → status: "passed" + // - FailingTest → status: "failed" + `message` + // - FlakyTest (retried) → first attempt failed, retried successfully: + // final status: "passed", retries: 1, + // retryAttempts[].status: "failed", flaky: true string actual = File.ReadAllText(filePath); string normalized = NormalizeCtrfReport(actual); @@ -144,13 +154,13 @@ private static void AssertCtrfReportShape(string filePath) } }, "summary": { - "tests": 1, - "passed": 1, - "failed": 0, + "tests": 3, + "passed": 2, + "failed": 1, "skipped": 0, "pending": 0, "other": 0, - "flaky": 0, + "flaky": 1, "start": , "stop": , "duration": @@ -161,7 +171,7 @@ private static void AssertCtrfReportShape(string filePath) "extra": { "user": "", "machine": "", - "exitCode": 0, + "exitCode": , "testApplication": "" } }, @@ -173,6 +183,33 @@ private static void AssertCtrfReportShape(string filePath) "extra": { "uid": "test-1" } + }, + { + "name": "FailingTest", + "status": "failed", + "duration": , + "message": "Expected 1 but got 2", + "extra": { + "uid": "test-2" + } + }, + { + "name": "FlakyTest", + "status": "passed", + "duration": , + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "duration": , + "message": "Transient failure" + } + ], + "flaky": true, + "extra": { + "uid": "test-3" + } } ] } @@ -200,6 +237,7 @@ private static string NormalizeCtrfReport(string actual) normalized = Regex.Replace(normalized, @"""osVersion"": ""[^""]*""", @"""osVersion"": """""); normalized = Regex.Replace(normalized, @"""user"": ""[^""]*""", @"""user"": """""); normalized = Regex.Replace(normalized, @"""machine"": ""[^""]*""", @"""machine"": """""); + normalized = Regex.Replace(normalized, @"""exitCode"": -?\d+", @"""exitCode"": "); normalized = Regex.Replace(normalized, @"""testApplication"": ""[^""]*""", @"""testApplication"": """""); return normalized; } @@ -266,6 +304,7 @@ public Task CloseTestSessionAsync(CloseTestSessionContex public async Task ExecuteRequestAsync(ExecuteRequestContext context) { + // 1) A plain passing test. await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage( context.Request.Session.SessionUid, new TestNode() @@ -274,6 +313,38 @@ public async Task ExecuteRequestAsync(ExecuteRequestContext context) DisplayName = "PassingTest", Properties = new PropertyBag(PassedTestNodeStateProperty.CachedInstance), })); + + // 2) A plain failing test (no Exception object so no stack trace / exception type + // are emitted; only the explanation propagates as CTRF `message`). + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage( + context.Request.Session.SessionUid, + new TestNode() + { + Uid = "test-2", + DisplayName = "FailingTest", + Properties = new PropertyBag(new FailedTestNodeStateProperty("Expected 1 but got 2")), + })); + + // 3) A flaky test: same Uid published twice — first failing, then passing. + // The CTRF engine collapses these into a single test entry with retries=1, + // retryAttempts[0].status=failed, and flaky=true on the final passing record. + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage( + context.Request.Session.SessionUid, + new TestNode() + { + Uid = "test-3", + DisplayName = "FlakyTest", + Properties = new PropertyBag(new FailedTestNodeStateProperty("Transient failure")), + })); + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage( + context.Request.Session.SessionUid, + new TestNode() + { + Uid = "test-3", + DisplayName = "FlakyTest", + Properties = new PropertyBag(PassedTestNodeStateProperty.CachedInstance), + })); + context.Complete(); } } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs index 31a07859fa..b67835cbc3 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs @@ -455,6 +455,21 @@ The file makes it possible to identify the tests that were running at the time o Description: Specify the type of the dump. Valid values are 'Mini', 'Heap', 'Triage' or 'Full'. Default type is 'Full'. For more information visit https://learn.microsoft.com/dotnet/core/diagnostics/collect-dumps-crash#types-of-mini-dumps + CtrfReportGeneratorCommandLine + Name: CTRF report generator + Version: * + Description: Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) + Options: + --report-ctrf + Arity: 0 + Hidden: False + Description: Enable generating a CTRF (Common Test Report Format) JSON report + --report-ctrf-filename + Arity: 1 + Hidden: False + Description: The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. + Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). + Example: MyReport_{tfm}.ctrf.json HangDumpCommandLineProvider Name: Hang dump Version: * @@ -506,21 +521,6 @@ Default type is 'Full' Description: The name of the generated HTML report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). Example: MyReport_{tfm}.html - CtrfReportGeneratorCommandLine - Name: CTRF report generator - Version: * - Description: Produce a CTRF (Common Test Report Format) JSON report for the current test session (https://ctrf.io) - Options: - --report-ctrf - Arity: 0 - Hidden: False - Description: Enable generating a CTRF (Common Test Report Format) JSON report - --report-ctrf-filename - Arity: 1 - Hidden: False - Description: The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. - Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). - Example: MyReport_{tfm}.ctrf.json MSBuildCommandLineProvider Name: MSBuildCommandLineProvider Version: * diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs index d514a1db0a..34ada5d35a 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs @@ -18,7 +18,8 @@ public async Task Microsoft_Testing_Platform_Extensions_ShouldBe_Correctly_Regis nameof(Microsoft_Testing_Platform_Extensions_ShouldBe_Correctly_Registered), SourceCode .PatchCodeWithReplace("$TargetFrameworks$", tfm) - .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion)); + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) + .PatchCodeWithReplace("$MicrosoftTestingExtensionsCtrfReportVersion$", MicrosoftTestingExtensionsCtrfReportVersion)); DotnetMuxerResult result = await DotnetCli.RunAsync($"{(verb == Verb.publish ? $"publish -f {tfm}" : "build")} -c {compilationMode} -r {RID} {testAsset.TargetAssetPath} -v:n", cancellationToken: TestContext.CancellationToken); string binlogFile = result.BinlogPath!; @@ -102,7 +103,7 @@ public async Task TestingPlatformBuilderHook_With_Conflicting_Metadata_Fails_Bui - + From 32fbbb41132d0c96b137148c452995034c72f3e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Mon, 8 Jun 2026 17:29:56 +0200 Subject: [PATCH 11/17] Address CTRF PR feedback: split stdout/stderr lines, skip CtrfPlayground in CI * Skip samples/CtrfPlayground/{Mtp,XunitMtp} under 'dotnet test -p:UsingDotNetTest=true' (the playground intentionally contains failing/throwing/skipped tests for diffing against xunit.v3 CTRF output). Mirrors the WasiPlayground pattern. Fixes the Windows/MacOS Debug Test CI failures. * Split CTRF stdout/stderr into per-line array entries. The CTRF schema types these as 'lines of output'; we now split on LF (normalizing CRLF) and omit the trailing empty entry when input ends with a newline. Brad Wilson's xunit.v3 CTRF comparison item #9. * Replace the misleading 'we never deduplicate on TestNode.Uid' wording in CtrfReportGenerator with a comment pointing at CtrfReportEngine.CollapseAttempts (which DOES collapse same-UID captures into retryAttempts[]). Adds two unit tests for the new line-splitting behavior (CRLF normalization, trailing-newline handling, single-line output). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- samples/CtrfPlayground/Mtp/Mtp.csproj | 10 ++- .../CtrfPlayground/XunitMtp/XunitMtp.csproj | 9 ++- .../CtrfReportEngine.cs | 78 ++++++++++++------- .../CtrfReportGenerator.cs | 9 ++- .../CtrfReportEngineTests.cs | 66 ++++++++++++++++ 5 files changed, 136 insertions(+), 36 deletions(-) diff --git a/samples/CtrfPlayground/Mtp/Mtp.csproj b/samples/CtrfPlayground/Mtp/Mtp.csproj index 8c2ca53541..1f58392bc7 100644 --- a/samples/CtrfPlayground/Mtp/Mtp.csproj +++ b/samples/CtrfPlayground/Mtp/Mtp.csproj @@ -6,7 +6,15 @@ enable $(NoWarn);NETSDK1023;SA0001;EnableGenerateDocumentationFile;TPEXP - true + + true false false diff --git a/samples/CtrfPlayground/XunitMtp/XunitMtp.csproj b/samples/CtrfPlayground/XunitMtp/XunitMtp.csproj index 2ec72e77db..f88ff7c0dc 100644 --- a/samples/CtrfPlayground/XunitMtp/XunitMtp.csproj +++ b/samples/CtrfPlayground/XunitMtp/XunitMtp.csproj @@ -7,7 +7,14 @@ false $(NoWarn);NETSDK1023;SA0001;EnableGenerateDocumentationFile - true + + true + false diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs index 612ebc34f3..154f77c6fc 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs @@ -504,21 +504,8 @@ private static void WriteTest(Utf8JsonWriter writer, CollapsedTestResult c) writer.WriteBoolean("flaky", true); } - if (r.StandardOutput is not null) - { - writer.WritePropertyName("stdout"); - writer.WriteStartArray(); - writer.WriteStringValue(r.StandardOutput); - writer.WriteEndArray(); - } - - if (r.StandardError is not null) - { - writer.WritePropertyName("stderr"); - writer.WriteStartArray(); - writer.WriteStringValue(r.StandardError); - writer.WriteEndArray(); - } + WriteOutputLines(writer, "stdout", r.StandardOutput); + WriteOutputLines(writer, "stderr", r.StandardError); // CTRF `labels` is reserved for user-controlled, classification-style // metadata (priority, severity, external IDs, etc.). We only emit the @@ -590,21 +577,8 @@ private static void WriteRetryAttempt(Utf8JsonWriter writer, CapturedTestResult writer.WriteNumber("line", line); } - if (attempt.StandardOutput is not null) - { - writer.WritePropertyName("stdout"); - writer.WriteStartArray(); - writer.WriteStringValue(attempt.StandardOutput); - writer.WriteEndArray(); - } - - if (attempt.StandardError is not null) - { - writer.WritePropertyName("stderr"); - writer.WriteStartArray(); - writer.WriteStringValue(attempt.StandardError); - writer.WriteEndArray(); - } + WriteOutputLines(writer, "stdout", attempt.StandardOutput); + WriteOutputLines(writer, "stderr", attempt.StandardError); if (attempt.RawStatus is not null || attempt.ExceptionType is not null) { @@ -626,6 +600,50 @@ private static void WriteRetryAttempt(Utf8JsonWriter writer, CapturedTestResult writer.WriteEndObject(); } + // CTRF `stdout`/`stderr` are typed as an array of lines (each item is one line + // of captured output). Splitting on LF (handling optional CR) preserves the + // original line structure for consumers that present output per-line. + private static void WriteOutputLines(Utf8JsonWriter writer, string propertyName, string? output) + { + if (output is null) + { + return; + } + + writer.WritePropertyName(propertyName); + writer.WriteStartArray(); + int start = 0; + for (int i = 0; i < output.Length; i++) + { + if (output[i] == '\n') + { + int end = i; + if (end > start && output[end - 1] == '\r') + { + end--; + } + + writer.WriteStringValue(output.AsSpan(start, end - start)); + start = i + 1; + } + } + + if (start < output.Length) + { + // Emit the trailing segment after the last LF (no trailing entry when + // the input ends with LF — a trailing newline isn't an additional line). + int end = output.Length; + if (end > start && output[end - 1] == '\r') + { + end--; + } + + writer.WriteStringValue(output.AsSpan(start, end - start)); + } + + writer.WriteEndArray(); + } + private static List CollapseAttempts(CapturedTestResult[] results) { // For each UID, group all captures in arrival order: the latest entry becomes the diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportGenerator.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportGenerator.cs index a1e7c3979d..9a1d937bc2 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportGenerator.cs +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportGenerator.cs @@ -96,10 +96,11 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo { // Project to a capped DTO immediately so we don't retain the original // TestNode (and its potentially huge stdout/stderr/stack trace strings) - // for the whole session. We never deduplicate on TestNode.Uid: some - // frameworks emit several distinct results sharing the same UID - // (parameterized rows, theory data, in-process retries). The engine - // surfaces all of them so no data is dropped. + // for the whole session. We capture every TestNode update as-is here; the + // engine later collapses captures that share the same UID into a single + // CTRF test entry (earlier captures become `retryAttempts[]`, marking the + // test as `flaky` when an earlier attempt failed). See + // CtrfReportEngine.CollapseAttempts for the deduplication logic. CapturedTestResult? captured = TestResultCapture.TryCapture(update.TestNode); if (captured is not null) { diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs index 321b03644a..7e72a3b258 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs @@ -710,6 +710,72 @@ public async Task GenerateReportAsync_SpecialCharactersInName_AreSafelyEscaped() Assert.AreEqual("", test.GetProperty("name").GetString()); } + [TestMethod] + public async Task GenerateReportAsync_SplitsStandardOutputAndError_OnNewlines() + { + // CTRF schema types stdout/stderr as an array of "lines of output"; + // we must split on LF (handling CRLF) and not include a trailing empty + // entry for inputs that end with a newline. + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + new CapturedTestResult + { + Uid = "id-multiline", + DisplayName = "MultiLine", + Status = "passed", + Duration = TimeSpan.Zero, + StandardOutput = "line1\nline2\r\nline3\n", + StandardError = "errA\nerrB", + }, + ]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement test = document.RootElement.GetProperty("results").GetProperty("tests")[0]; + + JsonElement stdout = test.GetProperty("stdout"); + Assert.AreEqual(JsonValueKind.Array, stdout.ValueKind); + Assert.AreEqual(3, stdout.GetArrayLength(), "Trailing newline must not produce an extra empty entry."); + Assert.AreEqual("line1", stdout[0].GetString()); + Assert.AreEqual("line2", stdout[1].GetString(), "CR before LF must be stripped (CRLF normalization)."); + Assert.AreEqual("line3", stdout[2].GetString()); + + JsonElement stderr = test.GetProperty("stderr"); + Assert.AreEqual(2, stderr.GetArrayLength()); + Assert.AreEqual("errA", stderr[0].GetString()); + Assert.AreEqual("errB", stderr[1].GetString(), "Final segment without trailing newline must still be emitted."); + } + + [TestMethod] + public async Task GenerateReportAsync_SingleLineOutput_EmitsOneArrayEntry() + { + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + new CapturedTestResult + { + Uid = "id-single", + DisplayName = "SingleLine", + Status = "passed", + Duration = TimeSpan.Zero, + StandardOutput = "only-line", + }, + ]; + + await engine.GenerateReportAsync(tests); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + JsonElement test = document.RootElement.GetProperty("results").GetProperty("tests")[0]; + + JsonElement stdout = test.GetProperty("stdout"); + Assert.AreEqual(1, stdout.GetArrayLength()); + Assert.AreEqual("only-line", stdout[0].GetString()); + } + private CtrfReportEngine CreateEngine(MemoryFileStream stream) { _ = _fileSystem.Setup(x => x.ExistFile(It.IsAny())).Returns(false); From 7559b75726a782c735c8b8cac281272e1a0dd8d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Tue, 9 Jun 2026 11:30:32 +0200 Subject: [PATCH 12/17] Wire CtrfReport and HtmlReport extensions into MSTest.Sdk - CtrfReport: opt-in only via EnableMicrosoftTestingExtensionsCtrfReport (centralized 1.0.0-alpha version flowed through MSTest.Sdk template). - HtmlReport: auto-enabled by AllMicrosoft profile and opt-in via EnableMicrosoftTestingExtensionsHtmlReport (uses platform-aligned MicrosoftTestingExtensionsCommonVersion). - VSTest.targets: add Error guards for the new opt-ins and the previously missing AzureDevOpsReport one. - Acceptance: cover CtrfReport (--report-ctrf) and HtmlReport (--report-html) selective-enable rows and append --report-html to the AllMicrosoft EnableAll run. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Directory.Build.props | 7 ++++++ src/Package/MSTest.Sdk/MSTest.Sdk.csproj | 10 +++++++- .../Sdk/Runner/ClassicEngine.targets | 25 +++++++++++++++++++ src/Package/MSTest.Sdk/Sdk/Sdk.props.template | 1 + .../MSTest.Sdk/Sdk/VSTest/VSTest.targets | 3 +++ ...osoft.Testing.Extensions.CtrfReport.csproj | 4 +-- .../SdkTests.cs | 12 ++++++++- 7 files changed, 58 insertions(+), 4 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index ce51a5be1e..40c350ba14 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -56,6 +56,13 @@ This is a early preview package, keep 2.0.0-alpha or similar suffix even in official builds. --> 2.0.0 + + alpha + + + 1.0.0 diff --git a/src/Package/MSTest.Sdk/MSTest.Sdk.csproj b/src/Package/MSTest.Sdk/MSTest.Sdk.csproj index 3218486287..55fba4bbe3 100644 --- a/src/Package/MSTest.Sdk/MSTest.Sdk.csproj +++ b/src/Package/MSTest.Sdk/MSTest.Sdk.csproj @@ -42,7 +42,15 @@ <_MSTestSourceGenerationVersionSuffix>$(_MSTestSourceGenerationPreReleaseVersionLabel)$(_BuildNumberLabels) <_MSTestSourceGenerationVersion>$(MSTestSourceGenerationVersionPrefix) <_MSTestSourceGenerationVersion Condition="'$(_MSTestSourceGenerationVersionSuffix)' != ''">$(_MSTestSourceGenerationVersion)-$(_MSTestSourceGenerationVersionSuffix) - <_TemplateProperties>MSTestSourceGenerationVersion=$(_MSTestSourceGenerationVersion);MSTestVersion=$(Version);MicrosoftTestingPlatformVersion=$(Version.Replace('$(VersionPrefix)', '$(TestingPlatformVersionPrefix)'));MicrosoftNETTestSdkVersion=$(MicrosoftNETTestSdkVersion);MicrosoftTestingExtensionsCodeCoverageVersion=$(MicrosoftTestingExtensionsCodeCoverageVersion);MicrosoftPlaywrightVersion=$(MicrosoftPlaywrightVersion);AspireHostingTestingVersion=$(AspireHostingTestingVersion);MicrosoftTestingExtensionsFakesVersion=$(MicrosoftTestingExtensionsFakesVersion) + + <_MicrosoftTestingExtensionsCtrfReportPreReleaseVersionLabel>$(MicrosoftTestingExtensionsCtrfReportPreReleaseVersionLabel) + <_MicrosoftTestingExtensionsCtrfReportPreReleaseVersionLabel Condition="'$(ContinuousIntegrationBuild)' == 'true' and '$(OfficialBuild)' != 'true'">ci + <_MicrosoftTestingExtensionsCtrfReportPreReleaseVersionLabel Condition="'$(ContinuousIntegrationBuild)' != 'true' and '$(OfficialBuild)' != 'true'">dev + <_MicrosoftTestingExtensionsCtrfReportVersionSuffix>$(_MicrosoftTestingExtensionsCtrfReportPreReleaseVersionLabel)$(_BuildNumberLabels) + <_MicrosoftTestingExtensionsCtrfReportVersion>$(MicrosoftTestingExtensionsCtrfReportVersionPrefix) + <_MicrosoftTestingExtensionsCtrfReportVersion Condition="'$(_MicrosoftTestingExtensionsCtrfReportVersionSuffix)' != ''">$(_MicrosoftTestingExtensionsCtrfReportVersion)-$(_MicrosoftTestingExtensionsCtrfReportVersionSuffix) + + <_TemplateProperties>MSTestSourceGenerationVersion=$(_MSTestSourceGenerationVersion);MSTestVersion=$(Version);MicrosoftTestingPlatformVersion=$(Version.Replace('$(VersionPrefix)', '$(TestingPlatformVersionPrefix)'));MicrosoftNETTestSdkVersion=$(MicrosoftNETTestSdkVersion);MicrosoftTestingExtensionsCodeCoverageVersion=$(MicrosoftTestingExtensionsCodeCoverageVersion);MicrosoftPlaywrightVersion=$(MicrosoftPlaywrightVersion);AspireHostingTestingVersion=$(AspireHostingTestingVersion);MicrosoftTestingExtensionsFakesVersion=$(MicrosoftTestingExtensionsFakesVersion);MicrosoftTestingExtensionsCtrfReportVersion=$(_MicrosoftTestingExtensionsCtrfReportVersion) + true + $(MicrosoftTestingExtensionsCommonVersion) + + + $(MicrosoftTestingExtensionsCtrfReportVersion) + true $(MicrosoftTestingExtensionsFakesVersion) @@ -131,6 +142,20 @@ + + + $(MicrosoftTestingExtensionsHtmlReportVersion) + + + + + $(MicrosoftTestingExtensionsCtrfReportVersion) + + diff --git a/src/Package/MSTest.Sdk/Sdk/Sdk.props.template b/src/Package/MSTest.Sdk/Sdk/Sdk.props.template index 3703df287d..63e62f0f55 100644 --- a/src/Package/MSTest.Sdk/Sdk/Sdk.props.template +++ b/src/Package/MSTest.Sdk/Sdk/Sdk.props.template @@ -20,6 +20,7 @@ ${MicrosoftNETTestSdkVersion} ${MicrosoftPlaywrightVersion} ${MicrosoftTestingExtensionsCodeCoverageVersion} + ${MicrosoftTestingExtensionsCtrfReportVersion} ${MicrosoftTestingExtensionsFakesVersion} ${MicrosoftTestingPlatformVersion} ${MSTestSourceGenerationVersion} diff --git a/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets b/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets index 172f9de923..039708753a 100644 --- a/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets +++ b/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets @@ -2,10 +2,13 @@ + + + diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj index 8f74a9894d..b0e7ba2722 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj @@ -2,8 +2,8 @@ netstandard2.0;$(SupportedNetFrameworks) - 1.0.0 - alpha + $(MicrosoftTestingExtensionsCtrfReportVersionPrefix) + $(MicrosoftTestingExtensionsCtrfReportPreReleaseVersionLabel) true true $(RepoRoot)src\Platform\SharedExtensionHelpers\BuildInfo.cs.template diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs index d6abc5d1a6..b9258548f4 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs @@ -200,6 +200,16 @@ public async Task RunTests_With_CentralPackageManagement_Standalone(string multi "true", "--report-azdo", "--crashdump")); + + yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, + "true", + "--report-html", + "--crashdump")); + + yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, + "true", + "--report-ctrf", + "--crashdump")); } } @@ -246,7 +256,7 @@ public async Task RunTests_With_MSTestRunner_Standalone_EnableAll_Extensions(str foreach (string tfm in multiTfm.Split(";")) { var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, tfm, buildConfiguration: buildConfiguration); - TestHostResult testHostResult = await testHost.ExecuteAsync(command: "--coverage --retry-failed-tests 3 --report-trx --crashdump --hangdump --report-azdo", cancellationToken: TestContext.CancellationToken); + TestHostResult testHostResult = await testHost.ExecuteAsync(command: "--coverage --retry-failed-tests 3 --report-trx --crashdump --hangdump --report-azdo --report-html", cancellationToken: TestContext.CancellationToken); testHostResult.AssertOutputContainsSummary(0, 1, 0); } } From 6a53e5918a48d594025df253d1c0cce4c7b5f2c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Tue, 9 Jun 2026 12:15:48 +0200 Subject: [PATCH 13/17] Wire OpenTelemetry extension into MSTest.Sdk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Microsoft.Testing.Extensions.OpenTelemetry to the MSTest.Sdk wiring so users can opt-in via the EnableMicrosoftTestingExtensionsOpenTelemetry MSBuild property. OpenTelemetry is independently versioned (1.0.0-alpha, matching the CtrfReport pattern) because it is still experimental ([TPEXP]). The extension is API-only — there is no CLI flag — so the enable property only opts the package into the build; activation requires the user to call builder.AddOpenTelemetryProvider(...) from their Program.cs. - Directory.Build.props: centralized version definitions - src/Platform/.../Microsoft.Testing.Extensions.OpenTelemetry.csproj: consume centralized version properties - MSTest.Sdk.csproj: compute and propagate _MicrosoftTestingExtensionsOpenTelemetryVersion (ci/dev/official override) - Sdk.props.template / ClassicEngine.targets / VSTest.targets: wire the opt-in PackageReference, version line, and Error guard - SdkTests.cs: add an acceptance row that validates the package restores and runs cleanly (empty CLI enable arg; --crashdump asserts no other extensions leaked in) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Directory.Build.props | 7 +++++++ src/Package/MSTest.Sdk/MSTest.Sdk.csproj | 9 ++++++++- .../MSTest.Sdk/Sdk/Runner/ClassicEngine.targets | 16 ++++++++++++++++ src/Package/MSTest.Sdk/Sdk/Sdk.props.template | 1 + src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets | 1 + ...osoft.Testing.Extensions.OpenTelemetry.csproj | 4 ++-- .../SdkTests.cs | 8 ++++++++ 7 files changed, 43 insertions(+), 3 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 40c350ba14..147e4a02c1 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -63,6 +63,13 @@ This is an early preview package, keep 1.0.0-alpha or similar suffix even in official builds. --> 1.0.0 + + alpha + + + 1.0.0 diff --git a/src/Package/MSTest.Sdk/MSTest.Sdk.csproj b/src/Package/MSTest.Sdk/MSTest.Sdk.csproj index 55fba4bbe3..4f2aeefc44 100644 --- a/src/Package/MSTest.Sdk/MSTest.Sdk.csproj +++ b/src/Package/MSTest.Sdk/MSTest.Sdk.csproj @@ -50,7 +50,14 @@ <_MicrosoftTestingExtensionsCtrfReportVersion>$(MicrosoftTestingExtensionsCtrfReportVersionPrefix) <_MicrosoftTestingExtensionsCtrfReportVersion Condition="'$(_MicrosoftTestingExtensionsCtrfReportVersionSuffix)' != ''">$(_MicrosoftTestingExtensionsCtrfReportVersion)-$(_MicrosoftTestingExtensionsCtrfReportVersionSuffix) - <_TemplateProperties>MSTestSourceGenerationVersion=$(_MSTestSourceGenerationVersion);MSTestVersion=$(Version);MicrosoftTestingPlatformVersion=$(Version.Replace('$(VersionPrefix)', '$(TestingPlatformVersionPrefix)'));MicrosoftNETTestSdkVersion=$(MicrosoftNETTestSdkVersion);MicrosoftTestingExtensionsCodeCoverageVersion=$(MicrosoftTestingExtensionsCodeCoverageVersion);MicrosoftPlaywrightVersion=$(MicrosoftPlaywrightVersion);AspireHostingTestingVersion=$(AspireHostingTestingVersion);MicrosoftTestingExtensionsFakesVersion=$(MicrosoftTestingExtensionsFakesVersion);MicrosoftTestingExtensionsCtrfReportVersion=$(_MicrosoftTestingExtensionsCtrfReportVersion) + <_MicrosoftTestingExtensionsOpenTelemetryPreReleaseVersionLabel>$(MicrosoftTestingExtensionsOpenTelemetryPreReleaseVersionLabel) + <_MicrosoftTestingExtensionsOpenTelemetryPreReleaseVersionLabel Condition="'$(ContinuousIntegrationBuild)' == 'true' and '$(OfficialBuild)' != 'true'">ci + <_MicrosoftTestingExtensionsOpenTelemetryPreReleaseVersionLabel Condition="'$(ContinuousIntegrationBuild)' != 'true' and '$(OfficialBuild)' != 'true'">dev + <_MicrosoftTestingExtensionsOpenTelemetryVersionSuffix>$(_MicrosoftTestingExtensionsOpenTelemetryPreReleaseVersionLabel)$(_BuildNumberLabels) + <_MicrosoftTestingExtensionsOpenTelemetryVersion>$(MicrosoftTestingExtensionsOpenTelemetryVersionPrefix) + <_MicrosoftTestingExtensionsOpenTelemetryVersion Condition="'$(_MicrosoftTestingExtensionsOpenTelemetryVersionSuffix)' != ''">$(_MicrosoftTestingExtensionsOpenTelemetryVersion)-$(_MicrosoftTestingExtensionsOpenTelemetryVersionSuffix) + + <_TemplateProperties>MSTestSourceGenerationVersion=$(_MSTestSourceGenerationVersion);MSTestVersion=$(Version);MicrosoftTestingPlatformVersion=$(Version.Replace('$(VersionPrefix)', '$(TestingPlatformVersionPrefix)'));MicrosoftNETTestSdkVersion=$(MicrosoftNETTestSdkVersion);MicrosoftTestingExtensionsCodeCoverageVersion=$(MicrosoftTestingExtensionsCodeCoverageVersion);MicrosoftPlaywrightVersion=$(MicrosoftPlaywrightVersion);AspireHostingTestingVersion=$(AspireHostingTestingVersion);MicrosoftTestingExtensionsFakesVersion=$(MicrosoftTestingExtensionsFakesVersion);MicrosoftTestingExtensionsCtrfReportVersion=$(_MicrosoftTestingExtensionsCtrfReportVersion);MicrosoftTestingExtensionsOpenTelemetryVersion=$(_MicrosoftTestingExtensionsOpenTelemetryVersion) $(MicrosoftTestingExtensionsCtrfReportVersion) + + $(MicrosoftTestingExtensionsOpenTelemetryVersion) + true $(MicrosoftTestingExtensionsFakesVersion) @@ -156,6 +165,13 @@ + + + $(MicrosoftTestingExtensionsOpenTelemetryVersion) + + diff --git a/src/Package/MSTest.Sdk/Sdk/Sdk.props.template b/src/Package/MSTest.Sdk/Sdk/Sdk.props.template index 63e62f0f55..338c564e2d 100644 --- a/src/Package/MSTest.Sdk/Sdk/Sdk.props.template +++ b/src/Package/MSTest.Sdk/Sdk/Sdk.props.template @@ -22,6 +22,7 @@ ${MicrosoftTestingExtensionsCodeCoverageVersion} ${MicrosoftTestingExtensionsCtrfReportVersion} ${MicrosoftTestingExtensionsFakesVersion} + ${MicrosoftTestingExtensionsOpenTelemetryVersion} ${MicrosoftTestingPlatformVersion} ${MSTestSourceGenerationVersion} ${MSTestVersion} diff --git a/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets b/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets index 039708753a..c12e70101d 100644 --- a/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets +++ b/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets @@ -9,6 +9,7 @@ + diff --git a/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/Microsoft.Testing.Extensions.OpenTelemetry.csproj b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/Microsoft.Testing.Extensions.OpenTelemetry.csproj index 281970d978..bf8362f4d7 100644 --- a/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/Microsoft.Testing.Extensions.OpenTelemetry.csproj +++ b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/Microsoft.Testing.Extensions.OpenTelemetry.csproj @@ -2,8 +2,8 @@ netstandard2.0;$(SupportedNetFrameworks) - 1.0.0 - alpha + $(MicrosoftTestingExtensionsOpenTelemetryVersionPrefix) + $(MicrosoftTestingExtensionsOpenTelemetryPreReleaseVersionLabel) true true $(RepoRoot)src\Platform\SharedExtensionHelpers\BuildInfo.cs.template diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs index b9258548f4..fd27507edc 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs @@ -210,6 +210,14 @@ public async Task RunTests_With_CentralPackageManagement_Standalone(string multi "true", "--report-ctrf", "--crashdump")); + + // OpenTelemetry is API-only (no CLI flag); the enable property only opts the package into the build. + // We pass an empty enable arg to validate the package restores and the test host runs cleanly, and we + // assert that an unrelated extension's CLI arg ('--crashdump') is rejected to prove no other extensions leaked in. + yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, + "true", + string.Empty, + "--crashdump")); } } From 208e9eb590fbecf610d5d54dadd5928998c21f8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Tue, 9 Jun 2026 17:02:53 +0200 Subject: [PATCH 14/17] Address CTRF PR review: experimental APIs, dogfood, and Brad's labels feedback - Remove top-level 'labels' object (not in CTRF spec; emitted illegal JSON with duplicate keys for multi-value traits like [TestCategory]). Instead emit: * top-level 'tags[]' populated from TestCategory trait values (mirrors the CTRF spec field and xunit's Category mapping) * 'extra.traits' as a grouped object { key: [values...] } that safely handles non-adjacent duplicate keys Addresses review feedback from @bradwilson. - Mark TestingPlatformBuilderHook with [Experimental("TPEXP")] alongside CtrfReportExtensions, matching the user's request that all public APIs in this new extension ship as experimental. Updated PublicAPI.Unshipped.txt entries with the [TPEXP] prefix. - Dogfood the extension in unit and integration test projects: * register AddCtrfReportProvider() in every test Program.cs that already lists the other built-in providers * add --report-ctrf --report-ctrf-filename to test/Directory.Build.targets TestingPlatformCommandLineArguments * add --report-ctrf to azure-pipelines.yml Release script so CI emits CTRF reports alongside the existing TRX and AzDO reports - Update CtrfReportEngineTests for the new JSON shape (tags + extra.traits, no labels). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- azure-pipelines.yml | 5 +- .../CtrfReportEngine.cs | 82 ++++++++++++++++--- .../PublicAPI/PublicAPI.Unshipped.txt | 4 +- .../TestingPlatformBuilderHook.cs | 1 + test/Directory.Build.targets | 2 + .../Program.cs | 3 +- .../MSTest.IntegrationTests/Program.cs | 3 +- .../Program.cs | 3 +- .../Program.cs | 3 +- .../Program.cs | 3 +- .../MSTest.Analyzers.UnitTests/Program.cs | 3 +- .../Program.cs | 5 +- .../Program.cs | 1 + .../Program.cs | 3 +- .../MSTestAdapter.UnitTests/Program.cs | 3 +- .../CtrfReportEngineTests.cs | 52 +++++++++--- .../Program.cs | 3 +- .../Program.cs | 3 +- .../Program.cs | 3 +- .../Program.cs | 3 +- .../TestFramework.UnitTests/Program.cs | 3 +- 21 files changed, 153 insertions(+), 38 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index baaed4ed17..1e2d446aed 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -110,6 +110,9 @@ stages: # options that makes sense in Release: # --report-trx + --report-trx-filename "{asm}_{tfm}.trx" — TRX per (assembly x TFM), # resolved by Microsoft.Testing.Platform.Services.ArtifactNamingHelper. + # --report-ctrf + --report-ctrf-filename "{asm}_{tfm}.ctrf.json" — CTRF report per + # (assembly x TFM). Same placeholder resolver as TRX. Dogfoods the new CTRF extension + # so any regression in the report shape surfaces on every CI run. # --report-azdo — surfaces failed tests in the AzDO UI. # --report-azdo-progress — emits ##vso[task.logdetail ...;progress=...] timeline records so # long-running test sessions show live progress on the AzDO timeline. No-op outside AzDO @@ -127,7 +130,7 @@ stages: # and this glob mixes net4x and net8+/net9+ hosts. # --coverage — Codecov upload is Debug-only (see "Upload coverage to codecov.io" step # below), so collecting coverage in Release would just write unused .coverage files. - - script: dotnet test --no-build --test-modules "artifacts/bin/*UnitTests*/$(_BuildConfig)/**/*UnitTests.exe" --results-directory $(BUILD.SOURCESDIRECTORY)\artifacts\TestResults\$(_BuildConfig) --no-progress --output detailed --report-trx --report-trx-filename "{asm}_{tfm}.trx" --report-azdo --report-azdo-progress --report-azdo-summary --hangdump --hangdump-timeout 15m --diagnostic --diagnostic-output-directory $(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig) --diagnostic-verbosity trace + - script: dotnet test --no-build --test-modules "artifacts/bin/*UnitTests*/$(_BuildConfig)/**/*UnitTests.exe" --results-directory $(BUILD.SOURCESDIRECTORY)\artifacts\TestResults\$(_BuildConfig) --no-progress --output detailed --report-trx --report-trx-filename "{asm}_{tfm}.trx" --report-ctrf --report-ctrf-filename "{asm}_{tfm}.ctrf.json" --report-azdo --report-azdo-progress --report-azdo-summary --hangdump --hangdump-timeout 15m --diagnostic --diagnostic-output-directory $(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig) --diagnostic-verbosity trace name: TestRelease displayName: Test (unit tests only) condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs index 154f77c6fc..ad5393d882 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs @@ -507,22 +507,32 @@ private static void WriteTest(Utf8JsonWriter writer, CollapsedTestResult c) WriteOutputLines(writer, "stdout", r.StandardOutput); WriteOutputLines(writer, "stderr", r.StandardError); - // CTRF `labels` is reserved for user-controlled, classification-style - // metadata (priority, severity, external IDs, etc.). We only emit the - // traits collected from MTP TestMetadataProperty here. Synthetic - // framework-generated metadata (method name, exception type, MTP UID) - // lives in the per-test `extra` object instead so CTRF consumers can - // filter/group by labels without seeing our internals. + // CTRF spec: `tags` is a top-level string array on the Test object used for + // classification. We promote MSTest `[TestCategory("…")]` trait values here + // so CTRF consumers can filter/group by category without having to dig into + // the free-form `extra` object. Other traits remain under `extra.traits`. if (r.Traits is { Count: > 0 }) { - writer.WritePropertyName("labels"); - writer.WriteStartObject(); + bool tagsArrayStarted = false; foreach (KeyValuePair trait in r.Traits) { - writer.WriteString(trait.Key, trait.Value); + if (string.Equals(trait.Key, "TestCategory", StringComparison.Ordinal)) + { + if (!tagsArrayStarted) + { + writer.WritePropertyName("tags"); + writer.WriteStartArray(); + tagsArrayStarted = true; + } + + writer.WriteStringValue(trait.Value); + } } - writer.WriteEndObject(); + if (tagsArrayStarted) + { + writer.WriteEndArray(); + } } // CTRF `extra` (free-form object) — the CTRF spec doesn't define a @@ -541,11 +551,63 @@ private static void WriteTest(Utf8JsonWriter writer, CollapsedTestResult c) writer.WriteString("exceptionType", r.ExceptionType); } + // Traits live under `extra.traits` (not a top-level `labels`) because: + // 1. The CTRF spec has no `labels` field on the Test object. + // 2. A test can declare the same trait key multiple times (e.g. several + // [TestCategory] attributes on one MSTest method), so we must group + // values per key — emitting `{ key: value }` would produce duplicate + // JSON property names, which RFC 8259 calls out as not interoperable. + // We emit `{ key: [value, value, ...] }` so every trait key appears once + // and consumers always see a deterministic array shape. The keys appear + // in first-seen order; values keep their original declaration order. + if (r.Traits is { Count: > 0 }) + { + writer.WritePropertyName("traits"); + writer.WriteStartObject(); + for (int i = 0; i < r.Traits.Count; i++) + { + string key = r.Traits[i].Key; + if (HasSameKeyEarlier(r.Traits, i)) + { + continue; + } + + writer.WritePropertyName(key); + writer.WriteStartArray(); + writer.WriteStringValue(r.Traits[i].Value); + for (int j = i + 1; j < r.Traits.Count; j++) + { + if (string.Equals(r.Traits[j].Key, key, StringComparison.Ordinal)) + { + writer.WriteStringValue(r.Traits[j].Value); + } + } + + writer.WriteEndArray(); + } + + writer.WriteEndObject(); + } + writer.WriteEndObject(); writer.WriteEndObject(); } + private static bool HasSameKeyEarlier(IReadOnlyList> traits, int index) + { + string key = traits[index].Key; + for (int k = 0; k < index; k++) + { + if (string.Equals(traits[k].Key, key, StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } + private static void WriteRetryAttempt(Utf8JsonWriter writer, CapturedTestResult attempt, int attemptNumber) { writer.WriteStartObject(); diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PublicAPI/PublicAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PublicAPI/PublicAPI.Unshipped.txt index b3862b10ff..7221ec44cd 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/PublicAPI/PublicAPI.Unshipped.txt @@ -1,5 +1,5 @@ #nullable enable -Microsoft.Testing.Extensions.CtrfReport.TestingPlatformBuilderHook +[TPEXP]Microsoft.Testing.Extensions.CtrfReport.TestingPlatformBuilderHook [TPEXP]Microsoft.Testing.Extensions.CtrfReportExtensions -static Microsoft.Testing.Extensions.CtrfReport.TestingPlatformBuilderHook.AddExtensions(Microsoft.Testing.Platform.Builder.ITestApplicationBuilder! testApplicationBuilder, string![]! _) -> void +[TPEXP]static Microsoft.Testing.Extensions.CtrfReport.TestingPlatformBuilderHook.AddExtensions(Microsoft.Testing.Platform.Builder.ITestApplicationBuilder! testApplicationBuilder, string![]! _) -> void [TPEXP]static Microsoft.Testing.Extensions.CtrfReportExtensions.AddCtrfReportProvider(this Microsoft.Testing.Platform.Builder.ITestApplicationBuilder! builder) -> void diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/TestingPlatformBuilderHook.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/TestingPlatformBuilderHook.cs index a69da97b23..e7ba703b99 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/TestingPlatformBuilderHook.cs +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/TestingPlatformBuilderHook.cs @@ -9,6 +9,7 @@ namespace Microsoft.Testing.Extensions.CtrfReport; /// This class is used by Microsoft.Testing.Platform.MSBuild to hook into the Testing Platform Builder /// to add CTRF report support. /// +[Experimental("TPEXP", UrlFormat = "https://aka.ms/testingplatform/diagnostics#{0}")] public static class TestingPlatformBuilderHook { /// diff --git a/test/Directory.Build.targets b/test/Directory.Build.targets index a350f1e23a..48f3ba1a59 100644 --- a/test/Directory.Build.targets +++ b/test/Directory.Build.targets @@ -16,6 +16,7 @@ $(TestingPlatformCommandLineArguments) --crashdump $(TestingPlatformCommandLineArguments) --hangdump --hangdump-timeout 15m $(TestingPlatformCommandLineArguments) --report-azdo + $(TestingPlatformCommandLineArguments) --report-ctrf --report-ctrf-filename $(MSBuildProjectName)_$(TargetFramework).ctrf.json @@ -38,6 +39,7 @@ + diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/Program.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/Program.cs index f4868ec3d2..6a2b11a992 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/Program.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -20,6 +20,7 @@ builder.AddCrashDumpProvider(ignoreIfNotSupported: true); builder.AddTrxReportProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); // Custom suite tools CompositeExtensionFactory slowestTestCompositeServiceFactory diff --git a/test/IntegrationTests/MSTest.IntegrationTests/Program.cs b/test/IntegrationTests/MSTest.IntegrationTests/Program.cs index 55b5a5adf2..64d3227684 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/Program.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -15,6 +15,7 @@ builder.AddCrashDumpProvider(ignoreIfNotSupported: true); builder.AddRetryProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); builder.AddMSTest(() => [Assembly.GetEntryAssembly()!]); diff --git a/test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/Program.cs b/test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/Program.cs index e1b9e97e60..62b3dae0d9 100644 --- a/test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/Program.cs +++ b/test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -15,6 +15,7 @@ builder.AddCrashDumpProvider(ignoreIfNotSupported: true); builder.AddRetryProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); builder.AddMSTest(() => [Assembly.GetEntryAssembly()!]); diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Program.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Program.cs index a66316503d..11d3efc61e 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Program.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -21,6 +21,7 @@ builder.AddTrxReportProvider(); builder.AddRetryProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); // Custom suite tools CompositeExtensionFactory slowestTestCompositeServiceFactory diff --git a/test/IntegrationTests/PlatformServices.Desktop.IntegrationTests/Program.cs b/test/IntegrationTests/PlatformServices.Desktop.IntegrationTests/Program.cs index ce5734f730..b1f854c9f2 100644 --- a/test/IntegrationTests/PlatformServices.Desktop.IntegrationTests/Program.cs +++ b/test/IntegrationTests/PlatformServices.Desktop.IntegrationTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -15,6 +15,7 @@ builder.AddCrashDumpProvider(ignoreIfNotSupported: true); builder.AddRetryProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); builder.AddInternalTestFramework(); diff --git a/test/UnitTests/MSTest.Analyzers.UnitTests/Program.cs b/test/UnitTests/MSTest.Analyzers.UnitTests/Program.cs index 7ac654fa12..2e19ddded0 100644 --- a/test/UnitTests/MSTest.Analyzers.UnitTests/Program.cs +++ b/test/UnitTests/MSTest.Analyzers.UnitTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -18,6 +18,7 @@ builder.AddHangDumpProvider(); builder.AddTrxReportProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); ITestApplication app = await builder.BuildAsync(); return await app.RunAsync(); diff --git a/test/UnitTests/MSTest.SelfRealExamples.UnitTests/Program.cs b/test/UnitTests/MSTest.SelfRealExamples.UnitTests/Program.cs index cd7e68a209..1148e30788 100644 --- a/test/UnitTests/MSTest.SelfRealExamples.UnitTests/Program.cs +++ b/test/UnitTests/MSTest.SelfRealExamples.UnitTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -16,6 +16,7 @@ testApplicationBuilder.AddCodeCoverageProvider(); #endif -testApplicationBuilder.AddAzureDevOpsProvider(); +testApplicationbuilder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); using ITestApplication testApplication = await testApplicationBuilder.BuildAsync(); return await testApplication.RunAsync(); diff --git a/test/UnitTests/MSTest.SourceGeneration.UnitTests/Program.cs b/test/UnitTests/MSTest.SourceGeneration.UnitTests/Program.cs index 9fbdc73473..a6f1d5b912 100644 --- a/test/UnitTests/MSTest.SourceGeneration.UnitTests/Program.cs +++ b/test/UnitTests/MSTest.SourceGeneration.UnitTests/Program.cs @@ -14,6 +14,7 @@ builder.AddTrxReportProvider(); builder.AddAppInsightsTelemetryProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); using ITestApplication app = await builder.BuildAsync(); return await app.RunAsync(); diff --git a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Program.cs b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Program.cs index ce5734f730..b1f854c9f2 100644 --- a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Program.cs +++ b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -15,6 +15,7 @@ builder.AddCrashDumpProvider(ignoreIfNotSupported: true); builder.AddRetryProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); builder.AddInternalTestFramework(); diff --git a/test/UnitTests/MSTestAdapter.UnitTests/Program.cs b/test/UnitTests/MSTestAdapter.UnitTests/Program.cs index ce5734f730..b1f854c9f2 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/Program.cs +++ b/test/UnitTests/MSTestAdapter.UnitTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -15,6 +15,7 @@ builder.AddCrashDumpProvider(ignoreIfNotSupported: true); builder.AddRetryProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); builder.AddInternalTestFramework(); diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs index 7e72a3b258..c50a252711 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs @@ -315,7 +315,7 @@ public async Task GenerateReportAsync_RoundTripsErrorMessageAndStackTrace() } [TestMethod] - public async Task GenerateReportAsync_IncludesTraitsUnderLabels() + public async Task GenerateReportAsync_IncludesTraitsUnderExtraTraits_AndPromotesTestCategoryToTags() { using var memoryStream = new MemoryFileStream(); CtrfReportEngine engine = CreateEngine(memoryStream); @@ -329,7 +329,12 @@ public async Task GenerateReportAsync_IncludesTraitsUnderLabels() Duration = TimeSpan.Zero, Traits = [ - new KeyValuePair("Category", "FastTest"), + // Multiple [TestCategory] attributes on one MSTest method produce repeated + // trait entries with the same key. The engine must group them as an array + // under extra.traits (the CTRF spec has no `labels` field, and emitting + // duplicate JSON keys would be non-interoperable per RFC 8259). + new KeyValuePair("TestCategory", "Fast"), + new KeyValuePair("TestCategory", "Smoke"), new KeyValuePair("Owner", "alice"), ], }, @@ -338,9 +343,32 @@ public async Task GenerateReportAsync_IncludesTraitsUnderLabels() await engine.GenerateReportAsync(tests); using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); - JsonElement labels = document.RootElement.GetProperty("results").GetProperty("tests")[0].GetProperty("labels"); - Assert.AreEqual("FastTest", labels.GetProperty("Category").GetString()); - Assert.AreEqual("alice", labels.GetProperty("Owner").GetString()); + JsonElement test = document.RootElement.GetProperty("results").GetProperty("tests")[0]; + + // No top-level `labels` — the CTRF Test schema doesn't define one. + Assert.IsFalse(test.TryGetProperty("labels", out _), "labels is not part of the CTRF Test schema."); + + // TestCategory values are promoted to the CTRF top-level `tags` array so consumers + // can filter/group by category. The values are preserved in declaration order. + JsonElement tags = test.GetProperty("tags"); + Assert.AreEqual(JsonValueKind.Array, tags.ValueKind); + Assert.AreEqual(2, tags.GetArrayLength()); + Assert.AreEqual("Fast", tags[0].GetString()); + Assert.AreEqual("Smoke", tags[1].GetString()); + + // All traits (including TestCategory) round-trip under extra.traits as + // { key: [value, ...] } so multi-value traits remain valid JSON. + JsonElement traits = test.GetProperty("extra").GetProperty("traits"); + JsonElement testCategory = traits.GetProperty("TestCategory"); + Assert.AreEqual(JsonValueKind.Array, testCategory.ValueKind); + Assert.AreEqual(2, testCategory.GetArrayLength()); + Assert.AreEqual("Fast", testCategory[0].GetString()); + Assert.AreEqual("Smoke", testCategory[1].GetString()); + + JsonElement owner = traits.GetProperty("Owner"); + Assert.AreEqual(JsonValueKind.Array, owner.ValueKind); + Assert.AreEqual(1, owner.GetArrayLength()); + Assert.AreEqual("alice", owner[0].GetString()); } [TestMethod] @@ -578,8 +606,8 @@ public async Task GenerateReportAsync_Environment_HasSchemaCompliantShape() [TestMethod] public async Task GenerateReportAsync_TestExtra_CarriesMethodNameAndExceptionType() { - // method, exceptionType, and uid all live under `extra` (not `labels`), - // so `labels` is reserved for user-controlled trait metadata only. + // method, exceptionType, and uid all live under `extra` so the per-test object + // remains aligned with the CTRF Test schema (which has no `labels` field). using var memoryStream = new MemoryFileStream(); CtrfReportEngine engine = CreateEngine(memoryStream); CapturedTestResult[] tests = @@ -605,9 +633,13 @@ public async Task GenerateReportAsync_TestExtra_CarriesMethodNameAndExceptionTyp Assert.AreEqual("MyMethod", extra.GetProperty("method").GetString()); Assert.AreEqual("System.InvalidOperationException", extra.GetProperty("exceptionType").GetString()); - // No `labels` is emitted when there are no user traits — `labels` is for - // classification (priority/severity/owner), not framework internals. - Assert.IsFalse(test.TryGetProperty("labels", out _), "Synthetic framework metadata must not appear in labels."); + // No `labels` is ever emitted — it isn't part of the CTRF Test schema. + Assert.IsFalse(test.TryGetProperty("labels", out _), "labels is not part of the CTRF Test schema."); + + // No `tags` is emitted when there is no TestCategory trait, and no `traits` is + // emitted under `extra` when the test has no user-supplied traits at all. + Assert.IsFalse(test.TryGetProperty("tags", out _), "tags is only emitted when TestCategory traits exist."); + Assert.IsFalse(extra.TryGetProperty("traits", out _), "extra.traits is only emitted when traits exist."); } [TestMethod] diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Program.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Program.cs index 74e33e5f51..68fa9721d4 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Program.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -18,6 +18,7 @@ builder.AddTrxReportProvider(); builder.AddAppInsightsTelemetryProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); ITestApplication app = await builder.BuildAsync(); return await app.RunAsync(); diff --git a/test/UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests/Program.cs b/test/UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests/Program.cs index dce3a96962..803f65ae0a 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests/Program.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -29,6 +29,7 @@ builder.AddCrashDumpProvider(ignoreIfNotSupported: true); builder.AddTrxReportProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); using ITestApplication app = await builder.BuildAsync(); return await app.RunAsync(); diff --git a/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/Program.cs b/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/Program.cs index 8765258282..d970418bc2 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/Program.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -15,6 +15,7 @@ builder.AddCrashDumpProvider(ignoreIfNotSupported: true); builder.AddTrxReportProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); ITestApplication app = await builder.BuildAsync(); return await app.RunAsync(); diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Program.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Program.cs index b0395cb8e8..06fe10174d 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Program.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -29,6 +29,7 @@ builder.AddTrxReportProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); ITestApplication app = await builder.BuildAsync(); return await app.RunAsync(); diff --git a/test/UnitTests/TestFramework.UnitTests/Program.cs b/test/UnitTests/TestFramework.UnitTests/Program.cs index ce5734f730..b1f854c9f2 100644 --- a/test/UnitTests/TestFramework.UnitTests/Program.cs +++ b/test/UnitTests/TestFramework.UnitTests/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions; @@ -15,6 +15,7 @@ builder.AddCrashDumpProvider(ignoreIfNotSupported: true); builder.AddRetryProvider(); builder.AddAzureDevOpsProvider(); +builder.AddCtrfReportProvider(); builder.AddInternalTestFramework(); From 2dc38a3f77f7743e4821dd00d3a8ca27b46199ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 10 Jun 2026 11:19:54 +0200 Subject: [PATCH 15/17] Restore CTRF labels per spec 9.15 and fix build The CTRF spec maintainer (ctrf-io/ctrf#53) confirmed that `labels` is the intended top-level test property for arbitrary key/value test metadata, and that array-of-primitives values are accepted for multi-valued keys. - Switch trait emission from `extra.traits` to top-level `labels`, serializing single-valued keys as scalar strings and multi-valued keys as arrays of strings. - Keep promoting `[TestCategory]` values to the top-level `tags` array (CTRF spec 9.14) so consumers can filter by category without walking labels. - Remove the two no-op self-assignments in ClassicEngine.targets that were flagged in PR review (CTRF and OpenTelemetry version properties are already seeded by Sdk.props.template at pack time). - Fix duplicated yield-return line in SdkTests.cs that broke CI builds. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Sdk/Runner/ClassicEngine.targets | 8 +- .../CtrfReportEngine.cs | 83 +++++++++++-------- .../SdkTests.cs | 1 - .../CtrfReportEngineTests.cs | 55 +++++++----- 4 files changed, 86 insertions(+), 61 deletions(-) diff --git a/src/Package/MSTest.Sdk/Sdk/Runner/ClassicEngine.targets b/src/Package/MSTest.Sdk/Sdk/Runner/ClassicEngine.targets index e0a0a705b4..4dbaa3f102 100644 --- a/src/Package/MSTest.Sdk/Sdk/Runner/ClassicEngine.targets +++ b/src/Package/MSTest.Sdk/Sdk/Runner/ClassicEngine.targets @@ -45,18 +45,18 @@ - $(MicrosoftTestingExtensionsCtrfReportVersion) - $(MicrosoftTestingExtensionsOpenTelemetryVersion) true diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs index ad5393d882..49e1af7fb2 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.cs @@ -507,10 +507,11 @@ private static void WriteTest(Utf8JsonWriter writer, CollapsedTestResult c) WriteOutputLines(writer, "stdout", r.StandardOutput); WriteOutputLines(writer, "stderr", r.StandardError); - // CTRF spec: `tags` is a top-level string array on the Test object used for - // classification. We promote MSTest `[TestCategory("…")]` trait values here - // so CTRF consumers can filter/group by category without having to dig into - // the free-form `extra` object. Other traits remain under `extra.traits`. + // CTRF spec 9.14 (`tags`): top-level string array on the Test object used + // for keyless classification. We promote MSTest `[TestCategory("…")]` trait + // values here so CTRF consumers can filter/group by category without having + // to walk the structured `labels` object. The full set of traits (including + // TestCategory) is also emitted under `labels` below. if (r.Traits is { Count: > 0 }) { bool tagsArrayStarted = false; @@ -535,34 +536,17 @@ private static void WriteTest(Utf8JsonWriter writer, CollapsedTestResult c) } } - // CTRF `extra` (free-form object) — the CTRF spec doesn't define a - // dedicated stable identifier so we surface the MTP UID here for - // cross-tool correlation, alongside other framework metadata. - writer.WritePropertyName("extra"); - writer.WriteStartObject(); - writer.WriteString("uid", r.Uid); - if (r.MethodName is not null) - { - writer.WriteString("method", r.MethodName); - } - - if (r.ExceptionType is not null) - { - writer.WriteString("exceptionType", r.ExceptionType); - } - - // Traits live under `extra.traits` (not a top-level `labels`) because: - // 1. The CTRF spec has no `labels` field on the Test object. - // 2. A test can declare the same trait key multiple times (e.g. several - // [TestCategory] attributes on one MSTest method), so we must group - // values per key — emitting `{ key: value }` would produce duplicate - // JSON property names, which RFC 8259 calls out as not interoperable. - // We emit `{ key: [value, value, ...] }` so every trait key appears once - // and consumers always see a deterministic array shape. The keys appear + // CTRF spec 9.15 (`labels`): top-level object on the Test object for + // structured key/value test metadata. Per ctrf-io/ctrf#53, this is the + // intended home for arbitrary framework-defined traits (Option A confirmed + // by the spec maintainer), and array values are an accepted extension for + // multi-valued keys (e.g. multiple `[TestCategory]` attributes on the same + // MSTest method). We emit a scalar string for single-valued keys and an + // array of strings when the same key appears more than once. Keys appear // in first-seen order; values keep their original declaration order. if (r.Traits is { Count: > 0 }) { - writer.WritePropertyName("traits"); + writer.WritePropertyName("labels"); writer.WriteStartObject(); for (int i = 0; i < r.Traits.Count; i++) { @@ -573,22 +557,55 @@ private static void WriteTest(Utf8JsonWriter writer, CollapsedTestResult c) } writer.WritePropertyName(key); - writer.WriteStartArray(); - writer.WriteStringValue(r.Traits[i].Value); + + int duplicateCount = 0; for (int j = i + 1; j < r.Traits.Count; j++) { if (string.Equals(r.Traits[j].Key, key, StringComparison.Ordinal)) { - writer.WriteStringValue(r.Traits[j].Value); + duplicateCount++; } } - writer.WriteEndArray(); + if (duplicateCount == 0) + { + writer.WriteStringValue(r.Traits[i].Value); + } + else + { + writer.WriteStartArray(); + writer.WriteStringValue(r.Traits[i].Value); + for (int j = i + 1; j < r.Traits.Count; j++) + { + if (string.Equals(r.Traits[j].Key, key, StringComparison.Ordinal)) + { + writer.WriteStringValue(r.Traits[j].Value); + } + } + + writer.WriteEndArray(); + } } writer.WriteEndObject(); } + // CTRF `extra` (free-form object) — the CTRF spec doesn't define a + // dedicated stable identifier so we surface the MTP UID here for + // cross-tool correlation, alongside other framework metadata. + writer.WritePropertyName("extra"); + writer.WriteStartObject(); + writer.WriteString("uid", r.Uid); + if (r.MethodName is not null) + { + writer.WriteString("method", r.MethodName); + } + + if (r.ExceptionType is not null) + { + writer.WriteString("exceptionType", r.ExceptionType); + } + writer.WriteEndObject(); writer.WriteEndObject(); diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs index 5b32d1e09d..b2db816def 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs @@ -201,7 +201,6 @@ public async Task RunTests_With_CentralPackageManagement_Standalone(string multi "--report-azdo", "--crashdump")); - yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, "true", "--report-ctrf", diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs index c50a252711..1c0299ad7d 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs @@ -315,7 +315,7 @@ public async Task GenerateReportAsync_RoundTripsErrorMessageAndStackTrace() } [TestMethod] - public async Task GenerateReportAsync_IncludesTraitsUnderExtraTraits_AndPromotesTestCategoryToTags() + public async Task GenerateReportAsync_PromotesTraitsToLabelsAndTestCategoryToTags() { using var memoryStream = new MemoryFileStream(); CtrfReportEngine engine = CreateEngine(memoryStream); @@ -329,10 +329,11 @@ public async Task GenerateReportAsync_IncludesTraitsUnderExtraTraits_AndPromotes Duration = TimeSpan.Zero, Traits = [ - // Multiple [TestCategory] attributes on one MSTest method produce repeated - // trait entries with the same key. The engine must group them as an array - // under extra.traits (the CTRF spec has no `labels` field, and emitting - // duplicate JSON keys would be non-interoperable per RFC 8259). + // Multiple [TestCategory] attributes on the same MSTest method produce + // repeated trait entries with the same key. Per ctrf-io/ctrf#53 the + // CTRF maintainer confirmed array values for top-level `labels` + // (spec 9.15), so multi-valued keys serialize as JSON arrays and + // single-valued keys serialize as scalar strings. new KeyValuePair("TestCategory", "Fast"), new KeyValuePair("TestCategory", "Smoke"), new KeyValuePair("Owner", "alice"), @@ -345,30 +346,36 @@ public async Task GenerateReportAsync_IncludesTraitsUnderExtraTraits_AndPromotes using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); JsonElement test = document.RootElement.GetProperty("results").GetProperty("tests")[0]; - // No top-level `labels` — the CTRF Test schema doesn't define one. - Assert.IsFalse(test.TryGetProperty("labels", out _), "labels is not part of the CTRF Test schema."); - - // TestCategory values are promoted to the CTRF top-level `tags` array so consumers - // can filter/group by category. The values are preserved in declaration order. + // TestCategory values are promoted to the CTRF top-level `tags` array (spec 9.14) + // so consumers can filter/group by category without walking the labels object. + // The values are preserved in declaration order. JsonElement tags = test.GetProperty("tags"); Assert.AreEqual(JsonValueKind.Array, tags.ValueKind); Assert.AreEqual(2, tags.GetArrayLength()); Assert.AreEqual("Fast", tags[0].GetString()); Assert.AreEqual("Smoke", tags[1].GetString()); - // All traits (including TestCategory) round-trip under extra.traits as - // { key: [value, ...] } so multi-value traits remain valid JSON. - JsonElement traits = test.GetProperty("extra").GetProperty("traits"); - JsonElement testCategory = traits.GetProperty("TestCategory"); + // All traits — including TestCategory — round-trip under the CTRF top-level + // `labels` object (spec 9.15). Single-valued keys are emitted as scalar + // strings; multi-valued keys are emitted as arrays of strings. + JsonElement labels = test.GetProperty("labels"); + Assert.AreEqual(JsonValueKind.Object, labels.ValueKind); + + JsonElement testCategory = labels.GetProperty("TestCategory"); Assert.AreEqual(JsonValueKind.Array, testCategory.ValueKind); Assert.AreEqual(2, testCategory.GetArrayLength()); Assert.AreEqual("Fast", testCategory[0].GetString()); Assert.AreEqual("Smoke", testCategory[1].GetString()); - JsonElement owner = traits.GetProperty("Owner"); - Assert.AreEqual(JsonValueKind.Array, owner.ValueKind); - Assert.AreEqual(1, owner.GetArrayLength()); - Assert.AreEqual("alice", owner[0].GetString()); + JsonElement owner = labels.GetProperty("Owner"); + Assert.AreEqual(JsonValueKind.String, owner.ValueKind); + Assert.AreEqual("alice", owner.GetString()); + + // `extra.traits` was the previous home for these values; it is no longer + // emitted now that the spec-defined `labels` field is populated. + Assert.IsFalse( + test.GetProperty("extra").TryGetProperty("traits", out _), + "Traits are now emitted under top-level labels, not extra.traits."); } [TestMethod] @@ -607,7 +614,9 @@ public async Task GenerateReportAsync_Environment_HasSchemaCompliantShape() public async Task GenerateReportAsync_TestExtra_CarriesMethodNameAndExceptionType() { // method, exceptionType, and uid all live under `extra` so the per-test object - // remains aligned with the CTRF Test schema (which has no `labels` field). + // surfaces framework-defined metadata next to the CTRF-defined `labels` + // (spec 9.15) and `tags` (spec 9.14) fields. A test with no traits should + // emit none of those optional fields. using var memoryStream = new MemoryFileStream(); CtrfReportEngine engine = CreateEngine(memoryStream); CapturedTestResult[] tests = @@ -633,13 +642,13 @@ public async Task GenerateReportAsync_TestExtra_CarriesMethodNameAndExceptionTyp Assert.AreEqual("MyMethod", extra.GetProperty("method").GetString()); Assert.AreEqual("System.InvalidOperationException", extra.GetProperty("exceptionType").GetString()); - // No `labels` is ever emitted — it isn't part of the CTRF Test schema. - Assert.IsFalse(test.TryGetProperty("labels", out _), "labels is not part of the CTRF Test schema."); + // No `labels` is emitted when there are no traits at all (spec 9.15). + Assert.IsFalse(test.TryGetProperty("labels", out _), "labels is only emitted when traits exist."); // No `tags` is emitted when there is no TestCategory trait, and no `traits` is - // emitted under `extra` when the test has no user-supplied traits at all. + // emitted under `extra` (we no longer write `extra.traits` — see labels above). Assert.IsFalse(test.TryGetProperty("tags", out _), "tags is only emitted when TestCategory traits exist."); - Assert.IsFalse(extra.TryGetProperty("traits", out _), "extra.traits is only emitted when traits exist."); + Assert.IsFalse(extra.TryGetProperty("traits", out _), "extra.traits is no longer emitted; traits go to top-level labels."); } [TestMethod] From 23e686d4030b709eff85aadfcd9693fffa773b71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 10 Jun 2026 12:14:23 +0200 Subject: [PATCH 16/17] Address PR feedback: drop unneeded TestApplicationBuilder type check AddCtrfReportProvider only relies on the public ITestApplicationBuilder surface (TestHost and CommandLine properties), so restricting it to the concrete TestApplicationBuilder is unnecessary. Remove the runtime guard and the now-unused InvalidTestApplicationBuilderType resource (regenerated xlfs). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../CtrfReportExtensions.cs | 6 ------ .../Resources/ExtensionResources.resx | 3 --- .../Resources/xlf/ExtensionResources.cs.xlf | 5 ----- .../Resources/xlf/ExtensionResources.de.xlf | 5 ----- .../Resources/xlf/ExtensionResources.es.xlf | 5 ----- .../Resources/xlf/ExtensionResources.fr.xlf | 5 ----- .../Resources/xlf/ExtensionResources.it.xlf | 5 ----- .../Resources/xlf/ExtensionResources.ja.xlf | 5 ----- .../Resources/xlf/ExtensionResources.ko.xlf | 5 ----- .../Resources/xlf/ExtensionResources.pl.xlf | 5 ----- .../Resources/xlf/ExtensionResources.pt-BR.xlf | 5 ----- .../Resources/xlf/ExtensionResources.ru.xlf | 5 ----- .../Resources/xlf/ExtensionResources.tr.xlf | 5 ----- .../Resources/xlf/ExtensionResources.zh-Hans.xlf | 5 ----- .../Resources/xlf/ExtensionResources.zh-Hant.xlf | 5 ----- 15 files changed, 74 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportExtensions.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportExtensions.cs index 5d8a03502c..49eecdc5a0 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportExtensions.cs +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportExtensions.cs @@ -2,7 +2,6 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions.CtrfReport; -using Microsoft.Testing.Extensions.CtrfReport.Resources; using Microsoft.Testing.Platform.Builder; using Microsoft.Testing.Platform.Extensions; using Microsoft.Testing.Platform.Helpers; @@ -23,11 +22,6 @@ public static class CtrfReportExtensions /// The test application builder. public static void AddCtrfReportProvider(this ITestApplicationBuilder builder) { - if (builder is not TestApplicationBuilder) - { - throw new InvalidOperationException(ExtensionResources.InvalidTestApplicationBuilderType); - } - var commandLine = new CtrfReportGeneratorCommandLine(); var compositeCtrfReportGenerator = diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/ExtensionResources.resx b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/ExtensionResources.resx index 10b13c193b..83f65d6b09 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/ExtensionResources.resx +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/ExtensionResources.resx @@ -96,7 +96,4 @@ Example: MyReport_{tfm}.ctrf.json Enable generating a CTRF (Common Test Report Format) JSON report - - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.cs.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.cs.xlf index 665835d295..bfe0165a83 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.cs.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.cs.xlf @@ -66,11 +66,6 @@ Example: MyReport_{tfm}.ctrf.json Enable generating a CTRF (Common Test Report Format) JSON report - - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - - \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.de.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.de.xlf index 3b2fc606ff..15bbc8f994 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.de.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.de.xlf @@ -66,11 +66,6 @@ Example: MyReport_{tfm}.ctrf.json Enable generating a CTRF (Common Test Report Format) JSON report - - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - - \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.es.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.es.xlf index 7768fa8a2c..6caade8705 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.es.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.es.xlf @@ -66,11 +66,6 @@ Example: MyReport_{tfm}.ctrf.json Enable generating a CTRF (Common Test Report Format) JSON report - - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - - \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.fr.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.fr.xlf index 4edf2ed2dd..f58b40f646 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.fr.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.fr.xlf @@ -66,11 +66,6 @@ Example: MyReport_{tfm}.ctrf.json Enable generating a CTRF (Common Test Report Format) JSON report - - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - - \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.it.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.it.xlf index a56086c445..20c821c636 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.it.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.it.xlf @@ -66,11 +66,6 @@ Example: MyReport_{tfm}.ctrf.json Enable generating a CTRF (Common Test Report Format) JSON report - - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - - \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ja.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ja.xlf index 76e08daa3b..a6445f9550 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ja.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ja.xlf @@ -66,11 +66,6 @@ Example: MyReport_{tfm}.ctrf.json Enable generating a CTRF (Common Test Report Format) JSON report - - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - - \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ko.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ko.xlf index c0d69d1593..0fa0072875 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ko.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ko.xlf @@ -66,11 +66,6 @@ Example: MyReport_{tfm}.ctrf.json Enable generating a CTRF (Common Test Report Format) JSON report - - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - - \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pl.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pl.xlf index 1c148b3b63..1d376ab6e3 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pl.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pl.xlf @@ -66,11 +66,6 @@ Example: MyReport_{tfm}.ctrf.json Enable generating a CTRF (Common Test Report Format) JSON report - - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - - \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pt-BR.xlf index 95f41add47..53e9e5f64b 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pt-BR.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.pt-BR.xlf @@ -66,11 +66,6 @@ Example: MyReport_{tfm}.ctrf.json Enable generating a CTRF (Common Test Report Format) JSON report - - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - - \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ru.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ru.xlf index 4b2e3b3118..87f95c747d 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ru.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.ru.xlf @@ -66,11 +66,6 @@ Example: MyReport_{tfm}.ctrf.json Enable generating a CTRF (Common Test Report Format) JSON report - - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - - \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.tr.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.tr.xlf index 5896bb5b25..c64a501627 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.tr.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.tr.xlf @@ -66,11 +66,6 @@ Example: MyReport_{tfm}.ctrf.json Enable generating a CTRF (Common Test Report Format) JSON report - - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - - \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hans.xlf index 144ee1136a..d6145343b2 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hans.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hans.xlf @@ -66,11 +66,6 @@ Example: MyReport_{tfm}.ctrf.json Enable generating a CTRF (Common Test Report Format) JSON report - - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - - \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hant.xlf index c70140b47e..ea01d029b7 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hant.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Resources/xlf/ExtensionResources.zh-Hant.xlf @@ -66,11 +66,6 @@ Example: MyReport_{tfm}.ctrf.json Enable generating a CTRF (Common Test Report Format) JSON report - - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - CTRF report generator only works with builders of type 'Microsoft.Testing.Platform.Builder.TestApplicationBuilder' - - \ No newline at end of file From d5afdcebcde936a28aad6cec9e50e53b17d45950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 10 Jun 2026 12:55:25 +0200 Subject: [PATCH 17/17] Address PR feedback: fix CTRF indentation in MSBuild files The newly-added CTRF entries in Directory.Build.props, MSTest.Sdk.csproj and ClassicEngine.targets started at column 1 instead of matching the surrounding indentation. Reindent them to align with their neighbors. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Directory.Build.props | 2 +- src/Package/MSTest.Sdk/MSTest.Sdk.csproj | 2 +- src/Package/MSTest.Sdk/Sdk/Runner/ClassicEngine.targets | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 275fa146e4..45e417c62d 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -66,7 +66,7 @@ --> 2.0.0 -alpha + alpha