From 9b1f963102761f7f854f6a965cd33b1c95723e09 Mon Sep 17 00:00:00 2001 From: Evangelink Date: Sun, 7 Jun 2026 09:58:01 +0200 Subject: [PATCH 1/5] Add ITestFilter / [TestFilterProvider] for programmatic test filtering (#8894) Introduces a new opt-in extension point so users can plug in their own filtering logic without paying for [AssemblyInitialize] / [ClassInitialize] on tests that will be dropped or skipped: - New attribute Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterProviderAttribute (assembly-level, AllowMultiple = true) declares an ITestFilter implementation. - New interface ITestFilter with single method Filter(TestFilterContext) returning a readonly struct TestFilterResult (Run / Drop / Skip(reason)). - TestFilterContext exposes the data available before any type is loaded: FullyQualifiedName, DisplayName, TestClassName, TestMethodName, Categories, Traits and Priority. Discovery reuses the assembly-graph BFS already powering [AssemblyFixtureProvider] (TypeCache.ProviderDiscovery.cs). Filters are cached per source assembly via TypeCache.GetOrLoadTestFilters and only instantiated once per run. Invocation happens at the very top of UnitTestRunner.RunSingleTestAsync, BEFORE GetTestMethodInfo, so a Drop pays zero AssemblyInit/ClassInit/type-load cost. A Skip(reason) is surfaced as a regular Ignored test result. Filters are composed with AND (first non-Run wins). Filter exceptions become an Error test result so they can't silently affect test selection. Also adds an _assemblyInitializeWasExecuted flag on UnitTestRunner so end-of- assembly cleanup still runs when the last test of an assembly is filtered out. Refs https://github.com/microsoft/testfx/issues/8894 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Execution/TypeCache.FilterDiscovery.cs | 179 ++++++++++++++++++ .../Execution/UnitTestRunner.cs | 157 +++++++++++++++ .../Resources/Resource.resx | 18 ++ .../Resources/xlf/Resource.cs.xlf | 30 +++ .../Resources/xlf/Resource.de.xlf | 30 +++ .../Resources/xlf/Resource.es.xlf | 30 +++ .../Resources/xlf/Resource.fr.xlf | 30 +++ .../Resources/xlf/Resource.it.xlf | 30 +++ .../Resources/xlf/Resource.ja.xlf | 30 +++ .../Resources/xlf/Resource.ko.xlf | 30 +++ .../Resources/xlf/Resource.pl.xlf | 30 +++ .../Resources/xlf/Resource.pt-BR.xlf | 30 +++ .../Resources/xlf/Resource.ru.xlf | 30 +++ .../Resources/xlf/Resource.tr.xlf | 30 +++ .../Resources/xlf/Resource.zh-Hans.xlf | 30 +++ .../Resources/xlf/Resource.zh-Hant.xlf | 30 +++ .../Lifecycle/TestFilterProviderAttribute.cs | 67 +++++++ .../TestFramework/Filtering/ITestFilter.cs | 34 ++++ .../Filtering/TestFilterAction.cs | 31 +++ .../Filtering/TestFilterContext.cs | 69 +++++++ .../Filtering/TestFilterResult.cs | 85 +++++++++ .../PublicAPI/PublicAPI.Unshipped.txt | 30 +++ 22 files changed, 1060 insertions(+) create mode 100644 src/Adapter/MSTestAdapter.PlatformServices/Execution/TypeCache.FilterDiscovery.cs create mode 100644 src/TestFramework/TestFramework/Attributes/Lifecycle/TestFilterProviderAttribute.cs create mode 100644 src/TestFramework/TestFramework/Filtering/ITestFilter.cs create mode 100644 src/TestFramework/TestFramework/Filtering/TestFilterAction.cs create mode 100644 src/TestFramework/TestFramework/Filtering/TestFilterContext.cs create mode 100644 src/TestFramework/TestFramework/Filtering/TestFilterResult.cs diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Execution/TypeCache.FilterDiscovery.cs b/src/Adapter/MSTestAdapter.PlatformServices/Execution/TypeCache.FilterDiscovery.cs new file mode 100644 index 0000000000..130168883e --- /dev/null +++ b/src/Adapter/MSTestAdapter.PlatformServices/Execution/TypeCache.FilterDiscovery.cs @@ -0,0 +1,179 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution; + +internal sealed partial class TypeCache +{ + // Filter instances cached per test assembly source path. Computed lazily on the first request + // for that source so the cost is paid at most once per run, even when many tests target the + // same assembly. + private readonly ConcurrentDictionary> _testFiltersBySource = + new(StringComparer.Ordinal); + + /// + /// Returns the cached instances registered via + /// for the given test assembly source path. + /// + /// The test assembly source path (typically TestMethod.AssemblyName). + /// + /// Discovery is metadata-only for the probe step and never forces the test types of the + /// assembly to load. Filter types are loaded the first time the filter for a given + /// source is requested. + /// + internal IReadOnlyList GetOrLoadTestFilters(string assemblySource) + => _testFiltersBySource.TryGetValue(assemblySource, out IReadOnlyList? cached) + ? cached + : _testFiltersBySource.GetOrAdd(assemblySource, LoadTestFiltersForSource); + + private static IReadOnlyList LoadTestFiltersForSource(string assemblySource) + { + Assembly assembly; + try + { + assembly = PlatformServiceProvider.Instance.FileOperations.LoadAssembly(assemblySource); + } + catch (Exception ex) + { + if (PlatformServiceProvider.Instance.AdapterTraceLogger.IsWarningEnabled) + { + PlatformServiceProvider.Instance.AdapterTraceLogger.Warning( + "TypeCache: Could not load test assembly {0} for TestFilterProvider discovery. {1}", + assemblySource, + ex); + } + + return []; + } + + return DiscoverTestFiltersFromProviders(assembly); + } + + private static IReadOnlyList DiscoverTestFiltersFromProviders(Assembly currentAssembly) + { + List? filters = null; + var visitedFilterTypes = new HashSet(); + + foreach (Assembly candidate in EnumerateCandidateAssemblies(currentAssembly)) + { + bool hasMarker; + try + { + hasMarker = HasTestFilterProviderMarker(candidate); + } + catch (Exception ex) + { + if (PlatformServiceProvider.Instance.AdapterTraceLogger.IsWarningEnabled) + { + PlatformServiceProvider.Instance.AdapterTraceLogger.Warning( + "TypeCache: Exception occurred while probing TestFilterProviderAttribute metadata from assembly {0}. {1}", + SafeGetAssemblyName(candidate), + ex); + } + + continue; + } + + if (!hasMarker) + { + continue; + } + + object[] markers; + try + { + markers = PlatformServiceProvider.Instance.ReflectionOperations.GetCustomAttributes(candidate, typeof(TestFilterProviderAttribute)); + } + catch (Exception ex) + { + // Marker is present (CustomAttributeData saw it) but the attribute cannot be + // instantiated. This typically means the type referenced by typeof(...) cannot be + // loaded. [TestFilterProvider] is explicit opt-in: silently dropping the marker + // would let the user's filter logic disappear at runtime, which is a more + // dangerous failure mode than a clear diagnostic. + string message = string.Format( + CultureInfo.CurrentCulture, + Resource.UTA_TestFilterProviderLoadFailed, + SafeGetAssemblyName(candidate) ?? "", + ex.Message); + throw new TypeInspectionException(message, ex); + } + + if (markers is null || markers.Length == 0) + { + continue; + } + + foreach (object marker in markers) + { + if (marker is not TestFilterProviderAttribute providerAttribute) + { + continue; + } + + Type? filterType = providerAttribute.FilterType; + if (filterType is null || !visitedFilterTypes.Add(filterType)) + { + // De-dup so a filter type referenced from both the consumer assembly and a + // shared infrastructure library isn't applied twice. + continue; + } + + ITestFilter filter = InstantiateTestFilter(filterType); + (filters ??= []).Add(filter); + } + } + + return filters is null ? [] : filters.ToArray(); + } + + private static ITestFilter InstantiateTestFilter(Type filterType) + { + if (filterType.IsGenericType) + { + string message = string.Format(CultureInfo.CurrentCulture, Resource.UTA_TestFilterProviderTypeIsGeneric, filterType.FullName); + throw new TypeInspectionException(message); + } + + if (filterType.IsAbstract || filterType.IsInterface) + { + string message = string.Format(CultureInfo.CurrentCulture, Resource.UTA_TestFilterProviderTypeIsNotInstantiable, filterType.FullName); + throw new TypeInspectionException(message); + } + + if (!typeof(ITestFilter).IsAssignableFrom(filterType)) + { + string message = string.Format(CultureInfo.CurrentCulture, Resource.UTA_TestFilterProviderTypeDoesNotImplementInterface, filterType.FullName, typeof(ITestFilter).FullName); + throw new TypeInspectionException(message); + } + + try + { + return (ITestFilter)Activator.CreateInstance(filterType)!; + } + catch (Exception ex) + { + string message = string.Format(CultureInfo.CurrentCulture, Resource.UTA_TestFilterProviderInstantiationFailed, filterType.FullName, ex.Message); + throw new TypeInspectionException(message, ex); + } + } + + private static bool HasTestFilterProviderMarker(Assembly assembly) + { + // Compare on the attribute type's FullName so we don't trigger attribute construction, + // mirroring the AssemblyFixtureProvider probe. + string markerFullName = typeof(TestFilterProviderAttribute).FullName!; + foreach (CustomAttributeData data in assembly.GetCustomAttributesData()) + { + if (string.Equals(data.AttributeType.FullName, markerFullName, StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } +} diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Execution/UnitTestRunner.cs b/src/Adapter/MSTestAdapter.PlatformServices/Execution/UnitTestRunner.cs index 5e391fb043..dc68fdcfd3 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Execution/UnitTestRunner.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Execution/UnitTestRunner.cs @@ -36,6 +36,12 @@ internal sealed class UnitTestRunner // Used to attach assembly cleanup failures to the right test. private UnitTestElement? _lastRunnableTestInWholeAssembly; + // Tracks whether at least one test in this runner's lifetime triggered AssemblyInitialize. + // Needed so that end-of-assembly cleanup still runs when the very last test of the assembly + // was filtered out by a [TestFilterProvider] (in which case testMethodInfo is null for the + // cleanup decision in RunSingleTestAsync). + private bool _assemblyInitializeWasExecuted; + /// /// Initializes a new instance of the class. /// @@ -133,6 +139,21 @@ internal async Task RunSingleTestAsync(UnitTestElement unitTestEle { testContextForTestExecution = PlatformServiceProvider.Instance.GetTestContext(testMethod, null, testContextProperties, messageLogger, UnitTestOutcome.InProgress); + // Apply user-supplied [TestFilterProvider] filters BEFORE loading the test type, BEFORE + // running [AssemblyInitialize] and BEFORE [ClassInitialize]. This is the whole point of + // the feature: a Drop or Skip here pays none of those costs. See + // https://github.com/microsoft/testfx/issues/8894 for the design. + TestResult[]? filterResult = ApplyTestFilters(unitTestElement); + if (filterResult is not null) + { + return await FinishFilteredOutTestAsync( + testMethod, + testContextProperties, + messageLogger, + filterResult, + testContextForTestExecution).ConfigureAwait(false); + } + // Get the testMethod TestMethodInfo? testMethodInfo = _typeCache.GetTestMethodInfo(testMethod); @@ -148,6 +169,7 @@ internal async Task RunSingleTestAsync(UnitTestElement unitTestEle testContextForAssemblyInit = PlatformServiceProvider.Instance.GetTestContext(testMethod: null, null, testContextProperties, messageLogger, testContextForTestExecution.Context.CurrentTestOutcome); TestResult assemblyInitializeResult = await RunAssemblyInitializeIfNeededAsync(testMethodInfo, testContextForAssemblyInit).ConfigureAwait(false); + _assemblyInitializeWasExecuted |= testMethodInfo.Parent.Parent.IsAssemblyInitializeExecuted; if (assemblyInitializeResult.Outcome != UnitTestOutcome.Passed) { @@ -390,4 +412,139 @@ private static bool IsTestMethodRunnable( } internal void ForceCleanup(IDictionary sourceLevelParameters, IMessageLogger logger) => ClassCleanupManager.ForceCleanup(_typeCache, sourceLevelParameters, logger); + + /// + /// Invokes the chain of instances registered via + /// for the given test. Returns + /// if no filter dropped or skipped the test (test should run normally), + /// an empty array if any filter returned , or a single + /// Skipped if any filter returned . + /// + /// + /// Filters are composed with AND: the first non-Run result wins, the remaining filters + /// are not invoked for that test. A filter exception is surfaced as an Error test result so the + /// failure is visible to the user instead of silently affecting test selection. + /// + private TestResult[]? ApplyTestFilters(UnitTestElement unitTestElement) + { + IReadOnlyList filters = _typeCache.GetOrLoadTestFilters(unitTestElement.TestMethod.AssemblyName); + if (filters.Count == 0) + { + return null; + } + + TestFilterContext context = CreateFilterContext(unitTestElement); + + foreach (ITestFilter filter in filters) + { + TestFilterResult result; + try + { + result = filter.Filter(context); + } + catch (Exception ex) + { + string message = string.Format( + CultureInfo.CurrentCulture, + Resource.UTA_TestFilterProviderThrew, + filter.GetType().FullName, + context.FullyQualifiedName, + ex.Message); + return + [ + new TestResult + { + Outcome = UnitTestOutcome.Error, + TestFailureException = new TestFailedException(UnitTestOutcome.Error, message, ex.TryGetStackTraceInformation()), + } + ]; + } + + switch (result.Action) + { + case TestFilterAction.Run: + continue; + case TestFilterAction.Drop: + return []; + case TestFilterAction.Skip: + return [TestResult.CreateIgnoredResult(result.SkipReason)]; + } + } + + return null; + } + + private static TestFilterContext CreateFilterContext(UnitTestElement element) + { + TestMethod testMethod = element.TestMethod; + string[] categories = element.TestCategory ?? []; + + KeyValuePair[] traits; + if (element.Traits is { Length: > 0 } source) + { + traits = new KeyValuePair[source.Length]; + for (int i = 0; i < source.Length; i++) + { + traits[i] = new KeyValuePair(source[i].Name, source[i].Value); + } + } + else + { + traits = []; + } + + return new TestFilterContext( + fullyQualifiedName: $"{testMethod.FullClassName}.{testMethod.Name}", + displayName: testMethod.DisplayName, + testClassName: testMethod.FullClassName, + testMethodName: testMethod.Name, + categories: categories, + traits: traits, + priority: element.Priority); + } + + /// + /// Handles the bookkeeping (class-cleanup countdown, end-of-assembly cleanup) for a test that + /// was filtered out by a . Mirrors the tail of + /// without ever requiring testMethodInfo, since the filter ran before any type was loaded. + /// + private async Task FinishFilteredOutTestAsync( + TestMethod testMethod, + IDictionary testContextProperties, + IMessageLogger messageLogger, + TestResult[] filterResult, + ITestContext testContextForTestExecution) + { + _classCleanupManager.MarkTestComplete(testMethod, out bool isLastTestInClass); + if (isLastTestInClass) + { + // No class cleanup possible: we never loaded the test type, so there's nothing to + // execute. Still mark the class as complete so end-of-assembly cleanup is gated correctly. + _classCleanupManager.MarkClassComplete(testMethod.FullClassName); + } + + if (_assemblyInitializeWasExecuted && _classCleanupManager.ShouldRunEndOfAssemblyCleanup) + { + ITestContext? testContextForAssemblyCleanup = null; + try + { + testContextForAssemblyCleanup = PlatformServiceProvider.Instance.GetTestContext(testMethod: null, null, testContextProperties, messageLogger, testContextForTestExecution.Context.CurrentTestOutcome); + + TestResult? assemblyCleanupResult = await RunAssemblyCleanupAsync(testContextForAssemblyCleanup, _typeCache, filterResult).ConfigureAwait(false); + if (assemblyCleanupResult is not null) + { + // Current test was filtered (no result), so an assembly cleanup failure needs to + // be associated with the last real test that ran in the assembly. + assemblyCleanupResult.AssociatedUnitTestElement = _lastRunnableTestInWholeAssembly; + filterResult = [.. filterResult, assemblyCleanupResult]; + } + } + finally + { + (testContextForAssemblyCleanup as IDisposable)?.Dispose(); + } + } + + return filterResult; + } } diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/Resource.resx b/src/Adapter/MSTestAdapter.PlatformServices/Resources/Resource.resx index 3380abb157..0fc2dadd53 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/Resource.resx +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/Resource.resx @@ -379,6 +379,24 @@ but received {4} argument(s), with types '{5}'. UTA072: Failed to load [AssemblyFixtureProvider] marker from assembly '{0}'. {1} + + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + + + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + + + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + + + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + UTA026: {0}: Cannot define more than one method with the ClassCleanup attribute inside a class. diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.cs.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.cs.xlf index 5ca4624317..e04c06314b 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.cs.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.cs.xlf @@ -522,6 +522,36 @@ byl však přijat tento počet argumentů: {4} s typy {5}. {0}.TestContext má nesprávný typ. + + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + + + + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + + + + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + + + + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + + + + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + + Method {0}.{1} has wrong signature. The method must be non-static, public, does not return a value and should not take any parameter. Additionally, if you are using async-await in method then return-type must be 'Task' or 'ValueTask'. Metoda {0}.{1} má špatný podpis. Metoda nesmí být static nebo public, nevrací hodnotu a nesmí přijímat žádný parametr. Pokud navíc v metodě používáte operátor async-await, musí být návratový typ Task nebo ValueTask. diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.de.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.de.xlf index ebbffd3709..db0d0ada32 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.de.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.de.xlf @@ -522,6 +522,36 @@ aber empfing {4} Argument(e) mit den Typen „{5}“. "{0}.TestContext" weist einen falschen Typ auf. + + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + + + + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + + + + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + + + + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + + + + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + + Method {0}.{1} has wrong signature. The method must be non-static, public, does not return a value and should not take any parameter. Additionally, if you are using async-await in method then return-type must be 'Task' or 'ValueTask'. Die Methode „{0}.{1}“ weist eine falsche Signatur auf. Die Methode muss nicht statisch und öffentlich sein, und sie darf keinen Wert zurückgeben und keinen Parameter annehmen. Wenn Sie außerdem in der Methode „async-await“ verwenden, muss der Rückgabetyp „Task“ oder „ValueTask“ sein. diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.es.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.es.xlf index afe7ab8315..2d89ea2ae7 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.es.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.es.xlf @@ -522,6 +522,36 @@ pero recibió {4} argumentos, con los tipos '{5}'. Tipo {0}.TestContext no es correcto. + + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + + + + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + + + + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + + + + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + + + + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + + Method {0}.{1} has wrong signature. The method must be non-static, public, does not return a value and should not take any parameter. Additionally, if you are using async-await in method then return-type must be 'Task' or 'ValueTask'. El método {0}.{1} tiene una firma incorrecta. Debe ser un método no estático, público, no devolver ningún valor y no debe aceptar parámetros. Además, si está usando async-await en el método entonces el tipo de valor devuelto debe ser 'Task' o 'ValueTask'. diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.fr.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.fr.xlf index 0e67f56036..01a4b7cbcd 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.fr.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.fr.xlf @@ -522,6 +522,36 @@ mais a reçu {4} argument(s), avec les types « {5} ». {0}.TestContext possède un type incorrect. + + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + + + + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + + + + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + + + + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + + + + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + + Method {0}.{1} has wrong signature. The method must be non-static, public, does not return a value and should not take any parameter. Additionally, if you are using async-await in method then return-type must be 'Task' or 'ValueTask'. La méthode {0}.{1} présente une signature incorrecte. La méthode doit être non statique, publique et ne doit retourner aucune valeur ni accepter aucun paramètre. En outre, si vous utilisez async-await dans la méthode, return-type doit être « Task » ou « ValueTask ». diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.it.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.it.xlf index fc92844e1f..c064f6e35f 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.it.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.it.xlf @@ -522,6 +522,36 @@ ma ha ricevuto {4} argomenti, con tipi '{5}'. Il tipo di {0}.TestContext non è corretto. + + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + + + + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + + + + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + + + + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + + + + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + + Method {0}.{1} has wrong signature. The method must be non-static, public, does not return a value and should not take any parameter. Additionally, if you are using async-await in method then return-type must be 'Task' or 'ValueTask'. La firma del metodo {0}.{1}non è corretta. Il metodo deve essere non statico e pubblico, non deve restituire un valore né accettare parametri. Se inoltre si usa async-await nel metodo di test, il tipo restituito deve essere 'Task' o 'ValueTask'. diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ja.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ja.xlf index a849fd11cc..950b9eabe2 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ja.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ja.xlf @@ -523,6 +523,36 @@ but received {4} argument(s), with types '{5}'. {0}.TestContext は不適切な型を含んでいます。 + + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + + + + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + + + + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + + + + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + + + + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + + Method {0}.{1} has wrong signature. The method must be non-static, public, does not return a value and should not take any parameter. Additionally, if you are using async-await in method then return-type must be 'Task' or 'ValueTask'. メソッド {0}。{1} は不適切なシグネチャを含んでいます。メソッドは non-static および public である必要があり、値を返しません。また、パラメーターを受け取ることはできません。また、メソッドで async-await を使用している場合、戻り値の型は 'Task' または 'ValueTask' である必要があります。 diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ko.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ko.xlf index 74f8f366ca..6503b7c8a5 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ko.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ko.xlf @@ -522,6 +522,36 @@ but received {4} argument(s), with types '{5}'. {0}.TestContext의 형식이 잘못되었습니다. + + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + + + + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + + + + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + + + + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + + + + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + + Method {0}.{1} has wrong signature. The method must be non-static, public, does not return a value and should not take any parameter. Additionally, if you are using async-await in method then return-type must be 'Task' or 'ValueTask'. {0}.{1} 메서드의 서명이 잘못되었습니다. 메서드는 정적이 아니고 공용이어야 하며, 값을 반환하거나 매개 변수를 사용할 수 없습니다. 또한 메서드에서 비동기 대기를 사용하는 경우 반환 형식은 'Task' 또는 'ValueTask'여야 합니다. diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.pl.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.pl.xlf index 1af348a901..6aef1ee0d0 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.pl.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.pl.xlf @@ -522,6 +522,36 @@ ale odebrał argumenty {4} z typami „{5}”. Element {0}.TestContext ma niepoprawny typ. + + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + + + + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + + + + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + + + + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + + + + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + + Method {0}.{1} has wrong signature. The method must be non-static, public, does not return a value and should not take any parameter. Additionally, if you are using async-await in method then return-type must be 'Task' or 'ValueTask'. Metoda {0}.{1}ma nieprawidłową sygnaturę. Metoda musi być niestatyczna, publiczna, nie może zwracać wartości i nie powinna przyjmować żadnego parametru. Ponadto jeśli w metodzie jest używane oczekiwanie asynchroniczne, wtedy zwracanym typem musi być typ „Task” lub „ValueTask”. diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.pt-BR.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.pt-BR.xlf index ea757352ae..2691c69e18 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.pt-BR.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.pt-BR.xlf @@ -522,6 +522,36 @@ mas {4} argumentos recebidos, com tipos '{5}'. O {0}.TestContext é do tipo incorreto. + + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + + + + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + + + + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + + + + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + + + + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + + Method {0}.{1} has wrong signature. The method must be non-static, public, does not return a value and should not take any parameter. Additionally, if you are using async-await in method then return-type must be 'Task' or 'ValueTask'. O método {0}.{1} tem a assinatura incorreta. O método deve ser não estático, público, não deve retornar um valor e não deve receber nenhum parâmetro. Além disso, se você estiver usando async-await no método, o return-type deverá ser "Task" ou "ValueTask". diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ru.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ru.xlf index 11a224d39f..c0b26029f5 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ru.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ru.xlf @@ -522,6 +522,36 @@ but received {4} argument(s), with types '{5}'. Для свойства {0}.TestContext указан неправильный тип. + + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + + + + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + + + + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + + + + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + + + + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + + Method {0}.{1} has wrong signature. The method must be non-static, public, does not return a value and should not take any parameter. Additionally, if you are using async-await in method then return-type must be 'Task' or 'ValueTask'. Метод {0}.{1} имеет неправильную сигнатуру. Метод должен быть нестатическим и открытым, не должен возвращать значение и принимать параметры. Кроме того, при использовании async-await в методе возвращаемое значение должно иметь тип "Task" или "ValueTask". diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.tr.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.tr.xlf index b85acecf72..b19f4613d1 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.tr.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.tr.xlf @@ -522,6 +522,36 @@ ancak, '{5}' türünde {4} bağımsız değişken aldı. {0}.TestContext yanlış türe sahip. + + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + + + + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + + + + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + + + + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + + + + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + + Method {0}.{1} has wrong signature. The method must be non-static, public, does not return a value and should not take any parameter. Additionally, if you are using async-await in method then return-type must be 'Task' or 'ValueTask'. {0}.{1} yönteminin imzası yanlış. Yöntem statik olmayan, genel, değer döndürmeyen bir yöntem olmalı, hiçbir parametre almamalıdır. Bunlara ek olarak, yöntemde async-await kullanıyorsanız return-type değeri 'Task' veya 'ValueTask' olmalıdır. diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.zh-Hans.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.zh-Hans.xlf index e9a8319a66..eec9137b20 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.zh-Hans.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.zh-Hans.xlf @@ -522,6 +522,36 @@ but received {4} argument(s), with types '{5}'. {0}.TestContext 的类型不正确。 + + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + + + + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + + + + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + + + + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + + + + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + + Method {0}.{1} has wrong signature. The method must be non-static, public, does not return a value and should not take any parameter. Additionally, if you are using async-await in method then return-type must be 'Task' or 'ValueTask'. 方法 {0}。{1}的签名错误。该方法必须是非静态的公共方法、不返回值并且不应采用任何参数。此外，如果在方法中使用同步等待，则返回类型必须为“Task”或“Value Task”。 diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.zh-Hant.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.zh-Hant.xlf index 227de99936..0010861a4d 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.zh-Hant.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.zh-Hant.xlf @@ -522,6 +522,36 @@ but received {4} argument(s), with types '{5}'. {0}.TestContext 有不正確的類型。 + + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + UTA077: Failed to instantiate type '{0}' referenced by [TestFilterProvider]. The type must declare a public parameterless constructor. {1} + + + + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + + + + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + UTA076: Type '{0}' referenced by [TestFilterProvider] must implement '{1}'. + + + + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + UTA074: Type '{0}' referenced by [TestFilterProvider] must not be generic. + + + + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + UTA075: Type '{0}' referenced by [TestFilterProvider] must be a concrete class, not an interface or abstract class. + + Method {0}.{1} has wrong signature. The method must be non-static, public, does not return a value and should not take any parameter. Additionally, if you are using async-await in method then return-type must be 'Task' or 'ValueTask'. 方法 {0}.{1} 有錯誤的簽章。方法必須為非靜態、公用、不傳回值，並且不應該接受任何參數。此外，如果您在方法中使用 async-await，則傳回類型必須是 'Task' 或 'ValueTask'。 diff --git a/src/TestFramework/TestFramework/Attributes/Lifecycle/TestFilterProviderAttribute.cs b/src/TestFramework/TestFramework/Attributes/Lifecycle/TestFilterProviderAttribute.cs new file mode 100644 index 0000000000..5d609259ea --- /dev/null +++ b/src/TestFramework/TestFramework/Attributes/Lifecycle/TestFilterProviderAttribute.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Registers a user-supplied implementation that the MSTest adapter +/// invokes for every test it is about to run, after any command-line filter (--filter, +/// Test Explorer selection, etc.) has been applied. +/// +/// +/// +/// Apply this attribute at the assembly level on either the test assembly itself or on a referenced +/// infrastructure library. Every test assembly that ends up loading the marked assembly at runtime +/// will pick up the filter without the test project needing to declare anything itself. +/// +/// +/// The filter type must be a non-generic class with a public parameterless constructor that +/// implements . A single instance is created per test assembly per test +/// run and reused for every test of that assembly. +/// +/// +/// The attribute can be applied multiple times on the same assembly to register more than one +/// filter type. When multiple filters are registered, a test runs only if every filter returns +/// . The first non-Run result wins; the remaining filters +/// are not invoked for that test. +/// +/// +/// The filter runs before the test type is loaded, before [AssemblyInitialize], +/// before [ClassInitialize], and before the test constructor is invoked, so dropping or +/// skipping a test through avoids paying any of those costs. +/// +/// +/// +/// +/// // In Contoso.TestInfra.dll +/// [assembly: TestFilterProvider(typeof(NightlyFilter))] +/// +/// public sealed class NightlyFilter : ITestFilter +/// { +/// public TestFilterResult Filter(TestFilterContext context) +/// => context.Categories.Contains("Nightly") +/// && Environment.GetEnvironmentVariable("RUN_NIGHTLY") != "1" +/// ? TestFilterResult.Skip("Set RUN_NIGHTLY=1 to run nightly tests.") +/// : TestFilterResult.Run; +/// } +/// +/// +[AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true, Inherited = false)] +public sealed class TestFilterProviderAttribute : Attribute +{ + /// + /// Initializes a new instance of the class. + /// + /// + /// The implementation to instantiate and invoke for every test in + /// the consuming test assembly. Must be a non-generic class with a public parameterless + /// constructor. + /// + public TestFilterProviderAttribute(Type filterType) + => FilterType = filterType ?? throw new ArgumentNullException(nameof(filterType)); + + /// + /// Gets the implementation registered by this attribute. + /// + public Type FilterType { get; } +} diff --git a/src/TestFramework/TestFramework/Filtering/ITestFilter.cs b/src/TestFramework/TestFramework/Filtering/ITestFilter.cs new file mode 100644 index 0000000000..bdd55027e0 --- /dev/null +++ b/src/TestFramework/TestFramework/Filtering/ITestFilter.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Implemented by a user-supplied test filter that decides, on a per-test basis, whether the test +/// should run, be silently dropped, or be reported as skipped. +/// +/// +/// +/// Implementations are registered via at the assembly +/// level. The MSTest adapter creates one instance per assembly per test run (via the public +/// parameterless constructor) and invokes for every test that survived the +/// command-line filter, before that test's type is loaded. +/// +/// +/// Implementations should be allocation-free and thread-safe; they may be called concurrently for +/// tests in different classes. +/// +/// +public interface ITestFilter +{ + /// + /// Decides whether the test described by should run. + /// + /// Metadata describing the test under consideration. + /// + /// to let the test execute normally, + /// to silently drop the test (no result emitted), or + /// to report the test as Skipped with a reason. + /// + TestFilterResult Filter(TestFilterContext context); +} diff --git a/src/TestFramework/TestFramework/Filtering/TestFilterAction.cs b/src/TestFramework/TestFramework/Filtering/TestFilterAction.cs new file mode 100644 index 0000000000..235dc230ef --- /dev/null +++ b/src/TestFramework/TestFramework/Filtering/TestFilterAction.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Indicates how the MSTest adapter should treat a test for which an +/// returned a particular . +/// +public enum TestFilterAction +{ + /// + /// Run the test normally. This is the default action so that default(TestFilterResult) + /// is the safe choice when an implementation forgets to set a result explicitly. + /// + Run = 0, + + /// + /// Silently drop the test. No is emitted, the test does not appear in + /// the test count, and the declaring class's [ClassInitialize] is not invoked unless + /// another (non-dropped) test in the same class still has to run. This matches the semantics + /// of the platform --filter command-line option. + /// + Drop, + + /// + /// Report the test as Skipped, with the reason supplied to . + /// The test appears in the test count, the TRX/console output, and IDE test explorers. + /// + Skip, +} diff --git a/src/TestFramework/TestFramework/Filtering/TestFilterContext.cs b/src/TestFramework/TestFramework/Filtering/TestFilterContext.cs new file mode 100644 index 0000000000..10c9b9d2ba --- /dev/null +++ b/src/TestFramework/TestFramework/Filtering/TestFilterContext.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Read-only snapshot of the metadata MSTest exposes to an for a single +/// test under consideration. +/// +/// +/// Only metadata that is available without loading the test type is exposed; +/// must be able to decide using strings, categories, traits, and priority alone. This is what allows +/// the filter to drop tests before their declaring type is loaded and before +/// [AssemblyInitialize] / [ClassInitialize] run. +/// +public sealed class TestFilterContext +{ + /// + /// Initializes a new instance of the class. + /// + /// The fully qualified test name (Namespace.Class.Method). + /// The display name reported for the test (often equal to ). + /// The fully qualified name of the declaring test class. + /// The unqualified test method name. + /// The values declared on the test (and its class). + /// The traits attached to the test. Multiple traits can share the same key. + /// The value if any, otherwise . + public TestFilterContext( + string fullyQualifiedName, + string displayName, + string testClassName, + string testMethodName, + IReadOnlyList categories, + IReadOnlyList> traits, + int? priority) + { + FullyQualifiedName = fullyQualifiedName ?? throw new ArgumentNullException(nameof(fullyQualifiedName)); + DisplayName = displayName ?? throw new ArgumentNullException(nameof(displayName)); + TestClassName = testClassName ?? throw new ArgumentNullException(nameof(testClassName)); + TestMethodName = testMethodName ?? throw new ArgumentNullException(nameof(testMethodName)); + Categories = categories ?? throw new ArgumentNullException(nameof(categories)); + Traits = traits ?? throw new ArgumentNullException(nameof(traits)); + Priority = priority; + } + + /// Gets the fully qualified test name (Namespace.Class.Method). + public string FullyQualifiedName { get; } + + /// Gets the display name of the test. + public string DisplayName { get; } + + /// Gets the fully qualified name of the declaring test class. + public string TestClassName { get; } + + /// Gets the unqualified test method name. + public string TestMethodName { get; } + + /// Gets the test categories declared via . + public IReadOnlyList Categories { get; } + + /// + /// Gets the traits attached to this test. Multiple traits can share the same key; consumers + /// must therefore not assume the collection behaves like a dictionary. + /// + public IReadOnlyList> Traits { get; } + + /// Gets the value if any, otherwise . + public int? Priority { get; } +} diff --git a/src/TestFramework/TestFramework/Filtering/TestFilterResult.cs b/src/TestFramework/TestFramework/Filtering/TestFilterResult.cs new file mode 100644 index 0000000000..b5eb752610 --- /dev/null +++ b/src/TestFramework/TestFramework/Filtering/TestFilterResult.cs @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// The decision returned by an for a single test. +/// +/// +/// Designed as a so the filter hot path stays +/// allocation-free. The static / properties (one shared value +/// each) and the parameterized factory are the only ways to construct +/// a result; this keeps the surface evolvable. +/// +public readonly struct TestFilterResult : IEquatable +{ + private TestFilterResult(TestFilterAction action, string? skipReason) + { + Action = action; + SkipReason = skipReason; + } + + /// + /// Gets the action MSTest should apply for the test that produced this result. + /// + public TestFilterAction Action { get; } + + /// + /// Gets the reason supplied to . Non- only when + /// is . + /// + public string? SkipReason { get; } + + /// + /// Gets the result indicating that the test should run normally. + /// + /// + /// Equivalent to default(TestFilterResult); is the + /// default enum value so a filter that forgets to assign a result still defaults to running + /// the test. + /// + public static TestFilterResult Run { get; } = new(TestFilterAction.Run, null); + + /// + /// Gets the result indicating that the test should be silently dropped. Matches the semantics + /// of the command-line --filter option: no test result is emitted and the test is not + /// counted. + /// + public static TestFilterResult Drop { get; } = new(TestFilterAction.Drop, null); + + /// + /// Creates a result indicating that the test should be reported as Skipped with the given reason. + /// + /// A non-empty human-readable explanation surfaced in TRX / console / IDE output. + /// A with equal to . + /// Thrown when is . + public static TestFilterResult Skip(string reason) + => new(TestFilterAction.Skip, reason ?? throw new ArgumentNullException(nameof(reason))); + + /// + public bool Equals(TestFilterResult other) + => Action == other.Action && string.Equals(SkipReason, other.SkipReason, StringComparison.Ordinal); + + /// + public override bool Equals(object? obj) + => obj is TestFilterResult other && Equals(other); + + /// + public override int GetHashCode() + { + unchecked + { + int hash = 17; + hash = (hash * 31) + (int)Action; + hash = (hash * 31) + (SkipReason?.GetHashCode() ?? 0); + return hash; + } + } + + /// Equality operator. + public static bool operator ==(TestFilterResult left, TestFilterResult right) => left.Equals(right); + + /// Inequality operator. + public static bool operator !=(TestFilterResult left, TestFilterResult right) => !left.Equals(right); +} diff --git a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt index ea572077cb..f890ab2b2e 100644 --- a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt @@ -5,9 +5,39 @@ Microsoft.VisualStudio.TestTools.UnitTesting.AssemblyFixtureProviderAttribute.As Microsoft.VisualStudio.TestTools.UnitTesting.AssemblyFixtureProviderAttribute.FixtureType.get -> System.Type! Microsoft.VisualStudio.TestTools.UnitTesting.AssertFailedException.ActualText.get -> string? Microsoft.VisualStudio.TestTools.UnitTesting.AssertFailedException.ExpectedText.get -> string? +Microsoft.VisualStudio.TestTools.UnitTesting.ITestFilter +Microsoft.VisualStudio.TestTools.UnitTesting.ITestFilter.Filter(Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext! context) -> Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult Microsoft.VisualStudio.TestTools.UnitTesting.SequenceOrder Microsoft.VisualStudio.TestTools.UnitTesting.SequenceOrder.InAnyOrder = 1 -> Microsoft.VisualStudio.TestTools.UnitTesting.SequenceOrder Microsoft.VisualStudio.TestTools.UnitTesting.SequenceOrder.InOrder = 0 -> Microsoft.VisualStudio.TestTools.UnitTesting.SequenceOrder +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterAction +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterAction.Drop = 1 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterAction +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterAction.Run = 0 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterAction +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterAction.Skip = 2 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterAction +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.Categories.get -> System.Collections.Generic.IReadOnlyList! +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.DisplayName.get -> string! +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.FullyQualifiedName.get -> string! +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.Priority.get -> int? +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.TestClassName.get -> string! +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.TestFilterContext(string! fullyQualifiedName, string! displayName, string! testClassName, string! testMethodName, System.Collections.Generic.IReadOnlyList! categories, System.Collections.Generic.IReadOnlyList>! traits, int? priority) -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.TestMethodName.get -> string! +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.Traits.get -> System.Collections.Generic.IReadOnlyList>! +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterProviderAttribute +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterProviderAttribute.FilterType.get -> System.Type! +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterProviderAttribute.TestFilterProviderAttribute(System.Type! filterType) -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult.Action.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterAction +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult.Equals(Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult other) -> bool +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult.SkipReason.get -> string? +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult.TestFilterResult() -> void +override Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult.Equals(object? obj) -> bool +override Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult.GetHashCode() -> int +static Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult.Drop.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult +static Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult.operator !=(Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult left, Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult right) -> bool +static Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult.operator ==(Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult left, Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult right) -> bool +static Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult.Run.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult +static Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult.Skip(string! reason) -> Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterResult static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.AreAllDistinct(System.Collections.IEnumerable? collection, System.Collections.IEqualityComparer? comparer, string? message = "", string! collectionExpression = "") -> void static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.AreAllDistinct(System.Collections.IEnumerable? collection, string? message = "", string! collectionExpression = "") -> void static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.AreAllDistinct(System.Collections.Generic.IEnumerable? collection, System.Collections.Generic.IEqualityComparer? comparer, string? message = "", string! collectionExpression = "") -> void From 9f956e8fa805572ed0ff69b134a3c53c39ec7b6e Mon Sep 17 00:00:00 2001 From: Evangelink Date: Sun, 7 Jun 2026 10:11:54 +0200 Subject: [PATCH 2/5] Simplify [TestFilterProvider] to a single filter per assembly Removes multi-filter discovery (no BFS across referenced assemblies, no AND composition, no ordering). Each test assembly may declare at most one [assembly: TestFilterProvider(typeof(T))]; users compose multiple strategies inside their own ITestFilter implementation. This sidesteps ordering questions and dedup logic. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Execution/TypeCache.FilterDiscovery.cs | 142 +++++++----------- .../Execution/UnitTestRunner.cs | 88 +++++------ .../Lifecycle/TestFilterProviderAttribute.cs | 16 +- 3 files changed, 102 insertions(+), 144 deletions(-) diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Execution/TypeCache.FilterDiscovery.cs b/src/Adapter/MSTestAdapter.PlatformServices/Execution/TypeCache.FilterDiscovery.cs index 130168883e..34459b8fec 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Execution/TypeCache.FilterDiscovery.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Execution/TypeCache.FilterDiscovery.cs @@ -8,28 +8,31 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution; internal sealed partial class TypeCache { - // Filter instances cached per test assembly source path. Computed lazily on the first request - // for that source so the cost is paid at most once per run, even when many tests target the - // same assembly. - private readonly ConcurrentDictionary> _testFiltersBySource = + // Single filter instance cached per test assembly source path. Computed lazily on the first + // request for that source so the cost is paid at most once per run, even when many tests + // target the same assembly. Stored as a TestFilterBox so the dictionary can cache the + // "no filter" answer alongside real filter instances. + private readonly ConcurrentDictionary _testFilterBySource = new(StringComparer.Ordinal); /// - /// Returns the cached instances registered via - /// for the given test assembly source path. + /// Returns the cached instance registered via + /// on the given test assembly, or + /// if the assembly does not register one. /// /// The test assembly source path (typically TestMethod.AssemblyName). /// /// Discovery is metadata-only for the probe step and never forces the test types of the - /// assembly to load. Filter types are loaded the first time the filter for a given - /// source is requested. + /// assembly to load. The filter type is loaded the first time the filter for a + /// given source is requested. Only the test assembly itself is inspected — registering a + /// in a referenced library has no effect. /// - internal IReadOnlyList GetOrLoadTestFilters(string assemblySource) - => _testFiltersBySource.TryGetValue(assemblySource, out IReadOnlyList? cached) - ? cached - : _testFiltersBySource.GetOrAdd(assemblySource, LoadTestFiltersForSource); + internal ITestFilter? GetOrLoadTestFilter(string assemblySource) + => _testFilterBySource + .GetOrAdd(assemblySource, static src => new TestFilterBox(LoadTestFilterForSource(src))) + .Filter; - private static IReadOnlyList LoadTestFiltersForSource(string assemblySource) + private static ITestFilter? LoadTestFilterForSource(string assemblySource) { Assembly assembly; try @@ -46,88 +49,46 @@ private static IReadOnlyList LoadTestFiltersForSource(string assemb ex); } - return []; + return null; } - return DiscoverTestFiltersFromProviders(assembly); + return DiscoverTestFilterFromProvider(assembly); } - private static IReadOnlyList DiscoverTestFiltersFromProviders(Assembly currentAssembly) + private static ITestFilter? DiscoverTestFilterFromProvider(Assembly testAssembly) { - List? filters = null; - var visitedFilterTypes = new HashSet(); - - foreach (Assembly candidate in EnumerateCandidateAssemblies(currentAssembly)) + // Cheap metadata-only probe first: avoid loading the filter's Type unless the attribute is + // actually present. Mirrors the AssemblyFixtureProvider probe pattern. + if (!HasTestFilterProviderMarker(testAssembly)) { - bool hasMarker; - try - { - hasMarker = HasTestFilterProviderMarker(candidate); - } - catch (Exception ex) - { - if (PlatformServiceProvider.Instance.AdapterTraceLogger.IsWarningEnabled) - { - PlatformServiceProvider.Instance.AdapterTraceLogger.Warning( - "TypeCache: Exception occurred while probing TestFilterProviderAttribute metadata from assembly {0}. {1}", - SafeGetAssemblyName(candidate), - ex); - } - - continue; - } - - if (!hasMarker) - { - continue; - } - - object[] markers; - try - { - markers = PlatformServiceProvider.Instance.ReflectionOperations.GetCustomAttributes(candidate, typeof(TestFilterProviderAttribute)); - } - catch (Exception ex) - { - // Marker is present (CustomAttributeData saw it) but the attribute cannot be - // instantiated. This typically means the type referenced by typeof(...) cannot be - // loaded. [TestFilterProvider] is explicit opt-in: silently dropping the marker - // would let the user's filter logic disappear at runtime, which is a more - // dangerous failure mode than a clear diagnostic. - string message = string.Format( - CultureInfo.CurrentCulture, - Resource.UTA_TestFilterProviderLoadFailed, - SafeGetAssemblyName(candidate) ?? "", - ex.Message); - throw new TypeInspectionException(message, ex); - } - - if (markers is null || markers.Length == 0) - { - continue; - } + return null; + } - foreach (object marker in markers) - { - if (marker is not TestFilterProviderAttribute providerAttribute) - { - continue; - } - - Type? filterType = providerAttribute.FilterType; - if (filterType is null || !visitedFilterTypes.Add(filterType)) - { - // De-dup so a filter type referenced from both the consumer assembly and a - // shared infrastructure library isn't applied twice. - continue; - } - - ITestFilter filter = InstantiateTestFilter(filterType); - (filters ??= []).Add(filter); - } + object[] markers; + try + { + markers = PlatformServiceProvider.Instance.ReflectionOperations.GetCustomAttributes(testAssembly, typeof(TestFilterProviderAttribute)); + } + catch (Exception ex) + { + // Marker is present (CustomAttributeData saw it) but the attribute cannot be + // instantiated. This typically means the type referenced by typeof(...) cannot be + // loaded. [TestFilterProvider] is explicit opt-in: silently dropping the marker + // would let the user's filter logic disappear at runtime, which is a more + // dangerous failure mode than a clear diagnostic. + string message = string.Format( + CultureInfo.CurrentCulture, + Resource.UTA_TestFilterProviderLoadFailed, + SafeGetAssemblyName(testAssembly) ?? "", + ex.Message); + throw new TypeInspectionException(message, ex); } - return filters is null ? [] : filters.ToArray(); + return markers is { Length: > 0 } + && markers[0] is TestFilterProviderAttribute providerAttribute + && providerAttribute.FilterType is not null + ? InstantiateTestFilter(providerAttribute.FilterType) + : null; } private static ITestFilter InstantiateTestFilter(Type filterType) @@ -176,4 +137,13 @@ private static bool HasTestFilterProviderMarker(Assembly assembly) return false; } + + // Tiny holder so the cache can distinguish "not computed yet" (missing key) from + // "computed and result is no filter" (present key with Filter = null). + private sealed class TestFilterBox + { + public TestFilterBox(ITestFilter? filter) => Filter = filter; + + public ITestFilter? Filter { get; } + } } diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Execution/UnitTestRunner.cs b/src/Adapter/MSTestAdapter.PlatformServices/Execution/UnitTestRunner.cs index dc68fdcfd3..6f446f725c 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Execution/UnitTestRunner.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Execution/UnitTestRunner.cs @@ -139,11 +139,11 @@ internal async Task RunSingleTestAsync(UnitTestElement unitTestEle { testContextForTestExecution = PlatformServiceProvider.Instance.GetTestContext(testMethod, null, testContextProperties, messageLogger, UnitTestOutcome.InProgress); - // Apply user-supplied [TestFilterProvider] filters BEFORE loading the test type, BEFORE + // Apply user-supplied [TestFilterProvider] filter BEFORE loading the test type, BEFORE // running [AssemblyInitialize] and BEFORE [ClassInitialize]. This is the whole point of // the feature: a Drop or Skip here pays none of those costs. See // https://github.com/microsoft/testfx/issues/8894 for the design. - TestResult[]? filterResult = ApplyTestFilters(unitTestElement); + TestResult[]? filterResult = ApplyTestFilter(unitTestElement); if (filterResult is not null) { return await FinishFilteredOutTestAsync( @@ -414,64 +414,58 @@ private static bool IsTestMethodRunnable( internal void ForceCleanup(IDictionary sourceLevelParameters, IMessageLogger logger) => ClassCleanupManager.ForceCleanup(_typeCache, sourceLevelParameters, logger); /// - /// Invokes the chain of instances registered via - /// for the given test. Returns - /// if no filter dropped or skipped the test (test should run normally), - /// an empty array if any filter returned , or a single - /// Skipped if any filter returned . + /// Invokes the user-supplied registered via + /// for the test assembly, if any. Returns + /// if no filter is registered or the filter returned + /// (test should run normally), an empty array if the + /// filter returned , or a single Skipped + /// if the filter returned . /// /// - /// Filters are composed with AND: the first non-Run result wins, the remaining filters - /// are not invoked for that test. A filter exception is surfaced as an Error test result so the - /// failure is visible to the user instead of silently affecting test selection. + /// A filter exception is surfaced as an Error test result so the failure is visible to the + /// user instead of silently affecting test selection. + /// is single-per-assembly by design: callers that want to combine multiple strategies should + /// compose them explicitly inside their implementation. /// - private TestResult[]? ApplyTestFilters(UnitTestElement unitTestElement) + private TestResult[]? ApplyTestFilter(UnitTestElement unitTestElement) { - IReadOnlyList filters = _typeCache.GetOrLoadTestFilters(unitTestElement.TestMethod.AssemblyName); - if (filters.Count == 0) + ITestFilter? filter = _typeCache.GetOrLoadTestFilter(unitTestElement.TestMethod.AssemblyName); + if (filter is null) { return null; } TestFilterContext context = CreateFilterContext(unitTestElement); - foreach (ITestFilter filter in filters) + TestFilterResult result; + try { - TestFilterResult result; - try - { - result = filter.Filter(context); - } - catch (Exception ex) - { - string message = string.Format( - CultureInfo.CurrentCulture, - Resource.UTA_TestFilterProviderThrew, - filter.GetType().FullName, - context.FullyQualifiedName, - ex.Message); - return - [ - new TestResult - { - Outcome = UnitTestOutcome.Error, - TestFailureException = new TestFailedException(UnitTestOutcome.Error, message, ex.TryGetStackTraceInformation()), - } - ]; - } - - switch (result.Action) - { - case TestFilterAction.Run: - continue; - case TestFilterAction.Drop: - return []; - case TestFilterAction.Skip: - return [TestResult.CreateIgnoredResult(result.SkipReason)]; - } + result = filter.Filter(context); + } + catch (Exception ex) + { + string message = string.Format( + CultureInfo.CurrentCulture, + Resource.UTA_TestFilterProviderThrew, + filter.GetType().FullName, + context.FullyQualifiedName, + ex.Message); + return + [ + new TestResult + { + Outcome = UnitTestOutcome.Error, + TestFailureException = new TestFailedException(UnitTestOutcome.Error, message, ex.TryGetStackTraceInformation()), + } + ]; } - return null; + return result.Action switch + { + TestFilterAction.Drop => [], + TestFilterAction.Skip => [TestResult.CreateIgnoredResult(result.SkipReason)], + _ => null, + }; } private static TestFilterContext CreateFilterContext(UnitTestElement element) diff --git a/src/TestFramework/TestFramework/Attributes/Lifecycle/TestFilterProviderAttribute.cs b/src/TestFramework/TestFramework/Attributes/Lifecycle/TestFilterProviderAttribute.cs index 5d609259ea..02e3a5dc0c 100644 --- a/src/TestFramework/TestFramework/Attributes/Lifecycle/TestFilterProviderAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/Lifecycle/TestFilterProviderAttribute.cs @@ -10,9 +10,10 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting; /// /// /// -/// Apply this attribute at the assembly level on either the test assembly itself or on a referenced -/// infrastructure library. Every test assembly that ends up loading the marked assembly at runtime -/// will pick up the filter without the test project needing to declare anything itself. +/// Apply this attribute at the assembly level on the test assembly itself. At most one +/// may be applied per test assembly; this is intentional +/// so that filter ordering is not part of the public API. If multiple filtering strategies are +/// needed, compose them explicitly inside a single implementation. /// /// /// The filter type must be a non-generic class with a public parameterless constructor that @@ -20,12 +21,6 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting; /// run and reused for every test of that assembly. /// /// -/// The attribute can be applied multiple times on the same assembly to register more than one -/// filter type. When multiple filters are registered, a test runs only if every filter returns -/// . The first non-Run result wins; the remaining filters -/// are not invoked for that test. -/// -/// /// The filter runs before the test type is loaded, before [AssemblyInitialize], /// before [ClassInitialize], and before the test constructor is invoked, so dropping or /// skipping a test through avoids paying any of those costs. @@ -33,7 +28,6 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting; /// /// /// -/// // In Contoso.TestInfra.dll /// [assembly: TestFilterProvider(typeof(NightlyFilter))] /// /// public sealed class NightlyFilter : ITestFilter @@ -46,7 +40,7 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting; /// } /// /// -[AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true, Inherited = false)] +[AttributeUsage(AttributeTargets.Assembly, AllowMultiple = false, Inherited = false)] public sealed class TestFilterProviderAttribute : Attribute { /// From f603f9c0d4fbc2ece310f723ea418bc562ca484e Mon Sep 17 00:00:00 2001 From: Evangelink Date: Sun, 7 Jun 2026 13:17:10 +0200 Subject: [PATCH 3/5] Address code-review findings on [TestFilterProvider] - TestFilterResult.Skip: reject null/empty/whitespace reasons (matches XML doc). - TypeCache: enforce single [TestFilterProvider] per assembly (UTA079) instead of silently first-wins. - ITestFilter: document UTA078 Error contract when Filter throws, and clarify single-instance caching semantics. - Add unit tests for TestFilterResult validation and InstantiateTestFilter error paths (UTA074-077). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Execution/TypeCache.FilterDiscovery.cs | 22 ++- .../Resources/Resource.resx | 3 + .../Resources/xlf/Resource.cs.xlf | 5 + .../Resources/xlf/Resource.de.xlf | 5 + .../Resources/xlf/Resource.es.xlf | 5 + .../Resources/xlf/Resource.fr.xlf | 5 + .../Resources/xlf/Resource.it.xlf | 5 + .../Resources/xlf/Resource.ja.xlf | 5 + .../Resources/xlf/Resource.ko.xlf | 5 + .../Resources/xlf/Resource.pl.xlf | 5 + .../Resources/xlf/Resource.pt-BR.xlf | 5 + .../Resources/xlf/Resource.ru.xlf | 5 + .../Resources/xlf/Resource.tr.xlf | 5 + .../Resources/xlf/Resource.zh-Hans.xlf | 5 + .../Resources/xlf/Resource.zh-Hant.xlf | 5 + .../TestFramework/Filtering/ITestFilter.cs | 15 +- .../Filtering/TestFilterResult.cs | 7 +- .../TypeCacheTestFilterProviderTests.cs | 129 ++++++++++++++++++ .../Filtering/TestFilterResultTests.cs | 94 +++++++++++++ 19 files changed, 324 insertions(+), 11 deletions(-) create mode 100644 test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TypeCacheTestFilterProviderTests.cs create mode 100644 test/UnitTests/TestFramework.UnitTests/Filtering/TestFilterResultTests.cs diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Execution/TypeCache.FilterDiscovery.cs b/src/Adapter/MSTestAdapter.PlatformServices/Execution/TypeCache.FilterDiscovery.cs index 34459b8fec..6d1413fd88 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Execution/TypeCache.FilterDiscovery.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Execution/TypeCache.FilterDiscovery.cs @@ -84,14 +84,26 @@ internal sealed partial class TypeCache throw new TypeInspectionException(message, ex); } - return markers is { Length: > 0 } - && markers[0] is TestFilterProviderAttribute providerAttribute - && providerAttribute.FilterType is not null - ? InstantiateTestFilter(providerAttribute.FilterType) + if (markers is null || markers.Length == 0) + { + return null; + } + + if (markers.Length > 1) + { + string message = string.Format( + CultureInfo.CurrentCulture, + Resource.UTA_TestFilterProviderMultipleDeclared, + SafeGetAssemblyName(testAssembly) ?? ""); + throw new TypeInspectionException(message); + } + + return markers[0] is TestFilterProviderAttribute { FilterType: { } filterType } + ? InstantiateTestFilter(filterType) : null; } - private static ITestFilter InstantiateTestFilter(Type filterType) + internal static ITestFilter InstantiateTestFilter(Type filterType) { if (filterType.IsGenericType) { diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/Resource.resx b/src/Adapter/MSTestAdapter.PlatformServices/Resources/Resource.resx index 0fc2dadd53..e893ac5d3d 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/Resource.resx +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/Resource.resx @@ -397,6 +397,9 @@ but received {4} argument(s), with types '{5}'. UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} + + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + UTA026: {0}: Cannot define more than one method with the ClassCleanup attribute inside a class. diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.cs.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.cs.xlf index e04c06314b..ec4cec5aa1 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.cs.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.cs.xlf @@ -532,6 +532,11 @@ byl však přijat tento počet argumentů: {4} s typy {5}. UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.de.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.de.xlf index db0d0ada32..c2092af92c 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.de.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.de.xlf @@ -532,6 +532,11 @@ aber empfing {4} Argument(e) mit den Typen „{5}“. UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.es.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.es.xlf index 2d89ea2ae7..4b4e5ec262 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.es.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.es.xlf @@ -532,6 +532,11 @@ pero recibió {4} argumentos, con los tipos '{5}'. UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.fr.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.fr.xlf index 01a4b7cbcd..21f98e5167 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.fr.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.fr.xlf @@ -532,6 +532,11 @@ mais a reçu {4} argument(s), avec les types « {5} ». UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.it.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.it.xlf index c064f6e35f..6c356dd203 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.it.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.it.xlf @@ -532,6 +532,11 @@ ma ha ricevuto {4} argomenti, con tipi '{5}'. UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ja.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ja.xlf index 950b9eabe2..ec3659efb3 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ja.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ja.xlf @@ -533,6 +533,11 @@ but received {4} argument(s), with types '{5}'. UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ko.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ko.xlf index 6503b7c8a5..47abada2b9 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ko.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ko.xlf @@ -532,6 +532,11 @@ but received {4} argument(s), with types '{5}'. UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.pl.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.pl.xlf index 6aef1ee0d0..a74f7198a4 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.pl.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.pl.xlf @@ -532,6 +532,11 @@ ale odebrał argumenty {4} z typami „{5}”. UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.pt-BR.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.pt-BR.xlf index 2691c69e18..adede11bfa 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.pt-BR.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.pt-BR.xlf @@ -532,6 +532,11 @@ mas {4} argumentos recebidos, com tipos '{5}'. UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ru.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ru.xlf index c0b26029f5..57b6dece45 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ru.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.ru.xlf @@ -532,6 +532,11 @@ but received {4} argument(s), with types '{5}'. UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.tr.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.tr.xlf index b19f4613d1..94d6e1c8ec 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.tr.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.tr.xlf @@ -532,6 +532,11 @@ ancak, '{5}' türünde {4} bağımsız değişken aldı. UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.zh-Hans.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.zh-Hans.xlf index eec9137b20..86ff98d509 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.zh-Hans.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.zh-Hans.xlf @@ -532,6 +532,11 @@ but received {4} argument(s), with types '{5}'. UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.zh-Hant.xlf b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.zh-Hant.xlf index 0010861a4d..b63baf4c25 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.zh-Hant.xlf +++ b/src/Adapter/MSTestAdapter.PlatformServices/Resources/xlf/Resource.zh-Hant.xlf @@ -532,6 +532,11 @@ but received {4} argument(s), with types '{5}'. UTA073: Failed to load [TestFilterProvider] marker from assembly '{0}'. {1} + + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + UTA079: Assembly '{0}' declares more than one [TestFilterProvider] attribute. At most one is allowed per test assembly; compose multiple filtering strategies inside a single ITestFilter implementation. + + UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} UTA078: The [TestFilterProvider] filter '{0}' threw an exception while evaluating test '{1}'. {2} diff --git a/src/TestFramework/TestFramework/Filtering/ITestFilter.cs b/src/TestFramework/TestFramework/Filtering/ITestFilter.cs index bdd55027e0..e7b078b045 100644 --- a/src/TestFramework/TestFramework/Filtering/ITestFilter.cs +++ b/src/TestFramework/TestFramework/Filtering/ITestFilter.cs @@ -10,13 +10,13 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting; /// /// /// Implementations are registered via at the assembly -/// level. The MSTest adapter creates one instance per assembly per test run (via the public -/// parameterless constructor) and invokes for every test that survived the -/// command-line filter, before that test's type is loaded. +/// level. The MSTest adapter materializes the filter lazily on the first test invocation for that +/// assembly using the public parameterless constructor; the same instance is then reused for +/// every test in the assembly. /// /// -/// Implementations should be allocation-free and thread-safe; they may be called concurrently for -/// tests in different classes. +/// Implementations should be allocation-free and thread-safe; may be +/// invoked concurrently for tests in different classes. /// /// public interface ITestFilter @@ -30,5 +30,10 @@ public interface ITestFilter /// to silently drop the test (no result emitted), or /// to report the test as Skipped with a reason. /// + /// + /// If this method throws, the test is reported as Error with diagnostic UTA078; the + /// exception is never silently swallowed. Implementations that want to opt the test in to + /// running on failure should catch their own exceptions and return . + /// TestFilterResult Filter(TestFilterContext context); } diff --git a/src/TestFramework/TestFramework/Filtering/TestFilterResult.cs b/src/TestFramework/TestFramework/Filtering/TestFilterResult.cs index b5eb752610..5ed341cd8b 100644 --- a/src/TestFramework/TestFramework/Filtering/TestFilterResult.cs +++ b/src/TestFramework/TestFramework/Filtering/TestFilterResult.cs @@ -54,8 +54,13 @@ private TestFilterResult(TestFilterAction action, string? skipReason) /// A non-empty human-readable explanation surfaced in TRX / console / IDE output. /// A with equal to . /// Thrown when is . + /// Thrown when is empty or whitespace only. public static TestFilterResult Skip(string reason) - => new(TestFilterAction.Skip, reason ?? throw new ArgumentNullException(nameof(reason))); + => reason is null + ? throw new ArgumentNullException(nameof(reason)) + : string.IsNullOrWhiteSpace(reason) + ? throw new ArgumentException("Value cannot be empty or whitespace.", nameof(reason)) + : new TestFilterResult(TestFilterAction.Skip, reason); /// public bool Equals(TestFilterResult other) diff --git a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TypeCacheTestFilterProviderTests.cs b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TypeCacheTestFilterProviderTests.cs new file mode 100644 index 0000000000..4b73c89918 --- /dev/null +++ b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TypeCacheTestFilterProviderTests.cs @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AwesomeAssertions; + +using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; +using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution; +using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.TestableImplementations; + +using TestFramework.ForTestingMSTest; + +namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution; + +public class TypeCacheTestFilterProviderTests : TestContainer +{ + private readonly TestablePlatformServiceProvider _testablePlatformServiceProvider; + + public TypeCacheTestFilterProviderTests() + { + _testablePlatformServiceProvider = new TestablePlatformServiceProvider(); + PlatformServiceProvider.Instance = _testablePlatformServiceProvider; + } + + protected override void Dispose(bool disposing) + { + if (!IsDisposed) + { + base.Dispose(disposing); + PlatformServiceProvider.Instance = null; + MSTestSettings.Reset(); + } + } + + // Direct unit tests for the validation branches of InstantiateTestFilter. Driving these + // through GetOrLoadTestFilter would require polluting AssemblyAttributes.cs with broken + // markers that would then run for every test in this assembly. The helper is scoped, + // internal, and self-contained, so testing it directly is both safer and clearer. + public void InstantiateTestFilter_WhenTypeIsOpenGeneric_ThrowsUTA074() + { + Action act = () => TypeCache.InstantiateTestFilter(typeof(GenericFilter<>)); + act.Should().Throw().WithMessage("*UTA074*"); + } + + public void InstantiateTestFilter_WhenTypeIsClosedGeneric_ThrowsUTA074() + { + Action act = () => TypeCache.InstantiateTestFilter(typeof(GenericFilter)); + act.Should().Throw().WithMessage("*UTA074*"); + } + + public void InstantiateTestFilter_WhenTypeIsAbstract_ThrowsUTA075() + { + Action act = () => TypeCache.InstantiateTestFilter(typeof(AbstractFilter)); + act.Should().Throw().WithMessage("*UTA075*"); + } + + public void InstantiateTestFilter_WhenTypeIsInterface_ThrowsUTA075() + { + Action act = () => TypeCache.InstantiateTestFilter(typeof(IFilterInterface)); + act.Should().Throw().WithMessage("*UTA075*"); + } + + public void InstantiateTestFilter_WhenTypeDoesNotImplementITestFilter_ThrowsUTA076() + { + Action act = () => TypeCache.InstantiateTestFilter(typeof(NotAFilter)); + act.Should().Throw().WithMessage("*UTA076*"); + } + + public void InstantiateTestFilter_WhenConstructorThrows_ThrowsUTA077() + { + // Activator.CreateInstance wraps the constructor exception in TargetInvocationException; + // the diagnostic surfaces UTA077 either way and preserves the inner exception. + Action act = () => TypeCache.InstantiateTestFilter(typeof(ThrowingFilter)); + act.Should().Throw().WithMessage("*UTA077*"); + } + + public void InstantiateTestFilter_WhenTypeIsMissingPublicParameterlessConstructor_ThrowsUTA077() + { + Action act = () => TypeCache.InstantiateTestFilter(typeof(FilterWithoutPublicCtor)); + act.Should().Throw().WithMessage("*UTA077*"); + } + + public void InstantiateTestFilter_WhenTypeIsValid_ReturnsInstance() + { + ITestFilter filter = TypeCache.InstantiateTestFilter(typeof(NoOpFilter)); + filter.Should().NotBeNull().And.BeOfType(); + } + + // ----- test types ----- + public sealed class NoOpFilter : ITestFilter + { + public TestFilterResult Filter(TestFilterContext context) => TestFilterResult.Run; + } + + public sealed class GenericFilter : ITestFilter + { + public TestFilterResult Filter(TestFilterContext context) => TestFilterResult.Run; + } + + public abstract class AbstractFilter : ITestFilter + { + public abstract TestFilterResult Filter(TestFilterContext context); + } + + public interface IFilterInterface : ITestFilter + { + } + + // Intentionally does not implement ITestFilter. + public sealed class NotAFilter + { + } + + public sealed class ThrowingFilter : ITestFilter + { + public ThrowingFilter() => throw new InvalidOperationException("filter ctor blew up"); + + public TestFilterResult Filter(TestFilterContext context) => TestFilterResult.Run; + } + + public sealed class FilterWithoutPublicCtor : ITestFilter + { + private FilterWithoutPublicCtor() + { + } + + public TestFilterResult Filter(TestFilterContext context) => TestFilterResult.Run; + } +} diff --git a/test/UnitTests/TestFramework.UnitTests/Filtering/TestFilterResultTests.cs b/test/UnitTests/TestFramework.UnitTests/Filtering/TestFilterResultTests.cs new file mode 100644 index 0000000000..0548f66150 --- /dev/null +++ b/test/UnitTests/TestFramework.UnitTests/Filtering/TestFilterResultTests.cs @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AwesomeAssertions; + +using TestFramework.ForTestingMSTest; + +namespace Microsoft.VisualStudio.TestPlatform.TestFramework.UnitTests.Filtering; + +public class TestFilterResultTests : TestContainer +{ + public void Run_HasActionRunAndNullReason() + { + TestFilterResult result = TestFilterResult.Run; + result.Action.Should().Be(TestFilterAction.Run); + result.SkipReason.Should().BeNull(); + } + + public void Drop_HasActionDropAndNullReason() + { + TestFilterResult result = TestFilterResult.Drop; + result.Action.Should().Be(TestFilterAction.Drop); + result.SkipReason.Should().BeNull(); + } + + public void Default_DefaultsToRunAction() + { + // The XML doc on TestFilterResult.Run promises that a default(TestFilterResult) value + // behaves like Run. A filter that forgets to assign a result still runs the test. + var result = default(TestFilterResult); + result.Action.Should().Be(TestFilterAction.Run); + result.SkipReason.Should().BeNull(); + } + + public void Skip_WithValidReason_SetsActionAndReason() + { + var result = TestFilterResult.Skip("Skipped because of CI policy."); + result.Action.Should().Be(TestFilterAction.Skip); + result.SkipReason.Should().Be("Skipped because of CI policy."); + } + + public void Skip_WhenReasonIsNull_ThrowsArgumentNullException() + { + Action act = static () => TestFilterResult.Skip(null!); + act.Should().Throw().WithParameterName("reason"); + } + + public void Skip_WhenReasonIsEmpty_ThrowsArgumentException() + { + // The XML doc promises a "non-empty human-readable explanation" — empty strings would + // surface as unactionable skipped tests in TRX / IDE output, so they are rejected. + Action act = static () => TestFilterResult.Skip(string.Empty); + act.Should().Throw().WithParameterName("reason"); + } + + public void Skip_WhenReasonIsWhitespace_ThrowsArgumentException() + { + Action act = static () => TestFilterResult.Skip(" "); + act.Should().Throw().WithParameterName("reason"); + } + + public void Equality_TwoSkipResultsWithSameReasonAreEqual() + { + var first = TestFilterResult.Skip("reason"); + var second = TestFilterResult.Skip("reason"); + + first.Equals(second).Should().BeTrue(); + (first == second).Should().BeTrue(); + (first != second).Should().BeFalse(); + first.GetHashCode().Should().Be(second.GetHashCode()); + } + + public void Equality_TwoSkipResultsWithDifferentReasonsAreNotEqual() + { + var first = TestFilterResult.Skip("reason A"); + var second = TestFilterResult.Skip("reason B"); + + first.Equals(second).Should().BeFalse(); + (first == second).Should().BeFalse(); + (first != second).Should().BeTrue(); + } + + public void Equality_RunAndDropAreNotEqual() + { + (TestFilterResult.Run == TestFilterResult.Drop).Should().BeFalse(); + TestFilterResult.Run.Equals(TestFilterResult.Drop).Should().BeFalse(); + } + + public void Equality_BoxedComparisonAgainstNonResultIsFalse() + { + TestFilterResult.Run.Equals("not a result").Should().BeFalse(); + TestFilterResult.Run.Equals(null).Should().BeFalse(); + } +} From 0833921ec26467143f6d63f4b11ae30eb07618b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Sun, 7 Jun 2026 13:34:44 +0200 Subject: [PATCH 4/5] Redesign TestFilterContext for forward compatibility Pivot TestFilterContext from a 7-arg positional ctor to a parameterless ctor plus public mutable properties so new fields can be added in future releases without breaking source or binary callers. `init` is banned for new public API in this repo, so the properties use plain `set`. Surface structured identification in the same shape as `TestMethodIdentifier` (Namespace, ClassName, ManagedTypeName, ManagedMethodName, MethodArity, ParameterTypeFullNames) alongside the existing flat fields. The structured fields come from the Hierarchy slots that discovery already populates and from a cheap `ManagedNameParser.ParseManagedMethodName` call, so the test type is still not loaded. Rename `AssemblyName` to `Source`: the underlying TestMethod field is actually a file path (matches VSTest TestCase.Source). Document on ITestFilter.Filter that the filter runs before `[Ignore]` is evaluated; returning `Run` does not override a later `[Ignore]`. Decisions explicitly out of scope: - No bypass for built-in adapter filtering (--filter, [Ignore]): keeps the contracts CI scripts and IDEs depend on. - No "already-included / already-excluded" info: --filter runs before ITestFilter so "already-included" would be tautological. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Execution/UnitTestRunner.cs | 55 ++++++- .../TestFramework/Filtering/ITestFilter.cs | 10 ++ .../Filtering/TestFilterContext.cs | 149 +++++++++++++----- .../PublicAPI/PublicAPI.Unshipped.txt | 25 ++- 4 files changed, 188 insertions(+), 51 deletions(-) diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Execution/UnitTestRunner.cs b/src/Adapter/MSTestAdapter.PlatformServices/Execution/UnitTestRunner.cs index 6f446f725c..6636ac745e 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Execution/UnitTestRunner.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Execution/UnitTestRunner.cs @@ -10,6 +10,7 @@ using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Extensions; +using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -487,14 +488,52 @@ private static TestFilterContext CreateFilterContext(UnitTestElement element) traits = []; } - return new TestFilterContext( - fullyQualifiedName: $"{testMethod.FullClassName}.{testMethod.Name}", - displayName: testMethod.DisplayName, - testClassName: testMethod.FullClassName, - testMethodName: testMethod.Name, - categories: categories, - traits: traits, - priority: element.Priority); + // Pull namespace + simple class name from the hierarchy when available — this is the + // same source the IDE / Test Explorer uses, so it correctly handles nested types and + // generic classes (where naïve FullClassName splitting would lie). + string? hierarchyNamespace = null; + string? hierarchyClassName = null; + if (testMethod.Hierarchy is IReadOnlyList hierarchy && hierarchy.Count > HierarchyConstants.Levels.ClassIndex) + { + hierarchyNamespace = hierarchy[HierarchyConstants.Levels.NamespaceIndex]; + hierarchyClassName = hierarchy[HierarchyConstants.Levels.ClassIndex]; + } + + // ManagedMethodName is an ECMA-335 string like `MyMethod`1(System.Int32)` — parse it + // cheaply (no MethodInfo reflection) to surface arity and parameter type names. + int? methodArity = null; + IReadOnlyList? parameterTypeFullNames = null; + if (testMethod.ManagedMethodName is { } managedMethod) + { + try + { + ManagedNameParser.ParseManagedMethodName(managedMethod, out _, out int arity, out string[]? parameterTypes); + methodArity = arity; + parameterTypeFullNames = parameterTypes ?? (IReadOnlyList)[]; + } + catch + { + // Defensive: if the managed name is malformed for any reason, surface what we + // can via the flat strings rather than failing the filter. + } + } + + return new TestFilterContext + { + FullyQualifiedName = $"{testMethod.FullClassName}.{testMethod.Name}", + DisplayName = testMethod.DisplayName, + MethodName = testMethod.Name, + Source = testMethod.AssemblyName, + Namespace = hierarchyNamespace, + ClassName = hierarchyClassName, + ManagedTypeName = testMethod.ManagedTypeName, + ManagedMethodName = testMethod.ManagedMethodName, + MethodArity = methodArity, + ParameterTypeFullNames = parameterTypeFullNames, + Categories = categories, + Traits = traits, + Priority = element.Priority, + }; } /// diff --git a/src/TestFramework/TestFramework/Filtering/ITestFilter.cs b/src/TestFramework/TestFramework/Filtering/ITestFilter.cs index e7b078b045..aee36cd975 100644 --- a/src/TestFramework/TestFramework/Filtering/ITestFilter.cs +++ b/src/TestFramework/TestFramework/Filtering/ITestFilter.cs @@ -18,6 +18,16 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting; /// Implementations should be allocation-free and thread-safe; may be /// invoked concurrently for tests in different classes. /// +/// +/// Ordering with built-in filtering: is composed with — +/// not a replacement for — the adapter's default filtering. Adapter-level filters such as the +/// VSTest --filter command line or test-explorer selection run before +/// , so only ever sees tests that already survived +/// those gates. By contrast, [Ignore] is evaluated after +/// (it requires loading the declaring type, which is specifically designed +/// to avoid). Returning does not override a later +/// [Ignore]; an ignored test is still ignored. +/// /// public interface ITestFilter { diff --git a/src/TestFramework/TestFramework/Filtering/TestFilterContext.cs b/src/TestFramework/TestFramework/Filtering/TestFilterContext.cs index 10c9b9d2ba..9d15f457a7 100644 --- a/src/TestFramework/TestFramework/Filtering/TestFilterContext.cs +++ b/src/TestFramework/TestFramework/Filtering/TestFilterContext.cs @@ -8,62 +8,131 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting; /// test under consideration. /// /// +/// /// Only metadata that is available without loading the test type is exposed; /// must be able to decide using strings, categories, traits, and priority alone. This is what allows /// the filter to drop tests before their declaring type is loaded and before /// [AssemblyInitialize] / [ClassInitialize] run. +/// +/// +/// Each property describes a separate facet of the test. The flat string properties +/// (, ) are convenience views and are always +/// populated. The structured properties (, , +/// , , , +/// ) may be when the metadata is not +/// available — typically for tests discovered through code paths that don't surface ECMA-335 +/// managed names. +/// +/// +/// The type is designed to be extended over time: new properties can be added without breaking +/// existing implementations or their unit tests. Consumers construct +/// instances using an object initializer (e.g. new TestFilterContext { FullyQualifiedName = "…" }); +/// no positional constructor needs to be updated when new properties land. +/// /// public sealed class TestFilterContext { /// - /// Initializes a new instance of the class. + /// Gets or sets the fully qualified test name in Namespace.Class.Method form. /// - /// The fully qualified test name (Namespace.Class.Method). - /// The display name reported for the test (often equal to ). - /// The fully qualified name of the declaring test class. - /// The unqualified test method name. - /// The values declared on the test (and its class). - /// The traits attached to the test. Multiple traits can share the same key. - /// The value if any, otherwise . - public TestFilterContext( - string fullyQualifiedName, - string displayName, - string testClassName, - string testMethodName, - IReadOnlyList categories, - IReadOnlyList> traits, - int? priority) - { - FullyQualifiedName = fullyQualifiedName ?? throw new ArgumentNullException(nameof(fullyQualifiedName)); - DisplayName = displayName ?? throw new ArgumentNullException(nameof(displayName)); - TestClassName = testClassName ?? throw new ArgumentNullException(nameof(testClassName)); - TestMethodName = testMethodName ?? throw new ArgumentNullException(nameof(testMethodName)); - Categories = categories ?? throw new ArgumentNullException(nameof(categories)); - Traits = traits ?? throw new ArgumentNullException(nameof(traits)); - Priority = priority; - } + /// + /// This mirrors the historical VSTest TestCase.FullyQualifiedName shape and is intended + /// for filters that want a single string to match against. + /// + public string FullyQualifiedName { get; set; } = string.Empty; - /// Gets the fully qualified test name (Namespace.Class.Method). - public string FullyQualifiedName { get; } + /// + /// Gets or sets the display name of the test, as reported to the runner / IDE. + /// + /// + /// Often equal to ; differs for data-driven tests, tests with a + /// custom display name, or attributes that override it. + /// + public string DisplayName { get; set; } = string.Empty; - /// Gets the display name of the test. - public string DisplayName { get; } + /// + /// Gets or sets the unqualified test method name (i.e. without class or namespace). + /// + public string MethodName { get; set; } = string.Empty; + + /// + /// Gets or sets the path to the test assembly file containing this test. + /// + /// + /// Matches VSTest TestCase.Source: the absolute path to the .dll being run. + /// This is not a simple assembly name; see if you + /// need an ECMA-335-style identifier. + /// + public string Source { get; set; } = string.Empty; - /// Gets the fully qualified name of the declaring test class. - public string TestClassName { get; } + /// + /// Gets or sets the namespace of the declaring test class, or when + /// no managed metadata is available or the class is in the global namespace. + /// + public string? Namespace { get; set; } - /// Gets the unqualified test method name. - public string TestMethodName { get; } + /// + /// Gets or sets the simple class name (without namespace), or when + /// no managed metadata is available. Nested types are surfaced using their managed metadata + /// form (e.g. Outer+Inner); see for the fully escaped + /// ECMA-335 representation. + /// + public string? ClassName { get; set; } - /// Gets the test categories declared via . - public IReadOnlyList Categories { get; } + /// + /// Gets or sets the declaring type name in ECMA-335 metadata form, or + /// when no managed metadata is available. + /// + /// + /// Includes the namespace, uses + for nested types, and uses backtick + arity for + /// generics (e.g. Acme.MyOuter+MyInner`1). Matches the format defined by the + /// + /// Managed TestCase Properties RFC. + /// + public string? ManagedTypeName { get; set; } /// - /// Gets the traits attached to this test. Multiple traits can share the same key; consumers - /// must therefore not assume the collection behaves like a dictionary. + /// Gets or sets the method name in ECMA-335 metadata form, or when no + /// managed metadata is available. /// - public IReadOnlyList> Traits { get; } + /// + /// Includes the method name, generic arity suffix, and parameter type list + /// (e.g. MyMethod`1(System.Int32)). Matches the format defined by the + /// + /// Managed TestCase Properties RFC. + /// + public string? ManagedMethodName { get; set; } - /// Gets the value if any, otherwise . - public int? Priority { get; } + /// + /// Gets or sets the number of generic type parameters declared on the test method, + /// or when no managed metadata is available. Zero indicates a + /// non-generic method. + /// + public int? MethodArity { get; set; } + + /// + /// Gets or sets the ECMA-335-style fully qualified parameter type names of the test method, + /// or when no managed metadata is available. An empty list indicates + /// a parameterless method. + /// + public IReadOnlyList? ParameterTypeFullNames { get; set; } + + /// + /// Gets or sets the test categories declared via on the + /// method or its declaring class. Defaults to an empty list. + /// + public IReadOnlyList Categories { get; set; } = []; + + /// + /// Gets or sets the traits attached to this test. Multiple traits can share the same key; + /// consumers must therefore not assume the collection behaves like a dictionary. Defaults + /// to an empty list. + /// + public IReadOnlyList> Traits { get; set; } = []; + + /// + /// Gets or sets the value declared on the test, or + /// when no priority was declared. + /// + public int? Priority { get; set; } } diff --git a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt index f890ab2b2e..f214542bd4 100644 --- a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt @@ -16,13 +16,32 @@ Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterAction.Run = 0 -> Microso Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterAction.Skip = 2 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterAction Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.Categories.get -> System.Collections.Generic.IReadOnlyList! +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.Categories.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.ClassName.get -> string? +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.ClassName.set -> void Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.DisplayName.get -> string! +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.DisplayName.set -> void Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.FullyQualifiedName.get -> string! +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.FullyQualifiedName.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.ManagedMethodName.get -> string? +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.ManagedMethodName.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.ManagedTypeName.get -> string? +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.ManagedTypeName.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.MethodArity.get -> int? +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.MethodArity.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.MethodName.get -> string! +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.MethodName.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.Namespace.get -> string? +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.Namespace.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.ParameterTypeFullNames.get -> System.Collections.Generic.IReadOnlyList? +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.ParameterTypeFullNames.set -> void Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.Priority.get -> int? -Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.TestClassName.get -> string! -Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.TestFilterContext(string! fullyQualifiedName, string! displayName, string! testClassName, string! testMethodName, System.Collections.Generic.IReadOnlyList! categories, System.Collections.Generic.IReadOnlyList>! traits, int? priority) -> void -Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.TestMethodName.get -> string! +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.Priority.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.Source.get -> string! +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.Source.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.TestFilterContext() -> void Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.Traits.get -> System.Collections.Generic.IReadOnlyList>! +Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterContext.Traits.set -> void Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterProviderAttribute Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterProviderAttribute.FilterType.get -> System.Type! Microsoft.VisualStudio.TestTools.UnitTesting.TestFilterProviderAttribute.TestFilterProviderAttribute(System.Type! filterType) -> void From 9674a625d9b7ebe07b7c127e8bbba3cab8ce58e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Thu, 11 Jun 2026 10:42:06 +0200 Subject: [PATCH 5/5] Address test filter review feedback Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Execution/TestExecutionManager.cs | 6 ++++++ .../Execution/TypeCache.FilterDiscovery.cs | 16 ++++++++++------ .../Execution/UnitTestRunner.cs | 16 ++++++++-------- .../TestFramework/Filtering/TestFilterResult.cs | 6 +++--- .../Execution/TestExecutionManagerTests.cs | 11 +++++++++++ .../Filtering/TestFilterResultTests.cs | 7 +++++-- 6 files changed, 43 insertions(+), 19 deletions(-) diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestExecutionManager.cs b/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestExecutionManager.cs index 74d60443e0..65383ccefb 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestExecutionManager.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestExecutionManager.cs @@ -262,6 +262,12 @@ internal void SendTestResults( DateTimeOffset endTime, ITestExecutionRecorder testExecutionRecorder) { + if (unitTestResults.Length == 0) + { + testExecutionRecorder.RecordEnd(test, TestOutcome.None); + return; + } + foreach (TestTools.UnitTesting.TestResult unitTestResult in unitTestResults) { _testRunCancellationToken?.ThrowIfCancellationRequested(); diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Execution/TypeCache.FilterDiscovery.cs b/src/Adapter/MSTestAdapter.PlatformServices/Execution/TypeCache.FilterDiscovery.cs index 6d1413fd88..4697d8da4e 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Execution/TypeCache.FilterDiscovery.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Execution/TypeCache.FilterDiscovery.cs @@ -8,11 +8,11 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution; internal sealed partial class TypeCache { - // Single filter instance cached per test assembly source path. Computed lazily on the first - // request for that source so the cost is paid at most once per run, even when many tests - // target the same assembly. Stored as a TestFilterBox so the dictionary can cache the - // "no filter" answer alongside real filter instances. - private readonly ConcurrentDictionary _testFilterBySource = + // Single filter instance cached per test assembly source path. The Lazy ensures discovery + // (including any failure) is evaluated once per source per run, even under contention. + // Stored as a TestFilterBox so the dictionary can cache the "no filter" answer alongside + // real filter instances. + private readonly ConcurrentDictionary> _testFilterBySource = new(StringComparer.Ordinal); /// @@ -29,9 +29,13 @@ internal sealed partial class TypeCache /// internal ITestFilter? GetOrLoadTestFilter(string assemblySource) => _testFilterBySource - .GetOrAdd(assemblySource, static src => new TestFilterBox(LoadTestFilterForSource(src))) + .GetOrAdd(assemblySource, CreateTestFilterBox) + .Value .Filter; + private static Lazy CreateTestFilterBox(string assemblySource) + => new(() => new TestFilterBox(LoadTestFilterForSource(assemblySource)), isThreadSafe: true); + private static ITestFilter? LoadTestFilterForSource(string assemblySource) { Assembly assembly; diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Execution/UnitTestRunner.cs b/src/Adapter/MSTestAdapter.PlatformServices/Execution/UnitTestRunner.cs index 6636ac745e..ecf271d728 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Execution/UnitTestRunner.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Execution/UnitTestRunner.cs @@ -268,13 +268,6 @@ internal async Task RunSingleTestAsync(UnitTestElement unitTestEle { testContextForAssemblyCleanup = PlatformServiceProvider.Instance.GetTestContext(testMethod: null, null, testContextProperties, messageLogger, testContextForClassCleanup.Context.CurrentTestOutcome); - // Flow properties set during AssemblyInitialize so the AssemblyCleanup method - // observes them. Class-init properties are intentionally NOT flowed here because - // AssemblyCleanup is assembly-scoped and runs once across many classes; picking - // a single class's snapshot would be arbitrary. - // testMethodInfo is non-null inside this block thanks to the guard above. - ((TestContextImplementation)testContextForAssemblyCleanup.Context).MergeProperties(testMethodInfo.Parent.Parent.PostAssemblyInitProperties); - TestResult? assemblyCleanupResult = await RunAssemblyCleanupAsync(testContextForAssemblyCleanup, _typeCache, result).ConfigureAwait(false); if (assemblyCleanupResult is not null) { @@ -340,9 +333,16 @@ private static async Task RunAssemblyInitializeIfNeededAsync(TestMet private static async Task RunAssemblyCleanupAsync(ITestContext testContext, TypeCache typeCache, TestResult[] results) { var testContextImpl = testContext as TestContextImplementation; + var testContextForAssemblyCleanup = testContext.Context as TestContextImplementation; IEnumerable assemblyInfoCache = typeCache.AssemblyInfoListWithExecutableCleanupMethods; foreach (TestAssemblyInfo assemblyInfo in assemblyInfoCache) { + // Flow properties set during AssemblyInitialize so the AssemblyCleanup method observes + // them. Class-init properties are intentionally NOT flowed here because AssemblyCleanup + // is assembly-scoped and runs once across many classes; picking a single class's + // snapshot would be arbitrary. + testContextForAssemblyCleanup?.MergeProperties(assemblyInfo.PostAssemblyInitProperties); + TestFailedException? ex = await assemblyInfo.ExecuteAssemblyCleanupAsync(testContext.Context).ConfigureAwait(false); if (ex is not null) @@ -511,7 +511,7 @@ private static TestFilterContext CreateFilterContext(UnitTestElement element) methodArity = arity; parameterTypeFullNames = parameterTypes ?? (IReadOnlyList)[]; } - catch + catch (InvalidManagedNameException) { // Defensive: if the managed name is malformed for any reason, surface what we // can via the flat strings rather than failing the filter. diff --git a/src/TestFramework/TestFramework/Filtering/TestFilterResult.cs b/src/TestFramework/TestFramework/Filtering/TestFilterResult.cs index 5ed341cd8b..84706ad105 100644 --- a/src/TestFramework/TestFramework/Filtering/TestFilterResult.cs +++ b/src/TestFramework/TestFramework/Filtering/TestFilterResult.cs @@ -8,9 +8,9 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting; /// /// /// Designed as a so the filter hot path stays -/// allocation-free. The static / properties (one shared value -/// each) and the parameterized factory are the only ways to construct -/// a result; this keeps the surface evolvable. +/// allocation-free. Use the static / properties (one shared +/// value each) or the parameterized factory to create explicit results. +/// The default value also represents . /// public readonly struct TestFilterResult : IEquatable { diff --git a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestExecutionManagerTests.cs b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestExecutionManagerTests.cs index 649b3fe564..18cfc4734a 100644 --- a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestExecutionManagerTests.cs +++ b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestExecutionManagerTests.cs @@ -120,6 +120,17 @@ public async Task RunTestsForTestWithFilterShouldSendResultsForFilteredTests() expectedResultList.SequenceEqual(_frameworkHandle.ResultsList).Should().BeTrue(); } + public void SendTestResults_WhenUnitTestResultsIsEmpty_RecordsEndWithoutResult() + { + TestCase testCase = GetTestCase(typeof(DummyTestClass), "PassingTest"); + Microsoft.VisualStudio.TestTools.UnitTesting.TestResult[] unitTestResults = []; + + _testExecutionManager.SendTestResults(testCase, unitTestResults, DateTimeOffset.Now, DateTimeOffset.Now, _frameworkHandle); + + _frameworkHandle.TestCaseEndList.Should().Equal("PassingTest:None"); + _frameworkHandle.ResultsList.Should().BeEmpty(); + } + public async Task RunTestsForIgnoredTestShouldSendResultsMarkingIgnoredTestsAsSkipped() { TestCase testCase = GetTestCase(typeof(DummyTestClass), "IgnoredTest"); diff --git a/test/UnitTests/TestFramework.UnitTests/Filtering/TestFilterResultTests.cs b/test/UnitTests/TestFramework.UnitTests/Filtering/TestFilterResultTests.cs index 0548f66150..5130d12add 100644 --- a/test/UnitTests/TestFramework.UnitTests/Filtering/TestFilterResultTests.cs +++ b/test/UnitTests/TestFramework.UnitTests/Filtering/TestFilterResultTests.cs @@ -82,8 +82,11 @@ public void Equality_TwoSkipResultsWithDifferentReasonsAreNotEqual() public void Equality_RunAndDropAreNotEqual() { - (TestFilterResult.Run == TestFilterResult.Drop).Should().BeFalse(); - TestFilterResult.Run.Equals(TestFilterResult.Drop).Should().BeFalse(); + TestFilterResult run = TestFilterResult.Run; + TestFilterResult drop = TestFilterResult.Drop; + + (run == drop).Should().BeFalse(); + run.Equals(drop).Should().BeFalse(); } public void Equality_BoxedComparisonAgainstNonResultIsFalse()