From 21424720f88b15dbfbf22553de4adebcb219e55e Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Wed, 25 Mar 2026 17:12:34 +0000 Subject: [PATCH 1/4] feat: add test discovery Activity span for OpenTelemetry tracing Add a "test discovery" span that captures the time spent finding, building, and resolving dependencies for all tests. This gives users clear visibility into discovery vs execution time in their trace views. The span is a child of the session span, tagged with tunit.test.count on completion, and records exceptions on failure. Zero cost when no listeners are attached. Also removes dead AreAllDependenciesSatisfied method and deduplicates discovery logic between DiscoverTests and DiscoverTestsFullyStreamingAsync into a shared DiscoverAndResolveTestsAsync helper. --- TUnit.Engine/TestDiscoveryService.cs | 175 ++++++++++++++------------- docs/docs/examples/opentelemetry.md | 5 +- 2 files changed, 97 insertions(+), 83 deletions(-) diff --git a/TUnit.Engine/TestDiscoveryService.cs b/TUnit.Engine/TestDiscoveryService.cs index 0d7121abe4..601489aba4 100644 --- a/TUnit.Engine/TestDiscoveryService.cs +++ b/TUnit.Engine/TestDiscoveryService.cs @@ -60,61 +60,7 @@ public async Task DiscoverTests(string testSessionId, ITest var contextProvider = _testExecutor.GetContextProvider(); contextProvider.BeforeTestDiscoveryContext.RestoreExecutionContext(); - var allTests = new List(); - -#pragma warning disable TPEXP - var isNopFilter = filter is NopFilter; -#pragma warning restore TPEXP - if (filter == null || !isForExecution || isNopFilter) - { - var buildingContext = new Building.TestBuildingContext(isForExecution, Filter: null); - await foreach (var test in DiscoverTestsStreamAsync(testSessionId, buildingContext, cancellationToken).ConfigureAwait(false)) - { - allTests.Add(test); - } - } - else - { - // Use filter-aware collection to pre-filter by type before materializing all metadata - var allMetadata = await _testBuilderPipeline.CollectTestMetadataAsync(testSessionId, filter).ConfigureAwait(false); - var allMetadataList = allMetadata.ToList(); - - var metadataToInclude = _dependencyExpander.ExpandToIncludeDependencies(allMetadataList, filter); - - // Build tests directly from the pre-collected metadata (avoid re-collecting) - // Apply 5-minute discovery timeout matching the streaming path (#4715) - using var filterCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - filterCts.CancelAfter(EngineDefaults.DiscoveryTimeout); - - var buildingContext = new Building.TestBuildingContext(isForExecution, Filter: null); - var tests = await _testBuilderPipeline.BuildTestsFromMetadataAsync( - metadataToInclude, - buildingContext, - filterCts.Token).ConfigureAwait(false); - - var testsList = tests.ToList(); - - // Cache tests so ITestFinder can locate them - foreach (var test in testsList) - { - _cachedTests.Add(test); - } - - allTests.AddRange(testsList); - } - - foreach (var test in allTests) - { - _dependencyResolver.RegisterTest(test); - } - - // Resolve dependencies in parallel — registration is complete so lookup dictionaries - // are effectively read-only, and each test's Dependencies are written independently. - _dependencyResolver.BatchResolveDependencies(allTests); - - // Populate TestContext._dependencies for ALL tests before After(TestDiscovery) hooks run. - // This ensures hooks can access dependency information on any TestContext (including focused tests). - _testBuilderPipeline.PopulateAllDependencies(allTests); + var allTests = await DiscoverAndResolveTestsAsync(testSessionId, filter, cancellationToken, isForExecution).ConfigureAwait(false); // Add tests to context and run After(TestDiscovery) hooks before event receivers // This marks the end of the discovery phase, before registration begins @@ -155,6 +101,95 @@ public async Task DiscoverTests(string testSessionId, ITest return new TestDiscoveryResult(filteredTests, finalContext); } + private async Task> DiscoverAndResolveTestsAsync( + string testSessionId, + ITestExecutionFilter? filter, + CancellationToken cancellationToken, + bool isForExecution) + { +#if NET + Activity? discoveryActivity = null; + if (TUnitActivitySource.Source.HasListeners()) + { + var sessionActivity = _testExecutor.GetContextProvider().TestSessionContext.Activity; + discoveryActivity = TUnitActivitySource.StartActivity( + "test discovery", + ActivityKind.Internal, + sessionActivity?.Context ?? default); + } +#endif + + var allTests = new List(); + + try + { +#pragma warning disable TPEXP + var isNopFilter = filter is NopFilter; +#pragma warning restore TPEXP + if (filter == null || !isForExecution || isNopFilter) + { + var buildingContext = new Building.TestBuildingContext(isForExecution, Filter: null); + await foreach (var test in DiscoverTestsStreamAsync(testSessionId, buildingContext, cancellationToken).ConfigureAwait(false)) + { + allTests.Add(test); + } + } + else + { + // Use filter-aware collection to pre-filter by type before materializing all metadata + var allMetadata = await _testBuilderPipeline.CollectTestMetadataAsync(testSessionId, filter).ConfigureAwait(false); + var allMetadataList = allMetadata.ToList(); + + var metadataToInclude = _dependencyExpander.ExpandToIncludeDependencies(allMetadataList, filter); + + // Apply 5-minute discovery timeout matching the streaming path (#4715) + using var filterCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + filterCts.CancelAfter(EngineDefaults.DiscoveryTimeout); + + var buildingContext = new Building.TestBuildingContext(isForExecution, Filter: null); + var tests = await _testBuilderPipeline.BuildTestsFromMetadataAsync( + metadataToInclude, + buildingContext, + filterCts.Token).ConfigureAwait(false); + + var testsList = tests.ToList(); + + foreach (var test in testsList) + { + _cachedTests.Add(test); + } + + allTests.AddRange(testsList); + } + + foreach (var test in allTests) + { + _dependencyResolver.RegisterTest(test); + } + + _dependencyResolver.BatchResolveDependencies(allTests); + _testBuilderPipeline.PopulateAllDependencies(allTests); + + return allTests; + } + catch (Exception ex) + { +#if NET + TUnitActivitySource.RecordException(discoveryActivity, ex); +#else + _ = ex; +#endif + throw; + } + finally + { +#if NET + discoveryActivity?.SetTag("tunit.test.count", allTests.Count); + TUnitActivitySource.StopActivity(discoveryActivity); +#endif + } + } + private async IAsyncEnumerable DiscoverTestsStreamAsync( string testSessionId, Building.TestBuildingContext buildingContext, @@ -183,25 +218,13 @@ public async IAsyncEnumerable DiscoverTestsFullyStreamin { await _testExecutor.ExecuteBeforeTestDiscoveryHooksAsync(cancellationToken).ConfigureAwait(false); - var buildingContext = new Building.TestBuildingContext(IsForExecution: false, Filter: null); - - var allTests = new List(); - await foreach (var test in DiscoverTestsStreamAsync(testSessionId, buildingContext, cancellationToken).ConfigureAwait(false)) - { - allTests.Add(test); - } - - // Resolve dependencies in parallel — registration is complete so lookup dictionaries - // are effectively read-only, and each test's Dependencies are written independently. - _dependencyResolver.BatchResolveDependencies(allTests); + var contextProvider = _testExecutor.GetContextProvider(); - // Populate TestContext._dependencies for ALL tests before After(TestDiscovery) hooks run. - // This ensures hooks can access dependency information on any TestContext (including focused tests). - _testBuilderPipeline.PopulateAllDependencies(allTests); + var allTests = await DiscoverAndResolveTestsAsync( + testSessionId, filter: null, cancellationToken, isForExecution: false).ConfigureAwait(false); // Add tests to context and run After(TestDiscovery) hooks before event receivers // This marks the end of the discovery phase, before registration begins - var contextProvider = _testExecutor.GetContextProvider(); contextProvider.TestDiscoveryContext.AddTests(allTests.Select(static t => t.Context)); await _testExecutor.ExecuteAfterTestDiscoveryHooksAsync(cancellationToken).ConfigureAwait(false); contextProvider.TestDiscoveryContext.RestoreExecutionContext(); @@ -274,18 +297,6 @@ public async IAsyncEnumerable DiscoverTestsFullyStreamin } } - private bool AreAllDependenciesSatisfied(AbstractExecutableTest test, ConcurrentDictionary completedTests) - { - foreach (var dependency in test.Dependencies) - { - if (!completedTests.ContainsKey(dependency.Test.TestId)) - { - return false; - } - } - return true; - } - private async Task InvokePostResolutionEventsInParallelAsync(List allTests) diff --git a/docs/docs/examples/opentelemetry.md b/docs/docs/examples/opentelemetry.md index 852e649acc..3346a5e3e3 100644 --- a/docs/docs/examples/opentelemetry.md +++ b/docs/docs/examples/opentelemetry.md @@ -53,11 +53,14 @@ TUnit creates a nested span tree that mirrors the test lifecycle: ``` test session + ├── test discovery └── test assembly └── test suite (one per test class) └── test case (one per test method invocation) ``` +The **test discovery** span captures the time spent finding, building, and resolving dependencies for all tests. It appears as a sibling of the assembly spans, giving you a clear view of discovery vs execution time. + ## Attributes Each span carries tags that follow [OpenTelemetry semantic conventions](https://opentelemetry.io/docs/specs/semconv/) where applicable. @@ -86,7 +89,7 @@ Each span carries tags that follow [OpenTelemetry semantic conventions](https:// | `tunit.test.method` | test case | Method name | | `tunit.test.id` | test case | Unique test instance ID | | `tunit.test.categories` | test case | Test categories (string array) | -| `tunit.test.count` | session/assembly/suite | Total test count | +| `tunit.test.count` | session/assembly/suite/discovery | Total test count | | `tunit.test.retry_attempt` | test case | Current retry attempt (when retrying) | | `tunit.test.skip_reason` | test case | Reason the test was skipped | From 2f239c5184a612b4e5dec1063d0cf074c0935514 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 26 Mar 2026 07:50:54 +0000 Subject: [PATCH 2/4] fix: address Claude's review - parameter ordering, catch pattern, and double-registration Co-authored-by: thomhurst <30480171+thomhurst@users.noreply.github.com> Agent-Logs-Url: https://github.com/thomhurst/TUnit/sessions/6a075196-2653-409f-b8d8-fdb9a5b1b912 --- TUnit.Engine/TestDiscoveryService.cs | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/TUnit.Engine/TestDiscoveryService.cs b/TUnit.Engine/TestDiscoveryService.cs index 601489aba4..180fe27e81 100644 --- a/TUnit.Engine/TestDiscoveryService.cs +++ b/TUnit.Engine/TestDiscoveryService.cs @@ -60,7 +60,7 @@ public async Task DiscoverTests(string testSessionId, ITest var contextProvider = _testExecutor.GetContextProvider(); contextProvider.BeforeTestDiscoveryContext.RestoreExecutionContext(); - var allTests = await DiscoverAndResolveTestsAsync(testSessionId, filter, cancellationToken, isForExecution).ConfigureAwait(false); + var allTests = await DiscoverAndResolveTestsAsync(testSessionId, filter, isForExecution, cancellationToken).ConfigureAwait(false); // Add tests to context and run After(TestDiscovery) hooks before event receivers // This marks the end of the discovery phase, before registration begins @@ -104,8 +104,8 @@ public async Task DiscoverTests(string testSessionId, ITest private async Task> DiscoverAndResolveTestsAsync( string testSessionId, ITestExecutionFilter? filter, - CancellationToken cancellationToken, - bool isForExecution) + bool isForExecution, + CancellationToken cancellationToken) { #if NET Activity? discoveryActivity = null; @@ -157,30 +157,29 @@ private async Task> DiscoverAndResolveTestsAsync( foreach (var test in testsList) { _cachedTests.Add(test); + _dependencyResolver.RegisterTest(test); } allTests.AddRange(testsList); } - foreach (var test in allTests) - { - _dependencyResolver.RegisterTest(test); - } - _dependencyResolver.BatchResolveDependencies(allTests); _testBuilderPipeline.PopulateAllDependencies(allTests); return allTests; } +#if NET catch (Exception ex) { -#if NET TUnitActivitySource.RecordException(discoveryActivity, ex); + throw; + } #else - _ = ex; -#endif + catch + { throw; } +#endif finally { #if NET @@ -221,7 +220,7 @@ public async IAsyncEnumerable DiscoverTestsFullyStreamin var contextProvider = _testExecutor.GetContextProvider(); var allTests = await DiscoverAndResolveTestsAsync( - testSessionId, filter: null, cancellationToken, isForExecution: false).ConfigureAwait(false); + testSessionId, filter: null, isForExecution: false, cancellationToken).ConfigureAwait(false); // Add tests to context and run After(TestDiscovery) hooks before event receivers // This marks the end of the discovery phase, before registration begins From 329668b064e6fe813d8b08d8ce2a7b698ff810a4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 26 Mar 2026 08:05:47 +0000 Subject: [PATCH 3/4] docs: expand OpenTelemetry setup with raw ActivityListener example and timing guidance Co-authored-by: thomhurst <30480171+thomhurst@users.noreply.github.com> Agent-Logs-Url: https://github.com/thomhurst/TUnit/sessions/0f08f6a5-38b3-4073-b779-b9b21acaa6b9 --- docs/docs/examples/opentelemetry.md | 64 ++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 5 deletions(-) diff --git a/docs/docs/examples/opentelemetry.md b/docs/docs/examples/opentelemetry.md index 3346a5e3e3..0571ae10a6 100644 --- a/docs/docs/examples/opentelemetry.md +++ b/docs/docs/examples/opentelemetry.md @@ -8,6 +8,8 @@ Activity tracing requires .NET 8 or later. It is not available on .NET Framework ## Setup +### Option A: OpenTelemetry SDK (recommended) + Add the OpenTelemetry packages to your test project: ```bash @@ -15,7 +17,7 @@ dotnet add package OpenTelemetry dotnet add package OpenTelemetry.Exporter.Console ``` -Then subscribe to the `"TUnit"` ActivitySource in your test setup: +Then subscribe to the `"TUnit"` ActivitySource in a `[Before(TestSession)]` hook: ```csharp using System.Diagnostics; @@ -47,6 +49,50 @@ public class TraceSetup Replace `AddConsoleExporter()` with your preferred exporter (Jaeger, Zipkin, OTLP, etc.). +### Option B: Raw `ActivityListener` (no SDK dependency) + +If you don't want the OpenTelemetry SDK, you can subscribe directly with a `System.Diagnostics.ActivityListener`: + +```csharp +using System.Diagnostics; + +public class TraceSetup +{ + private static ActivityListener? _listener; + + [Before(TestSession)] + public static void SetupTracing() + { + _listener = new ActivityListener + { + ShouldListenTo = source => source.Name == "TUnit", + Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, + ActivityStarted = activity => Console.WriteLine($"▶ {activity.OperationName}"), + ActivityStopped = activity => Console.WriteLine($"■ {activity.OperationName} ({activity.Duration.TotalMilliseconds:F1}ms)") + }; + ActivitySource.AddActivityListener(_listener); + } + + [After(TestSession)] + public static void TeardownTracing() + { + _listener?.Dispose(); + } +} +``` + +### Why `[Before(TestSession)]`? + +The listener **must** be registered in a `[Before(TestSession)]` hook so it is active before test discovery begins. TUnit's hook execution order is: + +1. `[Before(TestSession)]` — register your listener here +2. `[Before(TestDiscovery)]` — discovery hooks run +3. **Test discovery** — the `"test discovery"` span is emitted here +4. Test execution — assembly, suite, and test case spans are emitted +5. `[After(TestSession)]` — dispose your listener here + +If you register the listener later (e.g., in `[Before(Assembly)]`), the discovery span will not be captured. + ## Span Hierarchy TUnit creates a nested span tree that mirrors the test lifecycle: @@ -107,17 +153,25 @@ When a test is configured with `[Retry]`, each failed attempt produces its own s ## Using with Jaeger, Zipkin, or OTLP -Swap the exporter in the setup: +Swap the exporter in the setup code above. Each exporter needs its own NuGet package. -```csharp -// OTLP (works with Jaeger, Tempo, Honeycomb, etc.) +### OTLP (works with Jaeger, Tempo, Honeycomb, etc.) + +```bash dotnet add package OpenTelemetry.Exporter.OpenTelemetryProtocol +``` +```csharp .AddOtlpExporter(opts => opts.Endpoint = new Uri("http://localhost:4317")) +``` -// Zipkin +### Zipkin + +```bash dotnet add package OpenTelemetry.Exporter.Zipkin +``` +```csharp .AddZipkinExporter(opts => opts.Endpoint = new Uri("http://localhost:9411/api/v2/spans")) ``` From fd4914bb2f2bbebc7c609a0c2bb061841448f382 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 26 Mar 2026 08:15:38 +0000 Subject: [PATCH 4/4] docs: use [Before(TestDiscovery)] for trace listener registration Co-authored-by: thomhurst <30480171+thomhurst@users.noreply.github.com> Agent-Logs-Url: https://github.com/thomhurst/TUnit/sessions/cb0bc698-4ced-4770-aeb2-4295bc1c4df7 --- docs/docs/examples/opentelemetry.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/docs/examples/opentelemetry.md b/docs/docs/examples/opentelemetry.md index 0571ae10a6..03a7462e8f 100644 --- a/docs/docs/examples/opentelemetry.md +++ b/docs/docs/examples/opentelemetry.md @@ -17,7 +17,7 @@ dotnet add package OpenTelemetry dotnet add package OpenTelemetry.Exporter.Console ``` -Then subscribe to the `"TUnit"` ActivitySource in a `[Before(TestSession)]` hook: +Then subscribe to the `"TUnit"` ActivitySource in a `[Before(TestDiscovery)]` hook: ```csharp using System.Diagnostics; @@ -29,7 +29,7 @@ public class TraceSetup { private static TracerProvider? _tracerProvider; - [Before(TestSession)] + [Before(TestDiscovery)] public static void SetupTracing() { _tracerProvider = Sdk.CreateTracerProviderBuilder() @@ -60,7 +60,7 @@ public class TraceSetup { private static ActivityListener? _listener; - [Before(TestSession)] + [Before(TestDiscovery)] public static void SetupTracing() { _listener = new ActivityListener @@ -81,12 +81,12 @@ public class TraceSetup } ``` -### Why `[Before(TestSession)]`? +### Why `[Before(TestDiscovery)]`? -The listener **must** be registered in a `[Before(TestSession)]` hook so it is active before test discovery begins. TUnit's hook execution order is: +The listener **must** be registered in a `[Before(TestDiscovery)]` hook (or earlier, e.g. `[Before(TestSession)]`) so it is active before the discovery span begins. TUnit's hook execution order is: -1. `[Before(TestSession)]` — register your listener here -2. `[Before(TestDiscovery)]` — discovery hooks run +1. `[Before(TestSession)]` — session-level setup +2. `[Before(TestDiscovery)]` — register your listener here 3. **Test discovery** — the `"test discovery"` span is emitted here 4. Test execution — assembly, suite, and test case spans are emitted 5. `[After(TestSession)]` — dispose your listener here