Fix for dashboard responsiveness during load testing.

Author: rhamlett_microsoft

src/PerfProblemSimulator/Services/LoadTestService.cs

@@ -301,7 +301,21 @@ public LoadTestService(
     /// Timer callback that broadcasts load test stats to event log every 60 seconds.
     /// Only broadcasts if there was activity during the period.
     /// </summary>
-    private async void BroadcastStats(object? state)
+    /// <remarks>
+    /// <para>
+    /// <strong>Thread Pool Independence:</strong>
+    /// </para>
+    /// <para>
+    /// Uses synchronous GetAwaiter().GetResult() instead of async/await to avoid
+    /// depending on thread pool threads for continuations. This ensures broadcasts
+    /// can still work during thread pool starvation from load tests.
+    /// </para>
+    /// <para>
+    /// Timer callbacks run on thread pool threads, but the broadcast itself doesn't
+    /// then need additional thread pool threads for async continuations.
+    /// </para>
+    /// </remarks>
+    private void BroadcastStats(object? state)
     {
         try
         {
@@ -347,7 +361,8 @@ private async void BroadcastStats(object? state)
                 "Broadcasting load test stats: {Requests} requests, {AvgMs}ms avg, {MaxMs}ms max, {RPS} RPS",
                 requestsCompleted, avgResponseTime, maxResponseTime, requestsPerSecond);
 
-            await _hubContext.Clients.All.ReceiveLoadTestStats(statsData);
+            // Use synchronous call to avoid thread pool dependency for continuations
+            _hubContext.Clients.All.ReceiveLoadTestStats(statsData).GetAwaiter().GetResult();
         }
         catch (Exception ex)
         {

src/PerfProblemSimulator/Services/MetricsBroadcastService.cs

@@ -2,6 +2,7 @@
 using PerfProblemSimulator.Hubs;
 using PerfProblemSimulator.Models;
 using PerfProblemSimulator.Services;
+using System.Collections.Concurrent;
 
 namespace PerfProblemSimulator.Services;
 
@@ -17,21 +18,32 @@ namespace PerfProblemSimulator.Services;
 /// It subscribes to metrics events and broadcasts them to all connected clients.
 /// </para>
 /// <para>
-/// <strong>Why a separate service?</strong>
+/// <strong>Thread Pool Independence (Critical for Load Testing):</strong>
+/// </para>
+/// <para>
+/// When the load test endpoint exhausts the thread pool, SignalR broadcasts would
+/// normally freeze because they rely on thread pool threads. To prevent this, we use:
 /// </para>
 /// <list type="bullet">
-/// <item>MetricsCollector is focused on data collection (single responsibility)</item>
-/// <item>SignalR hub context can be injected here safely</item>
-/// <item>Allows metrics collection to work even with no connected clients</item>
-/// <item>Graceful startup/shutdown via IHostedService lifecycle</item>
+/// <item>A dedicated broadcast thread (not from thread pool)</item>
+/// <item>A message queue (BlockingCollection) for thread-safe message passing</item>
+/// <item>Fire-and-forget semantics that don't await thread pool continuations</item>
 /// </list>
+/// <para>
+/// This ensures the dashboard continues updating even during severe thread pool starvation.
+/// </para>
 /// </remarks>
 public class MetricsBroadcastService : IHostedService
 {
     private readonly IMetricsCollector _metricsCollector;
     private readonly ISimulationTracker _simulationTracker;
     private readonly IHubContext<MetricsHub, IMetricsClient> _hubContext;
     private readonly ILogger<MetricsBroadcastService> _logger;
+    
+    // Message queue for thread-pool-independent broadcasting
+    private readonly BlockingCollection<BroadcastMessage> _messageQueue = new(boundedCapacity: 100);
+    private Thread? _broadcastThread;
+    private volatile bool _running;
 
     /// <summary>
     /// Initializes a new instance of the <see cref="MetricsBroadcastService"/> class.
@@ -51,62 +63,155 @@ public MetricsBroadcastService(
     /// <inheritdoc />
     public Task StartAsync(CancellationToken cancellationToken)
     {
+        _running = true;
+        
+        // Start dedicated broadcast thread (not from thread pool)
+        _broadcastThread = new Thread(BroadcastLoop)
+        {
+            Name = "SignalR-Broadcast",
+            IsBackground = true,
+            Priority = ThreadPriority.AboveNormal // Prioritize dashboard updates
+        };
+        _broadcastThread.Start();
+        
         _metricsCollector.MetricsCollected += OnMetricsCollected;
         _simulationTracker.SimulationStarted += OnSimulationStarted;
         _simulationTracker.SimulationCompleted += OnSimulationCompleted;
         _metricsCollector.Start();
 
-        _logger.LogInformation("Metrics broadcast service started");
+        _logger.LogInformation("Metrics broadcast service started with dedicated broadcast thread");
         return Task.CompletedTask;
     }
 
     /// <inheritdoc />
     public Task StopAsync(CancellationToken cancellationToken)
     {
+        _running = false;
+        _messageQueue.CompleteAdding();
+        
         _metricsCollector.MetricsCollected -= OnMetricsCollected;
         _simulationTracker.SimulationStarted -= OnSimulationStarted;
         _simulationTracker.SimulationCompleted -= OnSimulationCompleted;
         _metricsCollector.Stop();
+        
+        // Wait for broadcast thread to finish (with timeout)
+        _broadcastThread?.Join(TimeSpan.FromSeconds(5));
 
         _logger.LogInformation("Metrics broadcast service stopped");
         return Task.CompletedTask;
     }
-
-    private async void OnMetricsCollected(object? sender, MetricsSnapshot snapshot)
+    
+    /// <summary>
+    /// Dedicated thread loop that processes broadcast messages.
+    /// This runs independently of the thread pool.
+    /// </summary>
+    private void BroadcastLoop()
     {
-        try
-        {
-            await _hubContext.Clients.All.ReceiveMetrics(snapshot);
-        }
-        catch (Exception ex)
+        _logger.LogDebug("Broadcast thread started");
+        
+        while (_running || _messageQueue.Count > 0)
         {
-            _logger.LogError(ex, "Error broadcasting metrics to clients");
+            try
+            {
+                // TryTake with timeout to allow checking _running flag
+                if (_messageQueue.TryTake(out var message, TimeSpan.FromMilliseconds(100)))
+                {
+                    ProcessMessage(message);
+                }
+            }
+            catch (InvalidOperationException)
+            {
+                // Collection was marked as complete - exit loop
+                break;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Error in broadcast loop");
+            }
         }
+        
+        _logger.LogDebug("Broadcast thread exiting");
     }
-
-    private async void OnSimulationStarted(object? sender, SimulationEventArgs e)
+    
+    /// <summary>
+    /// Process a single broadcast message.
+    /// Uses GetAwaiter().GetResult() to avoid thread pool dependency.
+    /// </summary>
+    private void ProcessMessage(BroadcastMessage message)
     {
         try
         {
-            await _hubContext.Clients.All.SimulationStarted(e.Type.ToString(), e.SimulationId);
-            _logger.LogDebug("Broadcast SimulationStarted: {Type} {Id}", e.Type, e.SimulationId);
+            switch (message.Type)
+            {
+                case BroadcastType.Metrics:
+                    _hubContext.Clients.All.ReceiveMetrics((MetricsSnapshot)message.Data!).GetAwaiter().GetResult();
+                    break;
+                    
+                case BroadcastType.SimulationStarted:
+                    var startArgs = (SimulationEventArgs)message.Data!;
+                    _hubContext.Clients.All.SimulationStarted(startArgs.Type.ToString(), startArgs.SimulationId).GetAwaiter().GetResult();
+                    _logger.LogDebug("Broadcast SimulationStarted: {Type} {Id}", startArgs.Type, startArgs.SimulationId);
+                    break;
+                    
+                case BroadcastType.SimulationCompleted:
+                    var completeArgs = (SimulationEventArgs)message.Data!;
+                    _hubContext.Clients.All.SimulationCompleted(completeArgs.Type.ToString(), completeArgs.SimulationId).GetAwaiter().GetResult();
+                    _logger.LogDebug("Broadcast SimulationCompleted: {Type} {Id}", completeArgs.Type, completeArgs.SimulationId);
+                    break;
+                    
+                case BroadcastType.Latency:
+                    _hubContext.Clients.All.ReceiveLatency((LatencyMeasurement)message.Data!).GetAwaiter().GetResult();
+                    break;
+                    
+                case BroadcastType.SlowRequestLatency:
+                    _hubContext.Clients.All.ReceiveSlowRequestLatency((SlowRequestLatencyData)message.Data!).GetAwaiter().GetResult();
+                    break;
+                    
+                case BroadcastType.LoadTestStats:
+                    _hubContext.Clients.All.ReceiveLoadTestStats((LoadTestStatsData)message.Data!).GetAwaiter().GetResult();
+                    break;
+            }
         }
         catch (Exception ex)
         {
-            _logger.LogError(ex, "Error broadcasting simulation started event");
+            _logger.LogWarning(ex, "Error broadcasting {Type} message", message.Type);
         }
     }
 
-    private async void OnSimulationCompleted(object? sender, SimulationEventArgs e)
+    private void OnMetricsCollected(object? sender, MetricsSnapshot snapshot)
     {
-        try
+        // Queue message - don't block if queue is full (drop oldest metrics)
+        if (!_messageQueue.TryAdd(new BroadcastMessage(BroadcastType.Metrics, snapshot)))
         {
-            await _hubContext.Clients.All.SimulationCompleted(e.Type.ToString(), e.SimulationId);
-            _logger.LogDebug("Broadcast SimulationCompleted: {Type} {Id}", e.Type, e.SimulationId);
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError(ex, "Error broadcasting simulation completed event");
+            _logger.LogTrace("Broadcast queue full, dropping metrics update");
         }
     }
+
+    private void OnSimulationStarted(object? sender, SimulationEventArgs e)
+    {
+        _messageQueue.TryAdd(new BroadcastMessage(BroadcastType.SimulationStarted, e));
+    }
+
+    private void OnSimulationCompleted(object? sender, SimulationEventArgs e)
+    {
+        _messageQueue.TryAdd(new BroadcastMessage(BroadcastType.SimulationCompleted, e));
+    }
+    
+    /// <summary>
+    /// Message types for the broadcast queue.
+    /// </summary>
+    private enum BroadcastType
+    {
+        Metrics,
+        SimulationStarted,
+        SimulationCompleted,
+        Latency,
+        SlowRequestLatency,
+        LoadTestStats
+    }
+    
+    /// <summary>
+    /// Wrapper for broadcast messages in the queue.
+    /// </summary>
+    private record BroadcastMessage(BroadcastType Type, object? Data);
 }