-
Notifications
You must be signed in to change notification settings - Fork 12
Expand file tree
/
Copy pathDevFlowAgentService.cs
More file actions
4327 lines (3797 loc) · 164 KB
/
DevFlowAgentService.cs
File metadata and controls
4327 lines (3797 loc) · 164 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
using System.Text.Json;
using System.Reflection;
using System.Runtime.CompilerServices;
using Microsoft.Maui;
using Microsoft.Maui.ApplicationModel;
using Microsoft.Maui.Controls;
using Microsoft.Maui.Controls.Internals;
using Microsoft.Maui.Devices;
using Microsoft.Maui.Devices.Sensors;
using Microsoft.Maui.Dispatching;
using Microsoft.Maui.Networking;
using Microsoft.Maui.Storage;
using MauiDevFlow.Agent.Core.Profiling;
using MauiDevFlow.Logging;
using MauiDevFlow.Agent.Core.Network;
namespace MauiDevFlow.Agent.Core;
/// <summary>
/// The main agent service that hosts the HTTP API and coordinates
/// visual tree inspection and element interactions.
/// </summary>
public class DevFlowAgentService : IDisposable, IMarkerPublisher
{
private readonly AgentOptions _options;
private readonly AgentHttpServer _server;
private readonly VisualTreeWalker _treeWalker;
private FileLogProvider? _logProvider;
private BrokerRegistration? _brokerRegistration;
protected Application? _app;
protected IDispatcher? _dispatcher;
private bool _disposed;
/// <summary>
/// The network request store for capturing HTTP traffic.
/// </summary>
public NetworkRequestStore NetworkStore { get; }
/// <summary>
/// Manages sensor subscriptions and broadcasts readings to WebSocket clients.
/// </summary>
public SensorManager Sensors { get; }
private readonly IProfilerCollector _profilerCollector;
private readonly ProfilerSessionStore _profilerSessions;
private readonly SemaphoreSlim _profilerStateGate = new(1, 1);
private CancellationTokenSource? _profilerLoopCts;
private Task? _profilerLoopTask;
private DateTime _lastAutoJankSpanTsUtc = DateTime.MinValue;
private const int UiHookScanIntervalMs = 3000;
private readonly ConditionalWeakTable<BindableObject, UiHookState> _uiHookStates = new();
private readonly List<Action> _uiHookUnsubscribers = new();
private readonly object _uiHookGate = new();
private int _uiHookGeneration = 1;
private int _uiHookScanInFlight;
private DateTime _lastUiHookScanTsUtc = DateTime.MinValue;
private Shell? _hookedShell;
private DateTime? _navigationStartedAtUtc;
private string? _navigationTargetRoute;
private DateTime _lastUserActionTsUtc = DateTime.MinValue;
private string? _lastUserActionName;
private string? _lastUserActionElementPath;
private readonly ConditionalWeakTable<Page, PageLifecycleState> _pageLifecycleStates = new();
private readonly ConditionalWeakTable<VisualElement, ElementRenderState> _elementRenderStates = new();
private readonly ConditionalWeakTable<BindableObject, ScrollBatchState> _scrollBatchStates = new();
private sealed class UiHookState
{
public int Generation { get; set; }
public HashSet<string> HookKeys { get; } = new(StringComparer.Ordinal);
}
private sealed class PageLifecycleState
{
public DateTime AppearingAtUtc { get; set; }
public string? Route { get; set; }
public bool FirstLayoutPublished { get; set; }
public int SizeChangedCount { get; set; }
public int MeasureInvalidatedCount { get; set; }
}
private sealed class ElementRenderState
{
public DateTime TrackingStartedAtUtc { get; set; }
public string? Role { get; set; }
public bool FirstLayoutPublished { get; set; }
public int SizeChangedCount { get; set; }
public int MeasureInvalidatedCount { get; set; }
}
private sealed class ScrollBatchState
{
public bool IsActive { get; set; }
public DateTime StartedAtUtc { get; set; }
public DateTime LastEventAtUtc { get; set; }
public int EventCount { get; set; }
public int FlushVersion { get; set; }
public double StartOffsetX { get; set; }
public double StartOffsetY { get; set; }
public double LastOffsetX { get; set; }
public double LastOffsetY { get; set; }
public int? StartFirstVisibleIndex { get; set; }
public int? StartLastVisibleIndex { get; set; }
public int? LastFirstVisibleIndex { get; set; }
public int? LastLastVisibleIndex { get; set; }
}
/// <summary>
/// Delegate for sending CDP commands to the Blazor WebView.
/// Set by the Blazor package when both are registered.
/// Deprecated: use RegisterCdpWebView() for multi-WebView support.
/// Setting this property registers the handler as WebView index 0.
/// </summary>
public Func<string, Task<string>>? CdpCommandHandler
{
get => _cdpWebViews.Count > 0 ? _cdpWebViews[0].CommandHandler : null;
set
{
if (value == null)
{
if (_cdpWebViews.Count > 0)
_cdpWebViews.RemoveAt(0);
return;
}
if (_cdpWebViews.Count > 0)
_cdpWebViews[0].CommandHandler = value;
else
_cdpWebViews.Add(new CdpWebViewInfo { Index = 0, CommandHandler = value, ReadyCheck = () => true });
}
}
/// <summary>Whether the CDP handler is ready to process commands.
/// Deprecated: use RegisterCdpWebView() for multi-WebView support.</summary>
public Func<bool>? CdpReadyCheck
{
get => _cdpWebViews.Count > 0 ? _cdpWebViews[0].ReadyCheck : null;
set
{
if (_cdpWebViews.Count > 0 && value != null)
_cdpWebViews[0].ReadyCheck = value;
}
}
private readonly List<CdpWebViewInfo> _cdpWebViews = new();
private int _nextWebViewIndex = 0;
/// <summary>Register a CDP-capable WebView with the agent.</summary>
public int RegisterCdpWebView(Func<string, Task<string>> commandHandler, Func<bool> readyCheck,
string? automationId = null, string? elementId = null, string? url = null)
{
var index = _nextWebViewIndex++;
_cdpWebViews.Add(new CdpWebViewInfo
{
Index = index,
AutomationId = automationId,
ElementId = elementId,
Url = url,
CommandHandler = commandHandler,
ReadyCheck = readyCheck,
});
return index;
}
/// <summary>Unregister a CDP WebView by index.</summary>
public void UnregisterCdpWebView(int index)
{
_cdpWebViews.RemoveAll(w => w.Index == index);
}
/// <summary>Update metadata for a registered WebView.</summary>
public void UpdateCdpWebView(int index, string? automationId = null, string? elementId = null, string? url = null)
{
var wv = _cdpWebViews.FirstOrDefault(w => w.Index == index);
if (wv == null) return;
if (automationId != null) wv.AutomationId = automationId;
if (elementId != null) wv.ElementId = elementId;
if (url != null) wv.Url = url;
}
private CdpWebViewInfo? ResolveCdpWebView(string? webviewId)
{
if (_cdpWebViews.Count == 0) return null;
if (string.IsNullOrEmpty(webviewId)) return _cdpWebViews[0]; // default to first
// Try index
if (int.TryParse(webviewId, out var idx))
{
var byIndex = _cdpWebViews.FirstOrDefault(w => w.Index == idx);
if (byIndex != null) return byIndex;
}
// Try AutomationId
var byAutomationId = _cdpWebViews.FirstOrDefault(w =>
!string.IsNullOrEmpty(w.AutomationId) && w.AutomationId.Equals(webviewId, StringComparison.OrdinalIgnoreCase));
if (byAutomationId != null) return byAutomationId;
// Try ElementId
var byElementId = _cdpWebViews.FirstOrDefault(w =>
!string.IsNullOrEmpty(w.ElementId) && w.ElementId.Equals(webviewId, StringComparison.OrdinalIgnoreCase));
if (byElementId != null) return byElementId;
return null;
}
public bool IsRunning => _server.IsRunning;
public int Port => _options.Port;
public DevFlowAgentService(AgentOptions? options = null)
{
_options = options ?? new AgentOptions();
_server = new AgentHttpServer(_options.Port);
_treeWalker = CreateTreeWalker();
NetworkStore = new NetworkRequestStore(_options.MaxNetworkBufferSize);
Sensors = new SensorManager();
_profilerCollector = CreateProfilerCollector();
_profilerSessions = new ProfilerSessionStore(
Math.Max(1, _options.MaxProfilerSamples),
Math.Max(1, _options.MaxProfilerMarkers),
Math.Max(1, _options.MaxProfilerSpans));
if (_options.EnableNetworkMonitoring)
DevFlowHttp.SetStore(NetworkStore);
NetworkStore.OnRequestCaptured += HandleCapturedNetworkRequest;
RegisterRoutes();
}
/// <summary>
/// Parses the optional "window" query parameter as a 0-based window index.
/// Returns null when not specified (callers should default to first window).
/// </summary>
private static int? ParseWindowIndex(HttpRequest request)
{
if (request.QueryParams.TryGetValue("window", out var ws) && int.TryParse(ws, out var wi))
return wi;
return null;
}
/// <summary>
/// Gets the window at the given index, or the first window when index is null.
/// </summary>
private Window? GetWindow(int? index)
{
if (_app == null) return null;
if (index == null) return _app.Windows.FirstOrDefault() as Window;
if (index.Value < 0 || index.Value >= _app.Windows.Count) return null;
return _app.Windows[index.Value] as Window;
}
/// <summary>
/// Creates the visual tree walker. Override in platform-specific subclasses
/// to return a walker with native info population.
/// </summary>
protected virtual VisualTreeWalker CreateTreeWalker() => new VisualTreeWalker();
/// <summary>
/// Creates the profiler collector. Override in platform-specific subclasses
/// to provide native frame/CPU integrations.
/// </summary>
protected virtual IProfilerCollector CreateProfilerCollector() => new RuntimeProfilerCollector();
/// <summary>Platform name for status reporting. Override for platforms without DeviceInfo.</summary>
protected virtual string PlatformName => DeviceInfo.Current.Platform.ToString();
/// <summary>Device type for status reporting. Override for platforms without DeviceInfo.</summary>
protected virtual string DeviceTypeName => DeviceInfo.Current.DeviceType.ToString();
/// <summary>Device idiom for status reporting. Override for platforms without DeviceInfo.</summary>
protected virtual string IdiomName => DeviceInfo.Current.Idiom.ToString();
/// <summary>
/// Gets the display density (scale factor) for a specific window. Returns 1.0 for standard,
/// 2.0 for @2x (Retina), 3.0 for @3x (iPhone Pro Max), etc.
/// Used to auto-scale screenshots to 1x logical resolution.
/// Override in platform-specific agents to query the native window's actual screen density,
/// which may vary across displays in multi-monitor setups.
/// </summary>
protected virtual double GetWindowDisplayDensity(IWindow? window)
{
try { return DeviceDisplay.MainDisplayInfo.Density; }
catch { return 1.0; }
}
/// <summary>Gets native window dimensions when MAUI reports 0. Override for platform-specific access.</summary>
protected virtual (double width, double height) GetNativeWindowSize(IWindow window) => (0, 0);
private bool IsProfilerFeatureAvailable => _options.EnableProfiler;
/// <summary>
/// Sets the file log provider for serving logs via the API.
/// Called by AgentServiceExtensions during registration.
/// </summary>
public void SetLogProvider(FileLogProvider provider)
=> _logProvider = provider;
public void SetBrokerRegistration(BrokerRegistration registration)
=> _brokerRegistration = registration;
/// <summary>
/// Writes a log entry originating from the WebView/Blazor console.
/// Called by the Blazor package via reflection to route JS console output through ILogger.
/// </summary>
public void WriteWebViewLog(string level, string category, string message, string? exception = null)
{
if (_logProvider == null) return;
var entry = new Logging.FileLogEntry(
Timestamp: DateTime.UtcNow,
Level: level,
Category: category,
Message: message,
Exception: exception,
Source: "webview"
);
_logProvider.Writer.Write(entry);
}
/// <summary>
/// Starts the agent and binds to the running MAUI app.
/// </summary>
public void Start(Application app, IDispatcher dispatcher)
{
if (!_options.Enabled) return;
_app = app;
_dispatcher = dispatcher;
try
{
_server.Start();
Console.WriteLine($"[MauiDevFlow.Agent] HTTP server started on port {_options.Port}");
}
catch (Exception ex)
{
Console.WriteLine($"[MauiDevFlow.Agent] Failed to start HTTP server: {ex.Message}");
}
}
public async Task StopAsync()
{
await StopProfilerAsync();
await _server.StopAsync();
}
private void RegisterRoutes()
{
_server.MapGet("/api/status", HandleStatus);
_server.MapGet("/api/tree", HandleTree);
_server.MapGet("/api/element/{id}", HandleElement);
_server.MapGet("/api/query", HandleQuery);
_server.MapGet("/api/hittest", HandleHitTest);
_server.MapGet("/api/screenshot", HandleScreenshot);
_server.MapGet("/api/property/{id}/{name}", HandleProperty);
_server.MapPost("/api/property/{id}/{name}", HandleSetProperty);
_server.MapPost("/api/action/tap", HandleTap);
_server.MapPost("/api/action/fill", HandleFill);
_server.MapPost("/api/action/clear", HandleClear);
_server.MapPost("/api/action/focus", HandleFocus);
_server.MapPost("/api/action/navigate", HandleNavigate);
_server.MapPost("/api/action/resize", HandleResize);
_server.MapPost("/api/action/scroll", HandleScroll);
_server.MapGet("/api/logs", HandleLogs);
_server.MapPost("/api/cdp", HandleCdp);
_server.MapGet("/api/cdp/webviews", HandleCdpWebViews);
_server.MapGet("/api/cdp/source", HandleCdpSource);
_server.MapGet("/api/profiler/capabilities", HandleProfilerCapabilities);
_server.MapPost("/api/profiler/start", HandleProfilerStart);
_server.MapPost("/api/profiler/stop", HandleProfilerStop);
_server.MapGet("/api/profiler/samples", HandleProfilerSamples);
_server.MapPost("/api/profiler/marker", HandleProfilerMarker);
_server.MapPost("/api/profiler/span", HandleProfilerSpan);
_server.MapGet("/api/profiler/hotspots", HandleProfilerHotspots);
// Network monitoring
_server.MapGet("/api/network", HandleNetworkList);
_server.MapGet("/api/network/{id}", HandleNetworkDetail);
_server.MapPost("/api/network/clear", HandleNetworkClear);
// WebSocket: live network monitoring stream
_server.MapWebSocket("/ws/network", HandleNetworkWebSocket);
// WebSocket: live log streaming
_server.MapWebSocket("/ws/logs", HandleLogsWebSocket);
// Preferences (CRUD)
_server.MapGet("/api/preferences", HandlePreferencesList);
_server.MapGet("/api/preferences/{key}", HandlePreferencesGet);
_server.MapPost("/api/preferences/{key}", HandlePreferencesSet);
_server.MapDelete("/api/preferences/{key}", HandlePreferencesDelete);
_server.MapPost("/api/preferences/clear", HandlePreferencesClear);
// Secure Storage (CRUD)
_server.MapGet("/api/secure-storage/{key}", HandleSecureStorageGet);
_server.MapPost("/api/secure-storage/{key}", HandleSecureStorageSet);
_server.MapDelete("/api/secure-storage/{key}", HandleSecureStorageDelete);
_server.MapPost("/api/secure-storage/clear", HandleSecureStorageClear);
// Platform info (read-only)
_server.MapGet("/api/platform/app-info", HandlePlatformAppInfo);
_server.MapGet("/api/platform/device-info", HandlePlatformDeviceInfo);
_server.MapGet("/api/platform/device-display", HandlePlatformDeviceDisplay);
_server.MapGet("/api/platform/battery", HandlePlatformBattery);
_server.MapGet("/api/platform/connectivity", HandlePlatformConnectivity);
_server.MapGet("/api/platform/version-tracking", HandlePlatformVersionTracking);
_server.MapGet("/api/platform/permissions", HandlePlatformPermissions);
_server.MapGet("/api/platform/permissions/{permission}", HandlePlatformPermissionCheck);
_server.MapGet("/api/platform/geolocation", HandlePlatformGeolocation);
// Sensors
_server.MapGet("/api/sensors", HandleSensorsList);
_server.MapPost("/api/sensors/{sensor}/start", HandleSensorStart);
_server.MapPost("/api/sensors/{sensor}/stop", HandleSensorStop);
_server.MapWebSocket("/ws/sensors", HandleSensorWebSocket);
}
private async Task<HttpResponse> HandleStatus(HttpRequest request)
{
var windowIndex = ParseWindowIndex(request);
var result = await DispatchAsync(() =>
{
var window = GetWindow(windowIndex);
var w = window?.Width ?? 0;
var h = window?.Height ?? 0;
// Try getting window size from native platform view if MAUI reports invalid values
if (window != null && (!double.IsFinite(w) || !double.IsFinite(h) || w <= 0 || h <= 0))
{
var (nw, nh) = GetNativeWindowSize(window);
if (nw > 0) w = nw;
if (nh > 0) h = nh;
}
return new
{
agent = "MauiDevFlow.Agent",
version = typeof(DevFlowAgentService).Assembly
.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion ?? "unknown",
platform = PlatformName,
deviceType = DeviceTypeName,
idiom = IdiomName,
displayDensity = GetWindowDisplayDensity(window),
appName = _app?.GetType().Assembly.GetName().Name ?? "unknown",
running = _app != null,
cdpReady = _cdpWebViews.Any(w => w.IsReady),
cdpWebViewCount = _cdpWebViews.Count,
windowCount = _app?.Windows.Count ?? 0,
windowWidth = double.IsFinite(w) ? w : 0,
windowHeight = double.IsFinite(h) ? h : 0,
profiler = BuildProfilerCapabilitiesPayload(),
profilerSession = _profilerSessions.CurrentSession
};
});
return HttpResponse.Json(result!);
}
private async Task<HttpResponse> HandleTree(HttpRequest request)
{
if (_app == null) return HttpResponse.Error("Agent not bound to app");
int maxDepth = 0;
if (request.QueryParams.TryGetValue("depth", out var depthStr))
int.TryParse(depthStr, out maxDepth);
var windowIndex = ParseWindowIndex(request);
var tree = await DispatchAsync(() => _treeWalker.WalkTree(_app, maxDepth, windowIndex));
return HttpResponse.Json(tree);
}
private async Task<HttpResponse> HandleElement(HttpRequest request)
{
if (_app == null) return HttpResponse.Error("Agent not bound to app");
if (!request.RouteParams.TryGetValue("id", out var id))
return HttpResponse.Error("Element ID required");
var element = await DispatchAsync(() =>
{
var el = _treeWalker.GetElementById(id, _app);
if (el is IVisualTreeElement vte)
return (object?)_treeWalker.WalkElement(vte, null, 1, 2);
// Synthetic elements: build detail from marker
if (el != null)
return (object?)_treeWalker.BuildSyntheticElementInfo(id, el);
return null;
});
return element != null ? HttpResponse.Json(element) : HttpResponse.NotFound($"Element '{id}' not found");
}
private async Task<HttpResponse> HandleQuery(HttpRequest request)
{
if (_app == null) return HttpResponse.Error("Agent not bound to app");
// CSS selector takes precedence over simple filters
if (request.QueryParams.TryGetValue("selector", out var selector) && !string.IsNullOrWhiteSpace(selector))
{
try
{
var results = await DispatchAsync(() => _treeWalker.QueryCss(_app, selector));
return HttpResponse.Json(results);
}
catch (FormatException ex)
{
return HttpResponse.Error($"Invalid CSS selector: {ex.Message}");
}
}
request.QueryParams.TryGetValue("type", out var type);
request.QueryParams.TryGetValue("automationId", out var automationId);
request.QueryParams.TryGetValue("text", out var text);
if (type == null && automationId == null && text == null)
return HttpResponse.Error("At least one query parameter required: type, automationId, text, or selector");
var simpleResults = await DispatchAsync(() => _treeWalker.Query(_app, type, automationId, text));
return HttpResponse.Json(simpleResults);
}
private async Task<HttpResponse> HandleHitTest(HttpRequest request)
{
if (_app == null) return HttpResponse.Error("Agent not bound to app");
if (!request.QueryParams.TryGetValue("x", out var xStr) || !double.TryParse(xStr, out var x))
return HttpResponse.Error("x coordinate is required");
if (!request.QueryParams.TryGetValue("y", out var yStr) || !double.TryParse(yStr, out var y))
return HttpResponse.Error("y coordinate is required");
var windowIndex = ParseWindowIndex(request);
var result = await DispatchAsync(() =>
{
var window = GetWindow(windowIndex);
if (window == null) return (object?)null;
// Ensure tree is walked so element IDs are assigned and synthetic bounds are populated
_treeWalker.WalkTree(_app!, 0, windowIndex);
// Build active Shell context to filter out inactive ShellItem subtrees
var activeShellItemIds = BuildActiveShellItemIds(window);
var platformHits = VisualTreeElementExtensions.GetVisualTreeElements(window, x, y);
// Supplement with bounds-based hit testing — some platforms (e.g. macOS AppKit)
// don't traverse into all containers via GetVisualTreeElements
var boundsHits = _treeWalker.HitTestByBounds(x, y, _app!, windowIndex);
var seen = new HashSet<object>(ReferenceEqualityComparer.Instance);
var allHits = new List<IVisualTreeElement>();
foreach (var h in platformHits)
{
seen.Add(h);
allHits.Add(h);
}
foreach (var bh in boundsHits)
{
if (seen.Add(bh))
allHits.Add(bh);
}
var elements = new List<object>();
// Detect modal pages — elements behind the topmost modal should be excluded
var modalPage = window.Navigation?.ModalStack?.LastOrDefault();
// Check synthetic elements first — they represent visible nav chrome
// (nav bar, tab bar, flyout button) that sits on top of MAUI content.
var syntheticHits = _treeWalker.HitTestSynthetics(x, y);
foreach (var (synId, marker, bounds) in syntheticHits)
{
// If modal is active, only include synthetics belonging to the modal page
if (modalPage != null && !IsSyntheticForPage(marker, modalPage))
continue;
var synInfo = new Dictionary<string, object?>
{
["id"] = synId,
["type"] = _treeWalker.GetSyntheticTypeName(marker),
["bounds"] = bounds,
["windowBounds"] = bounds, // synthetic bounds are already window-absolute
["synthetic"] = true,
};
var text = _treeWalker.GetSyntheticText(marker);
if (text != null) synInfo["text"] = text;
elements.Add(synInfo);
}
foreach (var hit in allHits)
{
if (hit is not IVisualTreeElement vte) continue;
// Skip elements under inactive ShellItem subtrees
if (activeShellItemIds != null && IsUnderInactiveShellItem(hit, activeShellItemIds))
continue;
// Skip elements behind the modal page
if (modalPage != null && !IsDescendantOfPage(hit, modalPage))
continue;
var id = _treeWalker.GetIdForElement(vte);
if (id == null) continue;
var info = new Dictionary<string, object?> { ["id"] = id, ["type"] = hit.GetType().Name };
if (hit is VisualElement ve)
{
info["automationId"] = ve.AutomationId;
info["bounds"] = new BoundsInfo
{
X = double.IsFinite(ve.Frame.X) ? ve.Frame.X : 0,
Y = double.IsFinite(ve.Frame.Y) ? ve.Frame.Y : 0,
Width = double.IsFinite(ve.Frame.Width) ? ve.Frame.Width : 0,
Height = double.IsFinite(ve.Frame.Height) ? ve.Frame.Height : 0
};
var wb = _treeWalker.ResolveWindowBoundsPublic(ve);
if (wb != null) info["windowBounds"] = wb;
}
if (hit is Label l) info["text"] = l.Text;
else if (hit is Button b) info["text"] = b.Text;
elements.Add(info);
}
return (object?)new { x, y, window = windowIndex ?? 0, elements };
});
return result != null
? HttpResponse.Json(result)
: HttpResponse.Error($"Window {windowIndex ?? 0} not found");
}
/// <summary>
/// Builds a set of active ShellItem objects for filtering hit test results.
/// Returns null if the window doesn't contain a Shell (no filtering needed).
/// </summary>
private static HashSet<object>? BuildActiveShellItemIds(Window window)
{
var shell = window.Page as Shell;
if (shell == null) return null;
var currentItem = shell.CurrentItem;
if (currentItem == null) return null;
// Only the current ShellItem is active
return new HashSet<object>(ReferenceEqualityComparer.Instance) { currentItem };
}
/// <summary>
/// Checks if an element is under an inactive ShellItem subtree.
/// Walks up the parent chain to find the containing ShellItem.
/// </summary>
private static bool IsUnderInactiveShellItem(object element, HashSet<object> activeShellItems)
{
var current = element as Element;
while (current != null)
{
if (current is ShellItem si)
return !activeShellItems.Contains(si);
current = current.Parent;
}
return false;
}
/// <summary>
/// Checks if an element is a descendant of the given page (or the page itself).
/// Used to filter hit test results when a modal page is active.
/// </summary>
private static bool IsDescendantOfPage(object element, Page page)
{
var current = element as Element;
while (current != null)
{
if (ReferenceEquals(current, page)) return true;
current = current.Parent;
}
return false;
}
/// <summary>
/// Checks if a synthetic marker belongs to the given modal page context.
/// The modal page may be a NavigationPage, TabbedPage, or FlyoutPage wrapping
/// inner pages, so we use descendant checks rather than reference equality.
/// </summary>
private static bool IsSyntheticForPage(object marker, Page modalPage)
{
return marker switch
{
VisualTreeWalker.NavBarTitleMarker m => IsDescendantOfPage(m.Page, modalPage),
ToolbarItem ti => IsDescendantOfPage(ti, modalPage),
VisualTreeWalker.BackButtonMarker => true,
VisualTreeWalker.SearchHandlerMarker => false,
_ => false
};
}
protected virtual async Task<HttpResponse> HandleScreenshot(HttpRequest request)
{
if (_app == null) return HttpResponse.Error("Agent not bound to app");
int? maxWidth = null;
if (request.QueryParams.TryGetValue("maxWidth", out var mwStr) && int.TryParse(mwStr, out var mw) && mw > 0)
maxWidth = mw;
// Auto-scale to 1x by default on HiDPI displays. Override with scale=native to keep full resolution.
bool autoScale = true;
if (request.QueryParams.TryGetValue("scale", out var scaleParam))
{
autoScale = !scaleParam.Equals("native", StringComparison.OrdinalIgnoreCase)
&& !scaleParam.Equals("full", StringComparison.OrdinalIgnoreCase);
}
// Resolve the target window and its display density on the UI thread
var windowIndex = ParseWindowIndex(request);
var density = await DispatchAsync(() =>
{
var w = GetWindow(windowIndex);
return GetWindowDisplayDensity(w);
});
// Check for fullscreen mode (captures all windows including dialogs)
if (request.QueryParams.TryGetValue("fullscreen", out var fs) &&
fs.Equals("true", StringComparison.OrdinalIgnoreCase))
{
try
{
var pngData = await CaptureFullScreenAsync();
if (pngData != null)
return HttpResponse.Png(ResizePngIfNeeded(pngData, maxWidth, density, autoScale));
return HttpResponse.Error("Full-screen capture not supported on this platform");
}
catch (Exception ex)
{
return HttpResponse.Error($"Full-screen screenshot failed: {ex.Message}");
}
}
// Element-level screenshot by ID
if (request.QueryParams.TryGetValue("id", out var elementId) && !string.IsNullOrWhiteSpace(elementId))
{
try
{
var element = await DispatchAsync(() =>
{
var el = _treeWalker.GetElementById(elementId, _app);
return el as VisualElement;
});
if (element == null)
return HttpResponse.Error($"Element '{elementId}' not found or not a VisualElement");
var pngData = await DispatchAsync(() => CaptureElementScreenshotAsync(element));
if (pngData == null)
return HttpResponse.Error($"Capture returned null for element '{elementId}'");
return HttpResponse.Png(ResizePngIfNeeded(pngData, maxWidth, density, autoScale));
}
catch (Exception ex)
{
return HttpResponse.Error($"Element screenshot failed: {ex.Message}");
}
}
// Element-level screenshot by CSS selector (captures first match)
if (request.QueryParams.TryGetValue("selector", out var selector) && !string.IsNullOrWhiteSpace(selector))
{
try
{
var matchId = await DispatchAsync(() =>
{
var results = _treeWalker.QueryCss(_app, selector);
return results.Count > 0 ? results[0].Id : null;
});
if (matchId == null)
return HttpResponse.Error($"No elements matching selector '{selector}'");
var element = await DispatchAsync(() =>
{
var el = _treeWalker.GetElementById(matchId, _app);
return el as VisualElement;
});
if (element == null)
return HttpResponse.Error($"Element '{matchId}' not found or not a VisualElement");
var pngData = await DispatchAsync(() => CaptureElementScreenshotAsync(element));
if (pngData == null)
return HttpResponse.Error($"Capture returned null for element '{matchId}'");
return HttpResponse.Png(ResizePngIfNeeded(pngData, maxWidth, density, autoScale));
}
catch (FormatException ex)
{
return HttpResponse.Error($"Invalid CSS selector: {ex.Message}");
}
catch (Exception ex)
{
return HttpResponse.Error($"Element screenshot failed: {ex.Message}");
}
}
try
{
var pngData = await DispatchAsync(async () =>
{
var window = GetWindow(windowIndex);
if (window == null) return null;
// If a modal page is displayed, capture it instead of the underlying page
VisualElement? topModal = null;
try
{
var modalStack = window.Page?.Navigation?.ModalStack;
if (modalStack?.Count > 0 && modalStack[^1] is VisualElement ms)
topModal = ms;
}
catch { }
// Fallback: check Window's visual children for modal pages
// (on some platforms like GTK, modals appear as direct children of the Window)
if (topModal == null && window is IVisualTreeElement windowVte)
{
var children = windowVte.GetVisualChildren();
for (int i = children.Count - 1; i >= 0; i--)
{
if (children[i] is Page page && page != window.Page)
{
topModal = page;
break;
}
}
}
if (topModal != null)
return await CaptureScreenshotAsync(topModal);
if (window.Page is not VisualElement rootElement) return null;
return await CaptureScreenshotAsync(rootElement);
});
if (pngData == null)
return HttpResponse.Error("Failed to capture screenshot");
return HttpResponse.Png(ResizePngIfNeeded(pngData, maxWidth, density, autoScale));
}
catch (Exception ex)
{
return HttpResponse.Error($"Screenshot failed: {ex.Message}");
}
}
/// <summary>
/// Captures a screenshot of the given root element. Override in platform-specific subclasses.
/// </summary>
protected virtual async Task<byte[]?> CaptureScreenshotAsync(VisualElement rootElement)
{
return await VisualDiagnostics.CaptureAsPngAsync(rootElement);
}
/// <summary>
/// Captures a screenshot of a specific element in the visual tree.
/// Override in platform-specific subclasses when VisualDiagnostics.CaptureAsPngAsync
/// is not supported (e.g. macOS AppKit).
/// </summary>
protected virtual async Task<byte[]?> CaptureElementScreenshotAsync(VisualElement element)
{
return await VisualDiagnostics.CaptureAsPngAsync(element);
}
/// <summary>
/// Captures a full-screen screenshot including all windows (dialogs, popups, etc.).
/// Override in platform-specific subclasses for native support.
/// Returns null if not supported.
/// </summary>
protected virtual Task<byte[]?> CaptureFullScreenAsync()
{
return Task.FromResult<byte[]?>(null);
}
/// <summary>
/// Resizes a PNG image based on display density and/or max width constraint.
/// By default, HiDPI screenshots are scaled to 1x logical resolution (e.g., a 3x iPhone
/// screenshot of 1290px becomes 430px). An explicit maxWidth overrides density scaling.
/// </summary>
private static byte[] ResizePngIfNeeded(byte[] pngData, int? maxWidth, double density = 1.0, bool autoScale = true)
{
// Determine target width: explicit maxWidth takes priority, then auto-scale by density
int? targetWidth = maxWidth;
if (targetWidth == null && autoScale && density > 1.0)
{
try
{
using var probe = SkiaSharp.SKBitmap.Decode(pngData);
if (probe != null)
targetWidth = (int)(probe.Width / density);
}
catch { return pngData; }
}
if (targetWidth == null || targetWidth <= 0) return pngData;
try
{
using var original = SkiaSharp.SKBitmap.Decode(pngData);
if (original == null || original.Width <= targetWidth.Value) return pngData;
var scale = (float)targetWidth.Value / original.Width;
var newHeight = (int)(original.Height * scale);
using var resized = original.Resize(new SkiaSharp.SKImageInfo(targetWidth.Value, newHeight), SkiaSharp.SKFilterQuality.Medium);
if (resized == null) return pngData;
using var image = SkiaSharp.SKImage.FromBitmap(resized);
using var encoded = image.Encode(SkiaSharp.SKEncodedImageFormat.Png, 100);
return encoded.ToArray();
}
catch
{
return pngData;
}
}
private async Task<HttpResponse> HandleProperty(HttpRequest request)
{
if (_app == null) return HttpResponse.Error("Agent not bound to app");
if (!request.RouteParams.TryGetValue("id", out var id))
return HttpResponse.Error("Element ID required");
if (!request.RouteParams.TryGetValue("name", out var propName))
return HttpResponse.Error("Property name required");
var value = await DispatchAsync(() =>
{
var el = _treeWalker.GetElementById(id, _app);
if (el == null) return (object?)null;
// Support dot-path notation (e.g., "Shadow.Radius")
var parts = propName.Split('.');
object? current = el;
PropertyInfo? prop = null;
foreach (var part in parts)
{
if (current == null) return null;
var type = current.GetType();
prop = type.GetProperty(part, BindingFlags.Public | BindingFlags.Instance | BindingFlags.IgnoreCase);
if (prop == null) return null;
current = prop.GetValue(current);
}
return FormatPropertyValue(current);
});
return value != null
? HttpResponse.Json(new { id, property = propName, value })
: HttpResponse.NotFound($"Property '{propName}' not found on element '{id}'");
}
private static string? FormatPropertyValue(object? value)
{
if (value == null) return null;
if (value is string s) return s;
// Try TypeConverter first — handles Thickness, CornerRadius, Color, enums, etc.
var converter = System.ComponentModel.TypeDescriptor.GetConverter(value.GetType());
if (converter.CanConvertTo(typeof(string))
&& converter.GetType() != typeof(System.ComponentModel.TypeConverter)
&& converter is not System.ComponentModel.CollectionConverter)
{
try
{
var result = converter.ConvertToString(value);
if (result != null) return result;
}
catch { }
}
// Fallback for complex types that lack TypeConverter ConvertTo support
return value switch
{
Shadow shadow => FormatShadow(shadow),
SolidColorBrush scb => $"SolidColorBrush Color={scb.Color?.ToArgbHex() ?? "(null)"}",
LinearGradientBrush lgb => $"LinearGradientBrush StartPoint={lgb.StartPoint}, EndPoint={lgb.EndPoint}, Stops=[{FormatGradientStops(lgb.GradientStops)}]",
RadialGradientBrush rgb => $"RadialGradientBrush Center={rgb.Center}, Radius={rgb.Radius}, Stops=[{FormatGradientStops(rgb.GradientStops)}]",
Brush brush => brush.GetType().Name,
Microsoft.Maui.Controls.Shapes.RoundRectangle rr => $"RoundRectangle CornerRadius={FormatPropertyValue(rr.CornerRadius)}",
Microsoft.Maui.Controls.Shapes.Shape shape => shape.GetType().Name,
ColumnDefinitionCollection cols => string.Join(", ", cols.Select(c => FormatGridLength(c.Width))),
RowDefinitionCollection rows => string.Join(", ", rows.Select(r => FormatGridLength(r.Height))),
LayoutOptions lo => $"{lo.Alignment}{(lo.Expands ? ", Expands" : "")}",
LinearItemsLayout lin => $"LinearItemsLayout Orientation={lin.Orientation}, ItemSpacing={lin.ItemSpacing}",
GridItemsLayout grid => $"GridItemsLayout Span={grid.Span}, Orientation={grid.Orientation}, HorizontalSpacing={grid.HorizontalItemSpacing}, VerticalSpacing={grid.VerticalItemSpacing}",
FileImageSource fis => $"File: {fis.File}",
UriImageSource uis => $"Uri: {uis.Uri}",
FontImageSource fontIs => $"Font: {fontIs.Glyph} ({fontIs.FontFamily})",
ImageSource img => img.GetType().Name,
System.Collections.ICollection col => $"{col.GetType().Name} ({col.Count} items)",
IFormattable f => f.ToString(null, System.Globalization.CultureInfo.InvariantCulture),
_ => value.ToString() ?? value.GetType().Name,
};
}
private static string FormatGridLength(GridLength gl) => gl.IsStar
? (gl.Value == 1 ? "*" : $"{gl.Value}*")
: gl.IsAbsolute ? gl.Value.ToString(System.Globalization.CultureInfo.InvariantCulture)
: "Auto";