Improve Android inspector and stream pacing

Author: DjDeveloperr

client/src/api/types.ts

@@ -160,15 +160,23 @@ export interface AccessibilityNode {
   AXUniqueId?: string | null;
   AXValue?: string | null;
   alpha?: number | null;
+  androidClass?: string | null;
+  androidPackage?: string | null;
+  androidResourceId?: string | null;
   backgroundColor?: Record<string, unknown> | null;
   bounds?: AccessibilityFrame | null;
+  checkable?: boolean | null;
+  checked?: boolean | null;
   className?: string | null;
+  clickable?: boolean | null;
   children?: AccessibilityNode[];
   control?: Record<string, unknown> | null;
   content_required?: boolean | null;
   custom_actions?: string[] | null;
   debugDescription?: string | null;
   enabled?: boolean | null;
+  focusable?: boolean | null;
+  focused?: boolean | null;
   frame?: AccessibilityFrame | null;
   frameInScreen?: AccessibilityFrame | null;
   help?: string | null;
@@ -177,14 +185,18 @@ export interface AccessibilityNode {
   isHidden?: boolean | null;
   isOpaque?: boolean | null;
   isUserInteractionEnabled?: boolean | null;
+  longClickable?: boolean | null;
   moduleName?: string | null;
   nativeScript?: Record<string, unknown> | null;
+  password?: boolean | null;
   pid?: number | null;
   placeholder?: string | null;
   reactNative?: Record<string, unknown> | null;
   role?: string | null;
   role_description?: string | null;
   scroll?: Record<string, unknown> | null;
+  scrollable?: boolean | null;
+  selected?: boolean | null;
   source?:
     | "native-ax"
     | "in-app-inspector"
@@ -210,6 +222,7 @@ export interface AccessibilityNode {
 }
 
 export type AccessibilitySource =
+  | "android-uiautomator"
   | "native-ax"
   | "in-app-inspector"
   | "nativescript"

client/src/app/AppShell.tsx

@@ -1257,6 +1257,7 @@ export function AppShell({
     viewportChromeProfile,
     chromeScreenRect,
   );
+  const isAndroidViewport = isAndroidSimulator(selectedSimulator);
   const chromeScreenStyle =
     viewportChromeProfile && chromeScreenRect
       ? ({
@@ -1287,15 +1288,19 @@ export function AppShell({
         } satisfies CSSProperties)
       : null;
   const screenOnlyStyle =
-    !viewportChromeProfile && chromeProfile && chromeProfile.screenWidth > 0
+    !viewportChromeProfile && isAndroidViewport
       ? ({
-          borderRadius: `${Math.min(
-            chromeProfile.cornerRadius *
-              (DEVICE_SCREEN_WIDTH / chromeProfile.screenWidth),
-            DEVICE_SCREEN_WIDTH / 2,
-          )}px`,
+          borderRadius: "10px",
         } satisfies CSSProperties)
-      : null;
+      : !viewportChromeProfile && chromeProfile && chromeProfile.screenWidth > 0
+        ? ({
+            borderRadius: `${Math.min(
+              chromeProfile.cornerRadius *
+                (DEVICE_SCREEN_WIDTH / chromeProfile.screenWidth),
+              DEVICE_SCREEN_WIDTH / 2,
+            )}px`,
+          } satisfies CSSProperties)
+        : null;
   const viewportScreenStyle = chromeScreenStyle ?? screenOnlyStyle;
   const shellStyle = viewportChromeProfile
     ? {
@@ -1998,6 +2003,7 @@ export function AppShell({
         outerCanvasRef={handleOuterCanvasRef}
         rotationQuarterTurns={rotationQuarterTurns}
         screenAspect={screenAspect}
+        screenClassName={isAndroidViewport ? "android-screen" : undefined}
         selectedSimulator={selectedSimulator}
         shellStyle={shellStyle}
         streamCanvasRef={handleStreamCanvasRef}
@@ -2149,6 +2155,14 @@ function normalizeMaxEdge(
     : fallback;
 }
 
+function isAndroidSimulator(simulator: SimulatorMetadata | null): boolean {
+  return Boolean(
+    simulator?.platform === "android-emulator" ||
+    simulator?.deviceTypeIdentifier === "android-emulator" ||
+    simulator?.udid.startsWith("android:"),
+  );
+}
+
 function streamConfigsEqual(left: StreamConfig, right: StreamConfig): boolean {
   return (
     left.encoder === right.encoder &&

client/src/app/uiState.ts

@@ -36,6 +36,7 @@ const ACCESSIBILITY_SOURCE_ORDER: AccessibilitySource[] = [
   "react-native",
   "swiftui",
   "in-app-inspector",
+  "android-uiautomator",
   "native-ax",
 ];
 
@@ -160,6 +161,7 @@ export function isAccessibilitySource(
     value === "react-native" ||
     value === "swiftui" ||
     value === "in-app-inspector" ||
+    value === "android-uiautomator" ||
     value === "native-ax"
   );
 }

client/src/features/accessibility/AccessibilityInspector.tsx

@@ -458,16 +458,23 @@ function NodeDetails({
   node: AccessibilityNode;
   selectedSimulator: SimulatorMetadata | null;
 }) {
+  const isAndroid = isAndroidSimulator(selectedSimulator);
   const details = [
     ["Type", accessibilityKind(node)],
     ["Label", primaryAccessibilityText(node)],
     ["Source", sourceLocationText(node)],
-    ["Identifier", accessibilityIdentifier(node)],
+    [
+      isAndroid ? "Resource ID" : "Identifier",
+      isAndroid
+        ? (node.androidResourceId ?? "")
+        : accessibilityIdentifier(node),
+    ],
     ["Inspector ID", node.inspectorId ?? ""],
     ["Module", node.moduleName ?? ""],
     ["NativeScript", nativeScriptDescription(node.nativeScript)],
     ["React Native", reactNativeDescription(node.reactNative)],
-    ["UIKit Class", node.className ?? ""],
+    [isAndroid ? "Android Class" : "UIKit Class", node.className ?? ""],
+    ["Package", isAndroid ? (node.androidPackage ?? "") : ""],
     ["Last JS", lastUIKitScriptText(node)],
     ["Value", node.AXValue ?? ""],
     ["Role", node.role ?? ""],
@@ -476,6 +483,15 @@ function NodeDetails({
     ["SwiftUI", swiftUIDescription(node.swiftUI)],
     ["Enabled", node.enabled == null ? "" : node.enabled ? "true" : "false"],
     ["Hidden", node.isHidden == null ? "" : node.isHidden ? "true" : "false"],
+    ["Clickable", boolDetail(isAndroid, node.clickable)],
+    ["Long Clickable", boolDetail(isAndroid, node.longClickable)],
+    ["Focusable", boolDetail(isAndroid, node.focusable)],
+    ["Focused", boolDetail(isAndroid, node.focused)],
+    ["Scrollable", boolDetail(isAndroid, node.scrollable)],
+    ["Checkable", boolDetail(isAndroid, node.checkable)],
+    ["Checked", boolDetail(isAndroid, node.checked)],
+    ["Selected", boolDetail(isAndroid, node.selected)],
+    ["Password", boolDetail(isAndroid, node.password)],
     ["Alpha", node.alpha == null ? "" : String(round(node.alpha))],
     ["Frame", validFrame(node.frame) ? frameText(node.frame) : ""],
     ["PID", node.pid == null ? "" : String(node.pid)],
@@ -497,6 +513,18 @@ function NodeDetails({
   );
 }
 
+function isAndroidSimulator(simulator: SimulatorMetadata | null): boolean {
+  return Boolean(
+    simulator?.platform === "android-emulator" ||
+    simulator?.deviceTypeIdentifier === "android-emulator" ||
+    simulator?.udid.startsWith("android:"),
+  );
+}
+
+function boolDetail(include: boolean, value: boolean | null | undefined) {
+  return include && value != null ? (value ? "true" : "false") : "";
+}
+
 function UIKitScriptEditor({
   node,
   selectedSimulator,
@@ -729,6 +757,7 @@ const HIERARCHY_SOURCE_ORDER: AccessibilitySource[] = [
   "react-native",
   "swiftui",
   "in-app-inspector",
+  "android-uiautomator",
   "native-ax",
 ];
 
@@ -759,6 +788,9 @@ function sourceLabel(source: AccessibilitySource): string {
   if (source === "swiftui") {
     return "SwiftUI";
   }
+  if (source === "android-uiautomator") {
+    return "Android";
+  }
   return source === "in-app-inspector" ? "UIKit" : "Native AX";
 }
 

client/src/features/viewport/DeviceChrome.tsx

@@ -42,6 +42,7 @@ interface DeviceChromeProps {
   onStartPanning: (event: React.PointerEvent<HTMLElement>) => void;
   rotationQuarterTurns: number;
   screenAspect: string;
+  screenClassName?: string;
   shellStyle: CSSProperties | null;
   simulatorName: string;
   streamBackend: string;
@@ -80,6 +81,7 @@ export function DeviceChrome({
   onStartPanning,
   rotationQuarterTurns,
   screenAspect,
+  screenClassName,
   shellStyle,
   simulatorName,
   streamBackend,
@@ -138,6 +140,7 @@ export function DeviceChrome({
           onPickerSelect={onPickerSelect}
           rotationQuarterTurns={rotationQuarterTurns}
           simulatorName={simulatorName}
+          screenClassName={screenClassName}
           streamBackend={streamBackend}
           streamCanvasRef={streamCanvasRef}
           streamCanvasKey={streamCanvasKey}
@@ -179,6 +182,7 @@ export function DeviceChrome({
         onPickerSelect={onPickerSelect}
         rotationQuarterTurns={rotationQuarterTurns}
         simulatorName={simulatorName}
+        screenClassName={screenClassName}
         streamBackend={streamBackend}
         streamCanvasRef={streamCanvasRef}
         streamCanvasKey={streamCanvasKey}
@@ -418,6 +422,7 @@ interface ScreenLayerProps {
   onPickerHover: (id: string | null) => void;
   onPickerSelect: (id: string) => void;
   rotationQuarterTurns: number;
+  screenClassName?: string;
   simulatorName: string;
   streamBackend: string;
   streamCanvasRef: Ref<HTMLCanvasElement | null>;
@@ -445,6 +450,7 @@ function ScreenLayer({
   onPickerHover,
   onPickerSelect,
   rotationQuarterTurns,
+  screenClassName,
   simulatorName,
   streamBackend,
   streamCanvasRef,
@@ -456,7 +462,13 @@ function ScreenLayer({
 }: ScreenLayerProps) {
   return (
     <div
-      className={`device-screen ${useChromeProfile ? "chrome-screen" : ""}`}
+      className={[
+        "device-screen",
+        useChromeProfile ? "chrome-screen" : "",
+        screenClassName ?? "",
+      ]
+        .filter(Boolean)
+        .join(" ")}
       onPointerCancel={onScreenPointerCancel}
       onPointerDown={onScreenPointerDown}
       onPointerMove={onScreenPointerMove}

client/src/features/viewport/SimulatorViewport.tsx

@@ -56,6 +56,7 @@ interface SimulatorViewportProps {
   outerCanvasRef: Ref<HTMLDivElement | null>;
   rotationQuarterTurns: number;
   screenAspect: string;
+  screenClassName?: string;
   selectedSimulator: SimulatorMetadata | null;
   shellStyle: CSSProperties | null;
   streamBackend: string;
@@ -112,6 +113,7 @@ export function SimulatorViewport({
   outerCanvasRef,
   rotationQuarterTurns,
   screenAspect,
+  screenClassName,
   selectedSimulator,
   shellStyle,
   streamBackend,
@@ -197,6 +199,7 @@ export function SimulatorViewport({
                   onStartPanning={onStartPanning}
                   rotationQuarterTurns={rotationQuarterTurns}
                   screenAspect={screenAspect}
+                  screenClassName={screenClassName}
                   shellStyle={shellStyle}
                   simulatorName={selectedSimulator.name}
                   streamBackend={streamBackend}

client/src/styles/components.css

@@ -636,6 +636,12 @@
   color: color-mix(in srgb, #d7ba7d 82%, var(--text));
 }
 
+.hierarchy-source-pill.source-android-uiautomator {
+  border-color: color-mix(in srgb, #7fd97f 55%, var(--border));
+  background: color-mix(in srgb, #7fd97f 13%, transparent);
+  color: color-mix(in srgb, #7fd97f 82%, var(--text));
+}
+
 .hierarchy-source-pill.active {
   gap: 5px;
   padding-inline: 7px 8px;
@@ -1491,6 +1497,11 @@
   border-radius: 0;
 }
 
+.device-screen.android-screen {
+  background: transparent;
+  border-radius: 10px;
+}
+
 .stream-canvas {
   position: absolute;
   inset: 0;