Fix keyboard detection across Android versions and vendor keyboards

omnarayan · omnarayan · commit dec9d6ebd54d · 2026-03-09T20:40:26.000+05:30
Verified against AOSP source (Android 10, 11, 13) that isOnScreen=,
mViewVisibility=, touchable region=, and mGivenContentInsets= are
present on all versions. The parsing now uses a 3-strategy approach:

1. touchable region (most accurate, used by stock keyboards)
2. mFrame + mGivenContentInsets (for vendor keyboards like Samsung/Xiaomi
   that don't set touchable insets — content insets reveal keyboard top)
3. mFrame alone with 60% height sanity check (rejects full-screen
   InputMethod windows that would cause false positives)

Negative signals (isOnScreen=false, mViewVisibility=0x8) bail early.
diff --git a/pkg/driver/devicelab/keyboard.go b/pkg/driver/devicelab/keyboard.go
@@ -12,35 +12,72 @@ import (
 
 // Patterns for extracting keyboard bounds from "dumpsys window InputMethod".
 var (
-	// Android <=12: "mFrame=[left,top][right,bottom]"
+	// Android <=12: "mFrame=[left,top][right,bottom]" (not present on Android 13+)
 	mFrameRegex = regexp.MustCompile(`mFrame=\[(\d+),(\d+)\]\[(\d+),(\d+)\]`)
 
-	// Android 13+: "touchable region=SkRegion((left,top,right,bottom))"
+	// "touchable region=SkRegion((left,top,right,bottom))" — present on all versions
+	// when the keyboard sets mTouchableInsets (stock keyboards do; some vendor keyboards don't).
 	touchableRegionRegex = regexp.MustCompile(`touchable region=SkRegion\(\((\d+),(\d+),(\d+),(\d+)\)\)`)
+
+	// "mGivenContentInsets=[left,top][right,bottom]" — tells us where keyboard content
+	// starts within the InputMethod window. The top inset is the transparent gap above
+	// the keyboard. Present on all versions.
+	contentInsetsRegex = regexp.MustCompile(`mGivenContentInsets=\[(\d+),(\d+)\]\[(\d+),(\d+)\]`)
 )
 
 // parseKeyboardFrame extracts keyboard bounds from "dumpsys window InputMethod" output.
 // Returns nil if keyboard is not visible.
+//
+// Strategy order (verified against AOSP source for Android 10, 11, 13):
+//  1. touchable region — most accurate, gives actual keyboard area.
+//  2. mFrame + mGivenContentInsets — for vendor keyboards (Samsung, Xiaomi, etc.)
+//     that don't set touchable insets. Content insets reveal where keyboard starts.
+//  3. mFrame alone — only if the frame looks like a keyboard (not a full-screen window).
 func parseKeyboardFrame(dumpsysOutput string) *core.Bounds {
-	// Bail early if the window is explicitly not on screen.
+	// isOnScreen= is present on all Android versions (10+). mViewVisibility=0x8 means GONE.
 	if strings.Contains(dumpsysOutput, "isOnScreen=false") ||
 		strings.Contains(dumpsysOutput, "mViewVisibility=0x8") {
 		return nil
 	}
 
-	// Prefer touchable region — gives the actual keyboard area, not the full
-	// InputMethod window. Available on Android 11+ (SDK 30+). Safe to use here
-	// because the caller already checks mInputShown before calling this function.
+	// Strategy 1: touchable region — the actual keyboard touchable area.
+	// Printed when mTouchableInsets != 0, which stock keyboards set but some vendor keyboards don't.
 	if matches := touchableRegionRegex.FindStringSubmatch(dumpsysOutput); matches != nil {
 		return boundsFromMatches(matches)
 	}
 
-	// Fallback for older Android (<=10) that lacks touchable region.
-	if matches := mFrameRegex.FindStringSubmatch(dumpsysOutput); matches != nil {
-		return boundsFromMatches(matches)
+	// Strategy 2+3: mFrame-based fallback (Android <=12 only; Android 13+ uses Frames: format).
+	frameMatches := mFrameRegex.FindStringSubmatch(dumpsysOutput)
+	if frameMatches == nil {
+		return nil
+	}
+	bounds := boundsFromMatches(frameMatches)
+	if bounds == nil {
+		return nil
 	}
 
-	return nil
+	// Strategy 2: adjust mFrame by content insets. mGivenContentInsets.top tells us how many
+	// pixels from the window top are transparent (not keyboard). This handles vendor keyboards
+	// that use a full-screen InputMethod window but report content insets correctly.
+	if insetsMatches := contentInsetsRegex.FindStringSubmatch(dumpsysOutput); insetsMatches != nil {
+		topInset, _ := strconv.Atoi(insetsMatches[2])
+		if topInset > 0 {
+			bounds.Y += topInset
+			bounds.Height -= topInset
+			if bounds.Height <= 0 {
+				return nil
+			}
+			return bounds
+		}
+	}
+
+	// Strategy 3: bare mFrame. Sanity check — a real keyboard is at most ~60% of screen height.
+	// If the frame is taller, it's the full InputMethod window, not the keyboard.
+	screenBottom := bounds.Y + bounds.Height
+	if screenBottom > 0 && bounds.Height > screenBottom*6/10 {
+		return nil
+	}
+	return bounds
 }
 
 // boundsFromMatches converts regex matches [_, left, top, right, bottom] to Bounds.
diff --git a/pkg/driver/uiautomator2/keyboard.go b/pkg/driver/uiautomator2/keyboard.go
@@ -12,35 +12,72 @@ import (
 
 // Patterns for extracting keyboard bounds from "dumpsys window InputMethod".
 var (
-	// Android <=12: "mFrame=[left,top][right,bottom]"
+	// Android <=12: "mFrame=[left,top][right,bottom]" (not present on Android 13+)
 	mFrameRegex = regexp.MustCompile(`mFrame=\[(\d+),(\d+)\]\[(\d+),(\d+)\]`)
 
-	// Android 13+: "touchable region=SkRegion((left,top,right,bottom))"
+	// "touchable region=SkRegion((left,top,right,bottom))" — present on all versions
+	// when the keyboard sets mTouchableInsets (stock keyboards do; some vendor keyboards don't).
 	touchableRegionRegex = regexp.MustCompile(`touchable region=SkRegion\(\((\d+),(\d+),(\d+),(\d+)\)\)`)
+
+	// "mGivenContentInsets=[left,top][right,bottom]" — tells us where keyboard content
+	// starts within the InputMethod window. The top inset is the transparent gap above
+	// the keyboard. Present on all versions.
+	contentInsetsRegex = regexp.MustCompile(`mGivenContentInsets=\[(\d+),(\d+)\]\[(\d+),(\d+)\]`)
 )
 
 // parseKeyboardFrame extracts keyboard bounds from "dumpsys window InputMethod" output.
 // Returns nil if keyboard is not visible.
+//
+// Strategy order (verified against AOSP source for Android 10, 11, 13):
+//  1. touchable region — most accurate, gives actual keyboard area.
+//  2. mFrame + mGivenContentInsets — for vendor keyboards (Samsung, Xiaomi, etc.)
+//     that don't set touchable insets. Content insets reveal where keyboard starts.
+//  3. mFrame alone — only if the frame looks like a keyboard (not a full-screen window).
 func parseKeyboardFrame(dumpsysOutput string) *core.Bounds {
-	// Bail early if the window is explicitly not on screen.
+	// isOnScreen= is present on all Android versions (10+). mViewVisibility=0x8 means GONE.
 	if strings.Contains(dumpsysOutput, "isOnScreen=false") ||
 		strings.Contains(dumpsysOutput, "mViewVisibility=0x8") {
 		return nil
 	}
 
-	// Prefer touchable region — gives the actual keyboard area, not the full
-	// InputMethod window. Available on Android 11+ (SDK 30+). Safe to use here
-	// because the caller already checks mInputShown before calling this function.
+	// Strategy 1: touchable region — the actual keyboard touchable area.
+	// Printed when mTouchableInsets != 0, which stock keyboards set but some vendor keyboards don't.
 	if matches := touchableRegionRegex.FindStringSubmatch(dumpsysOutput); matches != nil {
 		return boundsFromMatches(matches)
 	}
 
-	// Fallback for older Android (<=10) that lacks touchable region.
-	if matches := mFrameRegex.FindStringSubmatch(dumpsysOutput); matches != nil {
-		return boundsFromMatches(matches)
+	// Strategy 2+3: mFrame-based fallback (Android <=12 only; Android 13+ uses Frames: format).
+	frameMatches := mFrameRegex.FindStringSubmatch(dumpsysOutput)
+	if frameMatches == nil {
+		return nil
+	}
+	bounds := boundsFromMatches(frameMatches)
+	if bounds == nil {
+		return nil
 	}
 
-	return nil
+	// Strategy 2: adjust mFrame by content insets. mGivenContentInsets.top tells us how many
+	// pixels from the window top are transparent (not keyboard). This handles vendor keyboards
+	// that use a full-screen InputMethod window but report content insets correctly.
+	if insetsMatches := contentInsetsRegex.FindStringSubmatch(dumpsysOutput); insetsMatches != nil {
+		topInset, _ := strconv.Atoi(insetsMatches[2])
+		if topInset > 0 {
+			bounds.Y += topInset
+			bounds.Height -= topInset
+			if bounds.Height <= 0 {
+				return nil
+			}
+			return bounds
+		}
+	}
+
+	// Strategy 3: bare mFrame. Sanity check — a real keyboard is at most ~60% of screen height.
+	// If the frame is taller, it's the full InputMethod window, not the keyboard.
+	screenBottom := bounds.Y + bounds.Height
+	if screenBottom > 0 && bounds.Height > screenBottom*6/10 {
+		return nil
+	}
+	return bounds
 }
 
 // boundsFromMatches converts regex matches [_, left, top, right, bottom] to Bounds.
diff --git a/pkg/driver/uiautomator2/keyboard_test.go b/pkg/driver/uiautomator2/keyboard_test.go
@@ -100,6 +100,25 @@ func TestParseKeyboardFrame(t *testing.T) {
     mHasSurface=true`,
 			want: &core.Bounds{X: 0, Y: 1428, Width: 1080, Height: 912},
 		},
+		{
+			name: "vendor keyboard — no touchable region, uses mFrame + content insets",
+			input: `  Window #1 Window{abcdef InputMethod}:
+    mFrame=[0,84][1080,2400]
+    mViewVisibility=0x0 mHaveFrame=true mObscured=false
+    mGivenContentInsets=[0,1292][0,0] mGivenVisibleInsets=[0,1292][0,0]
+    mHasSurface=true isReadyForDisplay()=true
+    isOnScreen=true`,
+			want: &core.Bounds{X: 0, Y: 1376, Width: 1080, Height: 1024},
+		},
+		{
+			name: "full-screen mFrame without insets or touchable region — rejected",
+			input: `  Window #1 Window{abcdef InputMethod}:
+    mFrame=[0,84][1080,2400]
+    mViewVisibility=0x0 mHaveFrame=true
+    mGivenContentInsets=[0,0][0,0]
+    isOnScreen=true`,
+			want: nil,
+		},
 	}
 
 	for _, tt := range tests {
