Support role=application web widgets

hugoholgersson · hugoholgersson · commit 1ed4eef02bad · 2021-12-08T09:51:39.000+01:00
After: The green rect follows web focus. Before: TalkBack tries to read TV apps as text documents. Background: Within <xxx role=application>...</xxx>, screen readers should not consume DPAD (arrow key) events. Web apps or widgets with role=application have, per the WAI-ARIA spec's contract, their own JavaScript logic for moving focus [1]. [1] w3c/aria#1049, where we discussed this. Problem: TalkBack does not handle role=application so such web apps lose their 4-way (up/down/left/right) navigation. TalkBack only moves forward/backward which breaks authors' pre-defined TV UX. Solution: Whenever accessibility focus (the green rect) goes to some web content with <body role=application> or anywhere within a role=application widget, we don't consume the DPAD events; we let them through. Testing done: Open a simple TV web app, hosted in an Android WebView, that has <body role=application>. Notice: Once the web view gets accessibilty focus, TalkBack won't eat (consume) DPAD key events and the the key events reach the web page's key handler in JavaScript.
diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/controller/TelevisionNavigationController.java b/talkback/src/main/java/com/google/android/accessibility/talkback/controller/TelevisionNavigationController.java
@@ -361,6 +361,13 @@ private boolean shouldHandleEvent(AccessibilityNodeInfoCompat cursor, KeyEvent e
         return false;
       }
     }
+
+    // Web applications and web widgets with role=application have, per the
+    // WAI-ARIA spec's contract, their own JavaScript logic for moving focus.
+    // TalkBack should not consume key events when such an app has accessibility focus.
+    boolean shouldProcessDPadKeyEvent = this.shouldProcessDPadKeyEvent &&
+            !AccessibilityNodeInfoUtils.isWebApplication(cursor);
+
     // TalkBack should always consume up/down/left/right on the d-pad, unless
     // shouldProcessDPadKeyEvent is false. Otherwise, strange things will happen when TalkBack
     // cannot navigate further.
diff --git a/utils/src/main/java/com/google/android/accessibility/utils/AccessibilityNodeInfoUtils.java b/utils/src/main/java/com/google/android/accessibility/utils/AccessibilityNodeInfoUtils.java
@@ -381,6 +381,39 @@ public boolean accept(AccessibilityNodeInfoCompat node) {
                 || (node != null && node.getCollectionInfo() != null);
           });
 
+  public static boolean hasApplicationWebRole(AccessibilityNodeInfoCompat node) {
+    return node != null && node.getExtras() != null
+            && node.getExtras().containsKey("AccessibilityNodeInfo.chromeRole")
+            && node.getExtras().get("AccessibilityNodeInfo.chromeRole").equals("application");
+  }
+
+  private static final Filter<AccessibilityNodeInfoCompat> FILTER_IN_WEB_APPLICATION =
+          new Filter<AccessibilityNodeInfoCompat>() {
+            @Override
+            public boolean accept(AccessibilityNodeInfoCompat node) {
+              return hasApplicationWebRole(node);
+            }
+          };
+
+  /**
+   * Returns true if |node| has role=application, i.e. |node| has JavaScript
+   * that handles key events.
+   */
+  public static boolean isWebApplication(AccessibilityNodeInfoCompat node) {
+      // When a WebView-like view (an actual WebView or a browser) has focus:
+      // Check the web content's accessibility tree's first node.
+      // If that node wants raw key event, instead of first "tabbing" the green
+      // rect to it, skip ahead and let the web app directly decide where to go.
+      boolean firstWebNode = WebInterfaceUtils.supportsWebActions(node)
+              && !WebInterfaceUtils.supportsWebActions(node.getParent());
+      boolean firstWebNodeWantsKeyEvents = firstWebNode
+              && node.getChildCount() > 0
+              && hasApplicationWebRole(node.getChild(0));
+
+      return firstWebNodeWantsKeyEvents
+              || getSelfOrMatchingAncestor(node, FILTER_IN_WEB_APPLICATION) != null;
+    }
+
   private AccessibilityNodeInfoUtils() {
     // This class is not instantiable.
   }

Original file line number	Diff line number	Diff line change
`@@ -361,6 +361,13 @@ private boolean shouldHandleEvent(AccessibilityNodeInfoCompat cursor, KeyEvent e`
`361`	`361`	`return false;`
`362`	`362`	`}`
`363`	`363`	`}`
	`364`	`+`
	`365`	`+ // Web applications and web widgets with role=application have, per the`
	`366`	`+ // WAI-ARIA spec's contract, their own JavaScript logic for moving focus.`
	`367`	`+ // TalkBack should not consume key events when such an app has accessibility focus.`
	`368`	`+ boolean shouldProcessDPadKeyEvent = this.shouldProcessDPadKeyEvent &&`
	`369`	`+ !AccessibilityNodeInfoUtils.isWebApplication(cursor);`
	`370`	`+`
`364`	`371`	`// TalkBack should always consume up/down/left/right on the d-pad, unless`
`365`	`372`	`// shouldProcessDPadKeyEvent is false. Otherwise, strange things will happen when TalkBack`
`366`	`373`	`// cannot navigate further.`