Skip to content

Commit dbda022

Browse files
Android-PowerUserAndroid-PowerUser
authored
Precise scrolling (#4)
* Add files via upload * Add files via upload * Add files via upload * Add files via upload * Add files via upload * Add files via upload * Add files via upload * Add files via upload * Add files via upload
1 parent b7e0f52 commit dbda022

File tree

6 files changed

+602
-18
lines changed

6 files changed

+602
-18
lines changed

‎app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt‎

Lines changed: 354 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,36 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
135135
showToast("Versuche nach oben zu scrollen", false)
136136
serviceInstance?.scrollUp()
137137
}
138+
is Command.ScrollLeft -> {
139+
Log.d(TAG, "Scrolling left")
140+
showToast("Versuche nach links zu scrollen", false)
141+
serviceInstance?.scrollLeft()
142+
}
143+
is Command.ScrollRight -> {
144+
Log.d(TAG, "Scrolling right")
145+
showToast("Versuche nach rechts zu scrollen", false)
146+
serviceInstance?.scrollRight()
147+
}
148+
is Command.ScrollDownFromCoordinates -> {
149+
Log.d(TAG, "Scrolling down from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms")
150+
showToast("Versuche von Position (${command.x}, ${command.y}) nach unten zu scrollen", false)
151+
serviceInstance?.scrollDown(command.x, command.y, command.distance, command.duration)
152+
}
153+
is Command.ScrollUpFromCoordinates -> {
154+
Log.d(TAG, "Scrolling up from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms")
155+
showToast("Versuche von Position (${command.x}, ${command.y}) nach oben zu scrollen", false)
156+
serviceInstance?.scrollUp(command.x, command.y, command.distance, command.duration)
157+
}
158+
is Command.ScrollLeftFromCoordinates -> {
159+
Log.d(TAG, "Scrolling left from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms")
160+
showToast("Versuche von Position (${command.x}, ${command.y}) nach links zu scrollen", false)
161+
serviceInstance?.scrollLeft(command.x, command.y, command.distance, command.duration)
162+
}
163+
is Command.ScrollRightFromCoordinates -> {
164+
Log.d(TAG, "Scrolling right from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms")
165+
showToast("Versuche von Position (${command.x}, ${command.y}) nach rechts zu scrollen", false)
166+
serviceInstance?.scrollRight(command.x, command.y, command.distance, command.duration)
167+
}
138168
}
139169
}
140170

@@ -1096,6 +1126,60 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
10961126
}
10971127
}
10981128

1129+
/**
1130+
* Scroll down from specific coordinates with custom distance and duration
1131+
*
1132+
* @param x Starting X coordinate
1133+
* @param y Starting Y coordinate
1134+
* @param distance Distance in pixels to scroll
1135+
* @param duration Duration of the scroll gesture in milliseconds
1136+
*/
1137+
fun scrollDown(x: Float, y: Float, distance: Float, duration: Long) {
1138+
Log.d(TAG, "Scrolling down from ($x, $y) with distance $distance and duration $duration ms")
1139+
showToast("Scrolle nach unten von bestimmter Position...", false)
1140+
1141+
try {
1142+
// Create a path for the gesture (swipe from specified position upward by the specified distance)
1143+
val swipePath = Path()
1144+
swipePath.moveTo(x, y) // Start from specified position
1145+
swipePath.lineTo(x, y - distance) // Move upward by the specified distance
1146+
1147+
// Create a gesture builder and add the swipe
1148+
val gestureBuilder = GestureDescription.Builder()
1149+
val gesture = GestureDescription.StrokeDescription(
1150+
swipePath,
1151+
0, // start time
1152+
duration // custom duration in milliseconds
1153+
)
1154+
gestureBuilder.addStroke(gesture)
1155+
1156+
// Dispatch the gesture
1157+
val result = dispatchGesture(
1158+
gestureBuilder.build(),
1159+
object : GestureResultCallback() {
1160+
override fun onCompleted(gestureDescription: GestureDescription) {
1161+
Log.d(TAG, "Coordinate-based scroll down gesture completed")
1162+
showToast("Erfolgreich nach unten gescrollt von Position ($x, $y)", false)
1163+
}
1164+
1165+
override fun onCancelled(gestureDescription: GestureDescription) {
1166+
Log.e(TAG, "Coordinate-based scroll down gesture cancelled")
1167+
showToast("Scrollen nach unten von Position ($x, $y) abgebrochen", true)
1168+
}
1169+
},
1170+
null // handler
1171+
)
1172+
1173+
if (!result) {
1174+
Log.e(TAG, "Failed to dispatch coordinate-based scroll down gesture")
1175+
showToast("Fehler beim Scrollen nach unten von Position ($x, $y)", true)
1176+
}
1177+
} catch (e: Exception) {
1178+
Log.e(TAG, "Error scrolling down from coordinates: ${e.message}")
1179+
showToast("Fehler beim Scrollen nach unten von Position ($x, $y): ${e.message}", true)
1180+
}
1181+
}
1182+
10991183
/**
11001184
* Scroll up on the screen using gesture
11011185
*/
@@ -1150,6 +1234,276 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
11501234
}
11511235
}
11521236

1237+
/**
1238+
* Scroll up from specific coordinates with custom distance and duration
1239+
*
1240+
* @param x Starting X coordinate
1241+
* @param y Starting Y coordinate
1242+
* @param distance Distance in pixels to scroll
1243+
* @param duration Duration of the scroll gesture in milliseconds
1244+
*/
1245+
fun scrollUp(x: Float, y: Float, distance: Float, duration: Long) {
1246+
Log.d(TAG, "Scrolling up from ($x, $y) with distance $distance and duration $duration ms")
1247+
showToast("Scrolle nach oben von bestimmter Position...", false)
1248+
1249+
try {
1250+
// Create a path for the gesture (swipe from specified position downward by the specified distance)
1251+
val swipePath = Path()
1252+
swipePath.moveTo(x, y) // Start from specified position
1253+
swipePath.lineTo(x, y + distance) // Move downward by the specified distance
1254+
1255+
// Create a gesture builder and add the swipe
1256+
val gestureBuilder = GestureDescription.Builder()
1257+
val gesture = GestureDescription.StrokeDescription(
1258+
swipePath,
1259+
0, // start time
1260+
duration // custom duration in milliseconds
1261+
)
1262+
gestureBuilder.addStroke(gesture)
1263+
1264+
// Dispatch the gesture
1265+
val result = dispatchGesture(
1266+
gestureBuilder.build(),
1267+
object : GestureResultCallback() {
1268+
override fun onCompleted(gestureDescription: GestureDescription) {
1269+
Log.d(TAG, "Coordinate-based scroll up gesture completed")
1270+
showToast("Erfolgreich nach oben gescrollt von Position ($x, $y)", false)
1271+
}
1272+
1273+
override fun onCancelled(gestureDescription: GestureDescription) {
1274+
Log.e(TAG, "Coordinate-based scroll up gesture cancelled")
1275+
showToast("Scrollen nach oben von Position ($x, $y) abgebrochen", true)
1276+
}
1277+
},
1278+
null // handler
1279+
)
1280+
1281+
if (!result) {
1282+
Log.e(TAG, "Failed to dispatch coordinate-based scroll up gesture")
1283+
showToast("Fehler beim Scrollen nach oben von Position ($x, $y)", true)
1284+
}
1285+
} catch (e: Exception) {
1286+
Log.e(TAG, "Error scrolling up from coordinates: ${e.message}")
1287+
showToast("Fehler beim Scrollen nach oben von Position ($x, $y): ${e.message}", true)
1288+
}
1289+
}
1290+
1291+
/**
1292+
* Scroll left on the screen using gesture
1293+
*/
1294+
fun scrollLeft() {
1295+
Log.d(TAG, "Scrolling left")
1296+
showToast("Scrolle nach links...", false)
1297+
1298+
try {
1299+
// Get display metrics to calculate swipe coordinates
1300+
val displayMetrics = resources.displayMetrics
1301+
val screenHeight = displayMetrics.heightPixels
1302+
val screenWidth = displayMetrics.widthPixels
1303+
1304+
// Create a path for the gesture (swipe from middle-right to middle-left)
1305+
val swipePath = Path()
1306+
swipePath.moveTo(screenWidth * 0.7f, screenHeight / 2f) // Start from 70% across the screen
1307+
swipePath.lineTo(screenWidth * 0.3f, screenHeight / 2f) // Move to 30% across the screen
1308+
1309+
// Create a gesture builder and add the swipe
1310+
val gestureBuilder = GestureDescription.Builder()
1311+
val gesture = GestureDescription.StrokeDescription(
1312+
swipePath,
1313+
0, // start time
1314+
300 // duration in milliseconds
1315+
)
1316+
gestureBuilder.addStroke(gesture)
1317+
1318+
// Dispatch the gesture
1319+
val result = dispatchGesture(
1320+
gestureBuilder.build(),
1321+
object : GestureResultCallback() {
1322+
override fun onCompleted(gestureDescription: GestureDescription) {
1323+
Log.d(TAG, "Scroll left gesture completed")
1324+
showToast("Erfolgreich nach links gescrollt", false)
1325+
}
1326+
1327+
override fun onCancelled(gestureDescription: GestureDescription) {
1328+
Log.e(TAG, "Scroll left gesture cancelled")
1329+
showToast("Scrollen nach links abgebrochen", true)
1330+
}
1331+
},
1332+
null // handler
1333+
)
1334+
1335+
if (!result) {
1336+
Log.e(TAG, "Failed to dispatch scroll left gesture")
1337+
showToast("Fehler beim Scrollen nach links", true)
1338+
}
1339+
} catch (e: Exception) {
1340+
Log.e(TAG, "Error scrolling left: ${e.message}")
1341+
showToast("Fehler beim Scrollen nach links: ${e.message}", true)
1342+
}
1343+
}
1344+
1345+
/**
1346+
* Scroll left from specific coordinates with custom distance and duration
1347+
*
1348+
* @param x Starting X coordinate
1349+
* @param y Starting Y coordinate
1350+
* @param distance Distance in pixels to scroll
1351+
* @param duration Duration of the scroll gesture in milliseconds
1352+
*/
1353+
fun scrollLeft(x: Float, y: Float, distance: Float, duration: Long) {
1354+
Log.d(TAG, "Scrolling left from ($x, $y) with distance $distance and duration $duration ms")
1355+
showToast("Scrolle nach links von bestimmter Position...", false)
1356+
1357+
try {
1358+
// Create a path for the gesture (swipe from specified position leftward by the specified distance)
1359+
val swipePath = Path()
1360+
swipePath.moveTo(x, y) // Start from specified position
1361+
swipePath.lineTo(x - distance, y) // Move leftward by the specified distance
1362+
1363+
// Create a gesture builder and add the swipe
1364+
val gestureBuilder = GestureDescription.Builder()
1365+
val gesture = GestureDescription.StrokeDescription(
1366+
swipePath,
1367+
0, // start time
1368+
duration // custom duration in milliseconds
1369+
)
1370+
gestureBuilder.addStroke(gesture)
1371+
1372+
// Dispatch the gesture
1373+
val result = dispatchGesture(
1374+
gestureBuilder.build(),
1375+
object : GestureResultCallback() {
1376+
override fun onCompleted(gestureDescription: GestureDescription) {
1377+
Log.d(TAG, "Coordinate-based scroll left gesture completed")
1378+
showToast("Erfolgreich nach links gescrollt von Position ($x, $y)", false)
1379+
}
1380+
1381+
override fun onCancelled(gestureDescription: GestureDescription) {
1382+
Log.e(TAG, "Coordinate-based scroll left gesture cancelled")
1383+
showToast("Scrollen nach links von Position ($x, $y) abgebrochen", true)
1384+
}
1385+
},
1386+
null // handler
1387+
)
1388+
1389+
if (!result) {
1390+
Log.e(TAG, "Failed to dispatch coordinate-based scroll left gesture")
1391+
showToast("Fehler beim Scrollen nach links von Position ($x, $y)", true)
1392+
}
1393+
} catch (e: Exception) {
1394+
Log.e(TAG, "Error scrolling left from coordinates: ${e.message}")
1395+
showToast("Fehler beim Scrollen nach links von Position ($x, $y): ${e.message}", true)
1396+
}
1397+
}
1398+
1399+
/**
1400+
* Scroll right on the screen using gesture
1401+
*/
1402+
fun scrollRight() {
1403+
Log.d(TAG, "Scrolling right")
1404+
showToast("Scrolle nach rechts...", false)
1405+
1406+
try {
1407+
// Get display metrics to calculate swipe coordinates
1408+
val displayMetrics = resources.displayMetrics
1409+
val screenHeight = displayMetrics.heightPixels
1410+
val screenWidth = displayMetrics.widthPixels
1411+
1412+
// Create a path for the gesture (swipe from middle-left to middle-right)
1413+
val swipePath = Path()
1414+
swipePath.moveTo(screenWidth * 0.3f, screenHeight / 2f) // Start from 30% across the screen
1415+
swipePath.lineTo(screenWidth * 0.7f, screenHeight / 2f) // Move to 70% across the screen
1416+
1417+
// Create a gesture builder and add the swipe
1418+
val gestureBuilder = GestureDescription.Builder()
1419+
val gesture = GestureDescription.StrokeDescription(
1420+
swipePath,
1421+
0, // start time
1422+
300 // duration in milliseconds
1423+
)
1424+
gestureBuilder.addStroke(gesture)
1425+
1426+
// Dispatch the gesture
1427+
val result = dispatchGesture(
1428+
gestureBuilder.build(),
1429+
object : GestureResultCallback() {
1430+
override fun onCompleted(gestureDescription: GestureDescription) {
1431+
Log.d(TAG, "Scroll right gesture completed")
1432+
showToast("Erfolgreich nach rechts gescrollt", false)
1433+
}
1434+
1435+
override fun onCancelled(gestureDescription: GestureDescription) {
1436+
Log.e(TAG, "Scroll right gesture cancelled")
1437+
showToast("Scrollen nach rechts abgebrochen", true)
1438+
}
1439+
},
1440+
null // handler
1441+
)
1442+
1443+
if (!result) {
1444+
Log.e(TAG, "Failed to dispatch scroll right gesture")
1445+
showToast("Fehler beim Scrollen nach rechts", true)
1446+
}
1447+
} catch (e: Exception) {
1448+
Log.e(TAG, "Error scrolling right: ${e.message}")
1449+
showToast("Fehler beim Scrollen nach rechts: ${e.message}", true)
1450+
}
1451+
}
1452+
1453+
/**
1454+
* Scroll right from specific coordinates with custom distance and duration
1455+
*
1456+
* @param x Starting X coordinate
1457+
* @param y Starting Y coordinate
1458+
* @param distance Distance in pixels to scroll
1459+
* @param duration Duration of the scroll gesture in milliseconds
1460+
*/
1461+
fun scrollRight(x: Float, y: Float, distance: Float, duration: Long) {
1462+
Log.d(TAG, "Scrolling right from ($x, $y) with distance $distance and duration $duration ms")
1463+
showToast("Scrolle nach rechts von bestimmter Position...", false)
1464+
1465+
try {
1466+
// Create a path for the gesture (swipe from specified position rightward by the specified distance)
1467+
val swipePath = Path()
1468+
swipePath.moveTo(x, y) // Start from specified position
1469+
swipePath.lineTo(x + distance, y) // Move rightward by the specified distance
1470+
1471+
// Create a gesture builder and add the swipe
1472+
val gestureBuilder = GestureDescription.Builder()
1473+
val gesture = GestureDescription.StrokeDescription(
1474+
swipePath,
1475+
0, // start time
1476+
duration // custom duration in milliseconds
1477+
)
1478+
gestureBuilder.addStroke(gesture)
1479+
1480+
// Dispatch the gesture
1481+
val result = dispatchGesture(
1482+
gestureBuilder.build(),
1483+
object : GestureResultCallback() {
1484+
override fun onCompleted(gestureDescription: GestureDescription) {
1485+
Log.d(TAG, "Coordinate-based scroll right gesture completed")
1486+
showToast("Erfolgreich nach rechts gescrollt von Position ($x, $y)", false)
1487+
}
1488+
1489+
override fun onCancelled(gestureDescription: GestureDescription) {
1490+
Log.e(TAG, "Coordinate-based scroll right gesture cancelled")
1491+
showToast("Scrollen nach rechts von Position ($x, $y) abgebrochen", true)
1492+
}
1493+
},
1494+
null // handler
1495+
)
1496+
1497+
if (!result) {
1498+
Log.e(TAG, "Failed to dispatch coordinate-based scroll right gesture")
1499+
showToast("Fehler beim Scrollen nach rechts von Position ($x, $y)", true)
1500+
}
1501+
} catch (e: Exception) {
1502+
Log.e(TAG, "Error scrolling right from coordinates: ${e.message}")
1503+
showToast("Fehler beim Scrollen nach rechts von Position ($x, $y): ${e.message}", true)
1504+
}
1505+
}
1506+
11531507
/**
11541508
* Retrieve the latest screenshot from the standard screenshot folder
11551509
*/

‎app/src/main/kotlin/com/google/ai/sample/feature/chat/ChatScreen.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,12 @@ fun ChatScreen(
200200
is Command.ShowRecentApps -> "Übersicht der letzten Apps öffnen"
201201
is Command.ScrollDown -> "Nach unten scrollen"
202202
is Command.ScrollUp -> "Nach oben scrollen"
203+
is Command.ScrollLeft -> "Nach links scrollen"
204+
is Command.ScrollRight -> "Nach rechts scrollen"
205+
is Command.ScrollDownFromCoordinates -> "Nach unten scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms"
206+
is Command.ScrollUpFromCoordinates -> "Nach oben scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms"
207+
is Command.ScrollLeftFromCoordinates -> "Nach links scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms"
208+
is Command.ScrollRightFromCoordinates -> "Nach rechts scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms"
203209
}
204210

205211
Text(

0 commit comments

Comments
 (0)