diff --git a/packages/scrollable_positioned_list/lib/src/scrollable_positioned_list.dart b/packages/scrollable_positioned_list/lib/src/scrollable_positioned_list.dart
index 82526a08..b73eaf97 100644
--- a/packages/scrollable_positioned_list/lib/src/scrollable_positioned_list.dart
+++ b/packages/scrollable_positioned_list/lib/src/scrollable_positioned_list.dart
@@ -338,9 +338,8 @@ class _ScrollablePositionedListState extends State<ScrollablePositionedList>
     return LayoutBuilder(
       builder: (context, constraints) {
         final cacheExtent = _cacheExtent(constraints);
-        return GestureDetector(
-          onPanDown: (_) => _stopScroll(canceled: true),
-          excludeFromSemantics: true,
+        return Listener(
+          onPointerDown: (_) => _stopScroll(canceled: true),
           child: Stack(
             children: <Widget>[
               PostMountCallback(
diff --git a/packages/scrollable_positioned_list/test/scrollable_positioned_list_test.dart b/packages/scrollable_positioned_list/test/scrollable_positioned_list_test.dart
index 7c275a70..abbfbb16 100644
--- a/packages/scrollable_positioned_list/test/scrollable_positioned_list_test.dart
+++ b/packages/scrollable_positioned_list/test/scrollable_positioned_list_test.dart
@@ -2266,6 +2266,25 @@ void main() {
     expect(find.text('Item 70'), findsOneWidget);
     expect(find.text('Item 50'), findsOneWidget);
   });
+
+  testWidgets(
+      'List positioned with 5 at top then scroll up so item 3 is at top',
+      (WidgetTester tester) async {
+    // debugPrintGestureArenaDiagnostics = true;
+    final itemPositionsListener = ItemPositionsListener.create();
+    await setUpWidgetTest(tester,
+        initialIndex: 5, itemPositionsListener: itemPositionsListener);
+
+    await tester.drag(
+        find.byType(ScrollablePositionedList), const Offset(0, 2 * itemHeight));
+    await tester.pumpAndSettle();
+
+    expect(
+        itemPositionsListener.itemPositions.value
+            .firstWhere((position) => position.index == 3)
+            .itemLeadingEdge,
+        0);
+  });
 }
 
 bool collectSemanticNodes(SemanticsNode root, List<SemanticsNode> nodes) {