ci: fix nightly builds

.github/workflows/all_plugins.yaml

@@ -91,7 +91,17 @@ jobs:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10
         with:
           fetch-depth: 0
-      - uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2
+      - id: setup-flutter-format
+        uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2
+        continue-on-error: true
+        with:
+          channel: 'stable'
+          cache: true
+          cache-key: "flutter-:os:-:channel:-:version:-:arch:-:hash:"
+          pub-cache-key: "flutter-pub-:os:-:channel:-:version:-:arch:-:hash:"
+      - name: Retry Flutter setup
+        if: steps.setup-flutter-format.outcome == 'failure'
+        uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2
         with:
           channel: 'stable'
           cache: true

.github/workflows/e2e_tests_fdc.yaml

@@ -324,16 +324,13 @@ jobs:
         # Web devices are not supported for the `flutter test` command yet. As a
         # workaround we can use the `flutter drive` command. Tracking issue:
         # https://github.com/flutter/flutter/issues/66264
+        # Chrome debug service can fail with AppConnectionException before tests
+        # connect. Retry only those infrastructure startup failures.
         run: |
-          chromedriver --port=4444 --trace-buffer-size=100000 &
-          flutter drive --target=./integration_test/e2e_test.dart --driver=./test_driver/integration_test.dart -d chrome --dart-define=CI=true | tee output.log
-          # We have to check the output for failed tests matching the string "[E]"
-          output=$(<output.log)
-          if [[ "$output" =~ \[E\] ]]; then
-          # You will see "All tests passed." in the logs even when tests failed.
-          echo "All tests did not pass. Please check the logs for more information."
-          exit 1
-          fi
+          FLUTTER_DRIVE_TARGET="./integration_test/e2e_test.dart" \
+          FLUTTER_DRIVE_DRIVER="./test_driver/integration_test.dart" \
+          FLUTTER_DRIVE_EXTRA_ARGS="--dart-define=CI=true" \
+          "${GITHUB_WORKSPACE}/.github/workflows/scripts/flutter-drive-web-retry.sh"
         shell: bash
       - name: Save Firestore Emulator Cache
         # Branches can read main cache but main cannot read branch cache. Avoid LRU eviction with main-only cache.

.github/workflows/e2e_tests_pipeline.yaml

@@ -158,14 +158,13 @@ jobs:
         # Web devices are not supported for the `flutter test` command yet. As a
         # workaround we use the `flutter drive` command. Tracking issue:
         # https://github.com/flutter/flutter/issues/66264
+        # Chrome debug service can fail with AppConnectionException before tests
+        # connect. Retry only those infrastructure startup failures.
         run: |
-          chromedriver --port=4444 --trace-buffer-size=100000 &
-          flutter drive --target=./integration_test/pipeline/pipeline_live_test.dart --driver=./test_driver/integration_test.dart -d chrome --dart-define=CI=true | tee output.log
-          output=$(<output.log)
-          if [[ "$output" =~ \[E\] ]]; then
-            echo "All tests did not pass. Please check the logs for more information."
-            exit 1
-          fi
+          FLUTTER_DRIVE_TARGET="./integration_test/pipeline/pipeline_live_test.dart" \
+          FLUTTER_DRIVE_DRIVER="./test_driver/integration_test.dart" \
+          FLUTTER_DRIVE_EXTRA_ARGS="--dart-define=CI=true" \
+          "${GITHUB_WORKSPACE}/.github/workflows/scripts/flutter-drive-web-retry.sh"
         shell: bash
 
   pipeline-e2e-ios:

.github/workflows/scripts/database.rules.json

@@ -32,6 +32,12 @@
         ".read": true,
         ".write": true,
         ".indexOn": [".value"]
+      },
+      "$test": {
+        ".indexOn": [
+          ".value",
+          "number"
+        ]
       }
     }
   }

.github/workflows/scripts/flutter-drive-web-retry.sh

@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${FLUTTER_DRIVE_TARGET:?FLUTTER_DRIVE_TARGET is required}"
+: "${FLUTTER_DRIVE_DRIVER:?FLUTTER_DRIVE_DRIVER is required}"
+
+FLUTTER_DRIVE_DEVICE="${FLUTTER_DRIVE_DEVICE:-chrome}"
+FLUTTER_DRIVE_TIMEOUT_SECONDS="${FLUTTER_DRIVE_TIMEOUT_SECONDS:-180}"
+FLUTTER_DRIVE_MAX_ATTEMPTS="${FLUTTER_DRIVE_MAX_ATTEMPTS:-4}"
+FLUTTER_DRIVE_EXTRA_ARGS="${FLUTTER_DRIVE_EXTRA_ARGS:-}"
+
+cleanup_web_processes() {
+  pkill -f "Google Chrome" || true
+  pkill -f chrome_crashpad || true
+  pkill -x chromedriver || true
+  pkill -x dartvm || true
+  pkill -x dartaotruntime || true
+}
+
+run_tests() {
+  rm -f output.log
+  cleanup_web_processes
+
+  chromedriver --port=4444 --trace-buffer-size=100000 &
+  chromedriver_pid=$!
+  sleep 2
+
+  set +e
+  python3 - <<'PY'
+import os
+import shlex
+import subprocess
+import sys
+
+
+def normalize_output(output):
+    if output is None:
+        return ''
+    if isinstance(output, bytes):
+        return output.decode(errors='replace')
+    return output
+
+
+command = [
+    'flutter',
+    'drive',
+    f"--target={os.environ['FLUTTER_DRIVE_TARGET']}",
+    f"--driver={os.environ['FLUTTER_DRIVE_DRIVER']}",
+    '-d',
+    os.environ['FLUTTER_DRIVE_DEVICE'],
+    *shlex.split(os.environ.get('FLUTTER_DRIVE_EXTRA_ARGS', '')),
+]
+
+try:
+    completed = subprocess.run(
+        command,
+        check=False,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        timeout=int(os.environ['FLUTTER_DRIVE_TIMEOUT_SECONDS']),
+    )
+except subprocess.TimeoutExpired as error:
+    output = normalize_output(error.stdout)
+    print(output, end='')
+    with open('output.log', 'w') as file:
+        file.write(output)
+    print('flutter drive timed out before tests completed.')
+    sys.exit(124)
+
+output = normalize_output(completed.stdout)
+print(output, end='')
+with open('output.log', 'w') as file:
+    file.write(output)
+sys.exit(completed.returncode)
+PY
+  exit_code=$?
+  set -e
+
+  kill "$chromedriver_pid" 2>/dev/null || true
+  wait "$chromedriver_pid" 2>/dev/null || true
+  cleanup_web_processes
+
+  output=$(<output.log)
+  if [[ "$output" =~ \[E\] ]]; then
+    # You will see "All tests passed." in the logs even when tests failed.
+    echo "All tests did not pass. Please check the logs for more information."
+    return 2
+  fi
+
+  if [[ "$exit_code" == "124" ]] ||
+     [[ "$output" == *"AppConnectionException"* ]] ||
+     [[ "$output" == *"Failed to exit Chromium"* ]]; then
+    return 3
+  fi
+
+  return "$exit_code"
+}
+
+for attempt in $(seq 1 "$FLUTTER_DRIVE_MAX_ATTEMPTS"); do
+  if run_tests; then
+    exit 0
+  fi
+
+  exit_code=$?
+  if [[ "$exit_code" != "3" || "$attempt" == "$FLUTTER_DRIVE_MAX_ATTEMPTS" ]]; then
+    exit "$exit_code"
+  fi
+
+  echo "Attempt $attempt failed before tests completed. Retrying with clean browser processes..."
+done