Implement basics for E2E tests driven by playwright (#6038)

* [PoC] Implement basics for E2E tests driven by playwright (JS)

* Update tracker test run instructions in README

* Update package config on e2e tests

* Set playwright reporter format to list

* Add mix task for running e2e tests in UI mode

* Add e2e test run job to CI config

* Add missing PG and CH services to e2e CI config

* Increase timeouts for e2e CI tasks

* Temporarily remove tzdata updater

* Revert "Temporarily remove tzdata updater"

This reverts commit 6105473.

* Add step downloading geo data

* Setup E2E seeds before running CI

* Run e2e tests via mix task to ensure setup env variables

* Set `BASE_URL` env var explicitly in CI config instead

* Reduce shards to 1

* Change how cwd is set for playwright server

* Show server log output in e2e tests

* Install asset dependencies during e2e setup

* Build assets as well

* Hide server output in e2e tests again

* Parallelize e2e tests again

* Reduce test sharding from 4 to 2

* Cache more fetched and compiled assets

* Remove redundant tests

* Clean up playwright config slightly

* Try reducing the time spent fetching system deps

* Output screenshots on failure

* Upload screenshots from failed tests to GH artifacts

* Make one test fail on purpose

* Revert "Make one test fail on purpose"

This reverts commit 3372a82.

* Update gitignore inside e2e to ignore outputDir

* Add notes about how to run e2e tests locally

* Add preferred envs for E2E mix tasks

* Don't dump screenshots and don't upload them as GH artifacts

* Rely on cached `tracker/node_modules` and not install tracker conditionally

* Remove no longer relevant comments from mix.exs

* Fix invalid playwright install option that surfaced with pruned cache

Author: zoldar

.github/workflows/elixir.yml

@@ -113,6 +113,147 @@ jobs:
         env:
           MIX_TEST_PARTITION: ${{ matrix.mix_test_partition }}
 
+  e2e:
+    name: End-to-end tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    env:
+      MIX_ENV: e2e_test
+      BASE_URL: "http://localhost:8000"
+    services:
+      postgres:
+        image: "postgres:18"
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_PASSWORD: postgres
+          options: >-
+            --health-cmd pg_isready
+            --health-interval 10s
+            --health-timeout 5s
+            --health-retries 5
+      clickhouse:
+        image: clickhouse/clickhouse-server:25.11.5.8-alpine
+        ports:
+          - 8123:8123
+        env:
+          CLICKHOUSE_SKIP_USER_SETUP: 1
+          options: >-
+            --health-cmd nc -zw3 localhost 8124
+            --health-interval 10s
+            --health-timeout 5s
+            --health-retries 5
+    strategy:
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2]
+        shardTotal: [2]
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - uses: marocchino/tool-versions-action@v1
+        id: versions
+
+      - uses: erlef/setup-beam@v1
+        with:
+          elixir-version: ${{ steps.versions.outputs.elixir }}
+          otp-version: ${{ steps.versions.outputs.erlang }}
+
+      - uses: actions/cache@v5
+        with:
+          path: |
+            deps
+            _build
+            assets/node_modules
+            tracker/node_modules
+            priv/tracker/js
+            priv/tracker/installation_support
+            ${{ env.PERSISTENT_CACHE_DIR }}
+          key: e2e-${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-${{ github.head_ref || github.ref }}-${{ hashFiles('**/mix.lock') }}
+          restore-keys: |
+            e2e-${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-${{ github.head_ref || github.ref }}-
+            e2e-${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-refs/heads/master-
+
+      - name: Cache E2E dependencies and Playwright browsers
+        uses: actions/cache@v5
+        id: playwright-cache
+        with:
+          path: |
+            e2e/node_modules
+            ~/.cache/ms-playwright
+            ~/.cache/ms-playwright-github
+          key: playwright-${{ runner.os }}-${{ hashFiles('e2e/package-lock.json') }}
+          restore-keys: |
+            playwright-${{ runner.os }}-
+
+      - name: Check for changes in tracker/**
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          filters: |
+            tracker:
+              - 'tracker/**'
+
+      - run: npm install --prefix ./tracker
+      - run: npm run deploy --prefix ./tracker
+      - run: mix deps.get --only $MIX_ENV
+      - run: mix compile --warnings-as-errors --all-warnings
+      - run: npm install --prefix ./assets
+      - run: mix assets.deploy
+      - run: mix do ecto.create, ecto.migrate
+      - run: mix download_country_database
+      - run: mix run -e "Tzdata.ReleaseUpdater.poll_for_update"
+
+      - name: Setup E2E seeds
+        run: mix run priv/repo/e2e_seeds.exs
+
+      - name: Install E2E dependencies
+        run: npm --prefix ./e2e ci
+
+      - name: Install E2E Playwright Browsers
+        if: steps.playwright-cache.outputs.cache-hit != 'true'
+        working-directory: ./e2e
+        run: npx playwright install --with-deps chromium
+
+      - name: Run E2E Playwright tests
+        run: npm --prefix ./e2e test -- --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --reporter=blob
+
+      - name: Upload E2E blob report to GitHub Actions Artifacts
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v6
+        with:
+          name: e2e-blob-report-${{ matrix.shardIndex }}
+          path: e2e/blob-report
+          retention-days: 1
+
+  merge-sharded-e2e-test-report:
+    if: ${{ !cancelled() }}
+    needs: [e2e]
+    timeout-minutes: 15
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-node@v6
+        with:
+          node-version: 23.2.0
+          cache: 'npm'
+          cache-dependency-path: e2e/package-lock.json
+      - name: Install dependencies
+        run: npm --prefix ./e2e ci
+
+      - name: Download blob reports from GitHub Actions Artifacts
+        uses: actions/download-artifact@v7
+        with:
+          path: all-e2e-blob-reports
+          pattern: e2e-blob-report-*
+          merge-multiple: true
+
+      - name: Merge into list report
+        working-directory: ./e2e
+        run: npx playwright merge-reports --reporter list ../all-e2e-blob-reports
+
   static:
     name: Static checks (format, credo, dialyzer)
     env:
@@ -125,6 +266,7 @@ jobs:
 
       - uses: marocchino/tool-versions-action@v1
         id: versions
+
       - uses: erlef/setup-beam@v1
         with:
           elixir-version: ${{ steps.versions.outputs.elixir }}

config/.env.e2e_test

@@ -0,0 +1,38 @@
+BASE_URL=http://localhost:8000
+SECURE_COOKIE=false
+DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/plausible_e2e
+CLICKHOUSE_DATABASE_URL=http://127.0.0.1:8123/plausible_e2e
+CLICKHOUSE_MAX_BUFFER_SIZE_BYTES=1000000
+SECRET_KEY_BASE=/njrhntbycvastyvtk1zycwfm981vpo/0xrvwjjvemdakc/vsvbrevlwsc6u8rcg
+TOTP_VAULT_KEY=Q3BD4nddbkVJIPXgHuo5NthGKSIH0yesRfG05J88HIo=
+ENVIRONMENT=dev
+MAILER_ADAPTER=Bamboo.LocalAdapter
+LOG_LEVEL=warning
+SELFHOST=false
+DISABLE_CRON=true
+ADMIN_USER_IDS=1
+SHOW_CITIES=true
+PADDLE_VENDOR_AUTH_CODE=895e20d4efaec0575bb857f44b183217b332d9592e76e69b8a
+PADDLE_VENDOR_ID=3942
+SSO_VERIFICATION_NAMESERVERS=0.0.0.0:5354
+
+GOOGLE_CLIENT_ID=875387135161-l8tp53dpt7fdhdg9m1pc3vl42si95rh0.apps.googleusercontent.com
+GOOGLE_CLIENT_SECRET=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
+
+PROMEX_DISABLED=false
+SITE_DEFAULT_INGEST_THRESHOLD=1000000
+
+S3_DISABLED=false
+S3_ACCESS_KEY_ID=minioadmin
+S3_SECRET_ACCESS_KEY=minioadmin
+S3_REGION=us-east-1
+S3_ENDPOINT=http://localhost:10000
+S3_EXPORTS_BUCKET=dev-exports
+S3_IMPORTS_BUCKET=dev-imports
+
+HELP_SCOUT_APP_ID=fake_app_id
+HELP_SCOUT_APP_SECRET=fake_app_secret
+HELP_SCOUT_SIGNATURE_KEY=fake_signature_key
+HELP_SCOUT_VAULT_KEY=ym9ZQg0KPNGCH3C2eD5y6KpL0tFzUqAhwxQO6uEv/ZM=
+
+VERIFICATION_ENABLED=true

config/e2e_test.exs

@@ -0,0 +1,18 @@
+import Config
+
+config :plausible, PlausibleWeb.Endpoint,
+  server: true,
+  check_origin: false
+
+config :plausible, paddle_api: Plausible.Billing.DevPaddleApiMock
+
+config :phoenix, :stacktrace_depth, 20
+config :phoenix, :plug_init_mode, :runtime
+
+config :bcrypt_elixir, :log_rounds, 4
+
+config :plausible, Plausible.Ingestion.Counters, enabled: false
+
+config :plausible, Oban, testing: :manual
+
+config :plausible, Plausible.Session.Salts, interval: :timer.hours(1)

config/runtime.exs

@@ -14,6 +14,10 @@ if config_env() == :ce_test do
   Envy.load(["config/.env.test"])
 end
 
+if config_env() == :e2e_test do
+  Envy.load(["config/.env.e2e_test"])
+end
+
 config_dir = System.get_env("CONFIG_DIR", "/run/secrets")
 
 log_format =

e2e/.gitignore

@@ -0,0 +1,9 @@
+
+# Playwright
+node_modules/
+/test-results/
+/playwright-report/
+/blob-report/
+/output/
+/playwright/.cache/
+/playwright/.auth/

e2e/package-lock.json

@@ -0,0 +1,12 @@
+{
+  "name": "e2e",
+  "scripts": {
+    "test": "npx playwright test",
+    "test:ui": "npx playwright test --ui"
+  },
+  "license": "MIT",
+  "devDependencies": {
+    "@playwright/test": "^1.58.0",
+    "@types/node": "^25.1.0"
+  }
+}