diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c7b7fe9..9ca2919 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,7 +1,11 @@ name: Tests -on: push + +# PER-8195: explicitly use `pull_request` only. `pull_request_target` is +# forbidden — it checks out attacker-controlled code with full secret access. +on: [push, pull_request] + jobs: - build: + basic: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -20,3 +24,46 @@ jobs: env: PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }} + advanced: + # PER-8195 advanced example. Runs in --testing mode so PR builds (including + # forks and Dependabot) don't require a real PERCY_TOKEN. The `--testing` + # flag is only valid on `percy exec` (not `exec:start`). + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + working-directory: advanced + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: 18 + - name: Install jq + yq + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq jq + sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + sudo chmod +x /usr/local/bin/yq + - name: Install advanced/ dependencies + run: npm install + - name: Fetch shared advanced-snapshot assertion helper + # TODO(PER-8195 D8): pin to a tagged commit once percy-public-repos-parent + # publishes the scripts/ dir to a stable URL or to an npm package. + run: | + curl -fsSL -o /tmp/assert-advanced-snapshots.sh \ + https://raw.githubusercontent.com/percy/percy-public-repos-parent/main/scripts/assert-advanced-snapshots.sh + chmod +x /tmp/assert-advanced-snapshots.sh + - name: Run mocha advanced tests (--testing) + capture /test/requests + env: + PERCY_TOKEN: fake_token + run: | + npx --no-install percy exec --testing -- bash -c ' + npx --no-install mocha tests/ --exit + ec=$? + curl -fsS http://localhost:5338/test/requests > advanced-requests.json || true + exit $ec + ' + - name: Assert matrix-row coverage + env: + PERCY_REQUESTS_FILE: ${{ github.workspace }}/advanced/advanced-requests.json + run: /tmp/assert-advanced-snapshots.sh ./matrix.yml diff --git a/README.md b/README.md index c4cfbce..14bf49a 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,18 @@ Example app showing integration of [Percy](https://percy.io/) visual testing into Selenium JavaScript tests. +> **New:** This repo ships an [`advanced/`](./advanced) example covering the full applicable Percy SDK feature surface for `@percy/selenium-webdriver`. See the [Percy SDK Feature Matrix](https://docs.percy.io/docs/sdk-feature-matrix) for cross-SDK coverage. + Based on the [TodoMVC](https://github.com/tastejs/todomvc) [VanillaJS](https://github.com/tastejs/todomvc/tree/master/examples/vanillajs) app, forked at commit [4e301c7014093505dcf6678c8f97a5e8dee2d250](https://github.com/tastejs/todomvc/tree/4e301c7014093505dcf6678c8f97a5e8dee2d250). +## Examples + +| Example | What it shows | Run command | +|---|---|---| +| `./` (basic, at repo root) | Minimum viable integration: a single `percySnapshot(driver, name)` call per test. Start here. | `npm test` | +| [`./advanced/`](./advanced) | Full applicable Percy SDK feature surface: widths, percyCSS, regions, readiness, responsive capture, etc. See [`advanced/README.md`](./advanced/README.md) for the matrix-row coverage table. | `cd advanced && npm install && npm run test:advanced` | + ## Selenium JavaScript Tutorial The tutorial assumes you're already familiar with JavaScript and diff --git a/advanced/.gitignore b/advanced/.gitignore new file mode 100644 index 0000000..bb93f4a --- /dev/null +++ b/advanced/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +advanced-requests.json +*.log diff --git a/advanced/.percy.yml b/advanced/.percy.yml new file mode 100644 index 0000000..651189f --- /dev/null +++ b/advanced/.percy.yml @@ -0,0 +1,17 @@ +# PER-8195 — advanced example global config. Demonstrates Percy CLI snapshot +# config that the @percy/selenium-webdriver SDK consumes as defaults across +# every snapshot. Per-snapshot options in tests/todomvc_advanced.spec.js +# override these. + +version: 2 + +snapshot: + widths: [375, 1280] + min-height: 1024 + percy-css: | + .new-todo::placeholder { color: #999 !important; } + +discovery: + allowed-hostnames: + - localhost + network-idle-timeout: 500 diff --git a/advanced/README.md b/advanced/README.md new file mode 100644 index 0000000..3e7f4c4 --- /dev/null +++ b/advanced/README.md @@ -0,0 +1,55 @@ +# Advanced Percy + Selenium-JS example + +This directory exercises the full applicable Percy SDK feature surface for `@percy/selenium-webdriver`. See the basic example at the repo root for the minimum integration. + +## What this example covers + +A single `mocha` spec (`tests/todomvc_advanced.spec.js`) where each `it(...)` block exercises one row of the [Percy SDK Advanced Feature Matrix](../../../docs/advanced-example-feature-matrix.md). Global SDK config — readiness preset, default widths, discovery — lives in `.percy.yml` and is consumed by every snapshot. + +## Run locally + +```bash +cd advanced +npm install +export PERCY_TOKEN="" # do NOT commit this +npm run test:advanced +``` + +To run without a real token (CI assertion mode): + +```bash +npm run test:advanced:ci # uses --testing + PERCY_TOKEN=fake_token +``` + +The CI variant asserts every matrix row appears in the captured POST bodies at the local `/test/requests` endpoint. No real Percy build is created. + +## Coverage matrix + +States: `Covered` / `N/A — ` / `Planned` / `Deprecated`. Source of truth is [`matrix.yml`](./matrix.yml). + +| Feature | State | Test | +|---|---|---| +| widths | Covered | `exercises widths` | +| minHeight | Covered | `exercises minHeight` | +| percyCSS | Covered | `exercises percyCSS` | +| enableJavaScript | Covered | `exercises enableJavaScript` | +| responsiveSnapshotCapture | Covered | `exercises responsiveSnapshotCapture` | +| readiness preset | Covered | `exercises readiness preset` | +| labels | Covered | `exercises labels` | +| testCase | Covered | `exercises testCase` | +| devicePixelRatio | Covered | `exercises devicePixelRatio` | +| regions | Covered | `exercises regions` | +| browsers override | Covered | `exercises browsers override` | +| sync mode | Covered | `exercises sync mode` | +| ignoreCanvasSerializationErrors | Covered | `exercises ignoreCanvasSerializationErrors` | +| ignoreStyleSheetSerializationErrors | Covered | `exercises ignoreStyleSheetSerializationErrors` | +| Chrome CDP resize on responsive | Covered | automatic via `@percy/selenium-webdriver >= 2.2.6` | +| cookie capture via `getCookies` | Covered | automatic via `@percy/selenium-webdriver` | +| `.percy.yml` global config | Covered | `.percy.yml` consumed at build start | +| environment info reporting | Covered | automatic via `@percy/selenium-webdriver` client info | +| PERCY_SERVER_ADDRESS via env | Covered | CI advanced job picks up `PERCY_SERVER_ADDRESS` | +| `createRegion` helper | Planned | — | +| `slowScrollToBottom` (lazy loading) | Planned | — | +| `scope` | N/A | Not exposed in SDK 2.2.6 | +| `domTransformation` | N/A | Not exposed in SDK 2.2.6 | +| `discovery` per-snapshot | N/A | discovery is per-build only | diff --git a/advanced/matrix.yml b/advanced/matrix.yml new file mode 100644 index 0000000..277ece0 --- /dev/null +++ b/advanced/matrix.yml @@ -0,0 +1,92 @@ +# PER-8195 Phase 1 — Selenium-JS matrix-row mapping. +# Test code: tests/todomvc_advanced.spec.js. + +sdk: selenium-javascript +package: '@percy/selenium-webdriver' +language: javascript +sdk_min_version: '2.2.6' +cli_min_version: '1.31.10' + +rows: + - id: widths + state: covered + test: 'TodoMVC Advanced > exercises widths' + - id: min_height + state: covered + test: 'TodoMVC Advanced > exercises minHeight' + - id: percy_css + state: covered + test: 'TodoMVC Advanced > exercises percyCSS' + - id: enable_javascript + state: covered + test: 'TodoMVC Advanced > exercises enableJavaScript' + - id: responsive_snapshot_capture + state: covered + test: 'TodoMVC Advanced > exercises responsiveSnapshotCapture' + - id: readiness_preset + state: covered + test: 'TodoMVC Advanced > exercises readiness preset' + - id: labels + state: covered + test: 'TodoMVC Advanced > exercises labels' + - id: test_case + state: covered + test: 'TodoMVC Advanced > exercises testCase' + - id: device_pixel_ratio + state: covered + test: 'TodoMVC Advanced > exercises devicePixelRatio' + - id: regions + state: covered + test: 'TodoMVC Advanced > exercises regions' + - id: browsers + state: covered + test: 'TodoMVC Advanced > exercises browsers override' + - id: sync + state: covered + test: 'TodoMVC Advanced > exercises sync mode' + - id: ignore_canvas_serialization_errors + state: covered + test: 'TodoMVC Advanced > exercises ignoreCanvasSerializationErrors' + notes: 'No assert-script predicate yet; reported as skip until extended.' + - id: ignore_stylesheet_serialization_errors + state: covered + test: 'TodoMVC Advanced > exercises ignoreStyleSheetSerializationErrors' + notes: 'No assert-script predicate yet; reported as skip until extended.' + + # Selenium-JS-specific helpers — exposed by SDK but not yet exercised in this + # phase. + - id: create_region_helper + state: planned + test: 'TodoMVC Advanced > exercises regions (via createRegion helper)' + - id: slow_scroll_to_bottom_lazy_loading + state: planned + test: 'TodoMVC Advanced > exercises slowScrollToBottom helper (PERCY_ENABLE_LAZY_LOADING_SCROLL=true)' + + # Automatic SDK behaviors. + - id: chrome_cdp_resize + state: covered + test: 'automatic via @percy/selenium-webdriver >= 2.2.6 when responsive_snapshot_capture is enabled' + - id: cookie_capture_via_getcookies + state: covered + test: 'automatic via @percy/selenium-webdriver' + + # Options not in the public Selenium-JS surface (per SDK index.d.ts). + - id: scope + state: n_a + reason: 'Not exposed in @percy/selenium-webdriver SnapshotOptions surface as of 2.2.6.' + - id: dom_transformation + state: n_a + reason: 'Not exposed in @percy/selenium-webdriver SnapshotOptions surface as of 2.2.6.' + - id: discovery + state: n_a + reason: 'discovery is per-build, not per-snapshot in this SDK.' + + - id: env_percy_server_address + state: covered + test: 'CI: advanced job sets PERCY_SERVER_ADDRESS via env' + - id: percy_yml_global_config + state: covered + test: 'global config consumed via .percy.yml' + - id: environment_info_reporting + state: covered + test: 'automatic via @percy/selenium-webdriver client info' diff --git a/advanced/package.json b/advanced/package.json new file mode 100644 index 0000000..1393a82 --- /dev/null +++ b/advanced/package.json @@ -0,0 +1,21 @@ +{ + "name": "example-percy-selenium-javascript-advanced", + "private": true, + "description": "Advanced Percy + Selenium-JS example. Exercises the full applicable SDK feature surface. See README.md for the matrix-row coverage.", + "scripts": { + "test:advanced": "percy exec -- mocha tests/ --exit", + "test:advanced:ci": "PERCY_TOKEN=fake_token percy exec --testing -- mocha tests/ --exit" + }, + "dependencies": { + "expect": "^30.2.0", + "todomvc-app-css": "^2.4.3" + }, + "devDependencies": { + "@percy/cli": "^1.31.13", + "@percy/selenium-webdriver": "^2.2.6", + "geckodriver": "^3.0.1", + "http-server": "^14.1.1", + "mocha": "^11.7.4", + "selenium-webdriver": "^4.0.0" + } +} diff --git a/advanced/tests/todomvc_advanced.spec.js b/advanced/tests/todomvc_advanced.spec.js new file mode 100644 index 0000000..b06ebdd --- /dev/null +++ b/advanced/tests/todomvc_advanced.spec.js @@ -0,0 +1,129 @@ +// PER-8195 Phase 1 — selenium-javascript advanced example. +// Each test exercises one row of the Advanced Feature Matrix. See ../matrix.yml +// for the canonical mapping of test name -> matrix row. + +const { Builder, By, Key, until } = require('selenium-webdriver'); +const firefox = require('selenium-webdriver/firefox'); +const percySnapshot = require('@percy/selenium-webdriver'); +const httpServer = require('http-server'); + +const PORT = process.env.PORT_NUMBER || 8003; +const TEST_URL = `http://localhost:${PORT}`; + +describe('TodoMVC Advanced', function () { + this.timeout(45000); + let driver; + let server; + + before(async () => { + server = httpServer.createServer({ root: `${__dirname}/../..` }); + server.listen(PORT); + + const options = new firefox.Options().headless(); + if (process.env.FIREFOX_BINARY) options.setBinary(process.env.FIREFOX_BINARY); + + driver = await new Builder() + .forBrowser('firefox') + .setFirefoxOptions(options) + .build(); + }); + + after(async () => { + if (driver) await driver.quit(); + if (server) server.close(); + }); + + beforeEach(async () => { + await driver.get(TEST_URL); + await driver.wait(until.titleIs('VanillaJS • TodoMVC'), 5000); + await driver + .findElement(By.className('new-todo')) + .sendKeys('Walk the dog', Key.ENTER); + }); + + it('exercises widths', async function () { + await percySnapshot(driver, this.test.fullTitle(), { + widths: [375, 768, 1280, 1920], + }); + }); + + it('exercises minHeight', async function () { + await percySnapshot(driver, this.test.fullTitle(), { minHeight: 2000 }); + }); + + it('exercises percyCSS', async function () { + await percySnapshot(driver, this.test.fullTitle(), { + percyCSS: '.todo-list li { background: #fffde7 !important; }', + }); + }); + + it('exercises enableJavaScript', async function () { + await percySnapshot(driver, this.test.fullTitle(), { + enableJavaScript: true, + }); + }); + + it('exercises responsiveSnapshotCapture', async function () { + await percySnapshot(driver, this.test.fullTitle(), { + responsiveSnapshotCapture: true, + widths: [375, 1280], + }); + }); + + it('exercises readiness preset', async function () { + await percySnapshot(driver, this.test.fullTitle(), { + readiness: { preset: 'strict', timeoutMs: 5000 }, + }); + }); + + it('exercises labels', async function () { + await percySnapshot(driver, this.test.fullTitle(), { + labels: 'smoke,sdk-selenium-js', + }); + }); + + it('exercises testCase', async function () { + await percySnapshot(driver, this.test.fullTitle(), { + testCase: 'todomvc-advanced-suite', + }); + }); + + it('exercises devicePixelRatio', async function () { + await percySnapshot(driver, this.test.fullTitle(), { devicePixelRatio: 2 }); + }); + + it('exercises regions', async function () { + await percySnapshot(driver, this.test.fullTitle(), { + regions: [ + { + algorithm: 'ignore', + elementSelector: { + boundingBox: { x: 0, y: 0, width: 200, height: 100 }, + }, + }, + ], + }); + }); + + it('exercises browsers override', async function () { + await percySnapshot(driver, this.test.fullTitle(), { + browsers: ['chrome', 'firefox'], + }); + }); + + it('exercises sync mode', async function () { + await percySnapshot(driver, this.test.fullTitle(), { sync: false }); + }); + + it('exercises ignoreCanvasSerializationErrors', async function () { + await percySnapshot(driver, this.test.fullTitle(), { + ignoreCanvasSerializationErrors: true, + }); + }); + + it('exercises ignoreStyleSheetSerializationErrors', async function () { + await percySnapshot(driver, this.test.fullTitle(), { + ignoreStyleSheetSerializationErrors: true, + }); + }); +});