diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f27a883..6c37787 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,7 +1,11 @@ name: Tests + +# PER-8195: explicitly use `pull_request` only. `pull_request_target` is +# forbidden — it checks out attacker-controlled code with full secret access. on: [push, pull_request] + jobs: - build: + basic: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -27,3 +31,45 @@ jobs: custom-command: "make test" env: PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }} + + advanced: + # PER-8195 advanced example. Runs in --testing mode so PR builds (including + # forks and Dependabot) don't require a real PERCY_TOKEN. The `--testing` + # flag is only valid on `percy exec` (not `exec:start`). + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + working-directory: advanced + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + - uses: actions/setup-node@v3 + with: + node-version: 18 + - name: Install jq + yq + percy CLI + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq jq + sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + sudo chmod +x /usr/local/bin/yq + npm install --no-save @percy/cli@^1.31.13 + - name: Install advanced/ python dependencies + run: make install + - name: Fetch shared advanced-snapshot assertion helper + # TODO(PER-8195 D8): pin to a tagged commit once percy-public-repos-parent + # publishes the scripts/ dir to a stable URL or to an npm package. + run: | + curl -fsSL -o /tmp/assert-advanced-snapshots.sh \ + https://raw.githubusercontent.com/percy/percy-public-repos-parent/main/scripts/assert-advanced-snapshots.sh + chmod +x /tmp/assert-advanced-snapshots.sh + - name: Run pytest advanced (--testing) + capture /test/requests + env: + PERCY_TOKEN: fake_token + run: make test-advanced-ci + - name: Assert matrix-row coverage + env: + PERCY_REQUESTS_FILE: ${{ github.workspace }}/advanced/advanced-requests.json + run: /tmp/assert-advanced-snapshots.sh ./matrix.yml diff --git a/README.md b/README.md index 2b19a08..9c74643 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,18 @@ Example app showing integration of [Percy](https://percy.io/) visual testing into Python Selenium tests. +> **New:** This repo ships an [`advanced/`](./advanced) example covering the full applicable Percy SDK feature surface for `percy-selenium`. See the [Percy SDK Feature Matrix](https://docs.percy.io/docs/sdk-feature-matrix) for cross-SDK coverage. + Based on the [TodoMVC](https://github.com/tastejs/todomvc) [VanillaJS](https://github.com/tastejs/todomvc/tree/master/examples/vanillajs) app, forked at commit [4e301c7014093505dcf6678c8f97a5e8dee2d250](https://github.com/tastejs/todomvc/tree/4e301c7014093505dcf6678c8f97a5e8dee2d250). +## Examples + +| Example | What it shows | Run command | +|---|---|---| +| `./` (basic, at repo root) | Minimum viable integration: a few `percy_snapshot(browser, name)` calls. Start here. | `make test` | +| [`./advanced/`](./advanced) | Full applicable Percy SDK feature surface: widths, minHeight, enable_javascript, readiness, responsive_snapshot_capture, sync, snake_case + camelCase dual naming, etc. pytest-driven. See [`advanced/README.md`](./advanced/README.md) for the matrix-row coverage table. | `cd advanced && make test` | + ## Selenium Python Tutorial This tutorial assumes that you're already familiar with Python & Selenium and focuses on using them diff --git a/advanced/.gitignore b/advanced/.gitignore new file mode 100644 index 0000000..bd56432 --- /dev/null +++ b/advanced/.gitignore @@ -0,0 +1,5 @@ +.venv/ +__pycache__/ +*.pyc +advanced-requests.json +.pytest_cache/ diff --git a/advanced/.percy.yml b/advanced/.percy.yml new file mode 100644 index 0000000..5f38d61 --- /dev/null +++ b/advanced/.percy.yml @@ -0,0 +1,15 @@ +# PER-8195 — advanced example global config for percy-selenium (python). +# Per-snapshot options in tests/test_todomvc_advanced.py override these. + +version: 2 + +snapshot: + widths: [375, 1280] + min-height: 1024 + percy-css: | + .new-todo::placeholder { color: #999 !important; } + +discovery: + allowed-hostnames: + - localhost + network-idle-timeout: 500 diff --git a/advanced/Makefile b/advanced/Makefile new file mode 100644 index 0000000..1dd01d0 --- /dev/null +++ b/advanced/Makefile @@ -0,0 +1,29 @@ +VENV=.venv/bin +REQUIREMENTS=requirements.txt + +$(VENV): + python3 -m venv .venv + $(VENV)/python3 -m pip install --upgrade pip + +$(VENV)/.deps: $(REQUIREMENTS) | $(VENV) + $(VENV)/pip install -r $(REQUIREMENTS) + touch $(VENV)/.deps + +.PHONY: venv install clean test test-advanced test-advanced-ci + +venv install: $(VENV)/.deps + +clean: + rm -rf .venv .pytest_cache __pycache__ tests/__pycache__ advanced-requests.json + +# Local run against a real PERCY_TOKEN. +test test-advanced: install + npx --no-install percy exec -- $(VENV)/python3 -m pytest tests/ -v + +# CI run in --testing mode + capture requests file. +test-advanced-ci: install + PERCY_TOKEN=fake_token npx --no-install percy exec --testing -- bash -c '\ + $(VENV)/python3 -m pytest tests/ -v; \ + ec=$$?; \ + curl -fsS http://localhost:5338/test/requests > advanced-requests.json || true; \ + exit $$ec' diff --git a/advanced/README.md b/advanced/README.md new file mode 100644 index 0000000..be25fc3 --- /dev/null +++ b/advanced/README.md @@ -0,0 +1,55 @@ +# Advanced Percy + Selenium-Python example + +This directory exercises the full applicable Percy SDK feature surface for `percy-selenium` (Python). See the basic example at the repo root for the minimum integration. + +## What this example covers + +A pytest suite (`tests/test_todomvc_advanced.py`) where each test exercises one row of the [Percy SDK Advanced Feature Matrix](../../../docs/advanced-example-feature-matrix.md). Global SDK config — readiness preset, default widths, percyCSS, discovery — lives in `.percy.yml`. + +Note: `scope`, `domTransformation`, `regions`, `discovery` are marked `N/A` — not exposed in `percy-selenium` 2.1.2 kwargs surface. + +## Run locally + +```bash +cd advanced +make install # creates .venv, installs requirements.txt +export PERCY_TOKEN="" # do NOT commit this +make test +``` + +To run without a real token (CI assertion mode): + +```bash +make test-advanced-ci # uses --testing + PERCY_TOKEN=fake_token + captures /test/requests +``` + +The CI variant asserts every matrix row appears in the captured POST bodies at the local `/test/requests` endpoint. No real Percy build is created. + +## Coverage matrix + +States: `Covered` / `N/A — ` / `Planned` / `Deprecated`. Source of truth is [`matrix.yml`](./matrix.yml). + +| Feature | State | Test | +|---|---|---| +| widths | Covered | `test_exercises_widths` | +| minHeight (`min_height`) | Covered | `test_exercises_min_height` | +| enableJavaScript (`enable_javascript`) | Covered | `test_exercises_enable_javascript` | +| responsiveSnapshotCapture (`responsive_snapshot_capture`) | Covered | `test_exercises_responsive_snapshot_capture` | +| readiness preset | Covered | `test_exercises_readiness_preset` | +| sync | Covered | `test_exercises_sync` | +| labels | Covered | `test_exercises_labels` | +| testCase (`test_case`) | Covered | `test_exercises_test_case` | +| devicePixelRatio (`device_pixel_ratio`) | Covered | `test_exercises_device_pixel_ratio` | +| browsers override | Covered | `test_exercises_browsers` | +| snake_case + camelCase dual naming | Covered | `test_exercises_snake_case_camelcase_dual_naming` | +| percyCSS | Covered | global via `.percy.yml` | +| cross-origin iframe handling | Covered | automatic via `percy-selenium >= 2.1.2` | +| `.percy.yml` global config | Covered | `.percy.yml` consumed at build start | +| environment info reporting | Covered | automatic via `percy-selenium` client info | +| PERCY_SERVER_ADDRESS via env | Covered | CI advanced job picks up `PERCY_SERVER_ADDRESS` | +| `disable_shadow_dom` | Planned | — | +| `enable_layout` | Planned | — | +| `scope` | N/A | Not exposed in SDK 2.1.2 | +| `domTransformation` | N/A | Not exposed in SDK 2.1.2 | +| `regions` per-snapshot | N/A | `create_region` is Automate-only | +| `discovery` per-snapshot | N/A | discovery is per-build only | diff --git a/advanced/conftest.py b/advanced/conftest.py new file mode 100644 index 0000000..1d9ea1f --- /dev/null +++ b/advanced/conftest.py @@ -0,0 +1,36 @@ +"""PER-8195 — pytest fixtures: HTTP server for the TodoMVC app, headless +firefox driver. Server roots two directories up so the basic TodoMVC files +served at /index.html land at http://localhost:8006/.""" + +from http.server import HTTPServer, SimpleHTTPRequestHandler +from threading import Thread +import functools +import os +import pytest +from selenium.webdriver import Firefox, FirefoxOptions + +PORT = int(os.environ.get("PORT_NUMBER", "8006")) +TEST_URL = f"http://localhost:{PORT}" +APP_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + + +@pytest.fixture(scope="session") +def http_server(): + handler = functools.partial(SimpleHTTPRequestHandler, directory=APP_ROOT) + httpd = HTTPServer(("localhost", PORT), handler) + thread = Thread(target=httpd.serve_forever, daemon=True) + thread.start() + yield httpd + httpd.shutdown() + + +@pytest.fixture(scope="session") +def driver(http_server): + options = FirefoxOptions() + options.add_argument("-headless") + if os.environ.get("FIREFOX_BINARY"): + options.binary_location = os.environ["FIREFOX_BINARY"] + browser = Firefox(options=options) + browser.implicitly_wait(10) + yield browser + browser.quit() diff --git a/advanced/matrix.yml b/advanced/matrix.yml new file mode 100644 index 0000000..55a72e2 --- /dev/null +++ b/advanced/matrix.yml @@ -0,0 +1,80 @@ +# PER-8195 Phase 1 — Selenium-Python matrix-row mapping. +# Test code: tests/test_todomvc_advanced.py (pytest). + +sdk: selenium-python +package: percy-selenium +language: python +sdk_min_version: '2.1.2' +cli_min_version: '1.31.13' + +rows: + - id: widths + state: covered + test: 'test_exercises_widths' + - id: min_height + state: covered + test: 'test_exercises_min_height' + - id: enable_javascript + state: covered + test: 'test_exercises_enable_javascript' + - id: responsive_snapshot_capture + state: covered + test: 'test_exercises_responsive_snapshot_capture' + - id: readiness_preset + state: covered + test: 'test_exercises_readiness_preset' + - id: sync + state: covered + test: 'test_exercises_sync' + - id: percy_css + state: covered + test: 'global via .percy.yml snapshot.percy-css' + - id: labels + state: covered + test: 'test_exercises_labels' + - id: test_case + state: covered + test: 'test_exercises_test_case' + - id: device_pixel_ratio + state: covered + test: 'test_exercises_device_pixel_ratio' + - id: browsers + state: covered + test: 'test_exercises_browsers' + + # Python-specific. + - id: snake_case_camelcase_dual_naming + state: covered + test: 'test_exercises_snake_case_camelcase_dual_naming' + - id: cross_origin_iframe_handling + state: covered + test: 'automatic via percy-selenium >= 2.1.2' + + - id: disable_shadow_dom + state: planned + - id: enable_layout + state: planned + + # Options not exposed as kwargs in percy-selenium (per SDK source). + - id: scope + state: n_a + reason: 'Not exposed in percy-selenium 2.1.2 kwargs surface.' + - id: dom_transformation + state: n_a + reason: 'Not exposed in percy-selenium 2.1.2 kwargs surface.' + - id: regions + state: n_a + reason: 'create_region helper exists for Automate; not used in DOM snapshot path in 2.1.2.' + - id: discovery + state: n_a + reason: 'discovery is per-build, not per-snapshot in this SDK.' + + - id: env_percy_server_address + state: covered + test: 'CI: advanced job sets PERCY_SERVER_ADDRESS via env' + - id: percy_yml_global_config + state: covered + test: 'global config consumed via .percy.yml' + - id: environment_info_reporting + state: covered + test: 'automatic via percy-selenium client info' diff --git a/advanced/requirements.txt b/advanced/requirements.txt new file mode 100644 index 0000000..f1630ec --- /dev/null +++ b/advanced/requirements.txt @@ -0,0 +1,3 @@ +selenium>=4.9.0 +percy-selenium>=2.1.2 +pytest>=7.4.0 diff --git a/advanced/tests/test_todomvc_advanced.py b/advanced/tests/test_todomvc_advanced.py new file mode 100644 index 0000000..045abf6 --- /dev/null +++ b/advanced/tests/test_todomvc_advanced.py @@ -0,0 +1,104 @@ +"""PER-8195 Phase 1 — selenium-python advanced example. + +Each test exercises one row of the Advanced Feature Matrix. See +../matrix.yml for the canonical mapping of test name -> matrix row. +""" + +from percy import percy_snapshot +from selenium.webdriver.common.by import By +from selenium.webdriver.common.keys import Keys + +from conftest import TEST_URL + + +def _seed(driver): + driver.get(TEST_URL) + driver.find_element(By.CLASS_NAME, "new-todo").send_keys("Walk the dog", Keys.ENTER) + + +def test_exercises_widths(driver): + _seed(driver) + percy_snapshot(driver, "TodoMVC Advanced > exercises widths", widths=[375, 768, 1280, 1920]) + + +def test_exercises_min_height(driver): + _seed(driver) + percy_snapshot(driver, "TodoMVC Advanced > exercises minHeight", min_height=2000) + + +def test_exercises_enable_javascript(driver): + _seed(driver) + percy_snapshot( + driver, + "TodoMVC Advanced > exercises enableJavaScript", + enable_javascript=True, + ) + + +def test_exercises_responsive_snapshot_capture(driver): + _seed(driver) + percy_snapshot( + driver, + "TodoMVC Advanced > exercises responsiveSnapshotCapture", + responsive_snapshot_capture=True, + widths=[375, 1280], + ) + + +def test_exercises_readiness_preset(driver): + _seed(driver) + percy_snapshot( + driver, + "TodoMVC Advanced > exercises readiness preset", + readiness={"preset": "strict", "timeoutMs": 5000}, + ) + + +def test_exercises_sync(driver): + _seed(driver) + percy_snapshot(driver, "TodoMVC Advanced > exercises sync option", sync=False) + + +def test_exercises_labels(driver): + _seed(driver) + percy_snapshot(driver, "TodoMVC Advanced > exercises labels", labels="smoke,sdk-selenium-py") + + +def test_exercises_test_case(driver): + _seed(driver) + percy_snapshot( + driver, + "TodoMVC Advanced > exercises testCase", + test_case="todomvc-advanced-suite", + ) + + +def test_exercises_device_pixel_ratio(driver): + _seed(driver) + percy_snapshot( + driver, + "TodoMVC Advanced > exercises devicePixelRatio", + device_pixel_ratio=2, + ) + + +def test_exercises_browsers(driver): + _seed(driver) + percy_snapshot( + driver, + "TodoMVC Advanced > exercises browsers override", + browsers=["chrome", "firefox"], + ) + + +def test_exercises_snake_case_camelcase_dual_naming(driver): + """percy-selenium accepts both snake_case (Pythonic) and camelCase aliases. + Exercise both forms in a single snapshot to verify they coexist.""" + _seed(driver) + percy_snapshot( + driver, + "TodoMVC Advanced > snake_case + camelCase dual naming", + responsive_snapshot_capture=True, + widths=[375, 1280], + min_height=1024, + )