diff --git a/CHANGES.rst b/CHANGES.rst index 2e29b245d4..cd025400db 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,6 +14,10 @@ Unreleased with a dedicated CI job. :pr:`3139` - Fix callable ``flag_value`` being instantiated when used as a default via ``default=True``. :issue:`3121` :pr:`3201` :pr:`3213` :pr:`3225` +- Add optional randomized parallel test execution using ``pytest-randomly`` and + ``pytest-xdist`` to detect test pollution and race conditions. :pr:`3151` +- Add contributor documentation for running stress tests, randomized + parallel tests, and Flask smoke tests. :pr:`3151` :pr:`3177` Version 8.3.1 -------------- diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 0000000000..1022606dc8 --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,65 @@ +# Contributing to Click + +These guidelines are particular to Click and complement the `general ones for all Pallets projects `_. + +## Running the Test Suite + +The default invocation runs the fast unit tests: + +```shell-session +$ tox +``` + +Or without tox: + +```shell-session +$ pytest +``` + +### Stress Tests + +These are a collection of long-running tests that reproduce race conditions in `CliRunner`. They are marked with `@pytest.mark.stress`. + +Run them with the dedicated tox environment: + +```shell-session +$ tox -e stress-py3.14 +``` + +Or directly with pytest: + +```shell-session +$ pytest tests/test_stream_lifecycle.py -m stress -x --override-ini="addopts=" +``` + +These tests run 30_000 iterations each and take a long time. Use `-x` to stop at the first failure. + +### Randomized & Parallel Tests + +Runs the full test suite in random order across multiple processes to detect test pollution and race conditions. This uses `pytest-randomly` and `pytest-xdist`. + +```shell-session +$ tox -e random +``` + +You can reproduce a specific ordering by passing the seed printed at the start of the run: + +```shell-session +$ tox -e random -- -p randomly -p no:randomly -p randomly --randomly-seed=12345 +``` + +### Flask Smoke Tests + +A CI workflow (`.github/workflows/test-flask.yaml`) runs Flask's own test suite against Click's `main` branch. This catches regressions that would break Flask, Click's primary downstream consumer. + +The workflow clones Flask, installs it, then overrides Click with the current branch. To replicate locally: + +```shell-session +$ git clone https://github.com/pallets/flask +$ cd flask +$ uv venv --python 3.14 +$ uv sync --all-extras +$ uv run --with "git+https://github.com/pallets/click.git@main" -- pytest +``` + +Replace `@main` with your branch or a local path (`-e /path/to/click`) to test local changes. diff --git a/docs/index.rst b/docs/index.rst index e705cc6d48..b97ffe2133 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -133,13 +133,14 @@ About Project * `Version Policy `_ -* `Contributing `_ +* `General Contributing Guidelines `_ * `Donate `_ .. toctree:: :maxdepth: 1 + contributing contrib license changes diff --git a/pyproject.toml b/pyproject.toml index 601bbe6804..be004b47c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,11 @@ pre-commit = [ tests = [ "pytest", ] +tests-random = [ + "pytest", + "pytest-randomly", + "pytest-xdist", +] typing = [ "mypy", "pyright", @@ -165,6 +170,15 @@ commands = [[ {replace = "posargs", default = [], extend = true}, ]] +[tool.tox.env.random] +description = "randomized parallel tests to detect test pollution and race conditions" +dependency_groups = ["tests-random"] +commands = [[ + "pytest", "-v", "--tb=short", "--numprocesses=auto", + "--basetemp={env_tmp_dir}", + {replace = "posargs", default = [], extend = true}, +]] + [tool.tox.env.stress] description = "stress tests for stream lifecycle race conditions" commands = [[ diff --git a/uv.lock b/uv.lock index 7b1929ff1b..2403511fc0 100644 --- a/uv.lock +++ b/uv.lock @@ -201,6 +201,11 @@ pre-commit = [ tests = [ { name = "pytest" }, ] +tests-random = [ + { name = "pytest" }, + { name = "pytest-randomly" }, + { name = "pytest-xdist" }, +] typing = [ { name = "mypy" }, { name = "pyright" }, @@ -230,6 +235,11 @@ pre-commit = [ { name = "pre-commit-uv" }, ] tests = [{ name = "pytest" }] +tests-random = [ + { name = "pytest" }, + { name = "pytest-randomly" }, + { name = "pytest-xdist" }, +] typing = [ { name = "mypy" }, { name = "pyright" }, @@ -275,6 +285,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "filelock" version = "3.20.0" @@ -714,6 +733,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/8b/6300fb80f858cda1c51ffa17075df5d846757081d11ab4aa35cef9e6258b/pytest-9.0.1-py3-none-any.whl", hash = "sha256:67be0030d194df2dfa7b556f2e56fb3c3315bd5c8822c6951162b92b32ce7dad", size = 373668, upload-time = "2025-11-12T13:05:07.379Z" }, ] +[[package]] +name = "pytest-randomly" +version = "4.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/1d/258a4bf1109258c00c35043f40433be5c16647387b6e7cd5582d638c116b/pytest_randomly-4.0.1.tar.gz", hash = "sha256:174e57bb12ac2c26f3578188490bd333f0e80620c3f47340158a86eca0593cd8", size = 14130, upload-time = "2025-09-12T15:23:00.085Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/3e/a4a9227807b56869790aad3e24472a554b585974fe7e551ea350f50897ae/pytest_randomly-4.0.1-py3-none-any.whl", hash = "sha256:e0dfad2fd4f35e07beff1e47c17fbafcf98f9bf4531fd369d9260e2f858bfcb7", size = 8304, upload-time = "2025-09-12T15:22:58.946Z" }, +] + +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3"