From 6059407fc635496ef5a732c0f05ada8762ed4917 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 3 Jun 2026 11:00:09 +0100 Subject: [PATCH 01/16] Add a test for pytest-cov with github coverage --- .github/workflows/test_tox.yml | 5 ++++- tox.ini | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_tox.yml b/.github/workflows/test_tox.yml index 5f135641..8e906354 100644 --- a/.github/workflows/test_tox.yml +++ b/.github/workflows/test_tox.yml @@ -162,8 +162,11 @@ jobs: with: coverage: github envs: | + # covcheck uses an explicit coverage run - linux: py313-covcheck - - linux: py314-covcheck + # Now also test it works with pytest-cov + - linux: py314 + posargs: --cov=test_package test_artifact_upload: uses: ./.github/workflows/tox.yml diff --git a/tox.ini b/tox.ini index cfe6b406..8598d260 100644 --- a/tox.ini +++ b/tox.ini @@ -62,7 +62,9 @@ commands = ruff check . [testenv:py3{10,11,12}{,-conda}] description = run pytest skip_install = false -dependency_groups = test +dependency_groups = + test + covcheck conda_deps = pytest commands = conda: python -c "import os, sys; assert os.path.exists(os.path.join(sys.prefix, 'conda-meta', 'history'))" From ad8caed27b9bd8e42eb78a400ad542258fae7718 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 3 Jun 2026 11:05:30 +0100 Subject: [PATCH 02/16] Don't require suffix on .coverage file --- .github/workflows/tox.yml | 6 ++++-- tox.ini | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 2fb6ae98..85f671a5 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -210,8 +210,8 @@ jobs: run: shell: bash -l {0} working-directory: ${{ inputs.working-directory }} - steps: + steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 @@ -313,7 +313,9 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverage-data-${{ github.run_id }}-${{ runner.os }}-${{ runner.arch }}-${{ matrix.toxenv }} - path: .coverage.* + path: | + .coverage + .coverage.* if-no-files-found: error include-hidden-files: true outputs: diff --git a/tox.ini b/tox.ini index 8598d260..1647129c 100644 --- a/tox.ini +++ b/tox.ini @@ -59,7 +59,7 @@ description = verify pep8 deps = ruff commands = ruff check . -[testenv:py3{10,11,12}{,-conda}] +[testenv:py3{10-16}{,-conda,-tmpdir}] description = run pytest skip_install = false dependency_groups = From 793a91eb1693951567822e9b2a5695b6af99dc68 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 3 Jun 2026 11:21:49 +0100 Subject: [PATCH 03/16] Add a test for running in a tmpdir --- .github/workflows/test_tox.yml | 3 +++ tox.ini | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/test_tox.yml b/.github/workflows/test_tox.yml index 8e906354..7585ceed 100644 --- a/.github/workflows/test_tox.yml +++ b/.github/workflows/test_tox.yml @@ -167,6 +167,9 @@ jobs: # Now also test it works with pytest-cov - linux: py314 posargs: --cov=test_package + # And test it works if the pytest command isn't run in the toxinidir + - linux: py314-tmpdir + posargs: --cov=test_package test_artifact_upload: uses: ./.github/workflows/tox.yml diff --git a/tox.ini b/tox.ini index 1647129c..952ddb1c 100644 --- a/tox.ini +++ b/tox.ini @@ -66,6 +66,8 @@ dependency_groups = test covcheck conda_deps = pytest +changedir = + tmpdir: .tmp/{envname} commands = conda: python -c "import os, sys; assert os.path.exists(os.path.join(sys.prefix, 'conda-meta', 'history'))" conda: micromamba list From 781f3a555a8db8ad1c3d5d738546854d77159334 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 3 Jun 2026 12:00:32 +0100 Subject: [PATCH 04/16] Improve documentation around coverage --- docs/source/tox.rst | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/docs/source/tox.rst b/docs/source/tox.rst index 73e1340a..5d06da2c 100644 --- a/docs/source/tox.rst +++ b/docs/source/tox.rst @@ -163,13 +163,37 @@ This option has no effect if ``pytest`` is ``false``. coverage ^^^^^^^^ -A space separated list of coverage providers to upload to, either -``codecov`` or ``github``. Default is to not upload coverage -reports. +The coverage option controls how coverage reports are made and processed after the tox job has completed. +This option has no effect if ``pytest`` is ``false``. +This option takes a space separated list of coverage providers to upload to, either ``codecov`` or ``github``. +Default is to not upload coverage reports. +If using ``codecov`` see :ref:`codecov-token`. -See also, ``CODECOV_TOKEN`` secret. +As the workflows do not control how your tests are run, configuring coverage correctly may require changes to your tox.ini. +The coverage collection is done inside the tox job, but the workflows handle generating reports and uploading to either codecov or github. -This option has no effect if ``pytest`` is ``false``. +Coverage Collection +################### + +There are two main ways to generate coverage for your test run(s): + +* Using `coverage.py `__ directly, which generally means prefixing your pytest command with ``coverage run -m ``. This will generate one or more ``.coverage`` files in the current directory (for parallel jobs it can generate one per process). +* Using `pytest-cov `__ which generally involves installing ``pytest-cov`` and adding ``--cov=`` to your pytest flags, this will also generate a ``.coverage`` file in the current directory as well as reporting coverage to the terminal by default. + +Coverage Reporting +################## + +After coverage has been collected the workflows work with collected ``.coverage`` database files that should have been generated as part of your test run. +If uploading to GitHub all these reports will be collected for all your jobs and combined together in a job at the end of the workflow. +If uploading to codecov they will be combined and uploaded at the end of each job. + +Both of these report uploads require the ``.coverage`` file(s) to be in the root of the GitHub Actions workspace for processing at the end of the job. +**If you are running your tests in a temporary directory**, the easiest way to achieve this is to set the following option in your tox.ini:: + + setenv = + COVERAGE_FILE={toxinidir}/.coverage + +This should work for both ``coverage.py`` and ``pytest-cov``. conda ^^^^^ @@ -612,6 +636,8 @@ like ``py311-test-cov`` and ``py312-test-cov`` instead of just ``py311`` and Secrets ~~~~~~~ +.. _codecov-token: + CODECOV_TOKEN ^^^^^^^^^^^^^ From 4265acef31fcd15690594a229fd15d3221796c90 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 3 Jun 2026 12:01:37 +0100 Subject: [PATCH 05/16] Set COVERAGE_FILE in tox.ini --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 952ddb1c..d80edb6c 100644 --- a/tox.ini +++ b/tox.ini @@ -68,6 +68,8 @@ dependency_groups = conda_deps = pytest changedir = tmpdir: .tmp/{envname} +setenv = + tmpdir: COVERAGE_FILE={toxinidir}/.coverage commands = conda: python -c "import os, sys; assert os.path.exists(os.path.join(sys.prefix, 'conda-meta', 'history'))" conda: micromamba list From 3db5cbc64ef28a4b3f6cba34f793a068ea014b94 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 3 Jun 2026 12:13:08 +0100 Subject: [PATCH 06/16] Add some codecov tests --- .github/workflows/test_tox.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/test_tox.yml b/.github/workflows/test_tox.yml index 7585ceed..9aca569e 100644 --- a/.github/workflows/test_tox.yml +++ b/.github/workflows/test_tox.yml @@ -171,6 +171,22 @@ jobs: - linux: py314-tmpdir posargs: --cov=test_package + test_coverage_codecov: + uses: ./.github/workflows/tox.yml + permissions: + id-token: write + with: + coverage: codecov + envs: | + # covcheck uses an explicit coverage run + - linux: py313-covcheck + # Now also test it works with pytest-cov + - linux: py314 + posargs: --cov=test_package + # And test it works if the pytest command isn't run in the toxinidir + - linux: py314-tmpdir + posargs: --cov=test_package + test_artifact_upload: uses: ./.github/workflows/tox.yml with: From 55d5f2fa57c86250ae7343f64fedc5ae9c6be427 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 3 Jun 2026 12:25:13 +0100 Subject: [PATCH 07/16] Fail CI if codecov upload fails --- .github/workflows/tox.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 85f671a5..de735158 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -306,6 +306,7 @@ jobs: uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} # zizmor: ignore[secrets-outside-env] + fail_ci_if_error: true - name: Upload coverage data to GitHub id: upload-coverage-gh From 881bb20c38d25292c9a6fd2ef3fa01e15fb93a11 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 3 Jun 2026 12:25:27 +0100 Subject: [PATCH 08/16] Explicitly generate an xml report for codecov --- .github/workflows/tox.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index de735158..04ef2079 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -300,6 +300,13 @@ jobs: with: paths: "**/results.xml" + - name: Generate xml report for Codecov + # Even if tox fails, upload coverage + if: ${{ (success() || failure()) && contains(matrix.coverage, 'codecov') && matrix.pytest == 'true' }} + run: | + uvx coverage combine + uvx coverage xml -i + - name: Upload to Codecov # Even if tox fails, upload coverage if: ${{ (success() || failure()) && contains(matrix.coverage, 'codecov') && matrix.pytest == 'true' }} From 6e95ce5d48f69e6f37b57d9945e040d57d307f30 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 3 Jun 2026 12:25:43 +0100 Subject: [PATCH 09/16] Support oidc auth for codecov --- .github/workflows/test_tox.yml | 2 +- .github/workflows/tox.yml | 1 + docs/source/tox.rst | 28 +++++++++++++--------------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test_tox.yml b/.github/workflows/test_tox.yml index 9aca569e..ad0b7d23 100644 --- a/.github/workflows/test_tox.yml +++ b/.github/workflows/test_tox.yml @@ -176,7 +176,7 @@ jobs: permissions: id-token: write with: - coverage: codecov + coverage: codecov-oidc envs: | # covcheck uses an explicit coverage run - linux: py313-covcheck diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 04ef2079..bc052eea 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -313,6 +313,7 @@ jobs: uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} # zizmor: ignore[secrets-outside-env] + use_oidc: ${{ contains(matrix.coverage, 'codecov-oidc') }} fail_ci_if_error: true - name: Upload coverage data to GitHub diff --git a/docs/source/tox.rst b/docs/source/tox.rst index 5d06da2c..75a7503b 100644 --- a/docs/source/tox.rst +++ b/docs/source/tox.rst @@ -164,10 +164,9 @@ coverage ^^^^^^^^ The coverage option controls how coverage reports are made and processed after the tox job has completed. +The default is to not upload coverage reports. This option has no effect if ``pytest`` is ``false``. -This option takes a space separated list of coverage providers to upload to, either ``codecov`` or ``github``. -Default is to not upload coverage reports. -If using ``codecov`` see :ref:`codecov-token`. +This option takes a space separated list of coverage providers to upload to, either ``codecov``, ``codecov-oidc`` or ``github``. As the workflows do not control how your tests are run, configuring coverage correctly may require changes to your tox.ini. The coverage collection is done inside the tox job, but the workflows handle generating reports and uploading to either codecov or github. @@ -195,6 +194,17 @@ Both of these report uploads require the ``.coverage`` file(s) to be in the root This should work for both ``coverage.py`` and ``pytest-cov``. +.. _codecov-auth: + +Authenticating with Codecov +########################### + +There are two supported ways to authenticate with Codecov, either using a ``CODECOV_TOKEN`` or using `OIDC `__. + +To use the token set the ``CODECOV_TOKEN`` environment variable or pass it as a secret to the workflow, and set ``coverage: codecov``. +To use oidc you need to give the job the ``id-token: write`` permission, we recommend you set this on the job level not the workflow level, and set ``coverage: codecov-oidc``. + + conda ^^^^^ @@ -632,15 +642,3 @@ This is a comma-separated list of factors. Default is none. If your package supports Python 3.11 and 3.12, this will generate environments like ``py311-test-cov`` and ``py312-test-cov`` instead of just ``py311`` and ``py312``. - -Secrets -~~~~~~~ - -.. _codecov-token: - -CODECOV_TOKEN -^^^^^^^^^^^^^ - -If your repository is private, in order to upload to Codecov you need to -set the ``CODECOV_TOKEN`` environment variable or pass it as a secret to -the workflow. From 33e0dade96abac32315cd05c4599820c966d21de Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 3 Jun 2026 12:47:20 +0100 Subject: [PATCH 10/16] BREAKING CHANGE: Remove --cov-report pytest flag --- .github/workflows/tox.yml | 2 +- tools/tox_matrix.py | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index bc052eea..5ea38684 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -173,7 +173,7 @@ jobs: shell: sh - run: echo $TOX_MATRIX_SCRIPT | base64 --decode > tox_matrix.py env: - TOX_MATRIX_SCRIPT: IyAvLy8gc2NyaXB0CiMgcmVxdWlyZXMtcHl0aG9uID0gIj09My4xMiIKIyBkZXBlbmRlbmNpZXMgPSBbCiMgICAgICJjbGljaz09OC4yLjEiLAojICAgICAicHl5YW1sPT02LjAuMiIsCiMgXQojIC8vLwppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHJlCmltcG9ydCB3YXJuaW5ncwoKaW1wb3J0IGNsaWNrCmltcG9ydCB5YW1sCgoKQGNsaWNrLmNvbW1hbmQoKQpAY2xpY2sub3B0aW9uKCItLWVudnMiLCBkZWZhdWx0PSIiKQpAY2xpY2sub3B0aW9uKCItLWxpYnJhcmllcyIsIGRlZmF1bHQ9IiIpCkBjbGljay5vcHRpb24oIi0tcG9zYXJncyIsIGRlZmF1bHQ9IiIpCkBjbGljay5vcHRpb24oIi0tdG94ZGVwcyIsIGRlZmF1bHQ9IiIpCkBjbGljay5vcHRpb24oIi0tdG94YXJncyIsIGRlZmF1bHQ9IiIpCkBjbGljay5vcHRpb24oIi0tcHl0ZXN0IiwgZGVmYXVsdD0idHJ1ZSIpCkBjbGljay5vcHRpb24oIi0tcHl0ZXN0LXJlc3VsdHMtc3VtbWFyeSIsIGRlZmF1bHQ9ImZhbHNlIikKQGNsaWNrLm9wdGlvbigiLS1jb3ZlcmFnZSIsIGRlZmF1bHQ9IiIpCkBjbGljay5vcHRpb24oIi0tY29uZGEiLCBkZWZhdWx0PSJhdXRvIikKQGNsaWNrLm9wdGlvbigiLS1zZXRlbnYiLCBkZWZhdWx0PSIiKQpAY2xpY2sub3B0aW9uKCItLWRpc3BsYXkiLCBkZWZhdWx0PSJmYWxzZSIpCkBjbGljay5vcHRpb24oIi0tY2FjaGUtcGF0aCIsIGRlZmF1bHQ9IiIpCkBjbGljay5vcHRpb24oIi0tY2FjaGUta2V5IiwgZGVmYXVsdD0iIikKQGNsaWNrLm9wdGlvbigiLS1jYWNoZS1yZXN0b3JlLWtleXMiLCBkZWZhdWx0PSIiKQpAY2xpY2sub3B0aW9uKCItLWFydGlmYWN0LXBhdGgiLCBkZWZhdWx0PSIiKQpAY2xpY2sub3B0aW9uKCItLWFydGlmYWN0LWFyY2hpdmUiLCBkZWZhdWx0PSJ0cnVlIikKQGNsaWNrLm9wdGlvbigiLS1hcnRpZmFjdC1pbmNsdWRlLWhpZGRlbi1maWxlcyIsIGRlZmF1bHQ9ImZhbHNlIikKQGNsaWNrLm9wdGlvbigiLS1hcnRpZmFjdC1pZi1uby1maWxlcy1mb3VuZCIsIGRlZmF1bHQ9Indhcm4iKQpAY2xpY2sub3B0aW9uKCItLXJ1bnMtb24iLCBkZWZhdWx0PSIiKQpAY2xpY2sub3B0aW9uKCItLWRlZmF1bHQtcHl0aG9uIiwgZGVmYXVsdD0iIikKQGNsaWNrLm9wdGlvbigiLS10aW1lb3V0LW1pbnV0ZXMiLCBkZWZhdWx0PSIzNjAiKQpkZWYgbG9hZF90b3hfdGFyZ2V0cygKICAgIGVudnMsCiAgICBsaWJyYXJpZXMsCiAgICBwb3NhcmdzLAogICAgdG94ZGVwcywKICAgIHRveGFyZ3MsCiAgICBweXRlc3QsCiAgICBweXRlc3RfcmVzdWx0c19zdW1tYXJ5LAogICAgY292ZXJhZ2UsCiAgICBjb25kYSwKICAgIHNldGVudiwKICAgIGRpc3BsYXksCiAgICBjYWNoZV9wYXRoLAogICAgY2FjaGVfa2V5LAogICAgY2FjaGVfcmVzdG9yZV9rZXlzLAogICAgYXJ0aWZhY3RfcGF0aCwKICAgIGFydGlmYWN0X2FyY2hpdmUsCiAgICBhcnRpZmFjdF9pbmNsdWRlX2hpZGRlbl9maWxlcywKICAgIGFydGlmYWN0X2lmX25vX2ZpbGVzX2ZvdW5kLAogICAgcnVuc19vbiwKICAgIGRlZmF1bHRfcHl0aG9uLAogICAgdGltZW91dF9taW51dGVzLAopOgogICAgIiIiU2NyaXB0IHRvIGxvYWQgdG94IHRhcmdldHMgZm9yIEdpdEh1YiBBY3Rpb25zIHdvcmtmbG93LiIiIgogICAgIyBMb2FkIGVudnMgY29uZmlnCiAgICBlbnZzID0geWFtbC5sb2FkKGVudnMucmVwbGFjZSgiXFxuIiwgIlxuIiksIExvYWRlcj15YW1sLkJhc2VMb2FkZXIpCiAgICBwcmludChqc29uLmR1bXBzKGVudnMsIGluZGVudD0yKSkKCiAgICAjIExvYWQgZ2xvYmFsIGxpYnJhcmllcyBjb25maWcKICAgIGdsb2JhbF9saWJyYXJpZXMgPSB7CiAgICAgICAgImJyZXciOiBbXSwKICAgICAgICAiYnJldy1jYXNrIjogW10sCiAgICAgICAgImFwdCI6IFtdLAogICAgICAgICJjaG9jbyI6IFtdLAogICAgfQogICAgbGlicmFyaWVzID0geWFtbC5sb2FkKGxpYnJhcmllcywgTG9hZGVyPXlhbWwuQmFzZUxvYWRlcikKICAgIGlmIGxpYnJhcmllcyBpcyBub3QgTm9uZToKICAgICAgICBnbG9iYWxfbGlicmFyaWVzLnVwZGF0ZShsaWJyYXJpZXMpCiAgICBwcmludChqc29uLmR1bXBzKGdsb2JhbF9saWJyYXJpZXMsIGluZGVudD0yKSkKCiAgICAjIERlZmF1bHQgaW1hZ2VzIHRvIHVzZSBmb3IgcnVubmVycwogICAgZGVmYXVsdF9ydW5zX29uID0gewogICAgICAgICJsaW51eCI6ICJ1YnVudHUtbGF0ZXN0IiwKICAgICAgICAibWFjb3MiOiAibWFjb3MtbGF0ZXN0IiwKICAgICAgICAid2luZG93cyI6ICJ3aW5kb3dzLWxhdGVzdCIsCiAgICB9CiAgICBjdXN0b21fcnVuc19vbiA9IHlhbWwubG9hZChydW5zX29uLCBMb2FkZXI9eWFtbC5CYXNlTG9hZGVyKQogICAgaWYgaXNpbnN0YW5jZShjdXN0b21fcnVuc19vbiwgZGljdCk6CiAgICAgICAgZGVmYXVsdF9ydW5zX29uLnVwZGF0ZShjdXN0b21fcnVuc19vbikKICAgIHByaW50KGpzb24uZHVtcHMoZGVmYXVsdF9ydW5zX29uLCBpbmRlbnQ9MikpCgogICAgIyBEZWZhdWx0IHN0cmluZyBwYXJhbWV0ZXJzIHdoaWNoIGNhbiBiZSBvdmVyd3JpdHRlbiBieSBlYWNoIGVudgogICAgc3RyaW5nX3BhcmFtZXRlcnMgPSB7CiAgICAgICAgInBvc2FyZ3MiOiBwb3NhcmdzLAogICAgICAgICJ0b3hkZXBzIjogdG94ZGVwcywKICAgICAgICAidG94YXJncyI6IHRveGFyZ3MsCiAgICAgICAgInB5dGVzdCI6IHB5dGVzdCwKICAgICAgICAicHl0ZXN0LXJlc3VsdHMtc3VtbWFyeSI6IHB5dGVzdF9yZXN1bHRzX3N1bW1hcnksCiAgICAgICAgImNvdmVyYWdlIjogY292ZXJhZ2UsCiAgICAgICAgImNvbmRhIjogY29uZGEsCiAgICAgICAgInNldGVudiI6IHNldGVudiwKICAgICAgICAiZGlzcGxheSI6IGRpc3BsYXksCiAgICAgICAgImNhY2hlLXBhdGgiOiBjYWNoZV9wYXRoLAogICAgICAgICJjYWNoZS1rZXkiOiBjYWNoZV9rZXksCiAgICAgICAgImNhY2hlLXJlc3RvcmUta2V5cyI6IGNhY2hlX3Jlc3RvcmVfa2V5cywKICAgICAgICAiYXJ0aWZhY3QtcGF0aCI6IGFydGlmYWN0X3BhdGgsCiAgICAgICAgImFydGlmYWN0LWFyY2hpdmUiOiBhcnRpZmFjdF9hcmNoaXZlLAogICAgICAgICJhcnRpZmFjdC1pbmNsdWRlLWhpZGRlbi1maWxlcyI6IGFydGlmYWN0X2luY2x1ZGVfaGlkZGVuX2ZpbGVzLAogICAgICAgICJhcnRpZmFjdC1pZi1uby1maWxlcy1mb3VuZCI6IGFydGlmYWN0X2lmX25vX2ZpbGVzX2ZvdW5kLAogICAgICAgICJ0aW1lb3V0LW1pbnV0ZXMiOiB0aW1lb3V0X21pbnV0ZXMsCiAgICB9CgogICAgIyBDcmVhdGUgbWF0cml4CiAgICBtYXRyaXggPSB7ImluY2x1ZGUiOiBbXX0KICAgIGZvciBlbnYgaW4gZW52czoKICAgICAgICBtYXRyaXhbImluY2x1ZGUiXS5hcHBlbmQoCiAgICAgICAgICAgIGdldF9tYXRyaXhfaXRlbSgKICAgICAgICAgICAgICAgIGVudiwKICAgICAgICAgICAgICAgIGdsb2JhbF9saWJyYXJpZXM9Z2xvYmFsX2xpYnJhcmllcywKICAgICAgICAgICAgICAgIGdsb2JhbF9zdHJpbmdfcGFyYW1ldGVycz1zdHJpbmdfcGFyYW1ldGVycywKICAgICAgICAgICAgICAgIHJ1bnNfb249ZGVmYXVsdF9ydW5zX29uLAogICAgICAgICAgICAgICAgZGVmYXVsdF9weXRob249ZGVmYXVsdF9weXRob24sCiAgICAgICAgICAgICkKICAgICAgICApCgogICAgIyBPdXRwdXQgbWF0cml4CiAgICBwcmludChqc29uLmR1bXBzKG1hdHJpeCwgaW5kZW50PTIpKQogICAgd2l0aCBvcGVuKG9zLmVudmlyb25bIkdJVEhVQl9PVVRQVVQiXSwgImEiKSBhcyBmOgogICAgICAgIGYud3JpdGUoZiJtYXRyaXg9e2pzb24uZHVtcHMobWF0cml4KX1cbiIpCgoKZGVmIGdldF9tYXRyaXhfaXRlbShlbnYsIGdsb2JhbF9saWJyYXJpZXMsIGdsb2JhbF9zdHJpbmdfcGFyYW1ldGVycywgcnVuc19vbiwgZGVmYXVsdF9weXRob24pOgoKICAgICMgZGVmaW5lIHNwZWMgZm9yIGVhY2ggbWF0cml4IGluY2x1ZGUgKCsgZ2xvYmFsX3N0cmluZ19wYXJhbWV0ZXJzKQogICAgaXRlbSA9IHsKICAgICAgICAib3MiOiBOb25lLAogICAgICAgICJ0b3hlbnYiOiBOb25lLAogICAgICAgICJweXRob25fdmVyc2lvbiI6IE5vbmUsCiAgICAgICAgIm5hbWUiOiBOb25lLAogICAgICAgICJweXRlc3RfZmxhZyI6IE5vbmUsCiAgICAgICAgImxpYnJhcmllc19icmV3IjogTm9uZSwKICAgICAgICAibGlicmFyaWVzX2JyZXdfY2FzayI6IE5vbmUsCiAgICAgICAgImxpYnJhcmllc19hcHQiOiBOb25lLAogICAgICAgICJsaWJyYXJpZXNfY2hvY28iOiBOb25lLAogICAgICAgICJjYWNoZS1wYXRoIjogTm9uZSwKICAgICAgICAiY2FjaGUta2V5IjogTm9uZSwKICAgICAgICAiY2FjaGUtcmVzdG9yZS1rZXlzIjogTm9uZSwKICAgICAgICAiYXJ0aWZhY3QtbmFtZSI6IE5vbmUsCiAgICAgICAgImFydGlmYWN0LXBhdGgiOiBOb25lLAogICAgICAgICJhcnRpZmFjdC1hcmNoaXZlIjogTm9uZSwKICAgICAgICAiYXJ0aWZhY3QtaW5jbHVkZS1oaWRkZW4tZmlsZXMiOiBOb25lLAogICAgICAgICJhcnRpZmFjdC1pZi1uby1maWxlcy1mb3VuZCI6IE5vbmUsCiAgICAgICAgInRpbWVvdXQtbWludXRlcyI6IE5vbmUsCiAgICB9CiAgICBmb3Igc3RyaW5nX3BhcmFtLCBkZWZhdWx0IGluIGdsb2JhbF9zdHJpbmdfcGFyYW1ldGVycy5pdGVtcygpOgogICAgICAgIGVudl92YWx1ZSA9IGVudi5nZXQoc3RyaW5nX3BhcmFtKQogICAgICAgIGl0ZW1bc3RyaW5nX3BhcmFtXSA9IGRlZmF1bHQgaWYgZW52X3ZhbHVlIGlzIE5vbmUgZWxzZSBlbnZfdmFsdWUKCiAgICAjIHNldCBvcyBhbmQgdG94ZW52CiAgICBmb3IgaywgdiBpbiBydW5zX29uLml0ZW1zKCk6CiAgICAgICAgaWYgayBpbiBlbnY6CiAgICAgICAgICAgIHBsYXRmb3JtID0gawogICAgICAgICAgICBpdGVtWyJvcyJdID0gZW52LmdldCgicnVucy1vbiIsIHYpCiAgICAgICAgICAgIGl0ZW1bInRveGVudiJdID0gZW52W2tdCiAgICBhc3NlcnQgaXRlbVsib3MiXSBpcyBub3QgTm9uZSBhbmQgaXRlbVsidG94ZW52Il0gaXMgbm90IE5vbmUKCiAgICAjIHNldCBweXRob25fdmVyc2lvbgogICAgcHl0aG9uX3ZlcnNpb24gPSBlbnYuZ2V0KCJweXRob24tdmVyc2lvbiIpCiAgICBtID0gcmUuc2VhcmNoKCJecHkoMnwzKShbMC05XSt0PykiLCBpdGVtWyJ0b3hlbnYiXSkKICAgIGlmIHB5dGhvbl92ZXJzaW9uIGlzIG5vdCBOb25lOgogICAgICAgIGl0ZW1bInB5dGhvbl92ZXJzaW9uIl0gPSBweXRob25fdmVyc2lvbgogICAgZWxpZiBtIGlzIG5vdCBOb25lOgogICAgICAgIG1ham9yLCBtaW5vciA9IG0uZ3JvdXBzKCkKICAgICAgICBpdGVtWyJweXRob25fdmVyc2lvbiJdID0gZiJ7bWFqb3J9LnttaW5vcn0iCiAgICBlbHNlOgogICAgICAgIGl0ZW1bInB5dGhvbl92ZXJzaW9uIl0gPSBlbnYuZ2V0KCJkZWZhdWx0X3B5dGhvbiIpIG9yIGRlZmF1bHRfcHl0aG9uCgogICAgIyBzZXQgbmFtZQogICAgaXRlbVsibmFtZSJdID0gZW52LmdldCgibmFtZSIpIG9yIGYie2l0ZW1bJ3RveGVudiddfSAoe2l0ZW1bJ29zJ119KSIKCiAgICAjIHNldCBhcnRpZmFjdC1uYW1lIChyZXBsYWNlIGludmFsaWQgcGF0aCBjaGFyYWN0ZXJzKQogICAgaXRlbVsiYXJ0aWZhY3QtbmFtZSJdID0gcmUuc3ViKHIiW1xcIC86PD58Kj9cIiddIiwgIi0iLCBpdGVtWyJuYW1lIl0pCiAgICBpdGVtWyJhcnRpZmFjdC1uYW1lIl0gPSByZS5zdWIociItKyIsICItIiwgaXRlbVsiYXJ0aWZhY3QtbmFtZSJdKQoKICAgICMgc2V0IHB5dGVzdF9mbGFnCiAgICBpdGVtWyJweXRlc3RfZmxhZyJdID0gIiIKICAgIHNlcCA9IHIiXFwiIGlmIHBsYXRmb3JtID09ICJ3aW5kb3dzIiBlbHNlICIvIgogICAgaWYgaXRlbVsicHl0ZXN0Il0gPT0gInRydWUiOgogICAgICAgIGlmICJjb2RlY292IiBpbiBpdGVtLmdldCgiY292ZXJhZ2UiLCAiIik6CiAgICAgICAgICAgICMgTm90ZSB0aGF0IHdlIGRvbid0IGluY2x1ZGUgLS1jb3YgaGVyZSBhcyBpZiBpdCdzIHByb3ZpZGVkIHRvIHB5dGVzdCB0d2ljZSBpdCBicmVha3MgY292IHJlcG9ydGluZy4KICAgICAgICAgICAgIyBMb3RzIG9mIHVzZXJzIG9mIHRoaXMgc3BlY2lmeSAtLWNvdiBpbiB0aGVpciB0b3guaW5pIHNvIGl0J3MgYmVlbiByZW1vdmVkIGZvciBiYWNrd2FyZHMgY29tcGF0aWJpbGl0eS4KICAgICAgICAgICAgIyBodHRwczovL2dpdGh1Yi5jb20vT3BlbkFzdHJvbm9teS9naXRodWItYWN0aW9ucy13b3JrZmxvd3MvaXNzdWVzLzM4MwogICAgICAgICAgICBpdGVtWyJweXRlc3RfZmxhZyJdICs9ICgKICAgICAgICAgICAgICAgIHJmIi0tY292LXJlcG9ydD14bWw6JHt7R0lUSFVCX1dPUktTUEFDRX19e3NlcH1jb3ZlcmFnZS54bWwgIgogICAgICAgICAgICApCgogICAgICAgIGlmIGl0ZW1bInB5dGVzdC1yZXN1bHRzLXN1bW1hcnkiXSA9PSAidHJ1ZSI6CiAgICAgICAgICAgIGl0ZW1bInB5dGVzdF9mbGFnIl0gKz0gcmYiLS1qdW5pdHhtbCAke3tHSVRIVUJfV09SS1NQQUNFfX17c2VwfXJlc3VsdHMueG1sICIKCiAgICAjIHNldCBsaWJyYXJpZXMKICAgIGVudl9saWJyYXJpZXMgPSBlbnYuZ2V0KCJsaWJyYXJpZXMiKQogICAgaWYgaXNpbnN0YW5jZShlbnZfbGlicmFyaWVzLCBzdHIpIGFuZCBsZW4oZW52X2xpYnJhcmllcy5zdHJpcCgpKSA9PSAwOgogICAgICAgIGVudl9saWJyYXJpZXMgPSB7fSAgIyBubyBsaWJyYXJpZXMgcmVxdWVzdGVkIGZvciBlbnZpcm9ubWVudAogICAgbGlicmFyaWVzID0gZ2xvYmFsX2xpYnJhcmllcyBpZiBlbnZfbGlicmFyaWVzIGlzIE5vbmUgZWxzZSBlbnZfbGlicmFyaWVzCiAgICBmb3IgbWFuYWdlciBpbiBbImJyZXciLCAiYnJld19jYXNrIiwgImFwdCIsICJjaG9jbyJdOgogICAgICAgIGl0ZW1bZiJsaWJyYXJpZXNfe21hbmFnZXJ9Il0gPSAiICIuam9pbihsaWJyYXJpZXMuZ2V0KG1hbmFnZXIsIFtdKSkKCiAgICBpZiBpdGVtWyJjb25kYSJdOgogICAgICAgIHdhcm5pbmdzLndhcm4oImBjb25kYWAgcGFyYW1ldGVyIGlzIGRlcHJlY2F0ZWQiKQoKICAgICAgICAjIHNldCAiYXV0byIgY29uZGEgdmFsdWUKICAgICAgICBpZiBpdGVtWyJjb25kYSJdID09ICJhdXRvIjoKICAgICAgICAgICAgaXRlbVsiY29uZGEiXSA9ICJ0cnVlIiBpZiAiY29uZGEiIGluIGl0ZW1bInRveGVudiJdIGVsc2UgImZhbHNlIgoKICAgICAgICAjIGluamVjdCB0b3hkZXBzIGZvciBjb25kYQogICAgICAgIGlmIGl0ZW1bImNvbmRhIl0gPT0gInRydWUiIGFuZCAidG94LWNvbmRhIiBub3QgaW4gaXRlbVsidG94ZGVwcyJdLmxvd2VyKCk6CiAgICAgICAgICAgIGl0ZW1bInRveGRlcHMiXSA9ICgidG94LWNvbmRhICIgKyBpdGVtWyJ0b3hkZXBzIl0pLnN0cmlwKCkKCiAgICAjIG1ha2UgdGltZW91dC1taW51dGVzIGEgbnVtYmVyCiAgICBpdGVtWyJ0aW1lb3V0LW1pbnV0ZXMiXSA9IGludChpdGVtWyJ0aW1lb3V0LW1pbnV0ZXMiXSkKCiAgICAjIHZlcmlmeSB2YWx1ZXMKICAgIGFzc2VydCBpdGVtWyJweXRlc3QiXSBpbiB7InRydWUiLCAiZmFsc2UifQogICAgYXNzZXJ0IGl0ZW1bImNvbmRhIl0gaW4geyJ0cnVlIiwgImZhbHNlIn0KICAgIGFzc2VydCBpdGVtWyJkaXNwbGF5Il0gaW4geyJ0cnVlIiwgImZhbHNlIn0KCiAgICByZXR1cm4gaXRlbQoKCmlmIF9fbmFtZV9fID09ICJfX21haW5fXyI6CiAgICBsb2FkX3RveF90YXJnZXRzKCkK + TOX_MATRIX_SCRIPT: IyAvLy8gc2NyaXB0CiMgcmVxdWlyZXMtcHl0aG9uID0gIj09My4xMiIKIyBkZXBlbmRlbmNpZXMgPSBbCiMgICAgICJjbGljaz09OC4yLjEiLAojICAgICAicHl5YW1sPT02LjAuMiIsCiMgXQojIC8vLwppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHJlCmltcG9ydCB3YXJuaW5ncwoKaW1wb3J0IGNsaWNrCmltcG9ydCB5YW1sCgoKQGNsaWNrLmNvbW1hbmQoKQpAY2xpY2sub3B0aW9uKCItLWVudnMiLCBkZWZhdWx0PSIiKQpAY2xpY2sub3B0aW9uKCItLWxpYnJhcmllcyIsIGRlZmF1bHQ9IiIpCkBjbGljay5vcHRpb24oIi0tcG9zYXJncyIsIGRlZmF1bHQ9IiIpCkBjbGljay5vcHRpb24oIi0tdG94ZGVwcyIsIGRlZmF1bHQ9IiIpCkBjbGljay5vcHRpb24oIi0tdG94YXJncyIsIGRlZmF1bHQ9IiIpCkBjbGljay5vcHRpb24oIi0tcHl0ZXN0IiwgZGVmYXVsdD0idHJ1ZSIpCkBjbGljay5vcHRpb24oIi0tcHl0ZXN0LXJlc3VsdHMtc3VtbWFyeSIsIGRlZmF1bHQ9ImZhbHNlIikKQGNsaWNrLm9wdGlvbigiLS1jb3ZlcmFnZSIsIGRlZmF1bHQ9IiIpCkBjbGljay5vcHRpb24oIi0tY29uZGEiLCBkZWZhdWx0PSJhdXRvIikKQGNsaWNrLm9wdGlvbigiLS1zZXRlbnYiLCBkZWZhdWx0PSIiKQpAY2xpY2sub3B0aW9uKCItLWRpc3BsYXkiLCBkZWZhdWx0PSJmYWxzZSIpCkBjbGljay5vcHRpb24oIi0tY2FjaGUtcGF0aCIsIGRlZmF1bHQ9IiIpCkBjbGljay5vcHRpb24oIi0tY2FjaGUta2V5IiwgZGVmYXVsdD0iIikKQGNsaWNrLm9wdGlvbigiLS1jYWNoZS1yZXN0b3JlLWtleXMiLCBkZWZhdWx0PSIiKQpAY2xpY2sub3B0aW9uKCItLWFydGlmYWN0LXBhdGgiLCBkZWZhdWx0PSIiKQpAY2xpY2sub3B0aW9uKCItLWFydGlmYWN0LWFyY2hpdmUiLCBkZWZhdWx0PSJ0cnVlIikKQGNsaWNrLm9wdGlvbigiLS1hcnRpZmFjdC1pbmNsdWRlLWhpZGRlbi1maWxlcyIsIGRlZmF1bHQ9ImZhbHNlIikKQGNsaWNrLm9wdGlvbigiLS1hcnRpZmFjdC1pZi1uby1maWxlcy1mb3VuZCIsIGRlZmF1bHQ9Indhcm4iKQpAY2xpY2sub3B0aW9uKCItLXJ1bnMtb24iLCBkZWZhdWx0PSIiKQpAY2xpY2sub3B0aW9uKCItLWRlZmF1bHQtcHl0aG9uIiwgZGVmYXVsdD0iIikKQGNsaWNrLm9wdGlvbigiLS10aW1lb3V0LW1pbnV0ZXMiLCBkZWZhdWx0PSIzNjAiKQpkZWYgbG9hZF90b3hfdGFyZ2V0cygKICAgIGVudnMsCiAgICBsaWJyYXJpZXMsCiAgICBwb3NhcmdzLAogICAgdG94ZGVwcywKICAgIHRveGFyZ3MsCiAgICBweXRlc3QsCiAgICBweXRlc3RfcmVzdWx0c19zdW1tYXJ5LAogICAgY292ZXJhZ2UsCiAgICBjb25kYSwKICAgIHNldGVudiwKICAgIGRpc3BsYXksCiAgICBjYWNoZV9wYXRoLAogICAgY2FjaGVfa2V5LAogICAgY2FjaGVfcmVzdG9yZV9rZXlzLAogICAgYXJ0aWZhY3RfcGF0aCwKICAgIGFydGlmYWN0X2FyY2hpdmUsCiAgICBhcnRpZmFjdF9pbmNsdWRlX2hpZGRlbl9maWxlcywKICAgIGFydGlmYWN0X2lmX25vX2ZpbGVzX2ZvdW5kLAogICAgcnVuc19vbiwKICAgIGRlZmF1bHRfcHl0aG9uLAogICAgdGltZW91dF9taW51dGVzLAopOgogICAgIiIiU2NyaXB0IHRvIGxvYWQgdG94IHRhcmdldHMgZm9yIEdpdEh1YiBBY3Rpb25zIHdvcmtmbG93LiIiIgogICAgIyBMb2FkIGVudnMgY29uZmlnCiAgICBlbnZzID0geWFtbC5sb2FkKGVudnMucmVwbGFjZSgiXFxuIiwgIlxuIiksIExvYWRlcj15YW1sLkJhc2VMb2FkZXIpCiAgICBwcmludChqc29uLmR1bXBzKGVudnMsIGluZGVudD0yKSkKCiAgICAjIExvYWQgZ2xvYmFsIGxpYnJhcmllcyBjb25maWcKICAgIGdsb2JhbF9saWJyYXJpZXMgPSB7CiAgICAgICAgImJyZXciOiBbXSwKICAgICAgICAiYnJldy1jYXNrIjogW10sCiAgICAgICAgImFwdCI6IFtdLAogICAgICAgICJjaG9jbyI6IFtdLAogICAgfQogICAgbGlicmFyaWVzID0geWFtbC5sb2FkKGxpYnJhcmllcywgTG9hZGVyPXlhbWwuQmFzZUxvYWRlcikKICAgIGlmIGxpYnJhcmllcyBpcyBub3QgTm9uZToKICAgICAgICBnbG9iYWxfbGlicmFyaWVzLnVwZGF0ZShsaWJyYXJpZXMpCiAgICBwcmludChqc29uLmR1bXBzKGdsb2JhbF9saWJyYXJpZXMsIGluZGVudD0yKSkKCiAgICAjIERlZmF1bHQgaW1hZ2VzIHRvIHVzZSBmb3IgcnVubmVycwogICAgZGVmYXVsdF9ydW5zX29uID0gewogICAgICAgICJsaW51eCI6ICJ1YnVudHUtbGF0ZXN0IiwKICAgICAgICAibWFjb3MiOiAibWFjb3MtbGF0ZXN0IiwKICAgICAgICAid2luZG93cyI6ICJ3aW5kb3dzLWxhdGVzdCIsCiAgICB9CiAgICBjdXN0b21fcnVuc19vbiA9IHlhbWwubG9hZChydW5zX29uLCBMb2FkZXI9eWFtbC5CYXNlTG9hZGVyKQogICAgaWYgaXNpbnN0YW5jZShjdXN0b21fcnVuc19vbiwgZGljdCk6CiAgICAgICAgZGVmYXVsdF9ydW5zX29uLnVwZGF0ZShjdXN0b21fcnVuc19vbikKICAgIHByaW50KGpzb24uZHVtcHMoZGVmYXVsdF9ydW5zX29uLCBpbmRlbnQ9MikpCgogICAgIyBEZWZhdWx0IHN0cmluZyBwYXJhbWV0ZXJzIHdoaWNoIGNhbiBiZSBvdmVyd3JpdHRlbiBieSBlYWNoIGVudgogICAgc3RyaW5nX3BhcmFtZXRlcnMgPSB7CiAgICAgICAgInBvc2FyZ3MiOiBwb3NhcmdzLAogICAgICAgICJ0b3hkZXBzIjogdG94ZGVwcywKICAgICAgICAidG94YXJncyI6IHRveGFyZ3MsCiAgICAgICAgInB5dGVzdCI6IHB5dGVzdCwKICAgICAgICAicHl0ZXN0LXJlc3VsdHMtc3VtbWFyeSI6IHB5dGVzdF9yZXN1bHRzX3N1bW1hcnksCiAgICAgICAgImNvdmVyYWdlIjogY292ZXJhZ2UsCiAgICAgICAgImNvbmRhIjogY29uZGEsCiAgICAgICAgInNldGVudiI6IHNldGVudiwKICAgICAgICAiZGlzcGxheSI6IGRpc3BsYXksCiAgICAgICAgImNhY2hlLXBhdGgiOiBjYWNoZV9wYXRoLAogICAgICAgICJjYWNoZS1rZXkiOiBjYWNoZV9rZXksCiAgICAgICAgImNhY2hlLXJlc3RvcmUta2V5cyI6IGNhY2hlX3Jlc3RvcmVfa2V5cywKICAgICAgICAiYXJ0aWZhY3QtcGF0aCI6IGFydGlmYWN0X3BhdGgsCiAgICAgICAgImFydGlmYWN0LWFyY2hpdmUiOiBhcnRpZmFjdF9hcmNoaXZlLAogICAgICAgICJhcnRpZmFjdC1pbmNsdWRlLWhpZGRlbi1maWxlcyI6IGFydGlmYWN0X2luY2x1ZGVfaGlkZGVuX2ZpbGVzLAogICAgICAgICJhcnRpZmFjdC1pZi1uby1maWxlcy1mb3VuZCI6IGFydGlmYWN0X2lmX25vX2ZpbGVzX2ZvdW5kLAogICAgICAgICJ0aW1lb3V0LW1pbnV0ZXMiOiB0aW1lb3V0X21pbnV0ZXMsCiAgICB9CgogICAgIyBDcmVhdGUgbWF0cml4CiAgICBtYXRyaXggPSB7ImluY2x1ZGUiOiBbXX0KICAgIGZvciBlbnYgaW4gZW52czoKICAgICAgICBtYXRyaXhbImluY2x1ZGUiXS5hcHBlbmQoCiAgICAgICAgICAgIGdldF9tYXRyaXhfaXRlbSgKICAgICAgICAgICAgICAgIGVudiwKICAgICAgICAgICAgICAgIGdsb2JhbF9saWJyYXJpZXM9Z2xvYmFsX2xpYnJhcmllcywKICAgICAgICAgICAgICAgIGdsb2JhbF9zdHJpbmdfcGFyYW1ldGVycz1zdHJpbmdfcGFyYW1ldGVycywKICAgICAgICAgICAgICAgIHJ1bnNfb249ZGVmYXVsdF9ydW5zX29uLAogICAgICAgICAgICAgICAgZGVmYXVsdF9weXRob249ZGVmYXVsdF9weXRob24sCiAgICAgICAgICAgICkKICAgICAgICApCgogICAgIyBPdXRwdXQgbWF0cml4CiAgICBwcmludChqc29uLmR1bXBzKG1hdHJpeCwgaW5kZW50PTIpKQogICAgd2l0aCBvcGVuKG9zLmVudmlyb25bIkdJVEhVQl9PVVRQVVQiXSwgImEiKSBhcyBmOgogICAgICAgIGYud3JpdGUoZiJtYXRyaXg9e2pzb24uZHVtcHMobWF0cml4KX1cbiIpCgoKZGVmIGdldF9tYXRyaXhfaXRlbShlbnYsIGdsb2JhbF9saWJyYXJpZXMsIGdsb2JhbF9zdHJpbmdfcGFyYW1ldGVycywgcnVuc19vbiwgZGVmYXVsdF9weXRob24pOgoKICAgICMgZGVmaW5lIHNwZWMgZm9yIGVhY2ggbWF0cml4IGluY2x1ZGUgKCsgZ2xvYmFsX3N0cmluZ19wYXJhbWV0ZXJzKQogICAgaXRlbSA9IHsKICAgICAgICAib3MiOiBOb25lLAogICAgICAgICJ0b3hlbnYiOiBOb25lLAogICAgICAgICJweXRob25fdmVyc2lvbiI6IE5vbmUsCiAgICAgICAgIm5hbWUiOiBOb25lLAogICAgICAgICJweXRlc3RfZmxhZyI6IE5vbmUsCiAgICAgICAgImxpYnJhcmllc19icmV3IjogTm9uZSwKICAgICAgICAibGlicmFyaWVzX2JyZXdfY2FzayI6IE5vbmUsCiAgICAgICAgImxpYnJhcmllc19hcHQiOiBOb25lLAogICAgICAgICJsaWJyYXJpZXNfY2hvY28iOiBOb25lLAogICAgICAgICJjYWNoZS1wYXRoIjogTm9uZSwKICAgICAgICAiY2FjaGUta2V5IjogTm9uZSwKICAgICAgICAiY2FjaGUtcmVzdG9yZS1rZXlzIjogTm9uZSwKICAgICAgICAiYXJ0aWZhY3QtbmFtZSI6IE5vbmUsCiAgICAgICAgImFydGlmYWN0LXBhdGgiOiBOb25lLAogICAgICAgICJhcnRpZmFjdC1hcmNoaXZlIjogTm9uZSwKICAgICAgICAiYXJ0aWZhY3QtaW5jbHVkZS1oaWRkZW4tZmlsZXMiOiBOb25lLAogICAgICAgICJhcnRpZmFjdC1pZi1uby1maWxlcy1mb3VuZCI6IE5vbmUsCiAgICAgICAgInRpbWVvdXQtbWludXRlcyI6IE5vbmUsCiAgICB9CiAgICBmb3Igc3RyaW5nX3BhcmFtLCBkZWZhdWx0IGluIGdsb2JhbF9zdHJpbmdfcGFyYW1ldGVycy5pdGVtcygpOgogICAgICAgIGVudl92YWx1ZSA9IGVudi5nZXQoc3RyaW5nX3BhcmFtKQogICAgICAgIGl0ZW1bc3RyaW5nX3BhcmFtXSA9IGRlZmF1bHQgaWYgZW52X3ZhbHVlIGlzIE5vbmUgZWxzZSBlbnZfdmFsdWUKCiAgICAjIHNldCBvcyBhbmQgdG94ZW52CiAgICBmb3IgaywgdiBpbiBydW5zX29uLml0ZW1zKCk6CiAgICAgICAgaWYgayBpbiBlbnY6CiAgICAgICAgICAgIHBsYXRmb3JtID0gawogICAgICAgICAgICBpdGVtWyJvcyJdID0gZW52LmdldCgicnVucy1vbiIsIHYpCiAgICAgICAgICAgIGl0ZW1bInRveGVudiJdID0gZW52W2tdCiAgICBhc3NlcnQgaXRlbVsib3MiXSBpcyBub3QgTm9uZSBhbmQgaXRlbVsidG94ZW52Il0gaXMgbm90IE5vbmUKCiAgICAjIHNldCBweXRob25fdmVyc2lvbgogICAgcHl0aG9uX3ZlcnNpb24gPSBlbnYuZ2V0KCJweXRob24tdmVyc2lvbiIpCiAgICBtID0gcmUuc2VhcmNoKCJecHkoMnwzKShbMC05XSt0PykiLCBpdGVtWyJ0b3hlbnYiXSkKICAgIGlmIHB5dGhvbl92ZXJzaW9uIGlzIG5vdCBOb25lOgogICAgICAgIGl0ZW1bInB5dGhvbl92ZXJzaW9uIl0gPSBweXRob25fdmVyc2lvbgogICAgZWxpZiBtIGlzIG5vdCBOb25lOgogICAgICAgIG1ham9yLCBtaW5vciA9IG0uZ3JvdXBzKCkKICAgICAgICBpdGVtWyJweXRob25fdmVyc2lvbiJdID0gZiJ7bWFqb3J9LnttaW5vcn0iCiAgICBlbHNlOgogICAgICAgIGl0ZW1bInB5dGhvbl92ZXJzaW9uIl0gPSBlbnYuZ2V0KCJkZWZhdWx0X3B5dGhvbiIpIG9yIGRlZmF1bHRfcHl0aG9uCgogICAgIyBzZXQgbmFtZQogICAgaXRlbVsibmFtZSJdID0gZW52LmdldCgibmFtZSIpIG9yIGYie2l0ZW1bJ3RveGVudiddfSAoe2l0ZW1bJ29zJ119KSIKCiAgICAjIHNldCBhcnRpZmFjdC1uYW1lIChyZXBsYWNlIGludmFsaWQgcGF0aCBjaGFyYWN0ZXJzKQogICAgaXRlbVsiYXJ0aWZhY3QtbmFtZSJdID0gcmUuc3ViKHIiW1xcIC86PD58Kj9cIiddIiwgIi0iLCBpdGVtWyJuYW1lIl0pCiAgICBpdGVtWyJhcnRpZmFjdC1uYW1lIl0gPSByZS5zdWIociItKyIsICItIiwgaXRlbVsiYXJ0aWZhY3QtbmFtZSJdKQoKICAgICMgc2V0IHB5dGVzdF9mbGFnCiAgICBpdGVtWyJweXRlc3RfZmxhZyJdID0gIiIKICAgIHNlcCA9IHIiXFwiIGlmIHBsYXRmb3JtID09ICJ3aW5kb3dzIiBlbHNlICIvIgogICAgaWYgaXRlbVsicHl0ZXN0Il0gPT0gInRydWUiOgogICAgICAgIGlmIGl0ZW1bInB5dGVzdC1yZXN1bHRzLXN1bW1hcnkiXSA9PSAidHJ1ZSI6CiAgICAgICAgICAgIGl0ZW1bInB5dGVzdF9mbGFnIl0gKz0gcmYiLS1qdW5pdHhtbCAke3tHSVRIVUJfV09SS1NQQUNFfX17c2VwfXJlc3VsdHMueG1sICIKCiAgICAjIHNldCBsaWJyYXJpZXMKICAgIGVudl9saWJyYXJpZXMgPSBlbnYuZ2V0KCJsaWJyYXJpZXMiKQogICAgaWYgaXNpbnN0YW5jZShlbnZfbGlicmFyaWVzLCBzdHIpIGFuZCBsZW4oZW52X2xpYnJhcmllcy5zdHJpcCgpKSA9PSAwOgogICAgICAgIGVudl9saWJyYXJpZXMgPSB7fSAgIyBubyBsaWJyYXJpZXMgcmVxdWVzdGVkIGZvciBlbnZpcm9ubWVudAogICAgbGlicmFyaWVzID0gZ2xvYmFsX2xpYnJhcmllcyBpZiBlbnZfbGlicmFyaWVzIGlzIE5vbmUgZWxzZSBlbnZfbGlicmFyaWVzCiAgICBmb3IgbWFuYWdlciBpbiBbImJyZXciLCAiYnJld19jYXNrIiwgImFwdCIsICJjaG9jbyJdOgogICAgICAgIGl0ZW1bZiJsaWJyYXJpZXNfe21hbmFnZXJ9Il0gPSAiICIuam9pbihsaWJyYXJpZXMuZ2V0KG1hbmFnZXIsIFtdKSkKCiAgICBpZiBpdGVtWyJjb25kYSJdOgogICAgICAgIHdhcm5pbmdzLndhcm4oImBjb25kYWAgcGFyYW1ldGVyIGlzIGRlcHJlY2F0ZWQiKQoKICAgICAgICAjIHNldCAiYXV0byIgY29uZGEgdmFsdWUKICAgICAgICBpZiBpdGVtWyJjb25kYSJdID09ICJhdXRvIjoKICAgICAgICAgICAgaXRlbVsiY29uZGEiXSA9ICJ0cnVlIiBpZiAiY29uZGEiIGluIGl0ZW1bInRveGVudiJdIGVsc2UgImZhbHNlIgoKICAgICAgICAjIGluamVjdCB0b3hkZXBzIGZvciBjb25kYQogICAgICAgIGlmIGl0ZW1bImNvbmRhIl0gPT0gInRydWUiIGFuZCAidG94LWNvbmRhIiBub3QgaW4gaXRlbVsidG94ZGVwcyJdLmxvd2VyKCk6CiAgICAgICAgICAgIGl0ZW1bInRveGRlcHMiXSA9ICgidG94LWNvbmRhICIgKyBpdGVtWyJ0b3hkZXBzIl0pLnN0cmlwKCkKCiAgICAjIG1ha2UgdGltZW91dC1taW51dGVzIGEgbnVtYmVyCiAgICBpdGVtWyJ0aW1lb3V0LW1pbnV0ZXMiXSA9IGludChpdGVtWyJ0aW1lb3V0LW1pbnV0ZXMiXSkKCiAgICAjIHZlcmlmeSB2YWx1ZXMKICAgIGFzc2VydCBpdGVtWyJweXRlc3QiXSBpbiB7InRydWUiLCAiZmFsc2UifQogICAgYXNzZXJ0IGl0ZW1bImNvbmRhIl0gaW4geyJ0cnVlIiwgImZhbHNlIn0KICAgIGFzc2VydCBpdGVtWyJkaXNwbGF5Il0gaW4geyJ0cnVlIiwgImZhbHNlIn0KCiAgICByZXR1cm4gaXRlbQoKCmlmIF9fbmFtZV9fID09ICJfX21haW5fXyI6CiAgICBsb2FkX3RveF90YXJnZXRzKCkK - run: cat tox_matrix.py - id: set-outputs run: | # zizmor: ignore[template-injection] diff --git a/tools/tox_matrix.py b/tools/tox_matrix.py index 9ceed1ad..a1153cf0 100644 --- a/tools/tox_matrix.py +++ b/tools/tox_matrix.py @@ -184,14 +184,6 @@ def get_matrix_item(env, global_libraries, global_string_parameters, runs_on, de item["pytest_flag"] = "" sep = r"\\" if platform == "windows" else "/" if item["pytest"] == "true": - if "codecov" in item.get("coverage", ""): - # Note that we don't include --cov here as if it's provided to pytest twice it breaks cov reporting. - # Lots of users of this specify --cov in their tox.ini so it's been removed for backwards compatibility. - # https://github.com/OpenAstronomy/github-actions-workflows/issues/383 - item["pytest_flag"] += ( - rf"--cov-report=xml:${{GITHUB_WORKSPACE}}{sep}coverage.xml " - ) - if item["pytest-results-summary"] == "true": item["pytest_flag"] += rf"--junitxml ${{GITHUB_WORKSPACE}}{sep}results.xml " From 63806a19fba1d8a0903f60b507b4664bef2630b4 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 3 Jun 2026 12:56:01 +0100 Subject: [PATCH 11/16] Try to reproduce sunpy fail --- .github/workflows/test_tox.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test_tox.yml b/.github/workflows/test_tox.yml index ad0b7d23..c718ef25 100644 --- a/.github/workflows/test_tox.yml +++ b/.github/workflows/test_tox.yml @@ -167,6 +167,12 @@ jobs: # Now also test it works with pytest-cov - linux: py314 posargs: --cov=test_package + + test_coverage_github_tmpdir: + uses: ./.github/workflows/tox.yml + with: + coverage: github + envs: | # And test it works if the pytest command isn't run in the toxinidir - linux: py314-tmpdir posargs: --cov=test_package From 5dfa618e02a72539c9c20084c00388883b9ced8b Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 3 Jun 2026 13:14:24 +0100 Subject: [PATCH 12/16] Add docs on source mapping for coverage: github --- docs/source/tox.rst | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/source/tox.rst b/docs/source/tox.rst index 75a7503b..0d0692ca 100644 --- a/docs/source/tox.rst +++ b/docs/source/tox.rst @@ -187,13 +187,26 @@ If uploading to GitHub all these reports will be collected for all your jobs and If uploading to codecov they will be combined and uploaded at the end of each job. Both of these report uploads require the ``.coverage`` file(s) to be in the root of the GitHub Actions workspace for processing at the end of the job. -**If you are running your tests in a temporary directory**, the easiest way to achieve this is to set the following option in your tox.ini:: + +**If you are running your tests in a temporary directory**, more configuration may be required to configure the coverage collection correctly. + +The first step is to write the ``.coverage`` file to the same dir as the ``tox.ini`` to do this set the following option in your tox.ini:: setenv = COVERAGE_FILE={toxinidir}/.coverage This should work for both ``coverage.py`` and ``pytest-cov``. +The next thing you may need to configure is `source mapping `__ to map the source code in the temporary directory to the source code in the repository checkout. +This is normally only needed when using ``coverage: github`` as the report is generated in a separate Github Actions job meaning the temporary directory is no longer present. + +An example of this is in the ``.coveragerc`` file:: + + [paths] + source = + test_package/ + .tox/**/test_package + .. _codecov-auth: Authenticating with Codecov From fc76dfac659d466e89713b63e511425673f4dc85 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 3 Jun 2026 13:20:00 +0100 Subject: [PATCH 13/16] Ignore errors in coverage combine No combining may be needed --- .github/workflows/tox.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 5ea38684..e09cbe56 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -304,7 +304,8 @@ jobs: # Even if tox fails, upload coverage if: ${{ (success() || failure()) && contains(matrix.coverage, 'codecov') && matrix.pytest == 'true' }} run: | - uvx coverage combine + # combine will exit 1 if it's a no-op which is ok for us + uvx coverage combine; uvx coverage xml -i - name: Upload to Codecov @@ -353,7 +354,8 @@ jobs: ignore-empty-workdir: "true" - name: generate coverage report run: | - uvx coverage combine + # combine will exit 1 if it's a no-op which is ok for us + uvx coverage combine; uvx coverage report -i -m --format=markdown >> $GITHUB_STEP_SUMMARY - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: From 9d321c76a49d39b2ac18dfcdfe5da4950a863626 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 3 Jun 2026 13:28:03 +0100 Subject: [PATCH 14/16] Split combine and report --- .github/workflows/tox.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index e09cbe56..bb88717b 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -343,20 +343,24 @@ jobs: submodules: ${{ inputs.submodules }} ref: ${{ inputs.checkout_ref }} persist-credentials: false - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - name: download coverage reports uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: coverage-data-${{ github.run_id }}-* merge-multiple: true + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # 8.1.0 with: ignore-empty-workdir: "true" - - name: generate coverage report - run: | - # combine will exit 1 if it's a no-op which is ok for us - uvx coverage combine; - uvx coverage report -i -m --format=markdown >> $GITHUB_STEP_SUMMARY + + - name: Combine Coverage + continue-on-error: true + run: uvx coverage combine + + - name: Combine Coverage + run: uvx coverage report -i -m --format=markdown >> $GITHUB_STEP_SUMMARY + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverage-report-${{ hashFiles('.coverage') }} From e0c57e958c5567d0120d49e55cd8ee8165b5c690 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 3 Jun 2026 17:45:42 +0100 Subject: [PATCH 15/16] Some more tweaks --- .github/workflows/test_tox.yml | 9 +++------ .github/workflows/tox.yml | 14 ++++++++++++-- tox.ini | 4 ++-- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test_tox.yml b/.github/workflows/test_tox.yml index c718ef25..66dc7f7b 100644 --- a/.github/workflows/test_tox.yml +++ b/.github/workflows/test_tox.yml @@ -167,12 +167,6 @@ jobs: # Now also test it works with pytest-cov - linux: py314 posargs: --cov=test_package - - test_coverage_github_tmpdir: - uses: ./.github/workflows/tox.yml - with: - coverage: github - envs: | # And test it works if the pytest command isn't run in the toxinidir - linux: py314-tmpdir posargs: --cov=test_package @@ -192,6 +186,9 @@ jobs: # And test it works if the pytest command isn't run in the toxinidir - linux: py314-tmpdir posargs: --cov=test_package + # This job should fail and print a warning + - linux: py314-tmpdir_warning + posargs: --cov=test_package test_artifact_upload: uses: ./.github/workflows/tox.yml diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index bb88717b..0aa7c38d 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -300,12 +300,22 @@ jobs: with: paths: "**/results.xml" + - name: Combine converage reports for Codecov + # Even if tox fails, upload coverage + if: ${{ (success() || failure()) && contains(matrix.coverage, 'codecov') && matrix.pytest == 'true' }} + # If there's not multiple files to combine then this will error but that's ok + continue-on-error: true + run: | + uvx coverage combine + - name: Generate xml report for Codecov # Even if tox fails, upload coverage if: ${{ (success() || failure()) && contains(matrix.coverage, 'codecov') && matrix.pytest == 'true' }} run: | - # combine will exit 1 if it's a no-op which is ok for us - uvx coverage combine; + if [ ! -f ".coverage" ]; then + echo "> [!WARNING]" >> $GITHUB_STEP_SUMMARY + echo "> No `.coverage` file was found in the root directory, this is now a requirement for uploading coverage to Codecov. See the documentation for more information: https://github-actions-workflows.openastronomy.org/en/stable/tox.html#coverage" >> $GITHUB_STEP_SUMMARY + fi uvx coverage xml -i - name: Upload to Codecov diff --git a/tox.ini b/tox.ini index d80edb6c..c4d2ae92 100644 --- a/tox.ini +++ b/tox.ini @@ -59,7 +59,7 @@ description = verify pep8 deps = ruff commands = ruff check . -[testenv:py3{10-16}{,-conda,-tmpdir}] +[testenv:py3{10-16}{,-conda,-tmpdir,-tmpdir_warning}] description = run pytest skip_install = false dependency_groups = @@ -67,7 +67,7 @@ dependency_groups = covcheck conda_deps = pytest changedir = - tmpdir: .tmp/{envname} + tmpdir,tmpdir_warning: .tmp/{envname} setenv = tmpdir: COVERAGE_FILE={toxinidir}/.coverage commands = From a1cc672d83450fa190fdcb4650e46728d98d176e Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 3 Jun 2026 17:12:21 +0000 Subject: [PATCH 16/16] Update docs/source/tox.rst Co-authored-by: P. L. Lim <2090236+pllim@users.noreply.github.com> --- docs/source/tox.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/tox.rst b/docs/source/tox.rst index 0d0692ca..56c53489 100644 --- a/docs/source/tox.rst +++ b/docs/source/tox.rst @@ -166,7 +166,7 @@ coverage The coverage option controls how coverage reports are made and processed after the tox job has completed. The default is to not upload coverage reports. This option has no effect if ``pytest`` is ``false``. -This option takes a space separated list of coverage providers to upload to, either ``codecov``, ``codecov-oidc`` or ``github``. +This option takes a space separated list of coverage providers to upload to, either ``codecov``, ``codecov-oidc``, or ``github``. As the workflows do not control how your tests are run, configuring coverage correctly may require changes to your tox.ini. The coverage collection is done inside the tox job, but the workflows handle generating reports and uploading to either codecov or github.