From 37016013fb09b278e60eb5f6b95eb93a2806d08f Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 4 Dec 2025 13:54:01 +0100 Subject: [PATCH 1/5] CI: refactor compile job into reusable workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Splits the compile matrix job into a reusable workflow that can be called with specific build configurations. This improves maintainability by centralizing build logic and enables parallel test execution by allowing jobs to depend on specific builds rather than all builds. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Changelog-None --- .github/workflows/build-cln.yaml | 83 ++++++++++++++++++++++ .github/workflows/ci.yaml | 118 +++++++++++++------------------ 2 files changed, 132 insertions(+), 69 deletions(-) create mode 100644 .github/workflows/build-cln.yaml diff --git a/.github/workflows/build-cln.yaml b/.github/workflows/build-cln.yaml new file mode 100644 index 000000000000..13d1343d4a85 --- /dev/null +++ b/.github/workflows/build-cln.yaml @@ -0,0 +1,83 @@ +--- +name: Build CLN +on: + workflow_call: + inputs: + cfg: + description: 'Configuration name (e.g., compile-gcc)' + required: true + type: string + compiler: + description: 'Compiler to use (gcc or clang)' + required: true + type: string + valgrind: + description: 'Enable valgrind (0 or 1)' + required: false + type: number + default: 1 + asan: + description: 'Enable address sanitizer (0 or 1)' + required: false + type: number + default: 0 + ubsan: + description: 'Enable undefined behavior sanitizer (0 or 1)' + required: false + type: number + default: 0 + coptflags: + description: 'Additional compiler optimization flags' + required: false + type: string + default: '' + +env: + RUST_PROFILE: release + SLOW_MACHINE: 1 + +jobs: + build: + name: Build CLN ${{ inputs.cfg }} + runs-on: ubuntu-22.04 + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Install dependencies + run: | + bash -x .github/scripts/setup.sh + + - name: Build + env: + COMPILER: ${{ inputs.compiler }} + ASAN: ${{ inputs.asan }} + UBSAN: ${{ inputs.ubsan }} + VALGRIND: ${{ inputs.valgrind }} + COMPAT: 1 + CFG: ${{ inputs.cfg }} + run: | + set -e + ./configure --enable-debugbuild CC="$COMPILER" ${{ inputs.coptflags }} + + uv run make -j $(nproc) testpack.tar.bz2 + + # Rename now so we don't clash + mv testpack.tar.bz2 cln-${CFG}.tar.bz2 + + - name: Check rust packages + run: cargo test --all + + - uses: actions/upload-artifact@v4 + with: + name: cln-${{ inputs.cfg }}.tar.bz2 + path: cln-${{ inputs.cfg }}.tar.bz2 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cd400b61eb26..3794d18a8e04 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -97,70 +97,48 @@ jobs: - name: Check docs run: uv run make check-doc - compile: - name: Compile CLN ${{ matrix.cfg }} - runs-on: ubuntu-22.04 - timeout-minutes: 30 + compile-gcc: + name: Compile CLN (GCC) needs: - prebuild - strategy: - fail-fast: true - matrix: - include: - - CFG: compile-gcc - VALGRIND: 1 - COMPILER: gcc - - CFG: compile-gcc-O3 - VALGRIND: 1 - COMPILER: gcc - COPTFLAGS_VAR: COPTFLAGS="-O3 -Werror" - # While we're at it let's try to compile with clang - - CFG: compile-clang - VALGRIND: 1 - COMPILER: clang - - CFG: compile-clang-sanitizers - COMPILER: clang - ASAN: 1 - UBSAN: 1 - VALGRIND: 0 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Install dependencies - run: | - bash -x .github/scripts/setup.sh - - - name: Build - env: - COMPILER: ${{ matrix.COMPILER }} - ASAN: ${{ matrix.ASAN }} - UBSAN: ${{ matrix.UBSAN }} - VALGRIND: ${{ matrix.VALGRIND }} - COMPAT: 1 - CFG: ${{ matrix.CFG }} - run: | - set -e - ./configure --enable-debugbuild CC="$COMPILER" ${{ matrix.COPTFLAGS_VAR }} - - uv run make -j $(nproc) testpack.tar.bz2 - - # Rename now so we don't clash - mv testpack.tar.bz2 cln-${CFG}.tar.bz2 - - name: Check rust packages - run: cargo test --all - - uses: actions/upload-artifact@v4 - with: - name: cln-${{ matrix.CFG }}.tar.bz2 - path: cln-${{ matrix.CFG }}.tar.bz2 + uses: ./.github/workflows/build-cln.yaml + with: + cfg: compile-gcc + compiler: gcc + valgrind: 1 + + compile-gcc-O3: + name: Compile CLN (GCC -O3) + needs: + - prebuild + uses: ./.github/workflows/build-cln.yaml + with: + cfg: compile-gcc-O3 + compiler: gcc + valgrind: 1 + coptflags: 'COPTFLAGS="-O3 -Werror"' + + compile-clang: + name: Compile CLN (Clang) + needs: + - prebuild + uses: ./.github/workflows/build-cln.yaml + with: + cfg: compile-clang + compiler: clang + valgrind: 1 + + compile-clang-sanitizers: + name: Compile CLN (Clang with sanitizers) + needs: + - prebuild + uses: ./.github/workflows/build-cln.yaml + with: + cfg: compile-clang-sanitizers + compiler: clang + valgrind: 0 + asan: 1 + ubsan: 1 check-units: # The unit test checks are not in the critical path (not dependent @@ -171,7 +149,8 @@ jobs: env: BOLTDIR: bolts needs: - - compile + - compile-gcc + - compile-clang-sanitizers strategy: fail-fast: true matrix: @@ -239,7 +218,7 @@ jobs: name: Check we can downgrade the node runs-on: ubuntu-22.04 needs: - - compile + - compile-gcc strategy: fail-fast: false matrix: @@ -323,7 +302,8 @@ jobs: RUST_PROFILE: release # Has to match the one in the compile step PYTEST_OPTS: --timeout=1200 --durations=10 needs: - - compile + - compile-gcc + - compile-clang strategy: fail-fast: false matrix: @@ -432,7 +412,7 @@ jobs: CFG: compile-gcc PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800 --durations=10 needs: - - compile + - compile-gcc strategy: fail-fast: false matrix: @@ -503,7 +483,7 @@ jobs: TEST_DEBUG: 1 PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800 --durations=10 needs: - - compile + - compile-clang-sanitizers strategy: fail-fast: false matrix: @@ -572,7 +552,7 @@ jobs: PYTEST_OPTS: --timeout=1200 --durations=10 TEST_NETWORK: regtest needs: - - compile + - compile-gcc steps: - name: Checkout uses: actions/checkout@v4 @@ -610,7 +590,7 @@ jobs: RUST_PROFILE: release # Has to match the one in the compile step PYTEST_OPTS: --timeout=1200 --durations=10 needs: - - compile + - compile-clang strategy: fail-fast: false matrix: From e2530e62a5ed190bb9ebb4e11483a18b88c34dda Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 4 Dec 2025 15:03:50 +0100 Subject: [PATCH 2/5] ci: Refactor ci.yaml to split out running tests --- .github/workflows/ci.yaml | 447 ++++++++++---------------------- .github/workflows/test-cln.yaml | 198 ++++++++++++++ 2 files changed, 341 insertions(+), 304 deletions(-) create mode 100644 .github/workflows/test-cln.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3794d18a8e04..6447ab16af03 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,6 +6,7 @@ on: - "master" pull_request: types: [opened, synchronize, edited] + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -214,330 +215,155 @@ jobs: ./configure --enable-debugbuild --enable-fuzzing --enable-address-sanitizer --enable-ub-sanitizer --disable-valgrind CC=clang uv run make -j $(nproc) check-fuzz - check-downgrade: - name: Check we can downgrade the node - runs-on: ubuntu-22.04 + check-downgrade-sqlite: + name: Check downgrade (SQLite) needs: - compile-gcc - strategy: - fail-fast: false - matrix: - include: - - CFG: compile-gcc - TEST_DB_PROVIDER: sqlite3 - TEST_NETWORK: regtest - VALGRIND: 1 - - CFG: compile-gcc - TEST_DB_PROVIDER: postgres - TEST_NETWORK: regtest - - CFG: compile-gcc - TEST_DB_PROVIDER: sqlite3 - TEST_NETWORK: liquid-regtest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Install dependencies - run: | - bash -x .github/scripts/setup.sh - - - name: Install bitcoind - env: - TEST_NETWORK: ${{ matrix.TEST_NETWORK }} - run: .github/scripts/install-bitcoind.sh - - - name: Download build - uses: actions/download-artifact@v4 - with: - name: cln-${{ matrix.CFG }}.tar.bz2 - - - name: Unpack pre-built CLN - env: - CFG: ${{ matrix.CFG }} - run: | - tar -xaf cln-${CFG}.tar.bz2 - - - name: Fetch and unpack previous CLN - run: | - mkdir /tmp/old-cln - cd /tmp/old-cln - wget https://github.com/ElementsProject/lightning/releases/download/v25.09/clightning-v25.09-Ubuntu-22.04-amd64.tar.xz - tar -xaf clightning-v25.09-Ubuntu-22.04-amd64.tar.xz - - - name: Switch network - if: ${{ matrix.TEST_NETWORK == 'liquid-regtest' }} - run: | - # Loading the network from config.vars rather than the envvar is a terrible idea... - sed -i 's/TEST_NETWORK=regtest/TEST_NETWORK=liquid-regtest/g' config.vars - cat config.vars - - - name: Test - env: - SLOW_MACHINE: 1 - PYTEST_PAR: 10 - TEST_DEBUG: 1 - TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }} - TEST_NETWORK: ${{ matrix.TEST_NETWORK }} - LIGHTNINGD_POSTGRES_NO_VACUUM: 1 - VALGRIND: ${{ matrix.VALGRIND }} - PREV_LIGHTNINGD: /tmp/old-cln/usr/bin/lightningd - run: | - env - cat config.vars - uv run eatmydata pytest tests/test_downgrade.py -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS} + uses: ./.github/workflows/test-cln.yaml + with: + name: Check downgrade (SQLite) + cfg: compile-gcc + test_path: tests/test_downgrade.py + test_db_provider: sqlite3 + valgrind: 1 + setup_old_cln: true - integration: - name: Test CLN ${{ matrix.name }} - runs-on: ubuntu-22.04 - timeout-minutes: 120 - env: - RUST_PROFILE: release # Has to match the one in the compile step - PYTEST_OPTS: --timeout=1200 --durations=10 + check-downgrade-postgres: + name: Check downgrade (Postgres) needs: - compile-gcc - - compile-clang - strategy: - fail-fast: false - matrix: - include: - - NAME: gcc - CFG: compile-gcc - TEST_DB_PROVIDER: sqlite3 - COMPILER: gcc - TEST_NETWORK: regtest - # While we're at it let's try to compile with clang - - NAME: clang - CFG: compile-clang - TEST_DB_PROVIDER: sqlite3 - COMPILER: clang - TEST_NETWORK: regtest - # And of course we want to test postgres too - - NAME: postgres - CFG: compile-gcc - COMPILER: gcc - TEST_DB_PROVIDER: postgres - TEST_NETWORK: regtest - # And don't forget about elements (like cdecker did when - # reworking the CI...) - - NAME: liquid - CFG: compile-gcc - COMPILER: gcc - TEST_NETWORK: liquid-regtest - TEST_DB_PROVIDER: sqlite3 - # And dual funding! - - NAME: dual-fund - CFG: compile-gcc - TEST_DB_PROVIDER: sqlite3 - COMPILER: gcc - TEST_NETWORK: regtest - EXPERIMENTAL_DUAL_FUND: 1 - # And splicing! - - NAME: splicing - CFG: compile-gcc - TEST_DB_PROVIDER: sqlite3 - COMPILER: gcc - TEST_NETWORK: regtest - EXPERIMENTAL_SPLICING: 1 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" + uses: ./.github/workflows/test-cln.yaml + with: + name: Check downgrade (Postgres) + cfg: compile-gcc + test_path: tests/test_downgrade.py + test_db_provider: postgres + setup_old_cln: true - - name: Install uv - uses: astral-sh/setup-uv@v5 + check-downgrade-liquid: + name: Check downgrade (Liquid) + needs: + - compile-gcc + uses: ./.github/workflows/test-cln.yaml + with: + name: Check downgrade (Liquid) + cfg: compile-gcc + test_path: tests/test_downgrade.py + test_db_provider: sqlite3 + test_network: liquid-regtest + setup_old_cln: true - - name: Install dependencies - run: | - bash -x .github/scripts/setup.sh + integration-gcc: + name: Test CLN (GCC) + needs: + - compile-gcc + uses: ./.github/workflows/test-cln.yaml + with: + name: Test CLN (GCC) + cfg: compile-gcc + test_db_provider: sqlite3 + compiler: gcc + compat: 1 - - name: Install bitcoind - env: - TEST_NETWORK: ${{ matrix.TEST_NETWORK }} - run: .github/scripts/install-bitcoind.sh + integration-clang: + name: Test CLN (Clang) + needs: + - compile-clang + uses: ./.github/workflows/test-cln.yaml + with: + name: Test CLN (Clang) + cfg: compile-clang + test_db_provider: sqlite3 + compiler: clang + compat: 1 - - name: Download build - uses: actions/download-artifact@v4 - with: - name: cln-${{ matrix.CFG }}.tar.bz2 + integration-postgres: + name: Test CLN (Postgres) + needs: + - compile-gcc + uses: ./.github/workflows/test-cln.yaml + with: + name: Test CLN (Postgres) + cfg: compile-gcc + test_db_provider: postgres + compiler: gcc + compat: 1 - - name: Unpack pre-built CLN - env: - CFG: ${{ matrix.CFG }} - run: | - tar -xaf cln-${CFG}.tar.bz2 + integration-liquid: + name: Test CLN (Liquid) + needs: + - compile-gcc + uses: ./.github/workflows/test-cln.yaml + with: + name: Test CLN (Liquid) + cfg: compile-gcc + test_db_provider: sqlite3 + compiler: gcc + test_network: liquid-regtest + compat: 1 - - name: Switch network - if: ${{ matrix.TEST_NETWORK == 'liquid-regtest' }} - run: | - # Loading the network from config.vars rather than the envvar is a terrible idea... - sed -i 's/TEST_NETWORK=regtest/TEST_NETWORK=liquid-regtest/g' config.vars - cat config.vars + integration-dual-fund: + name: Test CLN (Dual Fund) + needs: + - compile-gcc + uses: ./.github/workflows/test-cln.yaml + with: + name: Test CLN (Dual Fund) + cfg: compile-gcc + test_db_provider: sqlite3 + compiler: gcc + experimental_dual_fund: 1 + compat: 1 - - name: Test - env: - COMPILER: ${{ matrix.COMPILER }} - EXPERIMENTAL_DUAL_FUND: ${{ matrix.EXPERIMENTAL_DUAL_FUND }} - EXPERIMENTAL_SPLICING: ${{ matrix.EXPERIMENTAL_SPLICING }} - COMPAT: 1 - CFG: ${{ matrix.CFG }} - SLOW_MACHINE: 1 - PYTEST_PAR: 10 - TEST_DEBUG: 1 - TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }} - TEST_NETWORK: ${{ matrix.TEST_NETWORK }} - LIGHTNINGD_POSTGRES_NO_VACUUM: 1 - run: | - env - cat config.vars - VALGRIND=0 uv run eatmydata pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS} + integration-splicing: + name: Test CLN (Splicing) + needs: + - compile-gcc + uses: ./.github/workflows/test-cln.yaml + with: + name: Test CLN (Splicing) + cfg: compile-gcc + test_db_provider: sqlite3 + compiler: gcc + experimental_splicing: 1 + compat: 1 integration-valgrind: - name: Valgrind Test CLN ${{ matrix.name }} - runs-on: ubuntu-22.04 - timeout-minutes: 120 - env: - RUST_PROFILE: release # Has to match the one in the compile step - CFG: compile-gcc - PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800 --durations=10 + name: Valgrind Test CLN (${{ matrix.shard }}/10) needs: - compile-gcc strategy: fail-fast: false matrix: - include: - - NAME: Valgrind (01/10) - PYTEST_OPTS: --test-group=1 --test-group-count=10 - - NAME: Valgrind (02/10) - PYTEST_OPTS: --test-group=2 --test-group-count=10 - - NAME: Valgrind (03/10) - PYTEST_OPTS: --test-group=3 --test-group-count=10 - - NAME: Valgrind (04/10) - PYTEST_OPTS: --test-group=4 --test-group-count=10 - - NAME: Valgrind (05/10) - PYTEST_OPTS: --test-group=5 --test-group-count=10 - - NAME: Valgrind (06/10) - PYTEST_OPTS: --test-group=6 --test-group-count=10 - - NAME: Valgrind (07/10) - PYTEST_OPTS: --test-group=7 --test-group-count=10 - - NAME: Valgrind (08/10) - PYTEST_OPTS: --test-group=8 --test-group-count=10 - - NAME: Valgrind (09/10) - PYTEST_OPTS: --test-group=9 --test-group-count=10 - - NAME: Valgrind (10/10) - PYTEST_OPTS: --test-group=10 --test-group-count=10 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Install dependencies - run: | - sudo apt-get update -qq - sudo apt-get install -yyq valgrind - bash -x .github/scripts/setup.sh - - - name: Install bitcoind - run: .github/scripts/install-bitcoind.sh - - - name: Download build - uses: actions/download-artifact@v4 - with: - name: cln-compile-gcc.tar.bz2 - - - name: Unpack build - run: tar -xvjf cln-compile-gcc.tar.bz2 - - - name: Test - env: - SLOW_MACHINE: 1 - TEST_DEBUG: 1 - run: | - VALGRIND=1 uv run eatmydata pytest tests/ -vvv -n 3 ${PYTEST_OPTS} ${{ matrix.PYTEST_OPTS }} + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + uses: ./.github/workflows/test-cln.yaml + with: + name: Valgrind Test CLN + cfg: compile-gcc + pytest_opts: '--test-group-random-seed=42 --timeout=1800 --durations=10' + pytest_workers: 3 + valgrind: 1 + install_valgrind: true + shard_count: 10 + shard_index: ${{ matrix.shard }} + test_db_provider: sqlite3 integration-sanitizers: - name: Sanitizers Test CLN - runs-on: ubuntu-22.04 - timeout-minutes: 120 - env: - RUST_PROFILE: release - SLOW_MACHINE: 1 - TEST_DEBUG: 1 - PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800 --durations=10 + name: Sanitizers Test CLN (${{ matrix.shard }}/10) needs: - compile-clang-sanitizers strategy: fail-fast: false matrix: - include: - - NAME: ASan/UBSan (01/10) - PYTEST_OPTS: --test-group=1 --test-group-count=10 - - NAME: ASan/UBSan (02/10) - PYTEST_OPTS: --test-group=2 --test-group-count=10 -n 1 - - NAME: ASan/UBSan (03/10) - PYTEST_OPTS: --test-group=3 --test-group-count=10 - - NAME: ASan/UBSan (04/10) - PYTEST_OPTS: --test-group=4 --test-group-count=10 - - NAME: ASan/UBSan (05/10) - PYTEST_OPTS: --test-group=5 --test-group-count=10 - - NAME: ASan/UBSan (06/10) - PYTEST_OPTS: --test-group=6 --test-group-count=10 - - NAME: ASan/UBSan (07/10) - PYTEST_OPTS: --test-group=7 --test-group-count=10 - - NAME: ASan/UBSan (08/10) - PYTEST_OPTS: --test-group=8 --test-group-count=10 - - NAME: ASan/UBSan (09/10) - PYTEST_OPTS: --test-group=9 --test-group-count=10 - - NAME: ASan/UBSan (10/10) - PYTEST_OPTS: --test-group=10 --test-group-count=10 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Install dependencies - run: | - bash -x .github/scripts/setup.sh - - - name: Install bitcoind - run: .github/scripts/install-bitcoind.sh - - - name: Download build - uses: actions/download-artifact@v4 - with: - name: cln-compile-clang-sanitizers.tar.bz2 - - - name: Unpack build - run: tar -xvjf cln-compile-clang-sanitizers.tar.bz2 - - - name: Test - run: | - uv run eatmydata pytest tests/ -vvv -n 2 ${PYTEST_OPTS} ${{ matrix.PYTEST_OPTS }} + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + uses: ./.github/workflows/test-cln.yaml + with: + name: ASan/UBSan Test CLN + cfg: compile-clang-sanitizers + pytest_opts: '--test-group-random-seed=42 --timeout=1800 --durations=10' + pytest_workers: 2 + shard_count: 10 + shard_index: ${{ matrix.shard }} update-docs-examples: name: Update examples in doc schemas (disabled temporarily!) @@ -682,24 +508,37 @@ jobs: name: CI completion runs-on: ubuntu-22.04 needs: - - integration + - integration-gcc + - integration-clang + - integration-postgres + - integration-liquid + - integration-dual-fund + - integration-splicing - check-units - integration-valgrind - integration-sanitizers - min-btc-support - - check-downgrade + - check-downgrade-sqlite + - check-downgrade-postgres + - check-downgrade-liquid if: ${{ always() }} steps: - name: Complete env: - JOB_NAMES: "INTEGRATION CHECK_UNITS VALGRIND SANITIZERS BTC" - INTEGRATION: ${{ needs.integration.result }} + JOB_NAMES: "INTEGRATION_GCC INTEGRATION_CLANG INTEGRATION_POSTGRES INTEGRATION_LIQUID INTEGRATION_DUAL_FUND INTEGRATION_SPLICING CHECK_UNITS VALGRIND SANITIZERS BTC CHECK_DOWNGRADE_SQLITE CHECK_DOWNGRADE_POSTGRES CHECK_DOWNGRADE_LIQUID" + INTEGRATION_GCC: ${{ needs['integration-gcc'].result }} + INTEGRATION_CLANG: ${{ needs['integration-clang'].result }} + INTEGRATION_POSTGRES: ${{ needs['integration-postgres'].result }} + INTEGRATION_LIQUID: ${{ needs['integration-liquid'].result }} + INTEGRATION_DUAL_FUND: ${{ needs['integration-dual-fund'].result }} + INTEGRATION_SPLICING: ${{ needs['integration-splicing'].result }} CHECK_UNITS: ${{ needs['check-units'].result }} VALGRIND: ${{ needs['integration-valgrind'].result }} SANITIZERS: ${{ needs['integration-sanitizers'].result }} - DOCS: ${{ needs['update-docs-examples'].result }} BTC: ${{ needs['min-btc-support'].result }} - CHECK_DOWNGRADE: ${{ needs['check-downgrade'].result }} + CHECK_DOWNGRADE_SQLITE: ${{ needs['check-downgrade-sqlite'].result }} + CHECK_DOWNGRADE_POSTGRES: ${{ needs['check-downgrade-postgres'].result }} + CHECK_DOWNGRADE_LIQUID: ${{ needs['check-downgrade-liquid'].result }} run: | failed="" for name in $JOB_NAMES; do diff --git a/.github/workflows/test-cln.yaml b/.github/workflows/test-cln.yaml new file mode 100644 index 000000000000..5cdf949b65bc --- /dev/null +++ b/.github/workflows/test-cln.yaml @@ -0,0 +1,198 @@ +--- +name: Test CLN +on: + workflow_call: + inputs: + name: + description: 'Test name for display' + required: true + type: string + cfg: + description: 'Build configuration to use (e.g., compile-gcc)' + required: true + type: string + test_path: + description: 'Path/pattern for pytest' + required: false + type: string + default: 'tests/' + pytest_opts: + description: 'Base pytest options' + required: false + type: string + default: '--timeout=1200 --durations=10' + matrix_pytest_opts: + description: 'Additional pytest options from matrix (e.g., test groups)' + required: false + type: string + default: '' + pytest_workers: + description: 'Number of pytest parallel workers (-n flag)' + required: false + type: number + default: 10 + valgrind: + description: 'Enable valgrind (0 or 1)' + required: false + type: number + default: 0 + test_debug: + description: 'Enable test debug (0 or 1)' + required: false + type: number + default: 1 + test_db_provider: + description: 'Database provider (sqlite3, postgres, or empty)' + required: false + type: string + default: '' + test_network: + description: 'Test network (regtest, liquid-regtest)' + required: false + type: string + default: 'regtest' + experimental_dual_fund: + description: 'Enable experimental dual fund' + required: false + type: number + default: 0 + experimental_splicing: + description: 'Enable experimental splicing' + required: false + type: number + default: 0 + lightningd_postgres_no_vacuum: + description: 'Disable postgres vacuum' + required: false + type: number + default: 1 + slow_machine: + description: 'Enable slow machine mode' + required: false + type: number + default: 1 + compat: + description: 'Enable compatibility mode' + required: false + type: number + default: 0 + compiler: + description: 'Compiler used (for env var)' + required: false + type: string + default: '' + rust_profile: + description: 'Rust profile' + required: false + type: string + default: 'release' + install_valgrind: + description: 'Install valgrind package before tests' + required: false + type: boolean + default: false + install_bitcoind: + description: 'Install bitcoind (true) or skip (false for custom Bitcoin install)' + required: false + type: boolean + default: true + setup_old_cln: + description: 'Download and setup old CLN for downgrade tests' + required: false + type: boolean + default: false + old_cln_version: + description: 'Version of old CLN to download (e.g., v25.09)' + required: false + type: string + default: 'v25.09' + shard_count: + description: 'Number of test shards (1 = no sharding, >1 = enable sharding)' + required: false + type: number + default: 1 + shard_index: + description: 'Shard index for this run (1-based, used when shard_count > 1)' + required: false + type: number + default: 1 + +jobs: + test: + name: ${{ inputs.shard_count > 1 && format('{0} ({1}/{2})', inputs.name, inputs.shard_index, inputs.shard_count) || inputs.name }} + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + RUST_PROFILE: ${{ inputs.rust_profile }} + PYTEST_OPTS: ${{ inputs.pytest_opts }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Install dependencies + run: | + bash -x .github/scripts/setup.sh + + - name: Install valgrind + if: ${{ inputs.install_valgrind }} + run: | + sudo apt-get update -qq + sudo apt-get install -yyq valgrind + + - name: Install bitcoind + if: ${{ inputs.install_bitcoind }} + env: + TEST_NETWORK: ${{ inputs.test_network }} + run: .github/scripts/install-bitcoind.sh + + - name: Download build + uses: actions/download-artifact@v4 + with: + name: cln-${{ inputs.cfg }}.tar.bz2 + + - name: Unpack pre-built CLN + run: tar -xaf cln-${{ inputs.cfg }}.tar.bz2 + + - name: Fetch and unpack previous CLN + if: ${{ inputs.setup_old_cln }} + run: | + mkdir /tmp/old-cln + cd /tmp/old-cln + wget https://github.com/ElementsProject/lightning/releases/download/${{ inputs.old_cln_version }}/clightning-${{ inputs.old_cln_version }}-Ubuntu-22.04-amd64.tar.xz + tar -xaf clightning-${{ inputs.old_cln_version }}-Ubuntu-22.04-amd64.tar.xz + + - name: Switch network + if: ${{ inputs.test_network == 'liquid-regtest' }} + run: | + # Loading the network from config.vars rather than the envvar is a terrible idea... + sed -i 's/TEST_NETWORK=regtest/TEST_NETWORK=liquid-regtest/g' config.vars + cat config.vars + + - name: Test + env: + COMPILER: ${{ inputs.compiler }} + EXPERIMENTAL_DUAL_FUND: ${{ inputs.experimental_dual_fund }} + EXPERIMENTAL_SPLICING: ${{ inputs.experimental_splicing }} + COMPAT: ${{ inputs.compat }} + CFG: ${{ inputs.cfg }} + SLOW_MACHINE: ${{ inputs.slow_machine }} + PYTEST_PAR: ${{ inputs.pytest_workers }} + TEST_DEBUG: ${{ inputs.test_debug }} + TEST_DB_PROVIDER: ${{ inputs.test_db_provider }} + TEST_NETWORK: ${{ inputs.test_network }} + LIGHTNINGD_POSTGRES_NO_VACUUM: ${{ inputs.lightningd_postgres_no_vacuum }} + VALGRIND: ${{ inputs.valgrind }} + PREV_LIGHTNINGD: ${{ inputs.setup_old_cln && '/tmp/old-cln/usr/bin/lightningd' || '' }} + SHARD_OPTS: ${{ inputs.shard_count > 1 && format('--test-group={0} --test-group-count={1}', inputs.shard_index, inputs.shard_count) || '' }} + run: | + env + cat config.vars + uv run eatmydata pytest ${{ inputs.test_path }} -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS} ${SHARD_OPTS} ${{ inputs.matrix_pytest_opts }} From 3f58151351b068f7b9476f4bcb1a680a7d0eea68 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 4 Dec 2025 15:11:28 +0100 Subject: [PATCH 3/5] ci: Enable sccache compiler cache for Rust and C --- .github/workflows/build-cln.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-cln.yaml b/.github/workflows/build-cln.yaml index 13d1343d4a85..c2384209c863 100644 --- a/.github/workflows/build-cln.yaml +++ b/.github/workflows/build-cln.yaml @@ -57,6 +57,9 @@ jobs: run: | bash -x .github/scripts/setup.sh + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Build env: COMPILER: ${{ inputs.compiler }} @@ -65,9 +68,10 @@ jobs: VALGRIND: ${{ inputs.valgrind }} COMPAT: 1 CFG: ${{ inputs.cfg }} + RUSTC_WRAPPER: sccache run: | set -e - ./configure --enable-debugbuild CC="$COMPILER" ${{ inputs.coptflags }} + ./configure --enable-debugbuild CC="sccache $COMPILER" ${{ inputs.coptflags }} uv run make -j $(nproc) testpack.tar.bz2 @@ -75,6 +79,8 @@ jobs: mv testpack.tar.bz2 cln-${CFG}.tar.bz2 - name: Check rust packages + env: + RUSTC_WRAPPER: sccache run: cargo test --all - uses: actions/upload-artifact@v4 From d5718984788948ea3824daa1fdf68d1be3890eb1 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 4 Dec 2025 15:21:49 +0100 Subject: [PATCH 4/5] ci: Add macos.yaml using the build-cln.yaml flow --- .github/workflows/build-cln.yaml | 41 ++++++++++++++++++++++++++++---- .github/workflows/macos.yaml | 40 +++++++++++++------------------ 2 files changed, 53 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build-cln.yaml b/.github/workflows/build-cln.yaml index c2384209c863..c11365b3a0b8 100644 --- a/.github/workflows/build-cln.yaml +++ b/.github/workflows/build-cln.yaml @@ -31,6 +31,11 @@ on: required: false type: string default: '' + os: + description: 'Operating system (ubuntu-22.04 or macos-14)' + required: false + type: string + default: 'ubuntu-22.04' env: RUST_PROFILE: release @@ -39,7 +44,7 @@ env: jobs: build: name: Build CLN ${{ inputs.cfg }} - runs-on: ubuntu-22.04 + runs-on: ${{ inputs.os }} timeout-minutes: 30 steps: - name: Checkout @@ -53,10 +58,21 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 - - name: Install dependencies + - name: Install dependencies (Linux) + if: ${{ inputs.os == 'ubuntu-22.04' }} run: | bash -x .github/scripts/setup.sh + - name: Install dependencies (macOS) + if: ${{ startsWith(inputs.os, 'macos-') }} + env: + GRPC_PYTHON_BUILD_SYSTEM_OPENSSL: 1 + GRPC_PYTHON_BUILD_SYSTEM_ZLIB: 1 + run: | + export PATH="/usr/local/opt:/Users/runner/.local/bin:/opt/homebrew/bin/python3.10/bin:$PATH" + brew install gnu-sed autoconf automake libtool protobuf openssl lowdown libsodium + uv sync --all-groups + - name: Setup sccache uses: mozilla-actions/sccache-action@v0.0.9 @@ -69,9 +85,25 @@ jobs: COMPAT: 1 CFG: ${{ inputs.cfg }} RUSTC_WRAPPER: sccache + # macOS-specific environment variables + CPATH: ${{ startsWith(inputs.os, 'macos-') && '/opt/homebrew/include' || '' }} + LIBRARY_PATH: ${{ startsWith(inputs.os, 'macos-') && '/opt/homebrew/lib' || '' }} + GRPC_PYTHON_BUILD_SYSTEM_OPENSSL: ${{ startsWith(inputs.os, 'macos-') && '1' || '' }} + GRPC_PYTHON_BUILD_SYSTEM_ZLIB: ${{ startsWith(inputs.os, 'macos-') && '1' || '' }} run: | set -e - ./configure --enable-debugbuild CC="sccache $COMPILER" ${{ inputs.coptflags }} + + # Determine configure flags based on OS + if [[ "${{ inputs.os }}" == ubuntu-* ]]; then + CONFIGURE_FLAGS="--enable-debugbuild" + CC_WRAPPER="sccache $COMPILER" + else + # macOS: disable valgrind (not available) and compat + CONFIGURE_FLAGS="--disable-valgrind --disable-compat" + CC_WRAPPER="$COMPILER" + fi + + ./configure $CONFIGURE_FLAGS CC="$CC_WRAPPER" ${{ inputs.coptflags }} uv run make -j $(nproc) testpack.tar.bz2 @@ -83,7 +115,8 @@ jobs: RUSTC_WRAPPER: sccache run: cargo test --all - - uses: actions/upload-artifact@v4 + - name: Upload artifact + uses: actions/upload-artifact@v4 with: name: cln-${{ inputs.cfg }}.tar.bz2 path: cln-${{ inputs.cfg }}.tar.bz2 diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml index 71ca8e9d235f..ea41d2ea46ad 100644 --- a/.github/workflows/macos.yaml +++ b/.github/workflows/macos.yaml @@ -2,11 +2,22 @@ name: Mac OS pytest on: pull_request: + workflow_dispatch: jobs: + build-macos: + name: Build CLN (macOS) + uses: ./.github/workflows/build-cln.yaml + with: + cfg: macos-clang + compiler: clang + os: macos-14 + smoke-test: name: Smoke Test macOS runs-on: macos-14 timeout-minutes: 120 + needs: + - build-macos strategy: fail-fast: true matrix: @@ -25,32 +36,13 @@ jobs: sudo mv bitcoin-${BITCOIND_VERSION}/bin/* /usr/local/bin rm -rf bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz bitcoin-${BITCOIND_VERSION} - - name: Set up Python 3.10 - uses: actions/setup-python@v5 + - name: Download build + uses: actions/download-artifact@v4 with: - python-version: "3.10" - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Install dependencies - run: | - export PATH="/usr/local/opt:/Users/runner/.local/bin:/opt/homebrew/bin/python3.10/bin:$PATH" - - brew install gnu-sed autoconf automake libtool protobuf openssl lowdown libsodium - - # https://github.com/grpc/grpc/issues/31737#issuecomment-1323796842 - export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 - export GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1 - uv sync --all-groups - - - name: Build and install CLN - run: | - export CPATH=/opt/homebrew/include - export LIBRARY_PATH=/opt/homebrew/lib + name: cln-macos-clang.tar.bz2 - uv run ./configure --disable-valgrind --disable-compat - uv run make + - name: Unpack CLN + run: tar -xaf cln-macos-clang.tar.bz2 - name: Start bitcoind in regtest mode run: | From 728210c813ac5f5312d2e9261352f03d183eb466 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Fri, 5 Dec 2025 15:15:06 +0100 Subject: [PATCH 5/5] fixup! ci: Refactor ci.yaml to split out running tests --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6447ab16af03..ba2bf4c3da54 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -364,6 +364,7 @@ jobs: pytest_workers: 2 shard_count: 10 shard_index: ${{ matrix.shard }} + test_db_provider: sqlite3 update-docs-examples: name: Update examples in doc schemas (disabled temporarily!)