From ba1918b6b094b70c06a51511b18b156707ec864b Mon Sep 17 00:00:00 2001 From: fanquake Date: Tue, 2 May 2023 09:51:36 +0100 Subject: [PATCH 1/6] Merge bitcoin/bitcoin#27191: blockstorage: Adjust fastprune limit if block exceeds blockfile size 8f14fc86225d8fe77353f61ebd6b0bdda8d13aa9 test: cover fastprune with excessive block size (Matthew Zipkin) 271c23e87f61276d7acab74e115b25a35144c8b4 blockstorage: Adjust fastprune limit if block exceeds blockfile size (Martin Zumsande) Pull request description: The debug-only `-fastprune` option used in several tests is not always safe to use: If a `-fastprune` node receives a block larger than the maximum blockfile size of `64kb` bad things happen: The while loop in `BlockManager::FindBlockPos` never terminates, and the node runs oom because memory for `m_blockfile_info` is allocated in each iteration of the loop. The same would happen if a naive user used `-fastprune` on anything other than regtest (so this can be tested by syncing on signet for example, the first block that crashes the node is at height 2232). Change the approach by raising the blockfile size to the size of the block, if that block otherwise wouldn't fit (idea by TheCharlatan). ACKs for top commit: ryanofsky: Code review ACK 8f14fc86225d8fe77353f61ebd6b0bdda8d13aa9. Added new assert, test, and comment since last review TheCharlatan: ACK 8f14fc86225d8fe77353f61ebd6b0bdda8d13aa9 pinheadmz: ACK 8f14fc86225d8fe77353f61ebd6b0bdda8d13aa9 Tree-SHA512: df2fea30613ef9d40ebbc2416eacb574f6d7d96847db5c33dda22a29a2c61a8db831aa9552734ea4477e097f253dbcb6dcb1395d43d2a090cc0588c9ce66eac3 --- src/node/blockstorage.cpp | 13 ++++++++- test/functional/feature_fastprune.py | 42 ++++++++++++++++++++++++++++ test/functional/test_runner.py | 1 + 3 files changed, 55 insertions(+), 1 deletion(-) create mode 100755 test/functional/feature_fastprune.py diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 0a5ca022fa73..1589d1afbb2b 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -624,7 +624,18 @@ bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigne bool finalize_undo = false; if (!fKnown) { - while (m_blockfile_info[nFile].nSize + nAddSize >= (gArgs.GetBoolArg("-fastprune", false) ? 0x10000 /* 64kb */ : MAX_BLOCKFILE_SIZE)) { + unsigned int max_blockfile_size{MAX_BLOCKFILE_SIZE}; + // Use smaller blockfiles in test-only -fastprune mode - but avoid + // the possibility of having a block not fit into the block file. + if (gArgs.GetBoolArg("-fastprune", false)) { + max_blockfile_size = 0x10000; // 64kiB + if (nAddSize >= max_blockfile_size) { + // dynamically adjust the blockfile size to be larger than the added size + max_blockfile_size = nAddSize + 1; + } + } + assert(nAddSize < max_blockfile_size); + while (m_blockfile_info[nFile].nSize + nAddSize >= max_blockfile_size) { // when the undo file is keeping up with the block file, we want to flush it explicitly // when it is lagging behind (more blocks arrive than are being connected), we let the // undo block write case handle it diff --git a/test/functional/feature_fastprune.py b/test/functional/feature_fastprune.py new file mode 100755 index 000000000000..825de63e3d20 --- /dev/null +++ b/test/functional/feature_fastprune.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test fastprune mode.""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal +) +from test_framework.blocktools import ( + create_block, + create_coinbase, + add_witness_commitment +) +from test_framework.wallet import MiniWallet + + +class FeatureFastpruneTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.extra_args = [["-fastprune"]] + + def run_test(self): + self.log.info("ensure that large blocks don't crash or freeze in -fastprune") + wallet = MiniWallet(self.nodes[0]) + tx = wallet.create_self_transfer()['tx'] + annex = [0x50] + for _ in range(0x10000): + annex.append(0xff) + tx.wit.vtxinwit[0].scriptWitness.stack.append(bytes(annex)) + tip = int(self.nodes[0].getbestblockhash(), 16) + time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] + 1 + height = self.nodes[0].getblockcount() + 1 + block = create_block(hashprev=tip, ntime=time, txlist=[tx], coinbase=create_coinbase(height=height)) + add_witness_commitment(block) + block.solve() + self.nodes[0].submitblock(block.serialize().hex()) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + +if __name__ == '__main__': + FeatureFastpruneTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 2c1a8597f535..1c6ce86a9afe 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -365,6 +365,7 @@ 'p2p_message_capture.py', 'feature_addrman.py', 'feature_asmap.py', + 'feature_fastprune.py', 'feature_includeconf.py', 'mempool_unbroadcast.py', 'mempool_compatibility.py', From f0b677cce8d43147bdfdecd624583406832272c1 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Tue, 30 Sep 2025 11:30:09 -0700 Subject: [PATCH 2/6] dash: adapt feature_fastprune test for non-segwit Since Dash doesn't have segwit/witness support, adapt the test to create a large block using many regular transactions instead of using witness data to inflate block size. This maintains the test's purpose of ensuring -fastprune handles blocks larger than 64kb without crashing. --- test/functional/feature_fastprune.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/functional/feature_fastprune.py b/test/functional/feature_fastprune.py index 825de63e3d20..a19e5d00d3a8 100755 --- a/test/functional/feature_fastprune.py +++ b/test/functional/feature_fastprune.py @@ -10,7 +10,6 @@ from test_framework.blocktools import ( create_block, create_coinbase, - add_witness_commitment ) from test_framework.wallet import MiniWallet @@ -23,16 +22,18 @@ def set_test_params(self): def run_test(self): self.log.info("ensure that large blocks don't crash or freeze in -fastprune") wallet = MiniWallet(self.nodes[0]) - tx = wallet.create_self_transfer()['tx'] - annex = [0x50] - for _ in range(0x10000): - annex.append(0xff) - tx.wit.vtxinwit[0].scriptWitness.stack.append(bytes(annex)) + + # Create many transactions to make a large block (>64kb) + # Since Dash doesn't have witness data, we need to create many regular transactions + txs = [] + for _ in range(500): # Create enough transactions to exceed 64kb + tx = wallet.create_self_transfer()['tx'] + txs.append(tx) + tip = int(self.nodes[0].getbestblockhash(), 16) time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] + 1 height = self.nodes[0].getblockcount() + 1 - block = create_block(hashprev=tip, ntime=time, txlist=[tx], coinbase=create_coinbase(height=height)) - add_witness_commitment(block) + block = create_block(hashprev=tip, ntime=time, txlist=txs, coinbase=create_coinbase(height=height)) block.solve() self.nodes[0].submitblock(block.serialize().hex()) assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) From 3b3081b7f0923f384fe088dd94fb29ac54a01708 Mon Sep 17 00:00:00 2001 From: backport Date: Sat, 29 Nov 2025 11:35:54 -0600 Subject: [PATCH 3/6] validation: fix feature_fastprune test - generate UTXOs for MiniWallet The test was failing because MiniWallet had no UTXOs to create transactions. This fix generates 500 blocks to fund the wallet before creating 500 transactions. - Added self.generate(wallet, 500) before transaction creation - This ensures the wallet has sufficient UTXOs for all transactions Fixes CI failures in linux64-test and linux64_nowallet-test jobs. --- test/functional/feature_fastprune.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/functional/feature_fastprune.py b/test/functional/feature_fastprune.py index a19e5d00d3a8..86ce3a1b218d 100755 --- a/test/functional/feature_fastprune.py +++ b/test/functional/feature_fastprune.py @@ -23,6 +23,10 @@ def run_test(self): self.log.info("ensure that large blocks don't crash or freeze in -fastprune") wallet = MiniWallet(self.nodes[0]) + # Generate blocks to fund the wallet with UTXOs + # We need at least 500 UTXOs for 500 transactions + self.generate(wallet, 500) + # Create many transactions to make a large block (>64kb) # Since Dash doesn't have witness data, we need to create many regular transactions txs = [] From dfbbfef5c54ee8e723c722de5c950c1a23a9fecd Mon Sep 17 00:00:00 2001 From: Claude Code Date: Tue, 2 Dec 2025 08:49:33 -0600 Subject: [PATCH 4/6] validation: fix feature_fastprune test - properly create CbTx for Dash In Dash, after DIP4 activation, all blocks must have a proper CbTx (coinbase transaction) with extra payload containing masternode/quorum information. The previous fix attempted to work around this by generating UTXOs but still used a basic create_coinbase() call. This fix: - Creates coinbase with dip4_activated=True and v20_activated=True - Gets block template to obtain proper coinbase_payload - Sets vExtraPayload on the coinbase transaction - Adds quorum commitments from the block template - Recalculates merkle root after adding quorum transactions Based on the pattern used in feature_asset_locks.py test. --- test/functional/feature_fastprune.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test/functional/feature_fastprune.py b/test/functional/feature_fastprune.py index 86ce3a1b218d..ea78870e32fe 100755 --- a/test/functional/feature_fastprune.py +++ b/test/functional/feature_fastprune.py @@ -11,6 +11,7 @@ create_block, create_coinbase, ) +from test_framework.messages import tx_from_hex from test_framework.wallet import MiniWallet @@ -37,7 +38,22 @@ def run_test(self): tip = int(self.nodes[0].getbestblockhash(), 16) time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] + 1 height = self.nodes[0].getblockcount() + 1 - block = create_block(hashprev=tip, ntime=time, txlist=txs, coinbase=create_coinbase(height=height)) + + # Create proper CbTx for Dash (DIP4/v20 activated) + cbb = create_coinbase(height, dip4_activated=True, v20_activated=True) + gbt = self.nodes[0].getblocktemplate() + cbb.vExtraPayload = bytes.fromhex(gbt["coinbase_payload"]) + cbb.rehash() + + block = create_block(hashprev=tip, ntime=time, txlist=txs, coinbase=cbb, version=4) + + # Add quorum commitments from block template + for tx_obj in gbt["transactions"]: + tx = tx_from_hex(tx_obj["data"]) + if tx.nType == 6: # TRANSACTION_QUORUM_COMMITMENT + block.vtx.append(tx) + + block.hashMerkleRoot = block.calc_merkle_root() block.solve() self.nodes[0].submitblock(block.serialize().hex()) assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) From dc69d7275c09f36e23a1d64bfb372a92c9917695 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Tue, 2 Dec 2025 14:39:10 -0600 Subject: [PATCH 5/6] validation: simplify feature_fastprune test - use OP_RETURN instead of 500 txs - Replace 500 small transactions with single large OP_RETURN transaction - This avoids quorum commitment complexity and race conditions - Single 65kb OP_RETURN output exceeds 64kb fastprune limit as intended - Fixes test failure where multiple transactions caused block rejection --- test/functional/feature_fastprune.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/test/functional/feature_fastprune.py b/test/functional/feature_fastprune.py index ea78870e32fe..1c11d089e94c 100755 --- a/test/functional/feature_fastprune.py +++ b/test/functional/feature_fastprune.py @@ -11,7 +11,8 @@ create_block, create_coinbase, ) -from test_framework.messages import tx_from_hex +from test_framework.messages import CTxOut, tx_from_hex +from test_framework.script import CScript, OP_RETURN from test_framework.wallet import MiniWallet @@ -24,16 +25,13 @@ def run_test(self): self.log.info("ensure that large blocks don't crash or freeze in -fastprune") wallet = MiniWallet(self.nodes[0]) - # Generate blocks to fund the wallet with UTXOs - # We need at least 500 UTXOs for 500 transactions - self.generate(wallet, 500) - - # Create many transactions to make a large block (>64kb) - # Since Dash doesn't have witness data, we need to create many regular transactions - txs = [] - for _ in range(500): # Create enough transactions to exceed 64kb - tx = wallet.create_self_transfer()['tx'] - txs.append(tx) + # Create a single transaction with large OP_RETURN to make block >64kb + # We need to create a transaction that's large enough to exceed the fastprune limit + tx = wallet.create_self_transfer()['tx'] + # Add a large OP_RETURN output (65kb of data to exceed 64kb fastprune limit) + large_data = b'\x00' * 65536 + tx.vout.append(CTxOut(0, CScript([OP_RETURN, large_data]))) + tx.rehash() tip = int(self.nodes[0].getbestblockhash(), 16) time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] + 1 @@ -45,7 +43,7 @@ def run_test(self): cbb.vExtraPayload = bytes.fromhex(gbt["coinbase_payload"]) cbb.rehash() - block = create_block(hashprev=tip, ntime=time, txlist=txs, coinbase=cbb, version=4) + block = create_block(hashprev=tip, ntime=time, txlist=[tx], coinbase=cbb, version=4) # Add quorum commitments from block template for tx_obj in gbt["transactions"]: @@ -55,7 +53,10 @@ def run_test(self): block.hashMerkleRoot = block.calc_merkle_root() block.solve() - self.nodes[0].submitblock(block.serialize().hex()) + result = self.nodes[0].submitblock(block.serialize().hex()) + # submitblock returns None on success, error string on failure + if result is not None: + raise AssertionError(f"submitblock failed: {result}") assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) From 20f982e82fc4567b53867d5816146aa2d9cff179 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Tue, 9 Dec 2025 13:14:49 -0600 Subject: [PATCH 6/6] validation: fix feature_fastprune test - fund MiniWallet with UTXO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MiniWallet needs at least one UTXO before create_self_transfer() can be called. Generate one block to fund the wallet before creating the large transaction. This fixes the StopIteration error in CI where wallet.get_utxo() had no UTXOs. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- test/functional/feature_fastprune.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/functional/feature_fastprune.py b/test/functional/feature_fastprune.py index 1c11d089e94c..977bbb0f08be 100755 --- a/test/functional/feature_fastprune.py +++ b/test/functional/feature_fastprune.py @@ -24,6 +24,8 @@ def set_test_params(self): def run_test(self): self.log.info("ensure that large blocks don't crash or freeze in -fastprune") wallet = MiniWallet(self.nodes[0]) + # Generate a block to fund the wallet with a UTXO + self.generate(wallet, 1) # Create a single transaction with large OP_RETURN to make block >64kb # We need to create a transaction that's large enough to exceed the fastprune limit