From 6d4a06e4fd5ab3073294402afb6ffbcb7e5f2b58 Mon Sep 17 00:00:00 2001 From: freemans13 Date: Fri, 12 Dec 2025 23:05:04 +0000 Subject: [PATCH 1/5] allow partial/chunked serialise/deserialise subtree using io.Reader and io.Writer --- subtree_data.go | 80 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/subtree_data.go b/subtree_data.go index 97766cc..1e49e47 100644 --- a/subtree_data.go +++ b/subtree_data.go @@ -111,6 +111,86 @@ func (s *Data) Serialize() ([]byte, error) { return buf.Bytes(), nil } +// WriteTransactionsToWriter writes a range of transactions directly to a writer. +// +// This enables memory-efficient serialization by streaming transactions to disk as they are loaded, +// without requiring all transactions to be in memory simultaneously. Transactions in the specified +// range are written sequentially, skipping any nil entries. +// +// Parameters: +// - w: Writer to stream transactions to +// - startIdx: Starting index (inclusive) of transactions to write +// - endIdx: Ending index (exclusive) of transactions to write +// +// Returns an error if writing fails or if required transactions are missing (nil). +func (s *Data) WriteTransactionsToWriter(w io.Writer, startIdx, endIdx int) error { + if s.Subtree == nil { + return ErrCannotSerializeSubtreeNotSet + } + + for i := startIdx; i < endIdx; i++ { + // Skip coinbase placeholder if it's the first transaction + if i == 0 && s.Subtree.Nodes[0].Hash.Equal(*CoinbasePlaceholderHash) { + continue + } + + if s.Txs[i] == nil { + return fmt.Errorf("transaction at index %d is nil, cannot serialize", i) + } + + // Serialize and stream transaction bytes to writer + txBytes := s.Txs[i].SerializeBytes() + if _, err := w.Write(txBytes); err != nil { + return fmt.Errorf("error writing transaction at index %d: %w", i, err) + } + } + + return nil +} + +// ReadTransactionsFromReader reads a range of transactions from a reader. +// +// This enables memory-efficient deserialization by reading only a chunk of transactions +// from disk at a time, rather than loading all transactions into memory. +// +// Parameters: +// - r: Reader to read transactions from +// - startIdx: Starting index (inclusive) where transactions should be stored +// - endIdx: Ending index (exclusive) where transactions should be stored +// +// Returns the number of transactions read and any error encountered. +func (s *Data) ReadTransactionsFromReader(r io.Reader, startIdx, endIdx int) (int, error) { + if s.Subtree == nil || len(s.Subtree.Nodes) == 0 { + return 0, ErrSubtreeNodesEmpty + } + + txsRead := 0 + for i := startIdx; i < endIdx; i++ { + // Skip coinbase placeholder + if i == 0 && s.Subtree.Nodes[0].Hash.Equal(CoinbasePlaceholderHashValue) { + continue + } + + tx := &bt.Tx{} + if _, err := tx.ReadFrom(r); err != nil { + if errors.Is(err, io.EOF) { + break + } + return txsRead, fmt.Errorf("error reading transaction at index %d: %w", i, err) + } + + // Validate tx hash matches expected + if !s.Subtree.Nodes[i].Hash.Equal(*tx.TxIDChainHash()) { + return txsRead, ErrTxHashMismatch + } + + s.Txs[i] = tx + txsRead++ + } + + return txsRead, nil +} + // serializeFromReader reads transactions from the provided reader and populates the Txs field. func (s *Data) serializeFromReader(buf io.Reader) error { var ( From 9f821d9787de8d9e997838474b7799fc5c31440f Mon Sep 17 00:00:00 2001 From: freemans13 Date: Fri, 12 Dec 2025 23:10:16 +0000 Subject: [PATCH 2/5] tests --- subtree_data_test.go | 188 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) diff --git a/subtree_data_test.go b/subtree_data_test.go index f91d5fc..4e74b6a 100644 --- a/subtree_data_test.go +++ b/subtree_data_test.go @@ -391,3 +391,191 @@ type mockReader struct { func (r *mockReader) Read(_ []byte) (n int, err error) { return 0, r.err } + +func TestWriteTransactionsToWriter(t *testing.T) { + tx1 := tx.Clone() + tx1.Version = 1 + + tx2 := tx.Clone() + tx2.Version = 2 + + tx3 := tx.Clone() + tx3.Version = 3 + + tx4 := tx.Clone() + tx4.Version = 4 + + t.Run("write full range of transactions", func(t *testing.T) { + subtree, err := NewTree(2) + require.NoError(t, err) + + _ = subtree.AddNode(*tx1.TxIDChainHash(), 111, 0) + _ = subtree.AddNode(*tx2.TxIDChainHash(), 111, 0) + _ = subtree.AddNode(*tx3.TxIDChainHash(), 111, 0) + _ = subtree.AddNode(*tx4.TxIDChainHash(), 111, 0) + + subtreeData := NewSubtreeData(subtree) + require.NoError(t, subtreeData.AddTx(tx1, 0)) + require.NoError(t, subtreeData.AddTx(tx2, 1)) + require.NoError(t, subtreeData.AddTx(tx3, 2)) + require.NoError(t, subtreeData.AddTx(tx4, 3)) + + // Write to buffer + buf := &bytes.Buffer{} + err = subtreeData.WriteTransactionsToWriter(buf, 0, 4) + require.NoError(t, err) + + // Verify we can read back + newSubtreeData, err := NewSubtreeDataFromBytes(subtree, buf.Bytes()) + require.NoError(t, err) + assert.Equal(t, tx1.Version, newSubtreeData.Txs[0].Version) + assert.Equal(t, tx2.Version, newSubtreeData.Txs[1].Version) + assert.Equal(t, tx3.Version, newSubtreeData.Txs[2].Version) + assert.Equal(t, tx4.Version, newSubtreeData.Txs[3].Version) + }) + + t.Run("write partial range of transactions", func(t *testing.T) { + subtree, err := NewTree(2) + require.NoError(t, err) + + _ = subtree.AddNode(*tx1.TxIDChainHash(), 111, 0) + _ = subtree.AddNode(*tx2.TxIDChainHash(), 111, 0) + _ = subtree.AddNode(*tx3.TxIDChainHash(), 111, 0) + _ = subtree.AddNode(*tx4.TxIDChainHash(), 111, 0) + + subtreeData := NewSubtreeData(subtree) + require.NoError(t, subtreeData.AddTx(tx1, 0)) + require.NoError(t, subtreeData.AddTx(tx2, 1)) + require.NoError(t, subtreeData.AddTx(tx3, 2)) + require.NoError(t, subtreeData.AddTx(tx4, 3)) + + // Write only middle transactions (1-3) + buf := &bytes.Buffer{} + err = subtreeData.WriteTransactionsToWriter(buf, 1, 3) + require.NoError(t, err) + + // Verify size matches tx2 + tx3 + expectedSize := len(tx2.SerializeBytes()) + len(tx3.SerializeBytes()) + assert.Equal(t, expectedSize, buf.Len()) + }) + + t.Run("error on nil transaction", func(t *testing.T) { + subtree, err := NewTree(2) + require.NoError(t, err) + + _ = subtree.AddNode(*tx1.TxIDChainHash(), 111, 0) + _ = subtree.AddNode(*tx2.TxIDChainHash(), 111, 0) + + subtreeData := NewSubtreeData(subtree) + require.NoError(t, subtreeData.AddTx(tx1, 0)) + // Don't add tx2 - leave it nil + + buf := &bytes.Buffer{} + err = subtreeData.WriteTransactionsToWriter(buf, 0, 2) + require.Error(t, err) + assert.Contains(t, err.Error(), "transaction at index 1 is nil") + }) +} + +func TestReadTransactionsFromReader(t *testing.T) { + tx1 := tx.Clone() + tx1.Version = 1 + + tx2 := tx.Clone() + tx2.Version = 2 + + tx3 := tx.Clone() + tx3.Version = 3 + + tx4 := tx.Clone() + tx4.Version = 4 + + t.Run("read full range of transactions", func(t *testing.T) { + subtree, err := NewTree(2) + require.NoError(t, err) + + _ = subtree.AddNode(*tx1.TxIDChainHash(), 111, 0) + _ = subtree.AddNode(*tx2.TxIDChainHash(), 111, 0) + _ = subtree.AddNode(*tx3.TxIDChainHash(), 111, 0) + _ = subtree.AddNode(*tx4.TxIDChainHash(), 111, 0) + + // Create source subtreeData + sourceData := NewSubtreeData(subtree) + require.NoError(t, sourceData.AddTx(tx1, 0)) + require.NoError(t, sourceData.AddTx(tx2, 1)) + require.NoError(t, sourceData.AddTx(tx3, 2)) + require.NoError(t, sourceData.AddTx(tx4, 3)) + + // Serialize to bytes + serialized, err := sourceData.Serialize() + require.NoError(t, err) + + // Read back using chunked reader + reader := bytes.NewReader(serialized) + targetData := NewSubtreeData(subtree) + + numRead, err := targetData.ReadTransactionsFromReader(reader, 0, 4) + require.NoError(t, err) + assert.Equal(t, 4, numRead) + + assert.Equal(t, tx1.Version, targetData.Txs[0].Version) + assert.Equal(t, tx2.Version, targetData.Txs[1].Version) + assert.Equal(t, tx3.Version, targetData.Txs[2].Version) + assert.Equal(t, tx4.Version, targetData.Txs[3].Version) + }) + + t.Run("read partial range of transactions", func(t *testing.T) { + subtree, err := NewTree(2) + require.NoError(t, err) + + _ = subtree.AddNode(*tx1.TxIDChainHash(), 111, 0) + _ = subtree.AddNode(*tx2.TxIDChainHash(), 111, 0) + _ = subtree.AddNode(*tx3.TxIDChainHash(), 111, 0) + _ = subtree.AddNode(*tx4.TxIDChainHash(), 111, 0) + + sourceData := NewSubtreeData(subtree) + require.NoError(t, sourceData.AddTx(tx1, 0)) + require.NoError(t, sourceData.AddTx(tx2, 1)) + require.NoError(t, sourceData.AddTx(tx3, 2)) + require.NoError(t, sourceData.AddTx(tx4, 3)) + + serialized, err := sourceData.Serialize() + require.NoError(t, err) + + // Read only first 2 transactions + reader := bytes.NewReader(serialized) + targetData := NewSubtreeData(subtree) + + numRead, err := targetData.ReadTransactionsFromReader(reader, 0, 2) + require.NoError(t, err) + assert.Equal(t, 2, numRead) + + assert.Equal(t, tx1.Version, targetData.Txs[0].Version) + assert.Equal(t, tx2.Version, targetData.Txs[1].Version) + assert.Nil(t, targetData.Txs[2]) // Not read yet + assert.Nil(t, targetData.Txs[3]) // Not read yet + }) + + t.Run("read EOF gracefully", func(t *testing.T) { + subtree, err := NewTree(2) + require.NoError(t, err) + + _ = subtree.AddNode(*tx1.TxIDChainHash(), 111, 0) + _ = subtree.AddNode(*tx2.TxIDChainHash(), 111, 0) + + sourceData := NewSubtreeData(subtree) + require.NoError(t, sourceData.AddTx(tx1, 0)) + require.NoError(t, sourceData.AddTx(tx2, 1)) + + serialized, err := sourceData.Serialize() + require.NoError(t, err) + + reader := bytes.NewReader(serialized) + targetData := NewSubtreeData(subtree) + + // Try to read more transactions than available - should stop at EOF + numRead, err := targetData.ReadTransactionsFromReader(reader, 0, 10) + require.NoError(t, err) // EOF is not an error + assert.Equal(t, 2, numRead) // Only 2 were available + }) +} From 6270545876d5de1ffe5e3b0736cf93b8e6460315 Mon Sep 17 00:00:00 2001 From: freemans13 Date: Fri, 12 Dec 2025 23:21:31 +0000 Subject: [PATCH 3/5] reduce code duplication --- subtree_data_test.go | 180 ++++++++++++------------------------------- 1 file changed, 50 insertions(+), 130 deletions(-) diff --git a/subtree_data_test.go b/subtree_data_test.go index 4e74b6a..71a45e5 100644 --- a/subtree_data_test.go +++ b/subtree_data_test.go @@ -392,190 +392,110 @@ func (r *mockReader) Read(_ []byte) (n int, err error) { return 0, r.err } -func TestWriteTransactionsToWriter(t *testing.T) { - tx1 := tx.Clone() - tx1.Version = 1 - - tx2 := tx.Clone() - tx2.Version = 2 +// Helper to create test subtree with 4 versioned transactions +func setupTestSubtreeData(t *testing.T) (*Subtree, *Data, []*bt.Tx) { + txs := make([]*bt.Tx, 4) + for i := range txs { + txs[i] = tx.Clone() + txs[i].Version = uint32(i + 1) + } - tx3 := tx.Clone() - tx3.Version = 3 + subtree, err := NewTree(2) + require.NoError(t, err) - tx4 := tx.Clone() - tx4.Version = 4 + for _, tx := range txs { + _ = subtree.AddNode(*tx.TxIDChainHash(), 111, 0) + } - t.Run("write full range of transactions", func(t *testing.T) { - subtree, err := NewTree(2) - require.NoError(t, err) + subtreeData := NewSubtreeData(subtree) + for i, tx := range txs { + require.NoError(t, subtreeData.AddTx(tx, i)) + } - _ = subtree.AddNode(*tx1.TxIDChainHash(), 111, 0) - _ = subtree.AddNode(*tx2.TxIDChainHash(), 111, 0) - _ = subtree.AddNode(*tx3.TxIDChainHash(), 111, 0) - _ = subtree.AddNode(*tx4.TxIDChainHash(), 111, 0) + return subtree, subtreeData, txs +} - subtreeData := NewSubtreeData(subtree) - require.NoError(t, subtreeData.AddTx(tx1, 0)) - require.NoError(t, subtreeData.AddTx(tx2, 1)) - require.NoError(t, subtreeData.AddTx(tx3, 2)) - require.NoError(t, subtreeData.AddTx(tx4, 3)) +func TestWriteTransactionsToWriter(t *testing.T) { + t.Run("write full range of transactions", func(t *testing.T) { + subtree, subtreeData, txs := setupTestSubtreeData(t) - // Write to buffer buf := &bytes.Buffer{} - err = subtreeData.WriteTransactionsToWriter(buf, 0, 4) + err := subtreeData.WriteTransactionsToWriter(buf, 0, 4) require.NoError(t, err) - // Verify we can read back newSubtreeData, err := NewSubtreeDataFromBytes(subtree, buf.Bytes()) require.NoError(t, err) - assert.Equal(t, tx1.Version, newSubtreeData.Txs[0].Version) - assert.Equal(t, tx2.Version, newSubtreeData.Txs[1].Version) - assert.Equal(t, tx3.Version, newSubtreeData.Txs[2].Version) - assert.Equal(t, tx4.Version, newSubtreeData.Txs[3].Version) + for i, tx := range txs { + assert.Equal(t, tx.Version, newSubtreeData.Txs[i].Version) + } }) - t.Run("write partial range of transactions", func(t *testing.T) { - subtree, err := NewTree(2) - require.NoError(t, err) - - _ = subtree.AddNode(*tx1.TxIDChainHash(), 111, 0) - _ = subtree.AddNode(*tx2.TxIDChainHash(), 111, 0) - _ = subtree.AddNode(*tx3.TxIDChainHash(), 111, 0) - _ = subtree.AddNode(*tx4.TxIDChainHash(), 111, 0) - - subtreeData := NewSubtreeData(subtree) - require.NoError(t, subtreeData.AddTx(tx1, 0)) - require.NoError(t, subtreeData.AddTx(tx2, 1)) - require.NoError(t, subtreeData.AddTx(tx3, 2)) - require.NoError(t, subtreeData.AddTx(tx4, 3)) + t.Run("write partial range", func(t *testing.T) { + _, subtreeData, txs := setupTestSubtreeData(t) - // Write only middle transactions (1-3) buf := &bytes.Buffer{} - err = subtreeData.WriteTransactionsToWriter(buf, 1, 3) + err := subtreeData.WriteTransactionsToWriter(buf, 1, 3) require.NoError(t, err) - // Verify size matches tx2 + tx3 - expectedSize := len(tx2.SerializeBytes()) + len(tx3.SerializeBytes()) + expectedSize := len(txs[1].SerializeBytes()) + len(txs[2].SerializeBytes()) assert.Equal(t, expectedSize, buf.Len()) }) t.Run("error on nil transaction", func(t *testing.T) { - subtree, err := NewTree(2) - require.NoError(t, err) - - _ = subtree.AddNode(*tx1.TxIDChainHash(), 111, 0) - _ = subtree.AddNode(*tx2.TxIDChainHash(), 111, 0) - - subtreeData := NewSubtreeData(subtree) - require.NoError(t, subtreeData.AddTx(tx1, 0)) - // Don't add tx2 - leave it nil + _, subtreeData, _ := setupTestSubtreeData(t) + subtreeData.Txs[1] = nil // Null out one transaction buf := &bytes.Buffer{} - err = subtreeData.WriteTransactionsToWriter(buf, 0, 2) + err := subtreeData.WriteTransactionsToWriter(buf, 0, 2) require.Error(t, err) assert.Contains(t, err.Error(), "transaction at index 1 is nil") }) } func TestReadTransactionsFromReader(t *testing.T) { - tx1 := tx.Clone() - tx1.Version = 1 - - tx2 := tx.Clone() - tx2.Version = 2 - - tx3 := tx.Clone() - tx3.Version = 3 - - tx4 := tx.Clone() - tx4.Version = 4 - - t.Run("read full range of transactions", func(t *testing.T) { - subtree, err := NewTree(2) - require.NoError(t, err) - - _ = subtree.AddNode(*tx1.TxIDChainHash(), 111, 0) - _ = subtree.AddNode(*tx2.TxIDChainHash(), 111, 0) - _ = subtree.AddNode(*tx3.TxIDChainHash(), 111, 0) - _ = subtree.AddNode(*tx4.TxIDChainHash(), 111, 0) + t.Run("read full range", func(t *testing.T) { + subtree, sourceData, txs := setupTestSubtreeData(t) - // Create source subtreeData - sourceData := NewSubtreeData(subtree) - require.NoError(t, sourceData.AddTx(tx1, 0)) - require.NoError(t, sourceData.AddTx(tx2, 1)) - require.NoError(t, sourceData.AddTx(tx3, 2)) - require.NoError(t, sourceData.AddTx(tx4, 3)) - - // Serialize to bytes serialized, err := sourceData.Serialize() require.NoError(t, err) - // Read back using chunked reader - reader := bytes.NewReader(serialized) targetData := NewSubtreeData(subtree) - - numRead, err := targetData.ReadTransactionsFromReader(reader, 0, 4) + numRead, err := targetData.ReadTransactionsFromReader(bytes.NewReader(serialized), 0, 4) require.NoError(t, err) assert.Equal(t, 4, numRead) - assert.Equal(t, tx1.Version, targetData.Txs[0].Version) - assert.Equal(t, tx2.Version, targetData.Txs[1].Version) - assert.Equal(t, tx3.Version, targetData.Txs[2].Version) - assert.Equal(t, tx4.Version, targetData.Txs[3].Version) + for i, tx := range txs { + assert.Equal(t, tx.Version, targetData.Txs[i].Version) + } }) - t.Run("read partial range of transactions", func(t *testing.T) { - subtree, err := NewTree(2) - require.NoError(t, err) - - _ = subtree.AddNode(*tx1.TxIDChainHash(), 111, 0) - _ = subtree.AddNode(*tx2.TxIDChainHash(), 111, 0) - _ = subtree.AddNode(*tx3.TxIDChainHash(), 111, 0) - _ = subtree.AddNode(*tx4.TxIDChainHash(), 111, 0) - - sourceData := NewSubtreeData(subtree) - require.NoError(t, sourceData.AddTx(tx1, 0)) - require.NoError(t, sourceData.AddTx(tx2, 1)) - require.NoError(t, sourceData.AddTx(tx3, 2)) - require.NoError(t, sourceData.AddTx(tx4, 3)) + t.Run("read partial range", func(t *testing.T) { + subtree, sourceData, txs := setupTestSubtreeData(t) serialized, err := sourceData.Serialize() require.NoError(t, err) - // Read only first 2 transactions - reader := bytes.NewReader(serialized) targetData := NewSubtreeData(subtree) - - numRead, err := targetData.ReadTransactionsFromReader(reader, 0, 2) + numRead, err := targetData.ReadTransactionsFromReader(bytes.NewReader(serialized), 0, 2) require.NoError(t, err) assert.Equal(t, 2, numRead) - assert.Equal(t, tx1.Version, targetData.Txs[0].Version) - assert.Equal(t, tx2.Version, targetData.Txs[1].Version) - assert.Nil(t, targetData.Txs[2]) // Not read yet - assert.Nil(t, targetData.Txs[3]) // Not read yet + assert.Equal(t, txs[0].Version, targetData.Txs[0].Version) + assert.Equal(t, txs[1].Version, targetData.Txs[1].Version) + assert.Nil(t, targetData.Txs[2]) + assert.Nil(t, targetData.Txs[3]) }) t.Run("read EOF gracefully", func(t *testing.T) { - subtree, err := NewTree(2) - require.NoError(t, err) - - _ = subtree.AddNode(*tx1.TxIDChainHash(), 111, 0) - _ = subtree.AddNode(*tx2.TxIDChainHash(), 111, 0) - - sourceData := NewSubtreeData(subtree) - require.NoError(t, sourceData.AddTx(tx1, 0)) - require.NoError(t, sourceData.AddTx(tx2, 1)) + subtree, sourceData, _ := setupTestSubtreeData(t) serialized, err := sourceData.Serialize() require.NoError(t, err) - reader := bytes.NewReader(serialized) + // Try to read more transactions than available targetData := NewSubtreeData(subtree) - - // Try to read more transactions than available - should stop at EOF - numRead, err := targetData.ReadTransactionsFromReader(reader, 0, 10) - require.NoError(t, err) // EOF is not an error - assert.Equal(t, 2, numRead) // Only 2 were available + numRead, err := targetData.ReadTransactionsFromReader(bytes.NewReader(serialized), 0, 10) + require.NoError(t, err) // EOF not an error + assert.Equal(t, 4, numRead) // Only 4 available }) } From 7aedb5ae5b378c22a522395734b89ab45cd09b77 Mon Sep 17 00:00:00 2001 From: freemans13 Date: Fri, 12 Dec 2025 23:28:20 +0000 Subject: [PATCH 4/5] linting fixes --- errors.go | 9 +++++++++ subtree_data.go | 6 +++--- subtree_data_test.go | 7 +++---- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/errors.go b/errors.go index 52414b2..0136846 100644 --- a/errors.go +++ b/errors.go @@ -65,4 +65,13 @@ var ( // ErrReadError is a generic read error for testing ErrReadError = errors.New("read error") + + // ErrTransactionNil is returned when a transaction is nil during serialization + ErrTransactionNil = errors.New("transaction is nil, cannot serialize") + + // ErrTransactionWrite is returned when writing a transaction fails + ErrTransactionWrite = errors.New("error writing transaction") + + // ErrTransactionRead is returned when reading a transaction fails + ErrTransactionRead = errors.New("error reading transaction") ) diff --git a/subtree_data.go b/subtree_data.go index 1e49e47..9198038 100644 --- a/subtree_data.go +++ b/subtree_data.go @@ -135,13 +135,13 @@ func (s *Data) WriteTransactionsToWriter(w io.Writer, startIdx, endIdx int) erro } if s.Txs[i] == nil { - return fmt.Errorf("transaction at index %d is nil, cannot serialize", i) + return ErrTransactionNil } // Serialize and stream transaction bytes to writer txBytes := s.Txs[i].SerializeBytes() if _, err := w.Write(txBytes); err != nil { - return fmt.Errorf("error writing transaction at index %d: %w", i, err) + return fmt.Errorf("%w at index %d: %w", ErrTransactionWrite, i, err) } } @@ -176,7 +176,7 @@ func (s *Data) ReadTransactionsFromReader(r io.Reader, startIdx, endIdx int) (in if errors.Is(err, io.EOF) { break } - return txsRead, fmt.Errorf("error reading transaction at index %d: %w", i, err) + return txsRead, fmt.Errorf("%w at index %d: %w", ErrTransactionRead, i, err) } // Validate tx hash matches expected diff --git a/subtree_data_test.go b/subtree_data_test.go index 71a45e5..d974109 100644 --- a/subtree_data_test.go +++ b/subtree_data_test.go @@ -397,7 +397,7 @@ func setupTestSubtreeData(t *testing.T) (*Subtree, *Data, []*bt.Tx) { txs := make([]*bt.Tx, 4) for i := range txs { txs[i] = tx.Clone() - txs[i].Version = uint32(i + 1) + txs[i].Version = uint32(i + 1) //nolint:gosec // G115: test data, safe conversion } subtree, err := NewTree(2) @@ -447,8 +447,7 @@ func TestWriteTransactionsToWriter(t *testing.T) { buf := &bytes.Buffer{} err := subtreeData.WriteTransactionsToWriter(buf, 0, 2) - require.Error(t, err) - assert.Contains(t, err.Error(), "transaction at index 1 is nil") + require.ErrorIs(t, err, ErrTransactionNil) }) } @@ -495,7 +494,7 @@ func TestReadTransactionsFromReader(t *testing.T) { // Try to read more transactions than available targetData := NewSubtreeData(subtree) numRead, err := targetData.ReadTransactionsFromReader(bytes.NewReader(serialized), 0, 10) - require.NoError(t, err) // EOF not an error + require.NoError(t, err) // EOF not an error assert.Equal(t, 4, numRead) // Only 4 available }) } From ee9935987e8955d297cb632c3eb729152627fcb7 Mon Sep 17 00:00:00 2001 From: freemans13 Date: Fri, 12 Dec 2025 23:51:04 +0000 Subject: [PATCH 5/5] simpler implementation --- subtree_data.go | 77 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/subtree_data.go b/subtree_data.go index 9198038..d096804 100644 --- a/subtree_data.go +++ b/subtree_data.go @@ -148,6 +148,83 @@ func (s *Data) WriteTransactionsToWriter(w io.Writer, startIdx, endIdx int) erro return nil } +// WriteTransactionChunk writes a slice of transactions directly to a writer. +// +// This is a simplified streaming function that writes transactions without requiring a SubtreeData +// structure. It's useful for workflows where transactions are already loaded and just need to be +// streamed to disk. +// +// Parameters: +// - w: Writer to stream transactions to +// - txs: Slice of transactions to write +// +// Returns an error if writing fails. +func WriteTransactionChunk(w io.Writer, txs []*bt.Tx) error { + for _, tx := range txs { + if tx == nil { + continue // Skip nil transactions + } + + txBytes := tx.SerializeBytes() + if _, err := w.Write(txBytes); err != nil { + return fmt.Errorf("%w: %w", ErrTransactionWrite, err) + } + } + + return nil +} + +// ReadTransactionChunk reads and validates a chunk of transactions from a reader. +// +// This is a simplified streaming function that reads transactions directly into a new slice, +// validates them against the subtree structure, and returns the populated slice. This is more +// memory-efficient than ReadTransactionsFromReader for processing workflows where the SubtreeData +// array is not needed. +// +// Parameters: +// - r: Reader to read transactions from +// - subtree: Subtree structure for hash validation +// - startIdx: Starting index in subtree for validation +// - count: Number of transactions to read +// +// Returns a slice of transactions and any error encountered. +func ReadTransactionChunk(r io.Reader, subtree *Subtree, startIdx, count int) ([]*bt.Tx, error) { + if subtree == nil || len(subtree.Nodes) == 0 { + return nil, ErrSubtreeNodesEmpty + } + + txs := make([]*bt.Tx, 0, count) + + for i := 0; i < count; i++ { + idx := startIdx + i + if idx >= len(subtree.Nodes) { + break // Reached end of subtree + } + + // Skip coinbase placeholder + if idx == 0 && subtree.Nodes[0].Hash.Equal(CoinbasePlaceholderHashValue) { + continue + } + + tx := &bt.Tx{} + if _, err := tx.ReadFrom(r); err != nil { + if errors.Is(err, io.EOF) { + break + } + return txs, fmt.Errorf("%w at index %d: %w", ErrTransactionRead, idx, err) + } + + // Validate tx hash matches expected + if !subtree.Nodes[idx].Hash.Equal(*tx.TxIDChainHash()) { + return txs, ErrTxHashMismatch + } + + txs = append(txs, tx) + } + + return txs, nil +} + // ReadTransactionsFromReader reads a range of transactions from a reader. // // This enables memory-efficient deserialization by reading only a chunk of transactions