diff --git a/ss/pebbledb/db.go b/ss/pebbledb/db.go index b1fe4916..09c469a4 100644 --- a/ss/pebbledb/db.go +++ b/ss/pebbledb/db.go @@ -677,6 +677,7 @@ func prefixEnd(b []byte) []byte { } func (db *Database) ReverseIterator(storeKey string, version int64, start, end []byte) (types.DBIterator, error) { + fmt.Printf("[Debug] Creating reverset iterator for storeKey %s, version %d, start %X, end %X\n", storeKey, version, start, end) if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { return nil, errorutils.ErrKeyEmpty } @@ -689,15 +690,23 @@ func (db *Database) ReverseIterator(storeKey string, version int64, start, end [ var upperBound []byte if end != nil { - upperBound = MVCCEncode(prependStoreKey(storeKey, end), 0) + upperBound = MVCCEncode(prependStoreKey(storeKey, end), version+1) } else { - upperBound = MVCCEncode(prefixEnd(storePrefix(storeKey)), 0) + upperBound = MVCCEncode(prefixEnd(storePrefix(storeKey)), version+1) } itr, err := db.storage.NewIter(&pebble.IterOptions{LowerBound: lowerBound, UpperBound: upperBound}) if err != nil { return nil, fmt.Errorf("failed to create PebbleDB iterator: %w", err) } + valid := itr.SeekLT(upperBound) + if !valid { + fmt.Printf("Invalid reverse iterator\n") + return nil, errors.New("invalid range") + } + currKey, currKeyVersion, _ := SplitMVCCKey(itr.Key()) + curKeyVersionDecoded, _ := decodeUint64Ascending(currKeyVersion) + fmt.Printf("[Debug] After seekLT, currKey is %X, currKeyVersion is %d\n", currKey, curKeyVersionDecoded) return newPebbleDBIterator(itr, storePrefix(storeKey), start, end, version, db.earliestVersion, true), nil } diff --git a/ss/pebbledb/iterator.go b/ss/pebbledb/iterator.go index 36ca8f53..a0f65f64 100644 --- a/ss/pebbledb/iterator.go +++ b/ss/pebbledb/iterator.go @@ -42,14 +42,9 @@ func newPebbleDBIterator(src *pebble.Iterator, prefix, mvccStart, mvccEnd []byte } } - // move the underlying PebbleDB iterator to the first key - var valid bool - if reverse { - valid = src.Last() - } else { - valid = src.First() - } + valid := true + // move the underlying PebbleDB iterator to the first key itr := &iterator{ source: src, prefix: prefix, @@ -66,12 +61,12 @@ func newPebbleDBIterator(src *pebble.Iterator, prefix, mvccStart, mvccEnd []byte // XXX: This should not happen as that would indicate we have a malformed MVCC key. panic(fmt.Sprintf("invalid PebbleDB MVCC key: %s", itr.source.Key())) } - curKeyVersionDecoded, err := decodeUint64Ascending(currKeyVersion) if err != nil { itr.valid = false return itr } + fmt.Printf("[Debug] Intialize iterator with currKey %X, version %d\n", currKey, curKeyVersionDecoded) // We need to check whether initial key iterator visits has a version <= requested version // If larger version, call next to find another key which does @@ -136,12 +131,14 @@ func (itr *iterator) nextForward() { return } - currKey, _, ok := SplitMVCCKey(itr.source.Key()) + currKey, currKeyVersionBz, ok := SplitMVCCKey(itr.source.Key()) if !ok { // XXX: This should not happen as that would indicate we have a malformed // MVCC key. panic(fmt.Sprintf("invalid PebbleDB MVCC key: %s", itr.source.Key())) } + currKeyVersion, _ := decodeUint64Ascending(currKeyVersionBz) // Assuming decode won't fail here based on prior checks + fmt.Printf("nextForward: currKey=%s, currKeyVersion=%d\n", string(currKey), currKeyVersion) next := itr.source.NextPrefix() @@ -149,22 +146,33 @@ func (itr *iterator) nextForward() { // desired version for that key, e.g. if the key was written at a later version, // so we seek back to the latest desired version, s.t. the version is <= itr.version. if next { - nextKey, _, ok := SplitMVCCKey(itr.source.Key()) + nextKey, nextKeyVersionBz, ok := SplitMVCCKey(itr.source.Key()) if !ok { // XXX: This should not happen as that would indicate we have a malformed // MVCC key. itr.valid = false return } + nextKeyVersion, _ := decodeUint64Ascending(nextKeyVersionBz) // Assuming decode won't fail here + fmt.Printf("nextForward: nextPrefix moved to nextKey=%s, nextKeyVersion=%d\n", string(nextKey), nextKeyVersion) + if !bytes.HasPrefix(nextKey, itr.prefix) { // the next key must have itr.prefix as the prefix + fmt.Printf("nextForward: nextKey %s does not have prefix %s\n", string(nextKey), string(itr.prefix)) itr.valid = false return } // Move the iterator to the closest version to the desired version, so we // append the current iterator key to the prefix and seek to that key. - itr.valid = itr.source.SeekLT(MVCCEncode(nextKey, itr.version+1)) + seekTarget := MVCCEncode(nextKey, itr.version+1) + fmt.Printf("nextForward: seeking LT %s (key=%s, version=%d)\n", string(seekTarget), string(nextKey), itr.version+1) + itr.valid = itr.source.SeekLT(seekTarget) + + if !itr.valid { + fmt.Printf("nextForward: SeekLT returned invalid\n") + return + } tmpKey, tmpKeyVersion, ok := SplitMVCCKey(itr.source.Key()) if !ok { @@ -173,13 +181,24 @@ func (itr *iterator) nextForward() { itr.valid = false return } + tmpKeyVersionDecoded, err := decodeUint64Ascending(tmpKeyVersion) + if err != nil { + itr.valid = false + return + } + fmt.Printf("nextForward: SeekLT landed on tmpKey=%s, tmpKeyVersion=%d\n", string(tmpKey), tmpKeyVersionDecoded) // There exists cases where the SeekLT() call moved us back to the same key // we started at, so we must move to next key, i.e. two keys forward. if bytes.Equal(tmpKey, currKey) { + fmt.Printf("nextForward: SeekLT moved back to original key %s, calling NextPrefix and nextForward recursively\n", string(currKey)) if itr.source.NextPrefix() { itr.nextForward() + // Need to re-check key and version after recursive call returns + if !itr.valid { + return // Recursive call invalidated the iterator + } _, tmpKeyVersion, ok = SplitMVCCKey(itr.source.Key()) if !ok { // XXX: This should not happen as that would indicate we have a malformed @@ -189,6 +208,7 @@ func (itr *iterator) nextForward() { } } else { + fmt.Printf("nextForward: NextPrefix after SeekLT returned false\n") itr.valid = false return } @@ -196,7 +216,8 @@ func (itr *iterator) nextForward() { // We need to verify that every Next call either moves the iterator to a key whose version // is less than or equal to requested iterator version, or exhausts the iterator - tmpKeyVersionDecoded, err := decodeUint64Ascending(tmpKeyVersion) + // Re-decode version after potential recursive call + tmpKeyVersionDecoded, err = decodeUint64Ascending(tmpKeyVersion) if err != nil { itr.valid = false return @@ -204,17 +225,20 @@ func (itr *iterator) nextForward() { // If iterator is at a entry whose version is higher than requested version, call nextForward again if tmpKeyVersionDecoded > itr.version { + fmt.Printf("nextForward: tmpKeyVersion %d > target version %d, calling nextForward recursively\n", tmpKeyVersionDecoded, itr.version) itr.nextForward() } // The cursor might now be pointing at a key/value pair that is tombstoned. // If so, we must move the cursor. if itr.valid && itr.cursorTombstoned() { + fmt.Printf("nextForward: cursor is tombstoned, calling nextForward recursively\n") itr.nextForward() } return } + fmt.Printf("nextForward: NextPrefix returned false\n") itr.valid = false } @@ -225,37 +249,52 @@ func (itr *iterator) nextReverse() { return } - currKey, _, ok := SplitMVCCKey(itr.source.Key()) + currKey, currKeyVersionBz, ok := SplitMVCCKey(itr.source.Key()) if !ok { // XXX: This should not happen as that would indicate we have a malformed // MVCC key. - panic(fmt.Sprintf("invalid PebbleDB MVCC key: %s", itr.source.Key())) + panic(fmt.Sprintf("invalid PebbleDB MVCC key: %X", itr.source.Key())) } + currKeyVersion, _ := decodeUint64Ascending(currKeyVersionBz) // Assuming decode won't fail here + fmt.Printf("nextReverse: currKey=%X, currKeyVersion=%d\n", currKey, currKeyVersion) - next := itr.source.SeekLT(MVCCEncode(currKey, 0)) + seekTarget := MVCCEncode(currKey, 0) + fmt.Printf("nextReverse: seeking LT %s (key=%X, version=0)\n", string(seekTarget), currKey) + next := itr.source.SeekLT(seekTarget) // First move the iterator to the next prefix, which may not correspond to the // desired version for that key, e.g. if the key was written at a later version, // so we seek back to the latest desired version, s.t. the version is <= itr.version. if next { - nextKey, _, ok := SplitMVCCKey(itr.source.Key()) + nextKey, nextKeyVersionBz, ok := SplitMVCCKey(itr.source.Key()) if !ok { // XXX: This should not happen as that would indicate we have a malformed // MVCC key. itr.valid = false return } + nextKeyVersion, _ := decodeUint64Ascending(nextKeyVersionBz) // Assuming decode won't fail here + fmt.Printf("nextReverse: SeekLT(currKey, 0) moved to nextKey=%X, nextKeyVersion=%d\n", nextKey, nextKeyVersion) + if !bytes.HasPrefix(nextKey, itr.prefix) { // the next key must have itr.prefix as the prefix + fmt.Printf("nextReverse: nextKey %X does not have prefix %s\n", nextKey, string(itr.prefix)) itr.valid = false return } // Move the iterator to the closest version to the desired version, so we // append the current iterator key to the prefix and seek to that key. - itr.valid = itr.source.SeekLT(MVCCEncode(nextKey, itr.version+1)) + seekTargetClosest := MVCCEncode(nextKey, itr.version+1) + fmt.Printf("nextReverse: seeking LT %s (key=%X, version=%d)\n", string(seekTargetClosest), nextKey, itr.version+1) + itr.valid = itr.source.SeekLT(seekTargetClosest) + + if !itr.valid { + fmt.Printf("nextReverse: SeekLT(nextKey, version+1) returned invalid\n") + return + } - _, tmpKeyVersion, ok := SplitMVCCKey(itr.source.Key()) + tmpKey, tmpKeyVersion, ok := SplitMVCCKey(itr.source.Key()) // Renamed from _ to tmpKey for logging if !ok { // XXX: This should not happen as that would indicate we have a malformed // MVCC key. @@ -270,20 +309,24 @@ func (itr *iterator) nextReverse() { itr.valid = false return } + fmt.Printf("nextReverse: SeekLT(nextKey, version+1) landed on tmpKey=%X, tmpKeyVersion=%d\n", tmpKey, tmpKeyVersionDecoded) // If iterator is at a entry whose version is higher than requested version, call nextReverse again if tmpKeyVersionDecoded > itr.version { + fmt.Printf("nextReverse: tmpKeyVersion %d > target version %d, calling nextReverse recursively\n", tmpKeyVersionDecoded, itr.version) itr.nextReverse() } // The cursor might now be pointing at a key/value pair that is tombstoned. // If so, we must move the cursor. if itr.valid && itr.cursorTombstoned() { + fmt.Printf("nextReverse: cursor is tombstoned, calling nextReverse recursively\n") itr.nextReverse() } return } + fmt.Printf("nextReverse: SeekLT(currKey, 0) returned false\n") itr.valid = false } diff --git a/tools/cmd/seidb/main.go b/tools/cmd/seidb/main.go index cef02f38..6513fb98 100644 --- a/tools/cmd/seidb/main.go +++ b/tools/cmd/seidb/main.go @@ -25,6 +25,7 @@ func main() { operations.PruneCmd(), operations.DumpIAVLCmd(), operations.StateSizeCmd(), + operations.TestIteratorCmd(), operations.ReplayChangelogCmd()) if err := rootCmd.Execute(); err != nil { fmt.Println(err) diff --git a/tools/cmd/seidb/operations/test_iterator.go b/tools/cmd/seidb/operations/test_iterator.go new file mode 100644 index 00000000..01de1a9e --- /dev/null +++ b/tools/cmd/seidb/operations/test_iterator.go @@ -0,0 +1,57 @@ +package operations + +import ( + "encoding/hex" + "fmt" + + "github.com/sei-protocol/sei-db/common/logger" + "github.com/sei-protocol/sei-db/config" + "github.com/sei-protocol/sei-db/ss" + "github.com/spf13/cobra" +) + +func TestIteratorCmd() *cobra.Command { + iteratorCmd := &cobra.Command{ + Use: "test-iterator", + Short: "Test forward or reverse iterator", + Run: executeIterator, + } + + iteratorCmd.PersistentFlags().StringP("home-dir", "d", "/root/.sei", "Database Directory") + iteratorCmd.PersistentFlags().StringP("start", "s", "07", "Start key") + iteratorCmd.PersistentFlags().StringP("end", "e", "08", "End key") + + return iteratorCmd +} + +func executeIterator(cmd *cobra.Command, _ []string) { + homeDir, _ := cmd.Flags().GetString("home-dir") + start, _ := cmd.Flags().GetString("start") + end, _ := cmd.Flags().GetString("end") + IterateDbData(homeDir, start, end) +} + +func IterateDbData(homeDir string, start string, end string) { + ssConfig := config.DefaultStateStoreConfig() + ssConfig.KeepRecent = 0 + ssStore, err := ss.NewStateStore(logger.NewNopLogger(), homeDir, ssConfig) + if err != nil { + panic(err) + } + defer ssStore.Close() + if err != nil { + panic(err) + } + fmt.Printf("Start reverse iteration\n") + startPos, _ := hex.DecodeString(start) + endPos, _ := hex.DecodeString(end) + iter, err := ssStore.ReverseIterator("oracle", 98350313, startPos, endPos) + if err != nil { + panic(err) + } + for ; iter.Valid(); iter.Next() { + fmt.Printf("key: %X, value %X\n", iter.Key(), iter.Value()) + } + iter.Close() + fmt.Printf("Complete reverse iteration\n") +}