From cf96ed997651b2cd432fa87e481ce9112e6b454f Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 12 May 2025 05:24:48 -0700 Subject: [PATCH 01/15] Add log for creating reverse iterator --- ss/pebbledb/db.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ss/pebbledb/db.go b/ss/pebbledb/db.go index cd51f0f4..2080240e 100644 --- a/ss/pebbledb/db.go +++ b/ss/pebbledb/db.go @@ -514,6 +514,7 @@ func (db *Database) Iterator(storeKey string, version int64, start, end []byte) } func (db *Database) ReverseIterator(storeKey string, version int64, start, end []byte) (types.DBIterator, error) { + fmt.Printf("[Debug] Creating reverset iterator for storeKey %s, version %d, start %X, end %X\n", storeKey, version, start, end) if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { return nil, errorutils.ErrKeyEmpty } From 4005b00e6f39ad897a5c857f4e1b48c064ba822a Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 12 May 2025 08:31:36 -0700 Subject: [PATCH 02/15] Add test iterator --- tools/cmd/seidb/operations/test_iterator.go | 45 +++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tools/cmd/seidb/operations/test_iterator.go diff --git a/tools/cmd/seidb/operations/test_iterator.go b/tools/cmd/seidb/operations/test_iterator.go new file mode 100644 index 00000000..b39b5f74 --- /dev/null +++ b/tools/cmd/seidb/operations/test_iterator.go @@ -0,0 +1,45 @@ +package operations + +import ( + "fmt" + + "github.com/sei-protocol/sei-db/common/logger" + "github.com/sei-protocol/sei-db/config" + "github.com/sei-protocol/sei-db/ss" + "github.com/spf13/cobra" +) + +func TestIteratorCmd() *cobra.Command { + iteratorCmd := &cobra.Command{ + Use: "test-iterator", + Short: "Test forward or reverse iterator", + Run: executeIterator, + } + + iteratorCmd.PersistentFlags().StringP("home-dir", "d", "/root/.sei", "Database Directory") + return iteratorCmd +} + +func executeIterator(cmd *cobra.Command, _ []string) { + homeDir, _ := cmd.Flags().GetString("home-dir") + IterateDbData(homeDir) +} + +func IterateDbData(homeDir string) { + ssConfig := config.DefaultStateStoreConfig() + ssConfig.KeepRecent = 0 + ssStore, err := ss.NewStateStore(logger.NewNopLogger(), homeDir, ssConfig) + defer ssStore.Close() + if err != nil { + panic(err) + } + fmt.Printf("Start iteration") + iter, err := ssStore.ReverseIterator("oracle", 98350313, []byte("07"), []byte("08")) + if err != nil { + panic(err) + } + for ; iter.Valid(); iter.Next() { + fmt.Printf("key: %X, value %X\n", iter.Key(), iter.Value()) + } + fmt.Printf("Complete iteration") +} From 1a1c0fb5d763266a632a56f45d4a0508f014a759 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 12 May 2025 08:32:50 -0700 Subject: [PATCH 03/15] Add cmd --- tools/cmd/seidb/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/cmd/seidb/main.go b/tools/cmd/seidb/main.go index cef02f38..6513fb98 100644 --- a/tools/cmd/seidb/main.go +++ b/tools/cmd/seidb/main.go @@ -25,6 +25,7 @@ func main() { operations.PruneCmd(), operations.DumpIAVLCmd(), operations.StateSizeCmd(), + operations.TestIteratorCmd(), operations.ReplayChangelogCmd()) if err := rootCmd.Execute(); err != nil { fmt.Println(err) From c7bcf82c07a2287c89c4e94c088fa9dfc30e9251 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 12 May 2025 08:35:53 -0700 Subject: [PATCH 04/15] Fix start end --- tools/cmd/seidb/operations/test_iterator.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/cmd/seidb/operations/test_iterator.go b/tools/cmd/seidb/operations/test_iterator.go index b39b5f74..5d4a7751 100644 --- a/tools/cmd/seidb/operations/test_iterator.go +++ b/tools/cmd/seidb/operations/test_iterator.go @@ -1,6 +1,7 @@ package operations import ( + "encoding/hex" "fmt" "github.com/sei-protocol/sei-db/common/logger" @@ -33,13 +34,15 @@ func IterateDbData(homeDir string) { if err != nil { panic(err) } - fmt.Printf("Start iteration") - iter, err := ssStore.ReverseIterator("oracle", 98350313, []byte("07"), []byte("08")) + fmt.Printf("Start iteration\n") + start, _ := hex.DecodeString("07") + end, _ := hex.DecodeString("08") + iter, err := ssStore.ReverseIterator("oracle", 98350313, start, end) if err != nil { panic(err) } for ; iter.Valid(); iter.Next() { fmt.Printf("key: %X, value %X\n", iter.Key(), iter.Value()) } - fmt.Printf("Complete iteration") + fmt.Printf("Complete iteration\n") } From 6ac285b9a9e59fdc325f37027c0c9e099a5f6eb9 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 12 May 2025 08:47:41 -0700 Subject: [PATCH 05/15] Print out all --- tools/cmd/seidb/operations/test_iterator.go | 27 +++++++++++++++------ 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/tools/cmd/seidb/operations/test_iterator.go b/tools/cmd/seidb/operations/test_iterator.go index 5d4a7751..3d2fa494 100644 --- a/tools/cmd/seidb/operations/test_iterator.go +++ b/tools/cmd/seidb/operations/test_iterator.go @@ -18,15 +18,20 @@ func TestIteratorCmd() *cobra.Command { } iteratorCmd.PersistentFlags().StringP("home-dir", "d", "/root/.sei", "Database Directory") + iteratorCmd.PersistentFlags().StringP("start", "s", "07", "Start key") + iteratorCmd.PersistentFlags().StringP("end", "e", "08", "End key") + return iteratorCmd } func executeIterator(cmd *cobra.Command, _ []string) { homeDir, _ := cmd.Flags().GetString("home-dir") - IterateDbData(homeDir) + start, _ := cmd.Flags().GetString("start") + end, _ := cmd.Flags().GetString("end") + IterateDbData(homeDir, start, end) } -func IterateDbData(homeDir string) { +func IterateDbData(homeDir string, start string, end string) { ssConfig := config.DefaultStateStoreConfig() ssConfig.KeepRecent = 0 ssStore, err := ss.NewStateStore(logger.NewNopLogger(), homeDir, ssConfig) @@ -34,15 +39,23 @@ func IterateDbData(homeDir string) { if err != nil { panic(err) } - fmt.Printf("Start iteration\n") - start, _ := hex.DecodeString("07") - end, _ := hex.DecodeString("08") - iter, err := ssStore.ReverseIterator("oracle", 98350313, start, end) + fmt.Printf("Start forward iteration\n") + forwardIter, err := ssStore.Iterator("oracle", 98350313, nil, nil) + for ; forwardIter.Valid(); forwardIter.Next() { + fmt.Printf("key: %X, value %X\n", forwardIter.Key(), forwardIter.Value()) + } + forwardIter.Close() + + fmt.Printf("Start reverse iteration\n") + startPos, _ := hex.DecodeString(start) + endPos, _ := hex.DecodeString(end) + iter, err := ssStore.ReverseIterator("oracle", 98350313, startPos, endPos) if err != nil { panic(err) } for ; iter.Valid(); iter.Next() { fmt.Printf("key: %X, value %X\n", iter.Key(), iter.Value()) } - fmt.Printf("Complete iteration\n") + iter.Close() + fmt.Printf("Complete reverse iteration\n") } From 50dbd2991fbff539295631c08151fd4ec44448f8 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 12 May 2025 08:50:53 -0700 Subject: [PATCH 06/15] Add log --- tools/cmd/seidb/operations/test_iterator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cmd/seidb/operations/test_iterator.go b/tools/cmd/seidb/operations/test_iterator.go index 3d2fa494..7dc330a5 100644 --- a/tools/cmd/seidb/operations/test_iterator.go +++ b/tools/cmd/seidb/operations/test_iterator.go @@ -45,7 +45,7 @@ func IterateDbData(homeDir string, start string, end string) { fmt.Printf("key: %X, value %X\n", forwardIter.Key(), forwardIter.Value()) } forwardIter.Close() - + fmt.Printf("Finished forward iteration\n") fmt.Printf("Start reverse iteration\n") startPos, _ := hex.DecodeString(start) endPos, _ := hex.DecodeString(end) From 90aec4e4c91b5859bb22244302396d34447b0e21 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 12 May 2025 09:00:10 -0700 Subject: [PATCH 07/15] Check error --- tools/cmd/seidb/operations/test_iterator.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/cmd/seidb/operations/test_iterator.go b/tools/cmd/seidb/operations/test_iterator.go index 7dc330a5..bce8308f 100644 --- a/tools/cmd/seidb/operations/test_iterator.go +++ b/tools/cmd/seidb/operations/test_iterator.go @@ -35,6 +35,9 @@ func IterateDbData(homeDir string, start string, end string) { ssConfig := config.DefaultStateStoreConfig() ssConfig.KeepRecent = 0 ssStore, err := ss.NewStateStore(logger.NewNopLogger(), homeDir, ssConfig) + if err != nil { + panic(err) + } defer ssStore.Close() if err != nil { panic(err) From 0dade246673f85788d642a7a0840815414072edf Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 12 May 2025 09:08:19 -0700 Subject: [PATCH 08/15] remove forward --- tools/cmd/seidb/operations/test_iterator.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tools/cmd/seidb/operations/test_iterator.go b/tools/cmd/seidb/operations/test_iterator.go index bce8308f..01de1a9e 100644 --- a/tools/cmd/seidb/operations/test_iterator.go +++ b/tools/cmd/seidb/operations/test_iterator.go @@ -42,13 +42,6 @@ func IterateDbData(homeDir string, start string, end string) { if err != nil { panic(err) } - fmt.Printf("Start forward iteration\n") - forwardIter, err := ssStore.Iterator("oracle", 98350313, nil, nil) - for ; forwardIter.Valid(); forwardIter.Next() { - fmt.Printf("key: %X, value %X\n", forwardIter.Key(), forwardIter.Value()) - } - forwardIter.Close() - fmt.Printf("Finished forward iteration\n") fmt.Printf("Start reverse iteration\n") startPos, _ := hex.DecodeString(start) endPos, _ := hex.DecodeString(end) From a7b84c0af1f87e5807b03bbc1d9353bfb87c5b42 Mon Sep 17 00:00:00 2001 From: kbhat1 Date: Mon, 12 May 2025 12:42:01 -0400 Subject: [PATCH 09/15] Add iterator key / version logging --- ss/pebbledb/iterator.go | 66 +++++++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/ss/pebbledb/iterator.go b/ss/pebbledb/iterator.go index 36ca8f53..996333f5 100644 --- a/ss/pebbledb/iterator.go +++ b/ss/pebbledb/iterator.go @@ -136,12 +136,14 @@ func (itr *iterator) nextForward() { return } - currKey, _, ok := SplitMVCCKey(itr.source.Key()) + currKey, currKeyVersionBz, ok := SplitMVCCKey(itr.source.Key()) if !ok { // XXX: This should not happen as that would indicate we have a malformed // MVCC key. panic(fmt.Sprintf("invalid PebbleDB MVCC key: %s", itr.source.Key())) } + currKeyVersion, _ := decodeUint64Ascending(currKeyVersionBz) // Assuming decode won't fail here based on prior checks + fmt.Printf("nextForward: currKey=%s, currKeyVersion=%d\n", string(currKey), currKeyVersion) next := itr.source.NextPrefix() @@ -149,22 +151,33 @@ func (itr *iterator) nextForward() { // desired version for that key, e.g. if the key was written at a later version, // so we seek back to the latest desired version, s.t. the version is <= itr.version. if next { - nextKey, _, ok := SplitMVCCKey(itr.source.Key()) + nextKey, nextKeyVersionBz, ok := SplitMVCCKey(itr.source.Key()) if !ok { // XXX: This should not happen as that would indicate we have a malformed // MVCC key. itr.valid = false return } + nextKeyVersion, _ := decodeUint64Ascending(nextKeyVersionBz) // Assuming decode won't fail here + fmt.Printf("nextForward: nextPrefix moved to nextKey=%s, nextKeyVersion=%d\n", string(nextKey), nextKeyVersion) + if !bytes.HasPrefix(nextKey, itr.prefix) { // the next key must have itr.prefix as the prefix + fmt.Printf("nextForward: nextKey %s does not have prefix %s\n", string(nextKey), string(itr.prefix)) itr.valid = false return } // Move the iterator to the closest version to the desired version, so we // append the current iterator key to the prefix and seek to that key. - itr.valid = itr.source.SeekLT(MVCCEncode(nextKey, itr.version+1)) + seekTarget := MVCCEncode(nextKey, itr.version+1) + fmt.Printf("nextForward: seeking LT %s (key=%s, version=%d)\n", string(seekTarget), string(nextKey), itr.version+1) + itr.valid = itr.source.SeekLT(seekTarget) + + if !itr.valid { + fmt.Printf("nextForward: SeekLT returned invalid\n") + return + } tmpKey, tmpKeyVersion, ok := SplitMVCCKey(itr.source.Key()) if !ok { @@ -173,13 +186,24 @@ func (itr *iterator) nextForward() { itr.valid = false return } + tmpKeyVersionDecoded, err := decodeUint64Ascending(tmpKeyVersion) + if err != nil { + itr.valid = false + return + } + fmt.Printf("nextForward: SeekLT landed on tmpKey=%s, tmpKeyVersion=%d\n", string(tmpKey), tmpKeyVersionDecoded) // There exists cases where the SeekLT() call moved us back to the same key // we started at, so we must move to next key, i.e. two keys forward. if bytes.Equal(tmpKey, currKey) { + fmt.Printf("nextForward: SeekLT moved back to original key %s, calling NextPrefix and nextForward recursively\n", string(currKey)) if itr.source.NextPrefix() { itr.nextForward() + // Need to re-check key and version after recursive call returns + if !itr.valid { + return // Recursive call invalidated the iterator + } _, tmpKeyVersion, ok = SplitMVCCKey(itr.source.Key()) if !ok { // XXX: This should not happen as that would indicate we have a malformed @@ -189,6 +213,7 @@ func (itr *iterator) nextForward() { } } else { + fmt.Printf("nextForward: NextPrefix after SeekLT returned false\n") itr.valid = false return } @@ -196,7 +221,8 @@ func (itr *iterator) nextForward() { // We need to verify that every Next call either moves the iterator to a key whose version // is less than or equal to requested iterator version, or exhausts the iterator - tmpKeyVersionDecoded, err := decodeUint64Ascending(tmpKeyVersion) + // Re-decode version after potential recursive call + tmpKeyVersionDecoded, err = decodeUint64Ascending(tmpKeyVersion) if err != nil { itr.valid = false return @@ -204,17 +230,20 @@ func (itr *iterator) nextForward() { // If iterator is at a entry whose version is higher than requested version, call nextForward again if tmpKeyVersionDecoded > itr.version { + fmt.Printf("nextForward: tmpKeyVersion %d > target version %d, calling nextForward recursively\n", tmpKeyVersionDecoded, itr.version) itr.nextForward() } // The cursor might now be pointing at a key/value pair that is tombstoned. // If so, we must move the cursor. if itr.valid && itr.cursorTombstoned() { + fmt.Printf("nextForward: cursor is tombstoned, calling nextForward recursively\n") itr.nextForward() } return } + fmt.Printf("nextForward: NextPrefix returned false\n") itr.valid = false } @@ -225,37 +254,52 @@ func (itr *iterator) nextReverse() { return } - currKey, _, ok := SplitMVCCKey(itr.source.Key()) + currKey, currKeyVersionBz, ok := SplitMVCCKey(itr.source.Key()) if !ok { // XXX: This should not happen as that would indicate we have a malformed // MVCC key. panic(fmt.Sprintf("invalid PebbleDB MVCC key: %s", itr.source.Key())) } + currKeyVersion, _ := decodeUint64Ascending(currKeyVersionBz) // Assuming decode won't fail here + fmt.Printf("nextReverse: currKey=%s, currKeyVersion=%d\n", string(currKey), currKeyVersion) - next := itr.source.SeekLT(MVCCEncode(currKey, 0)) + seekTarget := MVCCEncode(currKey, 0) + fmt.Printf("nextReverse: seeking LT %s (key=%s, version=0)\n", string(seekTarget), string(currKey)) + next := itr.source.SeekLT(seekTarget) // First move the iterator to the next prefix, which may not correspond to the // desired version for that key, e.g. if the key was written at a later version, // so we seek back to the latest desired version, s.t. the version is <= itr.version. if next { - nextKey, _, ok := SplitMVCCKey(itr.source.Key()) + nextKey, nextKeyVersionBz, ok := SplitMVCCKey(itr.source.Key()) if !ok { // XXX: This should not happen as that would indicate we have a malformed // MVCC key. itr.valid = false return } + nextKeyVersion, _ := decodeUint64Ascending(nextKeyVersionBz) // Assuming decode won't fail here + fmt.Printf("nextReverse: SeekLT(currKey, 0) moved to nextKey=%s, nextKeyVersion=%d\n", string(nextKey), nextKeyVersion) + if !bytes.HasPrefix(nextKey, itr.prefix) { // the next key must have itr.prefix as the prefix + fmt.Printf("nextReverse: nextKey %s does not have prefix %s\n", string(nextKey), string(itr.prefix)) itr.valid = false return } // Move the iterator to the closest version to the desired version, so we // append the current iterator key to the prefix and seek to that key. - itr.valid = itr.source.SeekLT(MVCCEncode(nextKey, itr.version+1)) + seekTargetClosest := MVCCEncode(nextKey, itr.version+1) + fmt.Printf("nextReverse: seeking LT %s (key=%s, version=%d)\n", string(seekTargetClosest), string(nextKey), itr.version+1) + itr.valid = itr.source.SeekLT(seekTargetClosest) + + if !itr.valid { + fmt.Printf("nextReverse: SeekLT(nextKey, version+1) returned invalid\n") + return + } - _, tmpKeyVersion, ok := SplitMVCCKey(itr.source.Key()) + tmpKey, tmpKeyVersion, ok := SplitMVCCKey(itr.source.Key()) // Renamed from _ to tmpKey for logging if !ok { // XXX: This should not happen as that would indicate we have a malformed // MVCC key. @@ -270,20 +314,24 @@ func (itr *iterator) nextReverse() { itr.valid = false return } + fmt.Printf("nextReverse: SeekLT(nextKey, version+1) landed on tmpKey=%s, tmpKeyVersion=%d\n", string(tmpKey), tmpKeyVersionDecoded) // If iterator is at a entry whose version is higher than requested version, call nextReverse again if tmpKeyVersionDecoded > itr.version { + fmt.Printf("nextReverse: tmpKeyVersion %d > target version %d, calling nextReverse recursively\n", tmpKeyVersionDecoded, itr.version) itr.nextReverse() } // The cursor might now be pointing at a key/value pair that is tombstoned. // If so, we must move the cursor. if itr.valid && itr.cursorTombstoned() { + fmt.Printf("nextReverse: cursor is tombstoned, calling nextReverse recursively\n") itr.nextReverse() } return } + fmt.Printf("nextReverse: SeekLT(currKey, 0) returned false\n") itr.valid = false } From bb21bfae77679b3a34b251676459409ce82edd96 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 12 May 2025 10:57:32 -0700 Subject: [PATCH 10/15] Set upperbound to version+1 --- ss/pebbledb/db.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ss/pebbledb/db.go b/ss/pebbledb/db.go index 0ed9e5fe..488aebe4 100644 --- a/ss/pebbledb/db.go +++ b/ss/pebbledb/db.go @@ -690,9 +690,9 @@ func (db *Database) ReverseIterator(storeKey string, version int64, start, end [ var upperBound []byte if end != nil { - upperBound = MVCCEncode(prependStoreKey(storeKey, end), 0) + upperBound = MVCCEncode(prependStoreKey(storeKey, end), version+1) } else { - upperBound = MVCCEncode(prefixEnd(storePrefix(storeKey)), 0) + upperBound = MVCCEncode(prefixEnd(storePrefix(storeKey)), version+1) } itr, err := db.storage.NewIter(&pebble.IterOptions{LowerBound: lowerBound, UpperBound: upperBound}) From bad65c71ebfbef950316834666e345ecf19389b7 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 12 May 2025 11:15:30 -0700 Subject: [PATCH 11/15] Try iter.prev --- ss/pebbledb/iterator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ss/pebbledb/iterator.go b/ss/pebbledb/iterator.go index 996333f5..626dddcd 100644 --- a/ss/pebbledb/iterator.go +++ b/ss/pebbledb/iterator.go @@ -45,9 +45,9 @@ func newPebbleDBIterator(src *pebble.Iterator, prefix, mvccStart, mvccEnd []byte // move the underlying PebbleDB iterator to the first key var valid bool if reverse { - valid = src.Last() + valid = src.Prev() } else { - valid = src.First() + valid = src.Next() } itr := &iterator{ From ea7aea1f8d89120c848fcdbb3798e9d40315c578 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 12 May 2025 11:23:42 -0700 Subject: [PATCH 12/15] Seek to end first --- ss/pebbledb/db.go | 5 +++++ ss/pebbledb/iterator.go | 9 ++------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ss/pebbledb/db.go b/ss/pebbledb/db.go index 488aebe4..50db6a12 100644 --- a/ss/pebbledb/db.go +++ b/ss/pebbledb/db.go @@ -699,6 +699,11 @@ func (db *Database) ReverseIterator(storeKey string, version int64, start, end [ if err != nil { return nil, fmt.Errorf("failed to create PebbleDB iterator: %w", err) } + valid := itr.SeekLT(upperBound) + if !valid { + fmt.Printf("Invalid reverse iterator\n") + return nil, errors.New("invalid range") + } return newPebbleDBIterator(itr, storePrefix(storeKey), start, end, version, db.earliestVersion, true), nil } diff --git a/ss/pebbledb/iterator.go b/ss/pebbledb/iterator.go index 626dddcd..8cb00322 100644 --- a/ss/pebbledb/iterator.go +++ b/ss/pebbledb/iterator.go @@ -42,14 +42,9 @@ func newPebbleDBIterator(src *pebble.Iterator, prefix, mvccStart, mvccEnd []byte } } - // move the underlying PebbleDB iterator to the first key - var valid bool - if reverse { - valid = src.Prev() - } else { - valid = src.Next() - } + valid := true + // move the underlying PebbleDB iterator to the first key itr := &iterator{ source: src, prefix: prefix, From f87804ad7c040e3081d10305c8634d0000de711b Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 12 May 2025 11:27:05 -0700 Subject: [PATCH 13/15] Seek to end first --- ss/pebbledb/db.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ss/pebbledb/db.go b/ss/pebbledb/db.go index 50db6a12..09c469a4 100644 --- a/ss/pebbledb/db.go +++ b/ss/pebbledb/db.go @@ -704,6 +704,9 @@ func (db *Database) ReverseIterator(storeKey string, version int64, start, end [ fmt.Printf("Invalid reverse iterator\n") return nil, errors.New("invalid range") } + currKey, currKeyVersion, _ := SplitMVCCKey(itr.Key()) + curKeyVersionDecoded, _ := decodeUint64Ascending(currKeyVersion) + fmt.Printf("[Debug] After seekLT, currKey is %X, currKeyVersion is %d\n", currKey, curKeyVersionDecoded) return newPebbleDBIterator(itr, storePrefix(storeKey), start, end, version, db.earliestVersion, true), nil } From 312eef812709fda51a95fc0f95b94c501782cc06 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 12 May 2025 11:49:43 -0700 Subject: [PATCH 14/15] key hex --- ss/pebbledb/db.go | 7 +++++++ ss/pebbledb/iterator.go | 14 +++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/ss/pebbledb/db.go b/ss/pebbledb/db.go index 09c469a4..f0e37a70 100644 --- a/ss/pebbledb/db.go +++ b/ss/pebbledb/db.go @@ -694,6 +694,13 @@ func (db *Database) ReverseIterator(storeKey string, version int64, start, end [ } else { upperBound = MVCCEncode(prefixEnd(storePrefix(storeKey)), version+1) } + // key2 + // s/k:oracle/key1_98350314 + // s/k:oracle/key1_98350315 + // s/k:oracle/key1_98350316 + // ... + // s/k:oracle/key1_146607775 + // s/k:oracle/key1_146607777 itr, err := db.storage.NewIter(&pebble.IterOptions{LowerBound: lowerBound, UpperBound: upperBound}) if err != nil { diff --git a/ss/pebbledb/iterator.go b/ss/pebbledb/iterator.go index 8cb00322..2696e0e0 100644 --- a/ss/pebbledb/iterator.go +++ b/ss/pebbledb/iterator.go @@ -253,13 +253,13 @@ func (itr *iterator) nextReverse() { if !ok { // XXX: This should not happen as that would indicate we have a malformed // MVCC key. - panic(fmt.Sprintf("invalid PebbleDB MVCC key: %s", itr.source.Key())) + panic(fmt.Sprintf("invalid PebbleDB MVCC key: %X", itr.source.Key())) } currKeyVersion, _ := decodeUint64Ascending(currKeyVersionBz) // Assuming decode won't fail here - fmt.Printf("nextReverse: currKey=%s, currKeyVersion=%d\n", string(currKey), currKeyVersion) + fmt.Printf("nextReverse: currKey=%X, currKeyVersion=%d\n", currKey, currKeyVersion) seekTarget := MVCCEncode(currKey, 0) - fmt.Printf("nextReverse: seeking LT %s (key=%s, version=0)\n", string(seekTarget), string(currKey)) + fmt.Printf("nextReverse: seeking LT %s (key=%X, version=0)\n", string(seekTarget), currKey) next := itr.source.SeekLT(seekTarget) // First move the iterator to the next prefix, which may not correspond to the @@ -274,11 +274,11 @@ func (itr *iterator) nextReverse() { return } nextKeyVersion, _ := decodeUint64Ascending(nextKeyVersionBz) // Assuming decode won't fail here - fmt.Printf("nextReverse: SeekLT(currKey, 0) moved to nextKey=%s, nextKeyVersion=%d\n", string(nextKey), nextKeyVersion) + fmt.Printf("nextReverse: SeekLT(currKey, 0) moved to nextKey=%X, nextKeyVersion=%d\n", nextKey, nextKeyVersion) if !bytes.HasPrefix(nextKey, itr.prefix) { // the next key must have itr.prefix as the prefix - fmt.Printf("nextReverse: nextKey %s does not have prefix %s\n", string(nextKey), string(itr.prefix)) + fmt.Printf("nextReverse: nextKey %X does not have prefix %s\n", nextKey, string(itr.prefix)) itr.valid = false return } @@ -286,7 +286,7 @@ func (itr *iterator) nextReverse() { // Move the iterator to the closest version to the desired version, so we // append the current iterator key to the prefix and seek to that key. seekTargetClosest := MVCCEncode(nextKey, itr.version+1) - fmt.Printf("nextReverse: seeking LT %s (key=%s, version=%d)\n", string(seekTargetClosest), string(nextKey), itr.version+1) + fmt.Printf("nextReverse: seeking LT %s (key=%X, version=%d)\n", string(seekTargetClosest), nextKey, itr.version+1) itr.valid = itr.source.SeekLT(seekTargetClosest) if !itr.valid { @@ -309,7 +309,7 @@ func (itr *iterator) nextReverse() { itr.valid = false return } - fmt.Printf("nextReverse: SeekLT(nextKey, version+1) landed on tmpKey=%s, tmpKeyVersion=%d\n", string(tmpKey), tmpKeyVersionDecoded) + fmt.Printf("nextReverse: SeekLT(nextKey, version+1) landed on tmpKey=%X, tmpKeyVersion=%d\n", tmpKey, tmpKeyVersionDecoded) // If iterator is at a entry whose version is higher than requested version, call nextReverse again if tmpKeyVersionDecoded > itr.version { From c322777bb6303e2bbedc818acf0df01111b58eec Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 12 May 2025 12:10:08 -0700 Subject: [PATCH 15/15] log --- ss/pebbledb/db.go | 7 ------- ss/pebbledb/iterator.go | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/ss/pebbledb/db.go b/ss/pebbledb/db.go index f0e37a70..09c469a4 100644 --- a/ss/pebbledb/db.go +++ b/ss/pebbledb/db.go @@ -694,13 +694,6 @@ func (db *Database) ReverseIterator(storeKey string, version int64, start, end [ } else { upperBound = MVCCEncode(prefixEnd(storePrefix(storeKey)), version+1) } - // key2 - // s/k:oracle/key1_98350314 - // s/k:oracle/key1_98350315 - // s/k:oracle/key1_98350316 - // ... - // s/k:oracle/key1_146607775 - // s/k:oracle/key1_146607777 itr, err := db.storage.NewIter(&pebble.IterOptions{LowerBound: lowerBound, UpperBound: upperBound}) if err != nil { diff --git a/ss/pebbledb/iterator.go b/ss/pebbledb/iterator.go index 2696e0e0..a0f65f64 100644 --- a/ss/pebbledb/iterator.go +++ b/ss/pebbledb/iterator.go @@ -61,12 +61,12 @@ func newPebbleDBIterator(src *pebble.Iterator, prefix, mvccStart, mvccEnd []byte // XXX: This should not happen as that would indicate we have a malformed MVCC key. panic(fmt.Sprintf("invalid PebbleDB MVCC key: %s", itr.source.Key())) } - curKeyVersionDecoded, err := decodeUint64Ascending(currKeyVersion) if err != nil { itr.valid = false return itr } + fmt.Printf("[Debug] Intialize iterator with currKey %X, version %d\n", currKey, curKeyVersionDecoded) // We need to check whether initial key iterator visits has a version <= requested version // If larger version, call next to find another key which does