Skip to content

impr: indexing & filtering - to reduce sizes of indexes #17

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
marino39 merged 1 commit into from
Feb 5, 2025
Merged

impr: indexing & filtering - to reduce sizes of indexes #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
163 changes: 98 additions & 65 deletions filter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,27 +5,29 @@ import (
"context"
"fmt"
"path"
"reflect"

"github.com/0xsequence/ethwal/storage"
"github.com/0xsequence/ethwal/storage/local"
"github.com/RoaringBitmap/roaring/v2/roaring64"
)

type Filter interface {
Eval(ctx context.Context) FilterIterator
}
// Filter is an interface that defines the methods to filter blocks
// based on the index data.
type Filter[T any] interface {
// Filter blocks inner data based on the filter criteria.
Filter(block Block[T]) Block[T]

// IndexIterator returns the iterator for the filter.
IndexIterator(ctx context.Context) *IndexIterator

type FilterIterator interface {
HasNext() bool
Next() (uint64, uint16)
Peek() (uint64, uint16)
Bitmap() *roaring64.Bitmap
bitmap(block Block[T]) *roaring64.Bitmap
}

type FilterBuilder interface {
And(filters ...Filter) Filter
Or(filters ...Filter) Filter
Eq(index string, key string) Filter
type FilterBuilder[T any] interface {
And(filters ...Filter[T]) Filter[T]
Or(filters ...Filter[T]) Filter[T]
Eq(index string, key string) Filter[T]
}

type FilterBuilderOptions[T any] struct {
Expand All @@ -47,7 +49,7 @@ type filterBuilder[T any] struct {
fs storage.FS
}

func NewFilterBuilder[T any](opt FilterBuilderOptions[T]) (FilterBuilder, error) {
func NewFilterBuilder[T any](opt FilterBuilderOptions[T]) (FilterBuilder[T], error) {
// apply default options on uninitialized fields
opt = opt.WithDefaults()

Expand All @@ -60,29 +62,54 @@ func NewFilterBuilder[T any](opt FilterBuilderOptions[T]) (FilterBuilder, error)
}, nil
}

type filter struct {
resultSet func(ctx context.Context) *roaring64.Bitmap
type filter[T any] struct {
blockBitmap func(ctx context.Context) *roaring64.Bitmap
dataIndexBitmapFunc func(block Block[T]) *roaring64.Bitmap
}

func (c *filter) Eval(ctx context.Context) FilterIterator {
if c.resultSet == nil {
c.resultSet = func(ctx context.Context) *roaring64.Bitmap {
func (c *filter[T]) IndexIterator(ctx context.Context) *IndexIterator {
if c.blockBitmap == nil {
c.blockBitmap = func(ctx context.Context) *roaring64.Bitmap {
return roaring64.New()
}
}
return newFilterIterator(c.resultSet(ctx))
return NewIndexIterator(c.blockBitmap(ctx))
}

func (c *filter[T]) Filter(block Block[T]) Block[T] {
dataIndexesBitmap := c.dataIndexBitmapFunc(block)
dataIndexes := dataIndexesBitmap.ToArray()
if len(dataIndexes) == 1 && dataIndexes[0] == IndexAllDataIndexes {
return block
}

if dType := reflect.TypeOf(block.Data); dType.Kind() == reflect.Slice || dType.Kind() == reflect.Array {
newData := reflect.Indirect(reflect.New(dType))
for _, dataIndex := range dataIndexes {
newData = reflect.Append(newData, reflect.ValueOf(block.Data).Index(int(dataIndex)))
}
block.Data = newData.Interface().(T)
}
return block
}

func (c *filter[T]) bitmap(block Block[T]) *roaring64.Bitmap {
if c.dataIndexBitmapFunc == nil {
return roaring64.New()
}
return c.dataIndexBitmapFunc(block)
}

func (c *filterBuilder[T]) And(filters ...Filter) Filter {
return &filter{
resultSet: func(ctx context.Context) *roaring64.Bitmap {
func (c *filterBuilder[T]) And(conds ...Filter[T]) Filter[T] {
return &filter[T]{
blockBitmap: func(ctx context.Context) *roaring64.Bitmap {
var bmap *roaring64.Bitmap
for _, filter := range filters {
if filter == nil {
for _, cond := range conds {
if cond == nil {
continue
}

iter := filter.Eval(ctx)
iter := cond.IndexIterator(ctx)
if bmap == nil {
bmap = iter.Bitmap().Clone()
} else {
Expand All @@ -91,19 +118,31 @@ func (c *filterBuilder[T]) And(filters ...Filter) Filter {
}
return bmap
},
dataIndexBitmapFunc: func(block Block[T]) *roaring64.Bitmap {
var bmap *roaring64.Bitmap
for _, cond := range conds {
condBitmap := cond.bitmap(block)
if bmap == nil {
bmap = condBitmap.Clone()
} else {
bmap.And(condBitmap)
}
}
return bmap
},
}
}

func (c *filterBuilder[T]) Or(filters ...Filter) Filter {
return &filter{
resultSet: func(ctx context.Context) *roaring64.Bitmap {
func (c *filterBuilder[T]) Or(conds ...Filter[T]) Filter[T] {
return &filter[T]{
blockBitmap: func(ctx context.Context) *roaring64.Bitmap {
var bmap *roaring64.Bitmap
for _, filter := range filters {
if filter == nil {
for _, cond := range conds {
if cond == nil {
continue
}

iter := filter.Eval(ctx)
iter := cond.IndexIterator(ctx)
if bmap == nil {
bmap = iter.Bitmap().Clone()
} else {
Expand All @@ -112,13 +151,24 @@ func (c *filterBuilder[T]) Or(filters ...Filter) Filter {
}
return bmap
},
dataIndexBitmapFunc: func(block Block[T]) *roaring64.Bitmap {
var bmap *roaring64.Bitmap
for _, cond := range conds {
condBitmap := cond.bitmap(block)
if bmap == nil {
bmap = condBitmap.Clone()
} else {
bmap.Or(condBitmap)
}
}
return bmap
},
}
}

func (c *filterBuilder[T]) Eq(index string, key string) Filter {

return &filter{
resultSet: func(ctx context.Context) *roaring64.Bitmap {
func (c *filterBuilder[T]) Eq(index string, key string) Filter[T] {
return &filter[T]{
blockBitmap: func(ctx context.Context) *roaring64.Bitmap {
// fetch the index file and include it in the result set
index_ := IndexName(index).Normalize()
idx, ok := c.indexes[index_]
Expand All @@ -132,36 +182,19 @@ func (c *filterBuilder[T]) Eq(index string, key string) Filter {
}
return bitmap
},
}
}

type filterIterator struct {
iter roaring64.IntPeekable64
bitmap *roaring64.Bitmap
}
dataIndexBitmapFunc: func(block Block[T]) *roaring64.Bitmap {
index_ := IndexName(index).Normalize()
idx, ok := c.indexes[index_]
if !ok {
return roaring64.New()
}

func newFilterIterator(bmap *roaring64.Bitmap) FilterIterator {
return &filterIterator{
iter: bmap.Iterator(),
bitmap: bmap,
indexUpdate, _ := idx.IndexBlock(context.Background(), nil, block)
bitmap, ok := indexUpdate.DataIndexBitmap[IndexedValue(key)]
if !ok {
return roaring64.New()
}
return bitmap
},
}
}

func (f *filterIterator) HasNext() bool {
return f.iter.HasNext()
}

func (f *filterIterator) Next() (uint64, uint16) {
// TODO: how to handle if there's no next?
val := f.iter.Next()
return IndexCompoundID(val).Split()
}

func (f *filterIterator) Peek() (uint64, uint16) {
val := f.iter.PeekNext()
return IndexCompoundID(val).Split()
}

func (f *filterIterator) Bitmap() *roaring64.Bitmap {
return f.bitmap
}
45 changes: 18 additions & 27 deletions filter_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ func indexOnlyEvenBlocks(block Block[[]int]) (toIndex bool, indexValueMap map[In
}

if toIndex {
indexValueMap["true"] = []uint16{math.MaxUint16}
indexValueMap["true"] = []uint16{IndexAllDataIndexes}
}

return
Expand All @@ -202,7 +202,7 @@ func indexOnlyOddBlocks(block Block[[]int]) (toIndex bool, indexValueMap map[Ind
}

if toIndex {
indexValueMap["true"] = []uint16{math.MaxUint16}
indexValueMap["true"] = []uint16{IndexAllDataIndexes}
}

return
Expand Down Expand Up @@ -251,12 +251,6 @@ func indexNone(block Block[[]int]) (toIndex bool, indexValueMap map[IndexedValue
return false, nil, nil
}

func TestMaxMagicCompoundID(t *testing.T) {
id := NewIndexCompoundID(uint64(math.Exp2(48)-1), math.MaxUint16)
assert.Equal(t, uint64(math.Exp2(48)-1), id.BlockNumber())
assert.Equal(t, uint16(math.MaxUint16), id.DataIndex())
}

func TestIntMixFiltering(t *testing.T) {
_, indexes, _, cleanup, err := setupMockData(generateMixedIntIndexes, generateMixedIntBlocks)
assert.NoError(t, err)
Expand Down Expand Up @@ -284,7 +278,7 @@ func TestIntMixFiltering(t *testing.T) {
"555",
"111",
}
var numberFilter Filter
var numberFilter Filter[[]int]
for _, number := range numbersIdxs {
if numberFilter == nil {
numberFilter = f.Eq("all", number)
Expand All @@ -293,30 +287,27 @@ func TestIntMixFiltering(t *testing.T) {
}
}

onlyEvenResults := onlyEvenFilter.Eval(context.Background())
onlyEvenResults := onlyEvenFilter.IndexIterator(context.Background())
assert.Len(t, onlyEvenResults.Bitmap().ToArray(), 20)
for _, id := range onlyEvenResults.Bitmap().ToArray() {
block, _ := IndexCompoundID(id).Split()
for _, block := range onlyEvenResults.Bitmap().ToArray() {
assert.True(t, block <= 20)
}

onlyOddResults := onlyOddFilter.Eval(context.Background())
onlyOddResults := onlyOddFilter.IndexIterator(context.Background())
assert.Len(t, onlyOddResults.Bitmap().ToArray(), 20+20)
for _, id := range onlyOddResults.Bitmap().ToArray() {
block, _ := IndexCompoundID(id).Split()
for _, block := range onlyOddResults.Bitmap().ToArray() {
assert.True(t, (block > 20 && block < 41) || (block > 50 && block < 71))
}

numberAllResults := numberFilter.Eval(context.Background())
// 20*20
assert.Len(t, numberAllResults.Bitmap().ToArray(), 400)
for _, id := range numberAllResults.Bitmap().ToArray() {
block, _ := IndexCompoundID(id).Split()
numberAllResults := numberFilter.IndexIterator(context.Background())
// 20
assert.Len(t, numberAllResults.Bitmap().ToArray(), 20)
for _, block := range numberAllResults.Bitmap().ToArray() {
assert.True(t, block > 50 && block < 71)
}

allNumberAndOdd := f.And(numberFilter, oddFilter)
allNumberOddResults := allNumberAndOdd.Eval(context.Background())
allNumberOddResults := allNumberAndOdd.IndexIterator(context.Background())
assert.ElementsMatch(t, numberAllResults.Bitmap().ToArray(), allNumberOddResults.Bitmap().ToArray())
}

Expand All @@ -331,20 +322,20 @@ func TestFiltering(t *testing.T) {
})
assert.NoError(t, err)
assert.NotNil(t, f)
result := f.Or(f.And(f.Eq("all", "1"), f.Eq("all", "2")), f.Eq("all", "3")).Eval(context.Background())
result := f.Or(f.And(f.Eq("all", "1"), f.Eq("all", "2")), f.Eq("all", "3")).IndexIterator(context.Background())
// result should contain block 1, 2, 3
assert.Len(t, result.Bitmap().ToArray(), 3)
block, _ := result.Next()
block := result.Next()
assert.Equal(t, uint64(1), block)
block, _ = result.Next()
block = result.Next()
assert.Equal(t, uint64(2), block)
block, _ = result.Next()
block = result.Next()
assert.Equal(t, uint64(3), block)

result = f.And(f.Eq("all", "1"), f.Eq("all", "2")).Eval(context.Background())
result = f.And(f.Eq("all", "1"), f.Eq("all", "2")).IndexIterator(context.Background())
// result should contain block 1
assert.Len(t, result.Bitmap().ToArray(), 1)
block, _ = result.Next()
block = result.Next()
assert.Equal(t, uint64(1), block)
}

Expand Down
Loading