diff --git a/config/config.go b/config/config.go index 337e2e8a..9c067635 100644 --- a/config/config.go +++ b/config/config.go @@ -5,7 +5,7 @@ const ( DefaultSnapshotKeepRecent = 1 DefaultSnapshotWriterLimit = 1 DefaultAsyncCommitBuffer = 100 - DefaultCacheSize = 100000 + DefaultCacheSize = 0 DefaultSSKeepRecent = 100000 DefaultSSPruneInterval = 600 DefaultSSImportWorkers = 1 @@ -45,8 +45,10 @@ type StateCommitConfig struct { SnapshotWriterLimit int `mapstructure:"snapshot-writer-limit"` // CacheSize defines the size of the cache for each memiavl store. - // Deprecated: this is removed, we will just rely on mmap page cache CacheSize int `mapstructure:"cache-size"` + + // CacheModules defines list of module names that should enable LRU cache. + CacheModules string `mapstructure:"cache-modules"` } type StateStoreConfig struct { @@ -100,6 +102,7 @@ func DefaultStateCommitConfig() StateCommitConfig { CacheSize: DefaultCacheSize, SnapshotInterval: DefaultSnapshotInterval, SnapshotKeepRecent: DefaultSnapshotKeepRecent, + ZeroCopy: true, } } diff --git a/config/toml.go b/config/toml.go index 9ba79c20..12108745 100644 --- a/config/toml.go +++ b/config/toml.go @@ -18,6 +18,12 @@ sc-directory = "{{ .StateCommit.Directory }}" # the sdk address cache will be disabled if zero-copy is enabled. sc-zero-copy = {{ .StateCommit.ZeroCopy }} +# Defines the max LRU cache capacity for each module in MB. Default to 0 to disable cache. +sc-cache-size = {{ .StateCommit.CacheSize }} + +# Defines which module should be enabled with LRU cache. Default emtpy to disable cache. +sc-cache-modules = {{ .StateCommit.CacheModules }} + # AsyncCommitBuffer defines the size of asynchronous commit queue, this greatly improve block catching-up # performance, setting to 0 means synchronous commit. sc-async-commit-buffer = {{ .StateCommit.AsyncCommitBuffer }} diff --git a/go.mod b/go.mod index 4758fe00..99a8aa29 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/sei-protocol/sei-db -go 1.19 +go 1.23.0 + +toolchain go1.23.5 require ( github.com/alitto/pond v1.8.3 @@ -8,17 +10,18 @@ require ( github.com/cockroachdb/pebble v0.0.0-20230819001538-1798fbf5956c github.com/confio/ics23/go v0.9.0 github.com/cosmos/iavl v0.21.0-alpha.1.0.20230904092046-df3db2d96583 + github.com/dgraph-io/ristretto/v2 v2.2.0 github.com/gogo/protobuf v1.3.3 github.com/ledgerwatch/erigon-lib v0.0.0-20230210071639-db0e7ed11263 github.com/linxGnu/grocksdb v1.8.4 github.com/spf13/cobra v1.6.1 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.10.0 github.com/tendermint/tm-db v0.6.8-0.20220519162814-e24b96538a12 github.com/tidwall/btree v1.6.0 github.com/tidwall/gjson v1.10.2 github.com/tidwall/wal v1.1.7 github.com/zbiljic/go-filelock v0.0.0-20170914061330-1dbf7103ab7d - golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb + golang.org/x/exp v0.0.0-20230206171751-46f607a40771 modernc.org/sqlite v1.26.0 ) @@ -26,7 +29,7 @@ require ( github.com/DataDog/zstd v1.4.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cockroachdb/errors v1.8.1 // indirect github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect github.com/cockroachdb/redact v1.0.8 // indirect @@ -36,7 +39,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgraph-io/badger/v3 v3.2103.2 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect - github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/golang/glog v1.1.0 // indirect @@ -78,7 +80,7 @@ require ( golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.13.0 // indirect + golang.org/x/sys v0.31.0 // indirect golang.org/x/tools v0.6.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 89f72143..03d579fc 100644 --- a/go.sum +++ b/go.sum @@ -176,8 +176,8 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charithe/durationcheck v0.0.9/go.mod h1:SSbRIBVfMjCi/kEB6K65XEA83D6prSM8ap1UCpNKtgg= github.com/chavacava/garif v0.0.0-20220316182200-5cad0b5181d4/go.mod h1:W8EnPSQ8Nv4fUjc/v1/8tHFqhuOJXnRub0dTfuAQktU= github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= @@ -202,6 +202,7 @@ github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWH github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= @@ -257,10 +258,12 @@ github.com/dgraph-io/badger/v3 v3.2103.2/go.mod h1:RHo4/GmYcKKh5Lxu63wLEMHJ70Pac github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM= +github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= -github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38= +github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/cli v20.10.14+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v20.10.17+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= @@ -459,6 +462,7 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -484,6 +488,7 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/trillian v1.3.11/go.mod h1:0tPraVHrSDkA3BO6vKX67zgLXs6SsOAbHEivX+9mPgw= @@ -734,6 +739,7 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= +github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= @@ -1042,8 +1048,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= github.com/subosito/gotenv v1.4.0/go.mod h1:mZd6rFysKEcUhUHXJk0C/08wAgyDBFuwEYL7vWWGaGo= @@ -1220,8 +1226,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb h1:mIKbk8weKhSeLH2GmUTrvx8CjkyJmnU1wFmg59CUjFA= -golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg= +golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -1482,8 +1488,8 @@ golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1499,6 +1505,7 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1822,6 +1829,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -1870,7 +1878,9 @@ modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= +modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM= modernc.org/libc v1.24.1/go.mod h1:FmfO1RLrU3MHJfyi9eYYmZBfi/R+tqZ6+hQ3yQQUkak= modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= @@ -1884,9 +1894,11 @@ modernc.org/sqlite v1.26.0/go.mod h1:FL3pVXie73rg3Rii6V/u5BoHlSoyeZeIgKZEgHARyCU modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY= +modernc.org/tcl v1.15.2/go.mod h1:3+k/ZaEbKrC8ePv8zJWPtBSW0V7Gg9g8rkmhI1Kfs3c= modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg= modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY= +modernc.org/z v1.7.3/go.mod h1:Ipv4tsdxZRbQyLq9Q1M6gdbkxYzdlrciF2Hi/lS7nWE= mvdan.cc/gofumpt v0.3.1/go.mod h1:w3ymliuxvzVx8DAutBnVyDqYb1Niy/yCJt/lk821YCE= mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= diff --git a/sc/memiavl/cache.go b/sc/memiavl/cache.go new file mode 100644 index 00000000..d02f5695 --- /dev/null +++ b/sc/memiavl/cache.go @@ -0,0 +1,111 @@ +package memiavl + +import ( + "github.com/dgraph-io/ristretto/v2" +) + +// Cache is an in-memory structure to persist nodes for quick access. +// Please see lruCache for more details about why we need a custom +// cache implementation. +type Cache interface { + // Adds key value to cache. + Add(key []byte, value []byte) + + // Returns Node for the key, if exists. nil otherwise. + Get(key []byte) []byte + + // Has returns true if node with key exists in cache, false otherwise. + Has(key []byte) bool + + // Remove removes node with key from cache. The removed value is returned. + // if not in cache, return nil. + Remove(key []byte) +} + +type LRUCache struct { + cache *ristretto.Cache[[]byte, []byte] +} + +// NewLRUCache creates a new LRU cache with the given size +func NewLRUCache(capacity int) *LRUCache { + if capacity <= 0 { + return nil + } + cache, err := ristretto.NewCache(&ristretto.Config[[]byte, []byte]{ + NumCounters: 1e8, // number of keys to track frequency of (10M). + MaxCost: int64(capacity * 1024 * 1024), // maximum cost of cache (1GB). + BufferItems: 64, // number of keys per Get buffer. + }) + if err != nil { + panic(err) + } + + return &LRUCache{cache: cache} + //config := bigcache.Config{ + // // number of shards (must be a power of 2) + // Shards: 1024, + // + // // time after which entry can be evicted + // LifeWindow: 60 * time.Minute, + // + // // Interval between removing expired entries (clean up). + // // If set to <= 0 then no action is performed. + // // Setting to < 1 second is counterproductive — bigcache has a one second resolution. + // CleanWindow: 0, + // + // // rps * lifeWindow, used only in initial memory allocation + // MaxEntriesInWindow: 1000 * 10 * 60, + // + // // max entry size in bytes, used only in initial memory allocation + // MaxEntrySize: int(math.Min(16, float64(capacity))) * 1024 * 1024, + // + // // prints information about additional memory allocation + // Verbose: false, + // + // // cache will not allocate more memory than this limit, value in MB + // // if value is reached then the oldest entries can be overridden for the new ones + // // 0 value means no size limit + // HardMaxCacheSize: capacity, + // + // // callback fired when the oldest entry is removed because of its expiration time or no space left + // // for the new entry, or because delete was called. A bitmask representing the reason will be returned. + // // Default value is nil which means no callback and it prevents from unwrapping the oldest entry. + // OnRemove: nil, + // + // // OnRemoveWithReason is a callback fired when the oldest entry is removed because of its expiration time or no space left + // // for the new entry, or because delete was called. A constant representing the reason will be passed through. + // // Default value is nil which means no callback and it prevents from unwrapping the oldest entry. + // // Ignored if OnRemove is specified. + // OnRemoveWithReason: nil, + //} + //fmt.Printf("Creating LRU cache with capacity of %d\n", capacity) + //cache, err := bigcache.New(context.Background(), config) + //if err != nil { + // panic(err) + //} + //return &LRUCache{ + // cache: cache, + //} +} + +func (c *LRUCache) Add(key []byte, value []byte) { + _ = c.cache.Set(key, value, int64(len(key)+len(value))) + c.cache.Wait() +} + +func (c *LRUCache) Get(key []byte) []byte { + + value, found := c.cache.Get(key) + if !found { + return nil + } + return value +} + +func (c *LRUCache) Has(key []byte) bool { + return c.Get(key) == nil +} + +func (c *LRUCache) Remove(key []byte) { + c.cache.Del(key) +} diff --git a/sc/memiavl/db.go b/sc/memiavl/db.go index 47c4a9e4..69048785 100644 --- a/sc/memiavl/db.go +++ b/sc/memiavl/db.go @@ -77,6 +77,9 @@ type DB struct { mtx sync.Mutex // worker goroutine IdleTimeout = 5s snapshotWriterPool *pond.WorkerPool + + // Cache capacity for each module + cacheSetup map[string]int } const ( @@ -124,7 +127,13 @@ func OpenDB(logger logger.Logger, targetVersion int64, opts Options) (*DB, error } path := filepath.Join(opts.Dir, snapshot) - mtree, err := LoadMultiTree(path, opts.ZeroCopy, opts.CacheSize) + cacheSetup := make(map[string]int) + if opts.CacheSize > 0 { + for _, treeName := range opts.CacheModules { + cacheSetup[treeName] = opts.CacheSize + } + } + mtree, err := LoadMultiTree(path, opts.ZeroCopy, cacheSetup) if err != nil { return nil, err } @@ -197,6 +206,7 @@ func OpenDB(logger logger.Logger, targetVersion int64, opts Options) (*DB, error snapshotKeepRecent: opts.SnapshotKeepRecent, snapshotInterval: opts.SnapshotInterval, snapshotWriterPool: workerPool, + cacheSetup: cacheSetup, } if !db.readOnly && db.Version() == 0 && len(opts.InitialStores) > 0 { @@ -485,12 +495,11 @@ func (db *DB) Commit() (int64, error) { func (db *DB) Copy() *DB { db.mtx.Lock() defer db.mtx.Unlock() - - return db.copy(db.cacheSize) + return db.copy() } -func (db *DB) copy(cacheSize int) *DB { - mtree := db.MultiTree.Copy(cacheSize) +func (db *DB) copy() *DB { + mtree := db.MultiTree.Copy() return &DB{ MultiTree: *mtree, @@ -529,7 +538,7 @@ func (db *DB) Reload() error { } func (db *DB) reload() error { - mtree, err := LoadMultiTree(currentPath(db.dir), db.zeroCopy, db.cacheSize) + mtree, err := LoadMultiTree(currentPath(db.dir), db.zeroCopy, db.cacheSetup) if err != nil { return err } @@ -537,11 +546,12 @@ func (db *DB) reload() error { } func (db *DB) reloadMultiTree(mtree *MultiTree) error { - // catch-up the pending changes - if err := mtree.apply(db.pendingLogEntry); err != nil { + err := db.MultiTree.ReplaceWith(mtree) + if err != nil { return err } - return db.MultiTree.ReplaceWith(mtree) + // catch-up the pending changes + return db.MultiTree.apply(db.pendingLogEntry) } // rewriteIfApplicable execute the snapshot rewrite strategy according to current height @@ -583,17 +593,17 @@ func (db *DB) rewriteSnapshotBackground() error { db.snapshotRewriteChan = ch db.snapshotRewriteCancelFunc = cancel - cloned := db.copy(0) + clonedDB := db.copy() go func() { defer close(ch) - - cloned.logger.Info("start rewriting snapshot", "version", cloned.Version()) - if err := cloned.RewriteSnapshot(ctx); err != nil { + version := clonedDB.Version() + clonedDB.logger.Info("start rewriting snapshot", "version", version) + if err := clonedDB.RewriteSnapshot(ctx); err != nil { ch <- snapshotResult{err: err} return } - cloned.logger.Info("finished rewriting snapshot", "version", cloned.Version()) - mtree, err := LoadMultiTree(currentPath(cloned.dir), cloned.zeroCopy, 0) + clonedDB.logger.Info("finished rewriting snapshot", "version", version) + mtree, err := LoadMultiTree(currentPath(clonedDB.dir), clonedDB.zeroCopy, nil) if err != nil { ch <- snapshotResult{err: err} return @@ -605,7 +615,7 @@ func (db *DB) rewriteSnapshotBackground() error { return } - cloned.logger.Info("finished best-effort catchup", "version", cloned.Version(), "latest", mtree.Version()) + clonedDB.logger.Info("finished best-effort catchup", "version", version, "latest", mtree.Version()) ch <- snapshotResult{mtree: mtree} }() @@ -798,7 +808,7 @@ func GetEarliestVersion(root string) (int64, error) { // current -> snapshot-0 // ``` func initEmptyDB(dir string, initialVersion uint32) error { - tmp := NewEmptyMultiTree(initialVersion, 0) + tmp := NewEmptyMultiTree(initialVersion) snapshotDir := snapshotName(0) // create tmp worker pool pool := pond.New(config.DefaultSnapshotWriterLimit, config.DefaultSnapshotWriterLimit*10) diff --git a/sc/memiavl/export.go b/sc/memiavl/export.go index 392b39d4..8ee4aac2 100644 --- a/sc/memiavl/export.go +++ b/sc/memiavl/export.go @@ -49,7 +49,7 @@ func NewMultiTreeExporter(dir string, version uint32, supportExportNonSnapshotVe if int64(version) > curVersion { return nil, fmt.Errorf("snapshot is not created yet: height: %d", version) } - mtree, err = LoadMultiTree(filepath.Join(dir, snapshotName(int64(version))), true, 0) + mtree, err = LoadMultiTree(filepath.Join(dir, snapshotName(int64(version))), true, nil) if err != nil { return nil, fmt.Errorf("snapshot don't exists: height: %d, %w", version, err) } diff --git a/sc/memiavl/multitree.go b/sc/memiavl/multitree.go index 2789b913..2b1a830b 100644 --- a/sc/memiavl/multitree.go +++ b/sc/memiavl/multitree.go @@ -44,8 +44,8 @@ type MultiTree struct { // it always corresponds to the rlog entry with index 1. initialVersion uint32 - zeroCopy bool - cacheSize int + zeroCopy bool + cacheSetup map[string]int // cache size in MB for each tree trees []NamedTree // always ordered by tree name treesByName map[string]int // index of the trees by name @@ -55,16 +55,16 @@ type MultiTree struct { metadata proto.MultiTreeMetadata } -func NewEmptyMultiTree(initialVersion uint32, cacheSize int) *MultiTree { +func NewEmptyMultiTree(initialVersion uint32) *MultiTree { return &MultiTree{ initialVersion: initialVersion, treesByName: make(map[string]int), zeroCopy: true, - cacheSize: cacheSize, + cacheSetup: make(map[string]int), } } -func LoadMultiTree(dir string, zeroCopy bool, cacheSize int) (*MultiTree, error) { +func LoadMultiTree(dir string, zeroCopy bool, cacheSetup map[string]int) (*MultiTree, error) { metadata, err := readMetadata(dir) if err != nil { return nil, err @@ -87,7 +87,14 @@ func LoadMultiTree(dir string, zeroCopy bool, cacheSize int) (*MultiTree, error) if err != nil { return nil, err } - treeMap[name] = NewFromSnapshot(snapshot, zeroCopy, cacheSize) + // set the correct cache capacity for this module based on cache setup + var treeCacheSize = 0 + if cacheSetup != nil { + if value, ok := cacheSetup[name]; ok { + treeCacheSize = value + } + } + treeMap[name] = NewFromSnapshot(snapshot, zeroCopy, treeCacheSize) } slices.Sort(treeNames) @@ -106,7 +113,7 @@ func LoadMultiTree(dir string, zeroCopy bool, cacheSize int) (*MultiTree, error) lastCommitInfo: *metadata.CommitInfo, metadata: *metadata, zeroCopy: zeroCopy, - cacheSize: cacheSize, + cacheSetup: cacheSetup, } // initial version is necessary for rlog index conversion mtree.setInitialVersion(metadata.InitialVersion) @@ -160,11 +167,11 @@ func (t *MultiTree) SetZeroCopy(zeroCopy bool) { } // Copy returns a snapshot of the tree which won't be corrupted by further modifications on the main tree. -func (t *MultiTree) Copy(cacheSize int) *MultiTree { +func (t *MultiTree) Copy() *MultiTree { trees := make([]NamedTree, len(t.trees)) treesByName := make(map[string]int, len(t.trees)) for i, entry := range t.trees { - tree := entry.Tree.Copy(cacheSize) + tree := entry.Tree.Copy() trees[i] = NamedTree{Tree: tree, Name: entry.Name} treesByName[entry.Name] = i } @@ -441,6 +448,7 @@ func (t *MultiTree) ReplaceWith(other *MultiTree) error { t.treesByName = other.treesByName t.lastCommitInfo = other.lastCommitInfo t.metadata = other.metadata + t.initialVersion = other.initialVersion return errors.Join(errs...) } diff --git a/sc/memiavl/opts.go b/sc/memiavl/opts.go index abb90561..835935db 100644 --- a/sc/memiavl/opts.go +++ b/sc/memiavl/opts.go @@ -22,7 +22,9 @@ type Options struct { AsyncCommitBuffer int // ZeroCopy if true, the get and iterator methods could return a slice pointing to mmaped blob files. ZeroCopy bool - // CacheSize defines the cache's max entry size for each memiavl store. + // CacheModules defines list of modules that should enable LRU cache + CacheModules []string + // CacheSize defines the cache's max entry size in MB for each memiavl store. CacheSize int // LoadForOverwriting if true rollbacks the state, specifically the OpenDB method will // truncate the versions after the `TargetVersion`, the `TargetVersion` becomes the latest version. @@ -53,8 +55,4 @@ func (opts *Options) FillDefaults() { if opts.SnapshotWriterLimit <= 0 { opts.SnapshotWriterLimit = config.DefaultSnapshotWriterLimit } - - if opts.CacheSize < 0 { - opts.CacheSize = config.DefaultCacheSize - } } diff --git a/sc/memiavl/tree.go b/sc/memiavl/tree.go index f498ca61..5c534944 100644 --- a/sc/memiavl/tree.go +++ b/sc/memiavl/tree.go @@ -32,12 +32,15 @@ type Tree struct { // sync.RWMutex is used to protect the tree for thread safety during snapshot reload mtx *sync.RWMutex + // max size of the cache in MB + cache *LRUCache + pendingChanges chan iavl.ChangeSet pendingWg *sync.WaitGroup } // NewEmptyTree creates an empty tree at an arbitrary version. -func NewEmptyTree(version uint64, initialVersion uint32) *Tree { +func NewEmptyTree(version uint64, initialVersion uint32, cacheSize int) *Tree { if version >= math.MaxUint32 { panic("version overflows uint32") } @@ -47,30 +50,32 @@ func NewEmptyTree(version uint64, initialVersion uint32) *Tree { initialVersion: initialVersion, // no need to copy if the tree is not backed by snapshot zeroCopy: true, + cache: NewLRUCache(cacheSize), mtx: &sync.RWMutex{}, pendingWg: &sync.WaitGroup{}, } } // New creates an empty tree at genesis version -func New(_ int) *Tree { - return NewEmptyTree(0, 0) +func New(cacheSize int) *Tree { + return NewEmptyTree(0, 0, cacheSize) } // NewWithInitialVersion creates an empty tree with initial-version, // it happens when a new store created at the middle of the chain. func NewWithInitialVersion(initialVersion uint32) *Tree { - return NewEmptyTree(0, initialVersion) + return NewEmptyTree(0, initialVersion, 0) } // NewFromSnapshot mmap the blob files and create the root node. -func NewFromSnapshot(snapshot *Snapshot, zeroCopy bool, _ int) *Tree { +func NewFromSnapshot(snapshot *Snapshot, zeroCopy bool, cacheSize int) *Tree { tree := &Tree{ version: snapshot.Version(), snapshot: snapshot, zeroCopy: zeroCopy, mtx: &sync.RWMutex{}, pendingWg: &sync.WaitGroup{}, + cache: NewLRUCache(cacheSize), } if !snapshot.IsEmpty() { @@ -98,7 +103,7 @@ func (t *Tree) SetInitialVersion(initialVersion int64) error { // Copy returns a snapshot of the tree which won't be modified by further modifications on the main tree, // the returned new tree can be accessed concurrently with the main tree. -func (t *Tree) Copy(_ int) *Tree { +func (t *Tree) Copy() *Tree { t.mtx.RLock() defer t.mtx.RUnlock() if _, ok := t.root.(*MemNode); ok { @@ -154,12 +159,18 @@ func (t *Tree) Set(key, value []byte) { value = []byte{} } t.root, _ = setRecursive(t.root, key, value, t.version+1, t.cowVersion) + if t.cache != nil { + t.cache.Add(key, value) + } } func (t *Tree) Remove(key []byte) { t.mtx.Lock() defer t.mtx.Unlock() _, t.root, _ = removeRecursive(t.root, key, t.version+1, t.cowVersion) + if t.cache != nil { + t.cache.Remove(key) + } } // SaveVersion increases the version number and optionally updates the hashes @@ -226,7 +237,18 @@ func (t *Tree) GetByIndex(index int64) ([]byte, []byte) { } func (t *Tree) Get(key []byte) []byte { + if t.cache != nil { + if v := t.cache.Get(key); v != nil { + return v + } + } _, value := t.GetWithIndex(key) + if value == nil { + return nil + } + if t.cache != nil { + t.cache.Add(key, value) + } return value } @@ -311,16 +333,16 @@ func (t *Tree) Close() error { func (t *Tree) ReplaceWith(other *Tree) error { t.mtx.Lock() defer t.mtx.Unlock() - snapshot := t.snapshot + if t.snapshot != nil { + _ = t.snapshot.Close() + t.snapshot = nil + } t.version = other.version t.root = other.root t.snapshot = other.snapshot t.initialVersion = other.initialVersion t.cowVersion = other.cowVersion t.zeroCopy = other.zeroCopy - if snapshot != nil { - return snapshot.Close() - } return nil } @@ -360,5 +382,4 @@ func (t *Tree) GetProof(key []byte) *ics23.CommitmentProof { } return commitmentProof - } diff --git a/sc/memiavl/tree_test.go b/sc/memiavl/tree_test.go index aa55d6fb..a550d1f8 100644 --- a/sc/memiavl/tree_test.go +++ b/sc/memiavl/tree_test.go @@ -196,7 +196,7 @@ func TestTreeCopy(t *testing.T) { _, _, err := tree.SaveVersion(true) require.NoError(t, err) - snapshot := tree.Copy(0) + snapshot := tree.Copy() tree.ApplyChangeSet(iavl.ChangeSet{Pairs: []*iavl.KVPair{ {Key: []byte("hello"), Value: []byte("world1")}, diff --git a/sc/store.go b/sc/store.go index f2bda964..4d39f9e4 100644 --- a/sc/store.go +++ b/sc/store.go @@ -2,6 +2,7 @@ package sc import ( "fmt" + "strings" "github.com/sei-protocol/sei-db/common/logger" "github.com/sei-protocol/sei-db/common/utils" @@ -34,6 +35,11 @@ func NewCommitStore(homeDir string, logger logger.Logger, config config.StateCom CacheSize: config.CacheSize, CreateIfMissing: true, } + if config.CacheModules != "" { + opts.CacheModules = strings.Split(config.CacheModules, ",") + } else { + opts.CacheModules = make([]string, 0) + } commitStore := &CommitStore{ logger: logger, opts: opts,