diff --git a/go.mod b/go.mod index aadd1ab..6680595 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,14 @@ go 1.25.1 require ( github.com/dgraph-io/ristretto/v2 v2.3.0 github.com/stretchr/testify v1.11.1 + github.com/syndtr/goleveldb v1.0.0 ) require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/sys v0.35.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index ae7c996..68510c2 100644 --- a/go.sum +++ b/go.sum @@ -8,13 +8,38 @@ github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa5 github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/leveldb/leveldb.go b/leveldb/leveldb.go new file mode 100644 index 0000000..93da580 --- /dev/null +++ b/leveldb/leveldb.go @@ -0,0 +1,61 @@ +package leveldb + +import ( + "errors" + "log/slog" + + "github.com/syndtr/goleveldb/leveldb" + "go.rtnl.ai/httpcache" +) + +// Cache is an implementation of httpcache.Cache with leveldb storage +type Cache struct { + db *leveldb.DB +} + +// New returns a cache that will store cached data in a leveldb database at the path. +func New(path string) (_ *Cache, err error) { + cache := &Cache{} + if cache.db, err = leveldb.OpenFile(path, nil); err != nil { + return nil, err + } + return cache, nil +} + +// Make returns a cache using the specified db instance as the underlying storage. +func Make(db *leveldb.DB) *Cache { + return &Cache{db: db} +} + +// Get a value from the cache for the specified key. If any error other than +// ErrNotFound occurs it is logged and false is returned. +func (c *Cache) Get(key string) ([]byte, bool) { + data, err := c.db.Get([]byte(key), nil) + if err != nil { + if !errors.Is(err, leveldb.ErrNotFound) { + httpcache.GetLogger().Warn("failed to read from leveldb cache", slog.Any("error", err)) + } + return nil, false + } + return data, true +} + +// Put a value into the cache with the specified key. If an error occurs it is logged. +func (c *Cache) Put(key string, value []byte) { + if err := c.db.Put([]byte(key), value, nil); err != nil { + httpcache.GetLogger().Warn("failed to write to leveldb cache", slog.Any("error", err)) + } +} + +// Del removes a value from the cache for the specified key. If an error occurs it is logged. +func (c *Cache) Del(key string) { + if err := c.db.Delete([]byte(key), nil); err != nil { + httpcache.GetLogger().Warn("failed to delete from leveldb cache", slog.Any("error", err)) + } +} + +// Close closes the underlying leveldb database. +// Implements io.Closer. +func (c *Cache) Close() error { + return c.db.Close() +} diff --git a/leveldb/leveldb_bench_test.go b/leveldb/leveldb_bench_test.go new file mode 100644 index 0000000..c994d91 --- /dev/null +++ b/leveldb/leveldb_bench_test.go @@ -0,0 +1,108 @@ +package leveldb_test + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "go.rtnl.ai/httpcache/leveldb" +) + +func benchmarkGet(size int) func(b *testing.B) { + return func(b *testing.B) { + path := filepath.Join(b.TempDir(), "cache.db") + cache, err := leveldb.New(path) + require.NoError(b, err) + defer cache.Close() + + value := make([]byte, size) + + // Prepopulate the cache + for i := 0; i < 128; i++ { + key := string(rune('a' + i)) + cache.Put(key, value) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + cache.Get(string(rune('a' + i%192))) + } + } +} + +func BenchmarkLevelDBCacheGet(b *testing.B) { + b.Run("Small", benchmarkGet(512)) + b.Run("Realistic", benchmarkGet(2048)) + b.Run("Large", benchmarkGet(5.243e+6)) +} + +func benchmarkPut(size int) func(b *testing.B) { + return func(b *testing.B) { + path := filepath.Join(b.TempDir(), "cache.db") + cache, err := leveldb.New(path) + require.NoError(b, err) + defer cache.Close() + + value := make([]byte, size) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + cache.Put(string(rune('a'+i%192)), value) + } + } +} + +func BenchmarkLevelDBCachePut(b *testing.B) { + b.Run("Small", benchmarkPut(512)) + b.Run("Realistic", benchmarkPut(2048)) + b.Run("Large", benchmarkPut(5.243e+6)) +} + +// Benchmark mixed operations +func BenchmarkLevelDBCacheMixed(b *testing.B) { + path := filepath.Join(b.TempDir(), "cache.db") + cache, err := leveldb.New(path) + require.NoError(b, err) + defer cache.Close() + + value := make([]byte, 1024) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + key := string(rune('a' + i%128)) + switch i % 3 { + case 0: + cache.Put(key, value) + case 1: + cache.Get(key) + case 2: + cache.Del(key) + } + } +} + +// Benchmark concurrent mixed operations +func BenchmarkLevelDBCacheParallelMixed(b *testing.B) { + path := filepath.Join(b.TempDir(), "cache.db") + cache, err := leveldb.New(path) + require.NoError(b, err) + defer cache.Close() + + value := make([]byte, 1024) + + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + key := string(rune('a' + i%128)) + switch i % 3 { + case 0: + cache.Put(key, value) + case 1: + cache.Get(key) + case 2: + cache.Del(key) + } + i++ + } + }) +} diff --git a/leveldb/leveldb_test.go b/leveldb/leveldb_test.go new file mode 100644 index 0000000..0582bf2 --- /dev/null +++ b/leveldb/leveldb_test.go @@ -0,0 +1,60 @@ +package leveldb_test + +import ( + "math/rand/v2" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/require" + "go.rtnl.ai/httpcache/leveldb" +) + +func TestLevelDBCache(t *testing.T) { + path := filepath.Join(t.TempDir(), "cache.db") + + cache, err := leveldb.New(path) + require.NoError(t, err) + defer cache.Close() + + cache.Put("foo", []byte("bar")) + + val, ok := cache.Get("foo") + require.True(t, ok) + require.Equal(t, []byte("bar"), val) + + cache.Del("foo") + _, ok = cache.Get("foo") + require.False(t, ok) +} + +func TestLevelDBRace(t *testing.T) { + // Ensures no race conditions occur during concurrent access. + path := filepath.Join(t.TempDir(), "cache.db") + cache, err := leveldb.New(path) + require.NoError(t, err) + defer cache.Close() + + value := make([]byte, 2048) + + var wg sync.WaitGroup + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 512; j++ { + k := rand.IntN(64) + key := string(rune('a' + k%16)) + switch k % 3 { + case 0: + cache.Put(key, value) + case 1: + cache.Get(key) + case 2: + cache.Del(key) + } + } + }() + } + wg.Wait() +} diff --git a/ristretto/ristretto_test.go b/ristretto/ristretto_test.go index c9d636c..88f26ee 100644 --- a/ristretto/ristretto_test.go +++ b/ristretto/ristretto_test.go @@ -16,6 +16,7 @@ func TestRistrettoCache(t *testing.T) { BufferItems: 64, // number of keys per Get buffer. }) require.NoError(t, err) + defer cache.Close() cache.Put("foo", []byte("bar")) cache.Wait() @@ -37,6 +38,8 @@ func TestRistrettoRace(t *testing.T) { BufferItems: 64, // number of keys per Get buffer. }) require.NoError(t, err) + defer cache.Close() + value := make([]byte, 2048) var wg sync.WaitGroup