From 534c3ac339552799bad1972b31a98610c84ca977 Mon Sep 17 00:00:00 2001 From: Rogietg Date: Tue, 22 Nov 2022 14:25:34 +0800 Subject: [PATCH] =?UTF-8?q?Goon=E3=81=AE2nd=20gen=E7=A7=BB=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 23 +++++-- cache.go | 179 +++++++++++++++++++++++++++++++++++++++++++++++++ cache_test.go | 180 ++++++++++++++++++++++++++++++++++++++++++++++++++ doc.go | 8 +++ entity.go | 4 +- go.mod | 9 +++ go.sum | 34 ++++++++++ goon.go | 8 +-- goon_test.go | 8 +-- query.go | 2 +- 10 files changed, 439 insertions(+), 16 deletions(-) create mode 100644 cache.go create mode 100644 cache_test.go create mode 100644 go.mod create mode 100644 go.sum diff --git a/README.md b/README.md index f8f5fa6..2e1476c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,18 @@ -# goon - -An autocaching interface to the app engine datastore for Go. Designed to be similar to the python NDB package. - -Documentation: [http://godoc.org/github.com/mjibson/goon](http://godoc.org/github.com/mjibson/goon) +# goon + +An autocaching interface to the app engine datastore for Go. Designed to be similar to the python NDB package. + +## The different flavors of App Engine + +You must choose the goon major version based on which App Engine library you are using. + +| App Engine Go library | Include in your project | Correct goon version | +|----------------------------------|-----------------------------|------------------------------------------------------------------------| +| `google.golang.org/appengine/v2` |`github.com/mjibson/goon/v2` | [goon v2.0.1](https://github.com/mjibson/goon/releases/tag/v2.0.1) | +| `google.golang.org/appengine` |`github.com/mjibson/goon` | [goon v1.1.0](https://github.com/mjibson/goon/releases/tag/v1.1.0) | +| `appengine` |`github.com/mjibson/goon` | [goon v0.9.0](https://github.com/mjibson/goon/releases/tag/v0.9.0) | +| `cloud.google.com/go` |N/A | Not supported ([issue #74](https://github.com/mjibson/goon/issues/74)) | + +## Documentation + +[https://pkg.go.dev/github.com/mjibson/goon](https://pkg.go.dev/github.com/mjibson/goon) diff --git a/cache.go b/cache.go new file mode 100644 index 0000000..05a6280 --- /dev/null +++ b/cache.go @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2012 The Goon Authors + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +package goon + +import ( + "container/list" + "reflect" + "sync" +) + +var cachedValueOverhead int + +func init() { + // Calculate the platform dependant overhead size for keeping a value in cache + var elem list.Element + var ci cacheItem + cachedValueOverhead += int(reflect.TypeOf(elem).Size()) // list.Element in cache.accessed + cachedValueOverhead += int(reflect.TypeOf(&elem).Size()) // *list.Element in cache.elements + cachedValueOverhead += int(reflect.TypeOf(ci.key).Size()) // string in cache.elements as key + cachedValueOverhead += int(reflect.TypeOf(ci).Size()) // cacheItem pointed to by list.Element.Value + // In addition to the above overhead, the total cache value size must include + // the length of the key string in bytes and the cap of the value []byte +} + +type cacheItem struct { + key string + value []byte +} + +type cache struct { + lock sync.Mutex + elements map[string]*list.Element // access via key + accessed list.List // most recently accessed in front + size int // Total size of all the values in the cache + limit int // Maximum size allowed +} + +const defaultCacheLimit = 16 << 20 // 16 MiB + +func newCache(limit int) *cache { + return &cache{elements: map[string]*list.Element{}, limit: limit} +} + +func (c *cache) setLimit(limit int) { + c.lock.Lock() + c.limit = limit + c.meetLimitUnderLock() + c.lock.Unlock() +} + +// meetLimit must be called under cache.lock +func (c *cache) meetLimitUnderLock() { + for c.size > c.limit { + e := c.accessed.Back() + if e == nil { + break + } + c.deleteExistingUnderLock(e) + } +} + +// setUnderLock must be called under cache.lock +func (c *cache) setUnderLock(item *cacheItem) { + // Check if there's already an entry for this key + if e, ok := c.elements[item.key]; ok { + // There already exists a value for this key, so update it + ci := e.Value.(*cacheItem) + c.size += cap(item.value) - cap(ci.value) + // Make sure that item.key is the same pointer as the map key, + // as this will ensure faster map lookup via pointer equality. + // Not doing so would also lead to double memory usage, as the two key + // pointers would be pointing to the same contents in different places. + item.key = ci.key + e.Value = item + c.accessed.MoveToFront(e) + } else { + // Brand new key, so add it + c.size += cachedValueOverhead + len(item.key) + cap(item.value) + c.elements[item.key] = c.accessed.PushFront(item) + } +} + +// Set takes ownership of item and treats it as immutable +func (c *cache) Set(item *cacheItem) { + c.lock.Lock() + c.setUnderLock(item) + c.meetLimitUnderLock() + c.lock.Unlock() +} + +// SetMulti takes ownership of the individual items and treats them as immutable +func (c *cache) SetMulti(items []*cacheItem) { + c.lock.Lock() + for _, item := range items { + c.setUnderLock(item) + } + c.meetLimitUnderLock() + c.lock.Unlock() +} + +// deleteExistingUnderLock must be called under cache.lock +// The specified element must be non-nil and be guaranteed to exist in the cache +func (c *cache) deleteExistingUnderLock(e *list.Element) { + ci := e.Value.(*cacheItem) + c.size -= cachedValueOverhead + len(ci.key) + cap(ci.value) + delete(c.elements, ci.key) + c.accessed.Remove(e) +} + +// deleteUnderLock must be called under cache.lock +func (c *cache) deleteUnderLock(key string) { + if e, ok := c.elements[key]; ok { + c.deleteExistingUnderLock(e) + } +} + +func (c *cache) Delete(key string) { + c.lock.Lock() + c.deleteUnderLock(key) + c.lock.Unlock() +} + +func (c *cache) DeleteMulti(keys []string) { + c.lock.Lock() + for _, key := range keys { + c.deleteUnderLock(key) + } + c.lock.Unlock() +} + +// getUnderLock must be called under cache.lock +func (c *cache) getUnderLock(key string) []byte { + if e, ok := c.elements[key]; ok { + c.accessed.MoveToFront(e) + return (e.Value.(*cacheItem)).value + } + return nil +} + +// The cache retains ownership of the []byte, so consider it immutable +func (c *cache) Get(key string) []byte { + c.lock.Lock() + result := c.getUnderLock(key) + c.lock.Unlock() + return result +} + +// The cache retains ownership of the []byte, so consider it immutable +func (c *cache) GetMulti(keys []string) [][]byte { + c.lock.Lock() + result := make([][]byte, 0, len(keys)) + for _, key := range keys { + result = append(result, c.getUnderLock(key)) + } + c.lock.Unlock() + return result +} + +func (c *cache) Flush() { + c.lock.Lock() + c.size = 0 + c.elements = map[string]*list.Element{} + c.accessed.Init() + c.lock.Unlock() +} diff --git a/cache_test.go b/cache_test.go new file mode 100644 index 0000000..1024281 --- /dev/null +++ b/cache_test.go @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2012 The Goon Authors + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +package goon + +import ( + "bytes" + "reflect" + "testing" + "unsafe" +) + +func TestCacheBasics(t *testing.T) { + items := []*cacheItem{} + items = append(items, &cacheItem{key: "foo", value: []byte{1, 2, 3}}) + items = append(items, &cacheItem{key: "bar", value: []byte{4, 5, 6}}) + + keys := make([]string, 0, len(items)) + for _, item := range items { + keys = append(keys, item.key) + } + + c := newCache(defaultCacheLimit) + + for i := range items { + if v := c.Get(items[i].key); v != nil { + t.Fatalf("Expected nil for items[%d] but got %v", i, v) + } + + c.Set(items[i]) + + if v := c.Get(items[i].key); !bytes.Equal(v, items[i].value) { + t.Fatalf("Invalid bytes for items[%d]! %x vs %x", i, v, items[i].value) + } + + c.Delete(items[i].key) + + if v := c.Get(items[i].key); v != nil { + t.Fatalf("Expected nil for items[%d] but got %v", i, v) + } + + if c.size != 0 { + t.Fatalf("Expected size to be zero, but got %v", c.size) + } + } + + if vs := c.GetMulti(keys); !reflect.DeepEqual(vs, [][]byte{nil, nil}) { + t.Fatalf("Expected nils but got %+v", vs) + } + + c.SetMulti(items) + + if vs := c.GetMulti(keys); !reflect.DeepEqual(vs, [][]byte{items[0].value, items[1].value}) { + t.Fatalf("Invalid bytes for items! %+v", vs) + } + + c.DeleteMulti(keys) + + if vs := c.GetMulti(keys); !reflect.DeepEqual(vs, [][]byte{nil, nil}) { + t.Fatalf("Expected nils but got %+v", vs) + } + + if c.size != 0 { + t.Fatalf("Expected size to be zero, but got %v", c.size) + } + + c.Set(items[0]) + c.Flush() + if v := c.Get(items[0].key); v != nil { + t.Fatalf("Expected nil after flush but got %v", v) + } + + c.Set(items[0]) + c.Set(&cacheItem{key: items[0].key, value: []byte{7, 7, 7}}) + if v := c.Get(items[0].key); !bytes.Equal(v, []byte{7, 7, 7}) { + t.Fatalf("Invalid bytes for value change! Got %x", v) + } +} + +func TestCacheKeyLeak(t *testing.T) { + ak, bk := string([]byte{'f', 'o', 'o'}), string([]byte{'f', 'o', 'o'}) + av, bv := []byte{1, 2, 3}, []byte{4, 5, 6} + + c := newCache(defaultCacheLimit) + + // Set the original value + c.Set(&cacheItem{key: ak, value: av}) + if v := c.Get(ak); !bytes.Equal(v, av) { + t.Fatalf("Invalid bytes! %v", v) + } + + // Rewrite it with a different value, and also a different key but same key contents + c.Set(&cacheItem{key: bk, value: bv}) + if v := c.Get(bk); !bytes.Equal(v, bv) { + t.Fatalf("Invalid bytes! %v", v) + } + + // Modify the new key contents without changing the pointer + *(*byte)(unsafe.Pointer(*(*uintptr)(unsafe.Pointer(&bk)))) = 'g' + if bk != "goo" { + t.Fatalf("Expected key to be 'goo' but it's %v", bk) + } + + // Make sure that we can no longer retrieve the value with the new key, + // as that will only be possible via pointer equality, which means that + // the cache is still holding on to the new key, which doubles key storage + if v := c.Get(bk); v != nil { + t.Fatalf("Cache is leaking memory by keeping around the new key pointer! %v", v) + } + // Also make sure that we can retrieve the correct new value + // by using an unrelated key pointer that just matches the key contents + if v := c.Get("foo"); !bytes.Equal(v, bv) { + t.Fatalf("Invalid bytes! %v", v) + } + + // Inspect the internals of the cache too, which contains only a single entry with ak as key + keyAddr := *(*uintptr)(unsafe.Pointer(&ak)) + for key, elem := range c.elements { + if ka := *(*uintptr)(unsafe.Pointer(&key)); ka != keyAddr { + t.Fatalf("map key has wrong pointer! %x vs %x", ka, keyAddr) + } + if ka := *(*uintptr)(unsafe.Pointer(&(elem.Value.(*cacheItem)).key)); ka != keyAddr { + t.Fatalf("element key has wrong pointer! %x vs %x", ka, keyAddr) + } + } +} + +func TestCacheLimit(t *testing.T) { + c := newCache(defaultCacheLimit) + + items := []*cacheItem{} + items = append(items, &cacheItem{key: "foo", value: []byte{1, 2, 3}}) + items = append(items, &cacheItem{key: "bar", value: []byte{4, 5, 6}}) + + keys := make([]string, 0, len(items)) + for _, item := range items { + keys = append(keys, item.key) + } + + if vs := c.GetMulti(keys); !reflect.DeepEqual(vs, [][]byte{nil, nil}) { + t.Fatalf("Expected nils but got %+v", vs) + } + + c.SetMulti(items) + + if vs := c.GetMulti(keys); !reflect.DeepEqual(vs, [][]byte{items[0].value, items[1].value}) { + t.Fatalf("Invalid bytes for items! %+v", vs) + } + + c.setLimit(0) + + if vs := c.GetMulti(keys); !reflect.DeepEqual(vs, [][]byte{nil, nil}) { + t.Fatalf("Expected nils but got %+v", vs) + } + + if c.size != 0 { + t.Fatalf("Expected size to be zero, but got %v", c.size) + } + + c.setLimit(cachedValueOverhead + len(items[1].key) + cap(items[1].value)) + + c.SetMulti(items) + + if vs := c.GetMulti(keys); !reflect.DeepEqual(vs, [][]byte{nil, items[1].value}) { + t.Fatalf("Invalid bytes for items! %+v", vs) + } +} diff --git a/doc.go b/doc.go index 81a25fd..603345e 100644 --- a/doc.go +++ b/doc.go @@ -134,5 +134,13 @@ requests. The default settings were determined experimentally and should provide reasonable defaults for most applications. See: http://talks.golang.org/2013/highperf.slide#23 + +PropertyLoadSaver support + +Structs that implement the PropertyLoadSaver interface are guaranteed to call +the Save() method once and only once per Put/PutMulti call and never elsewhere. +Similarly the Load() method is guaranteed to be called once and only once per +Get/GetMulti/GetAll/Next call and never elsewhere. + */ package goon diff --git a/entity.go b/entity.go index be78aa4..2f41d86 100644 --- a/entity.go +++ b/entity.go @@ -27,8 +27,8 @@ import ( "sync" "time" - "google.golang.org/appengine" - "google.golang.org/appengine/datastore" + "google.golang.org/appengine/v2" + "google.golang.org/appengine/v2/datastore" ) type fieldInfo struct { diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..8c73684 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module github.com/topgate/goon + +go 1.12 + +require ( + github.com/golang/protobuf v1.3.1 + golang.org/x/net v0.2.0 + google.golang.org/appengine/v2 v2.0.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..9e36706 --- /dev/null +++ b/go.sum @@ -0,0 +1,34 @@ +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine/v2 v2.0.1 h1:jTGfiRmR5qoInpT3CXJ72GJEB4owDGEKN+xRDA6ekBY= +google.golang.org/appengine/v2 v2.0.1/go.mod h1:XgltgQxPOF3ShivrVrZyfvYCx8Dunh73bKjUuXUZb8Q= diff --git a/goon.go b/goon.go index e28d3e6..e8000ae 100644 --- a/goon.go +++ b/goon.go @@ -27,10 +27,10 @@ import ( "time" "golang.org/x/net/context" - "google.golang.org/appengine" - "google.golang.org/appengine/datastore" - "google.golang.org/appengine/log" - "google.golang.org/appengine/memcache" + "google.golang.org/appengine/v2" + "google.golang.org/appengine/v2/datastore" + "google.golang.org/appengine/v2/log" + "google.golang.org/appengine/v2/memcache" ) var ( diff --git a/goon_test.go b/goon_test.go index 4f44937..c896e5a 100644 --- a/goon_test.go +++ b/goon_test.go @@ -26,10 +26,10 @@ import ( "github.com/golang/protobuf/proto" "golang.org/x/net/context" - "google.golang.org/appengine" - "google.golang.org/appengine/aetest" - "google.golang.org/appengine/datastore" - "google.golang.org/appengine/memcache" + "google.golang.org/appengine/v2" + "google.golang.org/appengine/v2/aetest" + "google.golang.org/appengine/v2/datastore" + "google.golang.org/appengine/v2/memcache" ) // *[]S, *[]*S, *[]I, []S, []*S, []I diff --git a/query.go b/query.go index d4ed5e1..dbc6cd8 100644 --- a/query.go +++ b/query.go @@ -20,7 +20,7 @@ import ( "fmt" "reflect" - "google.golang.org/appengine/datastore" + "google.golang.org/appengine/v2/datastore" ) // Count returns the number of results for the query.