Goonの2nd gen移行

README.md

@@ -1,5 +1,18 @@
-# goon
-
-An autocaching interface to the app engine datastore for Go. Designed to be similar to the python NDB package.
-
-Documentation: [http://godoc.org/github.com/mjibson/goon](http://godoc.org/github.com/mjibson/goon)
+# goon
+
+An autocaching interface to the app engine datastore for Go. Designed to be similar to the python NDB package.
+
+## The different flavors of App Engine
+
+You must choose the goon major version based on which App Engine library you are using.
+
+| App Engine Go library            | Include in your project     | Correct goon version                                                   |
+|----------------------------------|-----------------------------|------------------------------------------------------------------------|
+| `google.golang.org/appengine/v2` |`github.com/mjibson/goon/v2` | [goon v2.0.1](https://github.com/mjibson/goon/releases/tag/v2.0.1)     |
+| `google.golang.org/appengine`    |`github.com/mjibson/goon`    | [goon v1.1.0](https://github.com/mjibson/goon/releases/tag/v1.1.0)     |
+| `appengine`                      |`github.com/mjibson/goon`    | [goon v0.9.0](https://github.com/mjibson/goon/releases/tag/v0.9.0)     |
+| `cloud.google.com/go`            |N/A                          | Not supported ([issue #74](https://github.com/mjibson/goon/issues/74)) |
+
+## Documentation
+
+[https://pkg.go.dev/github.com/mjibson/goon](https://pkg.go.dev/github.com/mjibson/goon)

cache.go

@@ -0,0 +1,179 @@
+/*
+ * Copyright (c) 2012 The Goon Authors
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+package goon
+
+import (
+	"container/list"
+	"reflect"
+	"sync"
+)
+
+var cachedValueOverhead int
+
+func init() {
+	// Calculate the platform dependant overhead size for keeping a value in cache
+	var elem list.Element
+	var ci cacheItem
+	cachedValueOverhead += int(reflect.TypeOf(elem).Size())   // list.Element in cache.accessed
+	cachedValueOverhead += int(reflect.TypeOf(&elem).Size())  // *list.Element in cache.elements
+	cachedValueOverhead += int(reflect.TypeOf(ci.key).Size()) // string in cache.elements as key
+	cachedValueOverhead += int(reflect.TypeOf(ci).Size())     // cacheItem pointed to by list.Element.Value
+	// In addition to the above overhead, the total cache value size must include
+	// the length of the key string in bytes and the cap of the value []byte
+}
+
+type cacheItem struct {
+	key   string
+	value []byte
+}
+
+type cache struct {
+	lock     sync.Mutex
+	elements map[string]*list.Element // access via key
+	accessed list.List                // most recently accessed in front
+	size     int                      // Total size of all the values in the cache
+	limit    int                      // Maximum size allowed
+}
+
+const defaultCacheLimit = 16 << 20 // 16 MiB
+
+func newCache(limit int) *cache {
+	return &cache{elements: map[string]*list.Element{}, limit: limit}
+}
+
+func (c *cache) setLimit(limit int) {
+	c.lock.Lock()
+	c.limit = limit
+	c.meetLimitUnderLock()
+	c.lock.Unlock()
+}
+
+// meetLimit must be called under cache.lock
+func (c *cache) meetLimitUnderLock() {
+	for c.size > c.limit {
+		e := c.accessed.Back()
+		if e == nil {
+			break
+		}
+		c.deleteExistingUnderLock(e)
+	}
+}
+
+// setUnderLock must be called under cache.lock
+func (c *cache) setUnderLock(item *cacheItem) {
+	// Check if there's already an entry for this key
+	if e, ok := c.elements[item.key]; ok {
+		// There already exists a value for this key, so update it
+		ci := e.Value.(*cacheItem)
+		c.size += cap(item.value) - cap(ci.value)
+		// Make sure that item.key is the same pointer as the map key,
+		// as this will ensure faster map lookup via pointer equality.
+		// Not doing so would also lead to double memory usage, as the two key
+		// pointers would be pointing to the same contents in different places.
+		item.key = ci.key
+		e.Value = item
+		c.accessed.MoveToFront(e)
+	} else {
+		// Brand new key, so add it
+		c.size += cachedValueOverhead + len(item.key) + cap(item.value)
+		c.elements[item.key] = c.accessed.PushFront(item)
+	}
+}
+
+// Set takes ownership of item and treats it as immutable
+func (c *cache) Set(item *cacheItem) {
+	c.lock.Lock()
+	c.setUnderLock(item)
+	c.meetLimitUnderLock()
+	c.lock.Unlock()
+}
+
+// SetMulti takes ownership of the individual items and treats them as immutable
+func (c *cache) SetMulti(items []*cacheItem) {
+	c.lock.Lock()
+	for _, item := range items {
+		c.setUnderLock(item)
+	}
+	c.meetLimitUnderLock()
+	c.lock.Unlock()
+}
+
+// deleteExistingUnderLock must be called under cache.lock
+// The specified element must be non-nil and be guaranteed to exist in the cache
+func (c *cache) deleteExistingUnderLock(e *list.Element) {
+	ci := e.Value.(*cacheItem)
+	c.size -= cachedValueOverhead + len(ci.key) + cap(ci.value)
+	delete(c.elements, ci.key)
+	c.accessed.Remove(e)
+}
+
+// deleteUnderLock must be called under cache.lock
+func (c *cache) deleteUnderLock(key string) {
+	if e, ok := c.elements[key]; ok {
+		c.deleteExistingUnderLock(e)
+	}
+}
+
+func (c *cache) Delete(key string) {
+	c.lock.Lock()
+	c.deleteUnderLock(key)
+	c.lock.Unlock()
+}
+
+func (c *cache) DeleteMulti(keys []string) {
+	c.lock.Lock()
+	for _, key := range keys {
+		c.deleteUnderLock(key)
+	}
+	c.lock.Unlock()
+}
+
+// getUnderLock must be called under cache.lock
+func (c *cache) getUnderLock(key string) []byte {
+	if e, ok := c.elements[key]; ok {
+		c.accessed.MoveToFront(e)
+		return (e.Value.(*cacheItem)).value
+	}
+	return nil
+}
+
+// The cache retains ownership of the []byte, so consider it immutable
+func (c *cache) Get(key string) []byte {
+	c.lock.Lock()
+	result := c.getUnderLock(key)
+	c.lock.Unlock()
+	return result
+}
+
+// The cache retains ownership of the []byte, so consider it immutable
+func (c *cache) GetMulti(keys []string) [][]byte {
+	c.lock.Lock()
+	result := make([][]byte, 0, len(keys))
+	for _, key := range keys {
+		result = append(result, c.getUnderLock(key))
+	}
+	c.lock.Unlock()
+	return result
+}
+
+func (c *cache) Flush() {
+	c.lock.Lock()
+	c.size = 0
+	c.elements = map[string]*list.Element{}
+	c.accessed.Init()
+	c.lock.Unlock()
+}

cache_test.go

@@ -0,0 +1,180 @@
+/*
+ * Copyright (c) 2012 The Goon Authors
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+package goon
+
+import (
+	"bytes"
+	"reflect"
+	"testing"
+	"unsafe"
+)
+
+func TestCacheBasics(t *testing.T) {
+	items := []*cacheItem{}
+	items = append(items, &cacheItem{key: "foo", value: []byte{1, 2, 3}})
+	items = append(items, &cacheItem{key: "bar", value: []byte{4, 5, 6}})
+
+	keys := make([]string, 0, len(items))
+	for _, item := range items {
+		keys = append(keys, item.key)
+	}
+
+	c := newCache(defaultCacheLimit)
+
+	for i := range items {
+		if v := c.Get(items[i].key); v != nil {
+			t.Fatalf("Expected nil for items[%d] but got %v", i, v)
+		}
+
+		c.Set(items[i])
+
+		if v := c.Get(items[i].key); !bytes.Equal(v, items[i].value) {
+			t.Fatalf("Invalid bytes for items[%d]! %x vs %x", i, v, items[i].value)
+		}
+
+		c.Delete(items[i].key)
+
+		if v := c.Get(items[i].key); v != nil {
+			t.Fatalf("Expected nil for items[%d] but got %v", i, v)
+		}
+
+		if c.size != 0 {
+			t.Fatalf("Expected size to be zero, but got %v", c.size)
+		}
+	}
+
+	if vs := c.GetMulti(keys); !reflect.DeepEqual(vs, [][]byte{nil, nil}) {
+		t.Fatalf("Expected nils but got %+v", vs)
+	}
+
+	c.SetMulti(items)
+
+	if vs := c.GetMulti(keys); !reflect.DeepEqual(vs, [][]byte{items[0].value, items[1].value}) {
+		t.Fatalf("Invalid bytes for items! %+v", vs)
+	}
+
+	c.DeleteMulti(keys)
+
+	if vs := c.GetMulti(keys); !reflect.DeepEqual(vs, [][]byte{nil, nil}) {
+		t.Fatalf("Expected nils but got %+v", vs)
+	}
+
+	if c.size != 0 {
+		t.Fatalf("Expected size to be zero, but got %v", c.size)
+	}
+
+	c.Set(items[0])
+	c.Flush()
+	if v := c.Get(items[0].key); v != nil {
+		t.Fatalf("Expected nil after flush but got %v", v)
+	}
+
+	c.Set(items[0])
+	c.Set(&cacheItem{key: items[0].key, value: []byte{7, 7, 7}})
+	if v := c.Get(items[0].key); !bytes.Equal(v, []byte{7, 7, 7}) {
+		t.Fatalf("Invalid bytes for value change! Got %x", v)
+	}
+}
+
+func TestCacheKeyLeak(t *testing.T) {
+	ak, bk := string([]byte{'f', 'o', 'o'}), string([]byte{'f', 'o', 'o'})
+	av, bv := []byte{1, 2, 3}, []byte{4, 5, 6}
+
+	c := newCache(defaultCacheLimit)
+
+	// Set the original value
+	c.Set(&cacheItem{key: ak, value: av})
+	if v := c.Get(ak); !bytes.Equal(v, av) {
+		t.Fatalf("Invalid bytes! %v", v)
+	}
+
+	// Rewrite it with a different value, and also a different key but same key contents
+	c.Set(&cacheItem{key: bk, value: bv})
+	if v := c.Get(bk); !bytes.Equal(v, bv) {
+		t.Fatalf("Invalid bytes! %v", v)
+	}
+
+	// Modify the new key contents without changing the pointer
+	*(*byte)(unsafe.Pointer(*(*uintptr)(unsafe.Pointer(&bk)))) = 'g'
+	if bk != "goo" {
+		t.Fatalf("Expected key to be 'goo' but it's %v", bk)
+	}
+
+	// Make sure that we can no longer retrieve the value with the new key,
+	// as that will only be possible via pointer equality, which means that
+	// the cache is still holding on to the new key, which doubles key storage
+	if v := c.Get(bk); v != nil {
+		t.Fatalf("Cache is leaking memory by keeping around the new key pointer! %v", v)
+	}
+	// Also make sure that we can retrieve the correct new value
+	// by using an unrelated key pointer that just matches the key contents
+	if v := c.Get("foo"); !bytes.Equal(v, bv) {
+		t.Fatalf("Invalid bytes! %v", v)
+	}
+
+	// Inspect the internals of the cache too, which contains only a single entry with ak as key
+	keyAddr := *(*uintptr)(unsafe.Pointer(&ak))
+	for key, elem := range c.elements {
+		if ka := *(*uintptr)(unsafe.Pointer(&key)); ka != keyAddr {
+			t.Fatalf("map key has wrong pointer! %x vs %x", ka, keyAddr)
+		}
+		if ka := *(*uintptr)(unsafe.Pointer(&(elem.Value.(*cacheItem)).key)); ka != keyAddr {
+			t.Fatalf("element key has wrong pointer! %x vs %x", ka, keyAddr)
+		}
+	}
+}
+
+func TestCacheLimit(t *testing.T) {
+	c := newCache(defaultCacheLimit)
+
+	items := []*cacheItem{}
+	items = append(items, &cacheItem{key: "foo", value: []byte{1, 2, 3}})
+	items = append(items, &cacheItem{key: "bar", value: []byte{4, 5, 6}})
+
+	keys := make([]string, 0, len(items))
+	for _, item := range items {
+		keys = append(keys, item.key)
+	}
+
+	if vs := c.GetMulti(keys); !reflect.DeepEqual(vs, [][]byte{nil, nil}) {
+		t.Fatalf("Expected nils but got %+v", vs)
+	}
+
+	c.SetMulti(items)
+
+	if vs := c.GetMulti(keys); !reflect.DeepEqual(vs, [][]byte{items[0].value, items[1].value}) {
+		t.Fatalf("Invalid bytes for items! %+v", vs)
+	}
+
+	c.setLimit(0)
+
+	if vs := c.GetMulti(keys); !reflect.DeepEqual(vs, [][]byte{nil, nil}) {
+		t.Fatalf("Expected nils but got %+v", vs)
+	}
+
+	if c.size != 0 {
+		t.Fatalf("Expected size to be zero, but got %v", c.size)
+	}
+
+	c.setLimit(cachedValueOverhead + len(items[1].key) + cap(items[1].value))
+
+	c.SetMulti(items)
+
+	if vs := c.GetMulti(keys); !reflect.DeepEqual(vs, [][]byte{nil, items[1].value}) {
+		t.Fatalf("Invalid bytes for items! %+v", vs)
+	}
+}

doc.go

@@ -134,5 +134,13 @@ requests. The default settings were determined experimentally and should
 provide reasonable defaults for most applications.
 
 See: http://talks.golang.org/2013/highperf.slide#23
+
+PropertyLoadSaver support
+
+Structs that implement the PropertyLoadSaver interface are guaranteed to call
+the Save() method once and only once per Put/PutMulti call and never elsewhere.
+Similarly the Load() method is guaranteed to be called once and only once per
+Get/GetMulti/GetAll/Next call and never elsewhere.
+
 */
 package goon