Initial commit

app/tool/cache/testdata/BUILD

@@ -0,0 +1,50 @@
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.cache.go",
+        "dao.go",
+        "multi.go",
+        "none.go",
+        "single.go",
+    ],
+    importpath = "go-common/app/tool/cache/testdata",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/stat/prom:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+        "//library/sync/pipeline/fanout:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "multi_test.go",
+        "none_test.go",
+        "single_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/tool/cache/testdata/dao.cache.go

@@ -0,0 +1,202 @@
+// Code generated by $GOPATH/src/go-common/app/tool/cache/gen. DO NOT EDIT.
+
+/*
+  Package testdata is a generated cache proxy package.
+  It is generated from:
+  type _cache interface {
+		// cache: -batch=10 -max_group=10 -batch_err=break -nullcache=&Article{ID:-1} -check_null_code=$.ID==-1
+		Articles(c context.Context, keys []int64) (map[int64]*Article, error)
+		// cache: -sync=true -nullcache=&Article{ID:-1} -check_null_code=$.ID==-1
+		Article(c context.Context, key int64) (*Article, error)
+		// cache: -paging=true
+		Article1(c context.Context, key int64, pn, ps int) (*Article, error)
+		// cache: -nullcache=&Article{ID:-1} -check_null_code=$.ID==-1
+		None(c context.Context) (*Article, error)
+	}
+*/
+
+package testdata
+
+import (
+	"context"
+	"sync"
+
+	"go-common/library/stat/prom"
+	"go-common/library/sync/errgroup"
+)
+
+var _ _cache
+
+// Articles get data from cache if miss will call source method, then add to cache.
+func (d *Dao) Articles(c context.Context, keys []int64) (res map[int64]*Article, err error) {
+	if len(keys) == 0 {
+		return
+	}
+	addCache := true
+	if res, err = d.CacheArticles(c, keys); err != nil {
+		addCache = false
+		res = nil
+		err = nil
+	}
+	var miss []int64
+	for _, key := range keys {
+		if (res == nil) || (res[key] == nil) {
+			miss = append(miss, key)
+		}
+	}
+	prom.CacheHit.Add("Articles", int64(len(keys)-len(miss)))
+	for k, v := range res {
+		if v.ID == -1 {
+			delete(res, k)
+		}
+	}
+	missLen := len(miss)
+	if missLen == 0 {
+		return
+	}
+	missData := make(map[int64]*Article, missLen)
+	prom.CacheMiss.Add("Articles", int64(missLen))
+	var mutex sync.Mutex
+	group, ctx := errgroup.WithContext(c)
+	if missLen > 10 {
+		group.GOMAXPROCS(10)
+	}
+	var run = func(ms []int64) {
+		group.Go(func() (err error) {
+			data, err := d.RawArticles(ctx, ms)
+			mutex.Lock()
+			for k, v := range data {
+				missData[k] = v
+			}
+			mutex.Unlock()
+			return
+		})
+	}
+	var (
+		i int
+		n = missLen / 10
+	)
+	for i = 0; i < n; i++ {
+		run(miss[i*n : (i+1)*n])
+	}
+	if len(miss[i*n:]) > 0 {
+		run(miss[i*n:])
+	}
+	err = group.Wait()
+	if res == nil {
+		res = make(map[int64]*Article, len(keys))
+	}
+	for k, v := range missData {
+		res[k] = v
+	}
+	if err != nil {
+		return
+	}
+	for _, key := range miss {
+		if res[key] == nil {
+			missData[key] = &Article{ID: -1}
+		}
+	}
+	if !addCache {
+		return
+	}
+	d.cache.Do(c, func(c context.Context) {
+		d.AddCacheArticles(c, missData)
+	})
+	return
+}
+
+// Article get data from cache if miss will call source method, then add to cache.
+func (d *Dao) Article(c context.Context, id int64) (res *Article, err error) {
+	addCache := true
+	res, err = d.CacheArticle(c, id)
+	if err != nil {
+		addCache = false
+		err = nil
+	}
+	defer func() {
+		if res.ID == -1 {
+			res = nil
+		}
+	}()
+	if res != nil {
+		prom.CacheHit.Incr("Article")
+		return
+	}
+	prom.CacheMiss.Incr("Article")
+	res, err = d.RawArticle(c, id)
+	if err != nil {
+		return
+	}
+	miss := res
+	if miss == nil {
+		miss = &Article{ID: -1}
+	}
+	if !addCache {
+		return
+	}
+	d.AddCacheArticle(c, id, miss)
+	return
+}
+
+// Article1 get data from cache if miss will call source method, then add to cache.
+func (d *Dao) Article1(c context.Context, id int64, pn, ps int) (res *Article, err error) {
+	addCache := true
+	res, err = d.CacheArticle1(c, id, pn, ps)
+	if err != nil {
+		addCache = false
+		err = nil
+	}
+	if res != nil {
+		prom.CacheHit.Incr("Article1")
+		return
+	}
+	var miss *Article
+	prom.CacheMiss.Incr("Article1")
+	res, miss, err = d.RawArticle1(c, id, pn, ps)
+	if err != nil {
+		return
+	}
+	if !addCache {
+		return
+	}
+	d.cache.Do(c, func(c context.Context) {
+		d.AddCacheArticle1(c, id, miss, pn, ps)
+	})
+	return
+}
+
+// None get data from cache if miss will call source method, then add to cache.
+func (d *Dao) None(c context.Context) (res *Article, err error) {
+	addCache := true
+	res, err = d.CacheNone(c)
+	if err != nil {
+		addCache = false
+		err = nil
+	}
+	defer func() {
+		if res.ID == -1 {
+			res = nil
+		}
+	}()
+	if res != nil {
+		prom.CacheHit.Incr("None")
+		return
+	}
+	prom.CacheMiss.Incr("None")
+	res, err = d.RawNone(c)
+	if err != nil {
+		return
+	}
+	var miss = res
+	if miss == nil {
+		miss = &Article{ID: -1}
+	}
+	if !addCache {
+		return
+	}
+	d.cache.Do(c, func(c context.Context) {
+		d.AddCacheNone(c, miss)
+	})
+	return
+}

app/tool/cache/testdata/dao.go

@@ -0,0 +1,35 @@
+package testdata
+
+import (
+	"context"
+
+	"go-common/library/sync/pipeline/fanout"
+)
+
+// Article test struct
+type Article struct {
+	ID    int64
+	Title string
+}
+
+// Dao .
+type Dao struct {
+	cache *fanout.Fanout
+}
+
+// New .
+func New() *Dao {
+	return &Dao{cache: fanout.New("cache")}
+}
+
+//go:generate $GOPATH/src/go-common/app/tool/cache/gen
+type _cache interface {
+	// cache: -batch=10 -max_group=10 -batch_err=break -nullcache=&Article{ID:-1} -check_null_code=$.ID==-1
+	Articles(c context.Context, keys []int64) (map[int64]*Article, error)
+	// cache: -sync=true -nullcache=&Article{ID:-1} -check_null_code=$.ID==-1
+	Article(c context.Context, key int64) (*Article, error)
+	// cache: -paging=true
+	Article1(c context.Context, key int64, pn, ps int) (*Article, error)
+	// cache: -nullcache=&Article{ID:-1} -check_null_code=$.ID==-1
+	None(c context.Context) (*Article, error)
+}

app/tool/cache/testdata/multi.go

@@ -0,0 +1,30 @@
+package testdata
+
+import (
+	"context"
+)
+
+// mock test
+var (
+	_multiCacheFunc    func(c context.Context, keys []int64) (map[int64]*Article, error)
+	_multiRawFunc      func(c context.Context, keys []int64) (map[int64]*Article, error)
+	_multiAddCacheFunc func(c context.Context, values map[int64]*Article) error
+)
+
+// CacheArticles .
+func (d *Dao) CacheArticles(c context.Context, keys []int64) (map[int64]*Article, error) {
+	// get data from cache
+	return _multiCacheFunc(c, keys)
+}
+
+// RawArticles .
+func (d *Dao) RawArticles(c context.Context, keys []int64) (map[int64]*Article, error) {
+	// get data from db
+	return _multiRawFunc(c, keys)
+}
+
+// AddCacheArticles .
+func (d *Dao) AddCacheArticles(c context.Context, values map[int64]*Article) error {
+	// add to cache
+	return _multiAddCacheFunc(c, values)
+}

app/tool/cache/testdata/multi_test.go

@@ -0,0 +1,67 @@
+package testdata
+
+import (
+	"context"
+	"errors"
+	"testing"
+)
+
+func TestMultiCache(t *testing.T) {
+	id := int64(1)
+	d := New()
+	meta := map[int64]*Article{id: {ID: id}}
+	getsFromCache := func(c context.Context, keys []int64) (map[int64]*Article, error) { return meta, nil }
+	notGetsFromCache := func(c context.Context, keys []int64) (map[int64]*Article, error) { return nil, errors.New("err") }
+	// 缓存返回了部分数据
+	partFromCache := func(c context.Context, keys []int64) (map[int64]*Article, error) { return meta, errors.New("err") }
+	getsFromSource := func(c context.Context, keys []int64) (map[int64]*Article, error) { return meta, nil }
+	notGetsFromSource := func(c context.Context, keys []int64) (map[int64]*Article, error) {
+		return meta, errors.New("err")
+	}
+	addToCache := func(c context.Context, values map[int64]*Article) error { return nil }
+	// gets from cache
+	_multiCacheFunc = getsFromCache
+	_multiRawFunc = notGetsFromSource
+	_multiAddCacheFunc = addToCache
+	res, err := d.Articles(context.TODO(), []int64{id})
+	if err != nil {
+		t.Fatalf("err should be nil, get: %v", err)
+	}
+	if res[1].ID != 1 {
+		t.Fatalf("id should be 1")
+	}
+	// get from source
+	_multiCacheFunc = notGetsFromCache
+	_multiRawFunc = getsFromSource
+	res, err = d.Articles(context.TODO(), []int64{id})
+	if err != nil {
+		t.Fatalf("err should be nil, get: %v", err)
+	}
+	if res[1].ID != 1 {
+		t.Fatalf("id should be 1")
+	}
+	// 缓存失败 返回部分数据 回源也失败的情况
+	_multiCacheFunc = partFromCache
+	_multiRawFunc = notGetsFromSource
+	res, err = d.Articles(context.TODO(), []int64{id})
+	if err == nil {
+		t.Fatalf("err should be nil, get: %v", err)
+	}
+	if res[1].ID != 1 {
+		t.Fatalf("id should be 1")
+	}
+	// with null cache
+	nullCache := &Article{ID: -1}
+	getNullFromCache := func(c context.Context, keys []int64) (map[int64]*Article, error) {
+		return map[int64]*Article{id: nullCache}, nil
+	}
+	_multiCacheFunc = getNullFromCache
+	_multiRawFunc = notGetsFromSource
+	res, err = d.Articles(context.TODO(), []int64{id})
+	if err != nil {
+		t.Fatalf("err should be nil, get: %v", err)
+	}
+	if res[id] != nil {
+		t.Fatalf("res should be nil")
+	}
+}

app/tool/cache/testdata/none.go

@@ -0,0 +1,30 @@
+package testdata
+
+import (
+	"context"
+)
+
+// mock test
+var (
+	_noneCacheFunc    func(c context.Context) (*Article, error)
+	_noneRawFunc      func(c context.Context) (*Article, error)
+	_noneAddCacheFunc func(c context.Context, value *Article) error
+)
+
+// CacheNone .
+func (d *Dao) CacheNone(c context.Context) (*Article, error) {
+	// get data from cache
+	return _noneCacheFunc(c)
+}
+
+// RawNone .
+func (d *Dao) RawNone(c context.Context) (*Article, error) {
+	// get data from db
+	return _noneRawFunc(c)
+}
+
+// AddCacheNone .
+func (d *Dao) AddCacheNone(c context.Context, value *Article) error {
+	// add to cache
+	return _noneAddCacheFunc(c, value)
+}

app/tool/cache/testdata/none_test.go

@@ -0,0 +1,50 @@
+package testdata
+
+import (
+	"context"
+	"errors"
+	"testing"
+)
+
+func TestNoneCache(t *testing.T) {
+	d := New()
+	meta := &Article{ID: 1}
+	getFromCache := func(c context.Context) (*Article, error) { return meta, nil }
+	notGetFromCache := func(c context.Context) (*Article, error) { return nil, errors.New("err") }
+	getFromSource := func(c context.Context) (*Article, error) { return meta, nil }
+	notGetFromSource := func(c context.Context) (*Article, error) { return meta, errors.New("err") }
+	addToCache := func(c context.Context, values *Article) error { return nil }
+	// get from cache
+	_noneCacheFunc = getFromCache
+	_noneRawFunc = notGetFromSource
+	_noneAddCacheFunc = addToCache
+	res, err := d.None(context.TODO())
+	if err != nil {
+		t.Fatalf("err should be nil, get: %v", err)
+	}
+	if res.ID != 1 {
+		t.Fatalf("id should be 1")
+	}
+	// get from source
+	_noneCacheFunc = notGetFromCache
+	_noneRawFunc = getFromSource
+	res, err = d.None(context.TODO())
+	if err != nil {
+		t.Fatalf("err should be nil, get: %v", err)
+	}
+	if res.ID != 1 {
+		t.Fatalf("id should be 1")
+	}
+	// with null cache
+	nullCache := &Article{ID: -1}
+	getNullFromCache := func(c context.Context) (*Article, error) { return nullCache, nil }
+	_noneCacheFunc = getNullFromCache
+	_noneRawFunc = notGetFromSource
+	res, err = d.None(context.TODO())
+	if err != nil {
+		t.Fatalf("err should be nil, get: %v", err)
+	}
+	if res != nil {
+		t.Fatalf("res should be nil")
+	}
+}

app/tool/cache/testdata/single.go

@@ -0,0 +1,48 @@
+package testdata
+
+import (
+	"context"
+)
+
+// mock test
+var (
+	_singleCacheFunc    func(c context.Context, key int64) (*Article, error)
+	_singleRawFunc      func(c context.Context, key int64) (*Article, error)
+	_singleAddCacheFunc func(c context.Context, key int64, value *Article) error
+)
+
+// CacheArticle .
+func (d *Dao) CacheArticle(c context.Context, key int64) (*Article, error) {
+	// get data from cache
+	return _singleCacheFunc(c, key)
+}
+
+// RawArticle .
+func (d *Dao) RawArticle(c context.Context, key int64) (*Article, error) {
+	// get data from db
+	return _singleRawFunc(c, key)
+}
+
+// AddCacheArticle .
+func (d *Dao) AddCacheArticle(c context.Context, key int64, value *Article) error {
+	// add to cache
+	return _singleAddCacheFunc(c, key, value)
+}
+
+// CacheArticle1 .
+func (d *Dao) CacheArticle1(c context.Context, key int64, pn, ps int) (*Article, error) {
+	// get data from cache
+	return nil, nil
+}
+
+// RawArticle1 .
+func (d *Dao) RawArticle1(c context.Context, key int64, pn, ps int) (*Article, *Article, error) {
+	// get data from db
+	return nil, nil, nil
+}
+
+// AddCacheArticle1 .
+func (d *Dao) AddCacheArticle1(c context.Context, key int64, value *Article, pn, ps int) error {
+	// add to cache
+	return nil
+}

app/tool/cache/testdata/single_test.go

@@ -0,0 +1,50 @@
+package testdata
+
+import (
+	"context"
+	"errors"
+	"testing"
+)
+
+func TestSingleCache(t *testing.T) {
+	d := New()
+	meta := &Article{ID: 1}
+	getFromCache := func(c context.Context, id int64) (*Article, error) { return meta, nil }
+	notGetFromCache := func(c context.Context, id int64) (*Article, error) { return nil, errors.New("err") }
+	getFromSource := func(c context.Context, id int64) (*Article, error) { return meta, nil }
+	notGetFromSource := func(c context.Context, id int64) (*Article, error) { return meta, errors.New("err") }
+	addToCache := func(c context.Context, id int64, values *Article) error { return nil }
+	// get from cache
+	_singleCacheFunc = getFromCache
+	_singleRawFunc = notGetFromSource
+	_singleAddCacheFunc = addToCache
+	res, err := d.Article(context.TODO(), 1)
+	if err != nil {
+		t.Fatalf("err should be nil, get: %v", err)
+	}
+	if res.ID != 1 {
+		t.Fatalf("id should be 1")
+	}
+	// get from source
+	_singleCacheFunc = notGetFromCache
+	_singleRawFunc = getFromSource
+	res, err = d.Article(context.TODO(), 1)
+	if err != nil {
+		t.Fatalf("err should be nil, get: %v", err)
+	}
+	if res.ID != 1 {
+		t.Fatalf("id should be 1")
+	}
+	// with null cache
+	nullCache := &Article{ID: -1}
+	getNullFromCache := func(c context.Context, id int64) (*Article, error) { return nullCache, nil }
+	_singleCacheFunc = getNullFromCache
+	_singleRawFunc = notGetFromSource
+	res, err = d.Article(context.TODO(), 1)
+	if err != nil {
+		t.Fatalf("err should be nil, get: %v", err)
+	}
+	if res != nil {
+		t.Fatalf("res should be nil")
+	}
+}