Initial commit

library/sync/errgroup.v2/BUILD.bazel

@@ -0,0 +1,41 @@
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "errgroup.go",
+    ],
+    importpath = "go-common/library/sync/errgroup.v2",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "errgroup_test.go",
+        "example_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

library/sync/errgroup.v2/CHANGELOG.md

@@ -0,0 +1,7 @@
+### errgroup
+
+#### Version 1.1.0
+> 1.支持 MaxProc 限制并发执行数
+
+#### Version 1.0.0
+> 1.提供带recover的errgroup，Wait()返回的err包含完整的堆栈信息  

library/sync/errgroup.v2/CONTRIBUTORS.md

@@ -0,0 +1,5 @@
+# Author
+weicheng
+ 
+# Reviewer
+haoguanwei

library/sync/errgroup.v2/OWNERS

@@ -0,0 +1,7 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- weicheng
+reviewers:
+- haoguanwei
+- weicheng

library/sync/errgroup.v2/README.md

@@ -0,0 +1,3 @@
+# go-common/errgroup.v2
+
+提供带recover的errgroup，err中包含详细堆栈信息

library/sync/errgroup.v2/doc.go

@@ -0,0 +1,47 @@
+// Package errgroup provides synchronization, error propagation, and Context
+// errgroup 包为一组子任务的 goroutine 提供了 goroutine 同步,错误取消功能.
+//
+//errgroup 包含三种常用方式
+//
+//1、直接使用 此时不会因为一个任务失败导致所有任务被 cancel:
+//		g := &errgroup.Group{}
+//		g.Go(func(ctx context.Context) {
+//			// NOTE: 此时 ctx 为 context.Background()
+//			// do something
+//		})
+//
+//2、WithContext 使用 WithContext 时不会因为一个任务失败导致所有任务被 cancel:
+//		g := errgroup.WithContext(ctx)
+//		g.Go(func(ctx context.Context) {
+//			// NOTE: 此时 ctx 为 errgroup.WithContext 传递的 ctx
+//			// do something
+//		})
+//
+//3、WithCancel 使用 WithCancel 时如果有一个人任务失败会导致所有*未进行或进行中*的任务被 cancel:
+//		g := errgroup.WithCancel(ctx)
+//		g.Go(func(ctx context.Context) {
+//			// NOTE: 此时 ctx 是从 errgroup.WithContext 传递的 ctx 派生出的 ctx
+//			// do something
+//		})
+//
+//设置最大并行数 GOMAXPROCS 对以上三种使用方式均起效
+//NOTE: 由于 errgroup 实现问题,设定 GOMAXPROCS 的 errgroup 需要立即调用 Wait() 例如:
+//
+//		g := errgroup.WithCancel(ctx)
+//		g.GOMAXPROCS(2)
+//		// task1
+//		g.Go(func(ctx context.Context) {
+//			fmt.Println("task1")
+//		})
+//		// task2
+//		g.Go(func(ctx context.Context) {
+//			fmt.Println("task2")
+//		})
+//		// task3
+//		g.Go(func(ctx context.Context) {
+//			fmt.Println("task3")
+//		})
+//		// NOTE: 此时设置的 GOMAXPROCS 为2, 添加了三个任务 task1, task2, task3 此时 task3 是不会运行的!
+//		// 只有调用了 Wait task3 才有运行的机会
+//		g.Wait() // task3 运行
+package errgroup

library/sync/errgroup.v2/errgroup.go

@@ -0,0 +1,125 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package errgroup provides synchronization, error propagation, and Context
+// cancelation for groups of goroutines working on subtasks of a common task.
+package errgroup
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+	"sync"
+)
+
+// A Group is a collection of goroutines working on subtasks that are part of
+// the same overall task.
+//
+// A zero Group is valid and does not cancel on error.
+type Group struct {
+	err     error
+	wg      sync.WaitGroup
+	errOnce sync.Once
+
+	workerOnce sync.Once
+	ch         chan func(ctx context.Context) error
+	chs        []func(ctx context.Context) error
+
+	ctx    context.Context
+	cancel func()
+}
+
+// WithContext create a Group.
+// given function from Go will receive this context,
+func WithContext(ctx context.Context) *Group {
+	return &Group{ctx: ctx}
+}
+
+// WithCancel create a new Group and an associated Context derived from ctx.
+//
+// given function from Go will receive context derived from this ctx,
+// The derived Context is canceled the first time a function passed to Go
+// returns a non-nil error or the first time Wait returns, whichever occurs
+// first.
+func WithCancel(ctx context.Context) *Group {
+	ctx, cancel := context.WithCancel(ctx)
+	return &Group{ctx: ctx, cancel: cancel}
+}
+
+func (g *Group) do(f func(ctx context.Context) error) {
+	ctx := g.ctx
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	var err error
+	defer func() {
+		if r := recover(); r != nil {
+			buf := make([]byte, 64<<10)
+			buf = buf[:runtime.Stack(buf, false)]
+			err = fmt.Errorf("errgroup: panic recovered: %s\n%s", r, buf)
+		}
+		if err != nil {
+			g.errOnce.Do(func() {
+				g.err = err
+				if g.cancel != nil {
+					g.cancel()
+				}
+			})
+		}
+		g.wg.Done()
+	}()
+	err = f(ctx)
+}
+
+// GOMAXPROCS set max goroutine to work.
+func (g *Group) GOMAXPROCS(n int) {
+	if n <= 0 {
+		panic("errgroup: GOMAXPROCS must great than 0")
+	}
+	g.workerOnce.Do(func() {
+		g.ch = make(chan func(context.Context) error, n)
+		for i := 0; i < n; i++ {
+			go func() {
+				for f := range g.ch {
+					g.do(f)
+				}
+			}()
+		}
+	})
+}
+
+// Go calls the given function in a new goroutine.
+//
+// The first call to return a non-nil error cancels the group; its error will be
+// returned by Wait.
+func (g *Group) Go(f func(ctx context.Context) error) {
+	g.wg.Add(1)
+	if g.ch != nil {
+		select {
+		case g.ch <- f:
+		default:
+			g.chs = append(g.chs, f)
+		}
+		return
+	}
+	go g.do(f)
+}
+
+// Wait blocks until all function calls from the Go method have returned, then
+// returns the first non-nil error (if any) from them.
+func (g *Group) Wait() error {
+	if g.ch != nil {
+		for _, f := range g.chs {
+			g.ch <- f
+		}
+	}
+	g.wg.Wait()
+	if g.ch != nil {
+		close(g.ch) // let all receiver exit
+	}
+	if g.cancel != nil {
+		g.cancel()
+	}
+	return g.err
+}

library/sync/errgroup.v2/errgroup_test.go

@@ -0,0 +1,266 @@
+package errgroup
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"net/http"
+	"os"
+	"testing"
+	"time"
+)
+
+type ABC struct {
+	CBA int
+}
+
+func TestNormal(t *testing.T) {
+	var (
+		abcs = make(map[int]*ABC)
+		g    Group
+		err  error
+	)
+	for i := 0; i < 10; i++ {
+		abcs[i] = &ABC{CBA: i}
+	}
+	g.Go(func(context.Context) (err error) {
+		abcs[1].CBA++
+		return
+	})
+	g.Go(func(context.Context) (err error) {
+		abcs[2].CBA++
+		return
+	})
+	if err = g.Wait(); err != nil {
+		t.Log(err)
+	}
+	t.Log(abcs)
+}
+
+func sleep1s(context.Context) error {
+	time.Sleep(time.Second)
+	return nil
+}
+
+func TestGOMAXPROCS(t *testing.T) {
+	// 没有并发数限制
+	g := Group{}
+	now := time.Now()
+	g.Go(sleep1s)
+	g.Go(sleep1s)
+	g.Go(sleep1s)
+	g.Go(sleep1s)
+	g.Wait()
+	sec := math.Round(time.Since(now).Seconds())
+	if sec != 1 {
+		t.FailNow()
+	}
+	// 限制并发数
+	g2 := Group{}
+	g2.GOMAXPROCS(2)
+	now = time.Now()
+	g2.Go(sleep1s)
+	g2.Go(sleep1s)
+	g2.Go(sleep1s)
+	g2.Go(sleep1s)
+	g2.Wait()
+	sec = math.Round(time.Since(now).Seconds())
+	if sec != 2 {
+		t.FailNow()
+	}
+	// context canceled
+	var canceled bool
+	g3 := WithCancel(context.Background())
+	g3.GOMAXPROCS(2)
+	g3.Go(func(context.Context) error {
+		return fmt.Errorf("error for testing errgroup context")
+	})
+	g3.Go(func(ctx context.Context) error {
+		time.Sleep(time.Second)
+		select {
+		case <-ctx.Done():
+			canceled = true
+		default:
+		}
+		return nil
+	})
+	g3.Wait()
+	if !canceled {
+		t.FailNow()
+	}
+}
+
+func TestRecover(t *testing.T) {
+	var (
+		abcs = make(map[int]*ABC)
+		g    Group
+		err  error
+	)
+	g.Go(func(context.Context) (err error) {
+		abcs[1].CBA++
+		return
+	})
+	g.Go(func(context.Context) (err error) {
+		abcs[2].CBA++
+		return
+	})
+	if err = g.Wait(); err != nil {
+		t.Logf("error:%+v", err)
+		return
+	}
+	t.FailNow()
+}
+
+func TestRecover2(t *testing.T) {
+	var (
+		g   Group
+		err error
+	)
+	g.Go(func(context.Context) (err error) {
+		panic("2233")
+	})
+	if err = g.Wait(); err != nil {
+		t.Logf("error:%+v", err)
+		return
+	}
+	t.FailNow()
+}
+
+var (
+	Web   = fakeSearch("web")
+	Image = fakeSearch("image")
+	Video = fakeSearch("video")
+)
+
+type Result string
+type Search func(ctx context.Context, query string) (Result, error)
+
+func fakeSearch(kind string) Search {
+	return func(_ context.Context, query string) (Result, error) {
+		return Result(fmt.Sprintf("%s result for %q", kind, query)), nil
+	}
+}
+
+// JustErrors illustrates the use of a Group in place of a sync.WaitGroup to
+// simplify goroutine counting and error handling. This example is derived from
+// the sync.WaitGroup example at https://golang.org/pkg/sync/#example_WaitGroup.
+func ExampleGroup_justErrors() {
+	var g Group
+	var urls = []string{
+		"http://www.golang.org/",
+		"http://www.google.com/",
+		"http://www.somestupidname.com/",
+	}
+	for _, url := range urls {
+		// Launch a goroutine to fetch the URL.
+		url := url // https://golang.org/doc/faq#closures_and_goroutines
+		g.Go(func(context.Context) error {
+			// Fetch the URL.
+			resp, err := http.Get(url)
+			if err == nil {
+				resp.Body.Close()
+			}
+			return err
+		})
+	}
+	// Wait for all HTTP fetches to complete.
+	if err := g.Wait(); err == nil {
+		fmt.Println("Successfully fetched all URLs.")
+	}
+}
+
+// Parallel illustrates the use of a Group for synchronizing a simple parallel
+// task: the "Google Search 2.0" function from
+// https://talks.golang.org/2012/concurrency.slide#46, augmented with a Context
+// and error-handling.
+func ExampleGroup_parallel() {
+	Google := func(ctx context.Context, query string) ([]Result, error) {
+		g := WithContext(ctx)
+
+		searches := []Search{Web, Image, Video}
+		results := make([]Result, len(searches))
+		for i, search := range searches {
+			i, search := i, search // https://golang.org/doc/faq#closures_and_goroutines
+			g.Go(func(context.Context) error {
+				result, err := search(ctx, query)
+				if err == nil {
+					results[i] = result
+				}
+				return err
+			})
+		}
+		if err := g.Wait(); err != nil {
+			return nil, err
+		}
+		return results, nil
+	}
+
+	results, err := Google(context.Background(), "golang")
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		return
+	}
+	for _, result := range results {
+		fmt.Println(result)
+	}
+
+	// Output:
+	// web result for "golang"
+	// image result for "golang"
+	// video result for "golang"
+}
+
+func TestZeroGroup(t *testing.T) {
+	err1 := errors.New("errgroup_test: 1")
+	err2 := errors.New("errgroup_test: 2")
+
+	cases := []struct {
+		errs []error
+	}{
+		{errs: []error{}},
+		{errs: []error{nil}},
+		{errs: []error{err1}},
+		{errs: []error{err1, nil}},
+		{errs: []error{err1, nil, err2}},
+	}
+
+	for _, tc := range cases {
+		var g Group
+
+		var firstErr error
+		for i, err := range tc.errs {
+			err := err
+			g.Go(func(context.Context) error { return err })
+
+			if firstErr == nil && err != nil {
+				firstErr = err
+			}
+
+			if gErr := g.Wait(); gErr != firstErr {
+				t.Errorf("after g.Go(func() error { return err }) for err in %v\n"+
+					"g.Wait() = %v; want %v", tc.errs[:i+1], err, firstErr)
+			}
+		}
+	}
+}
+
+func TestWithCancel(t *testing.T) {
+	g := WithCancel(context.Background())
+	g.Go(func(ctx context.Context) error {
+		time.Sleep(100 * time.Millisecond)
+		return fmt.Errorf("boom")
+	})
+	var doneErr error
+	g.Go(func(ctx context.Context) error {
+		select {
+		case <-ctx.Done():
+			doneErr = ctx.Err()
+		}
+		return doneErr
+	})
+	g.Wait()
+	if doneErr != context.Canceled {
+		t.Error("error should be Canceled")
+	}
+}

library/sync/errgroup.v2/example_test.go

@@ -0,0 +1,63 @@
+package errgroup
+
+import (
+	"context"
+)
+
+func fakeRunTask(ctx context.Context) error {
+	return nil
+}
+
+func ExampleGroup_group() {
+	g := Group{}
+	g.Go(func(context.Context) error {
+		return fakeRunTask(context.Background())
+	})
+	g.Go(func(context.Context) error {
+		return fakeRunTask(context.Background())
+	})
+	if err := g.Wait(); err != nil {
+		// handle err
+	}
+}
+
+func ExampleGroup_ctx() {
+	g := WithContext(context.Background())
+	g.Go(func(ctx context.Context) error {
+		return fakeRunTask(ctx)
+	})
+	g.Go(func(ctx context.Context) error {
+		return fakeRunTask(ctx)
+	})
+	if err := g.Wait(); err != nil {
+		// handle err
+	}
+}
+
+func ExampleGroup_cancel() {
+	g := WithCancel(context.Background())
+	g.Go(func(ctx context.Context) error {
+		return fakeRunTask(ctx)
+	})
+	g.Go(func(ctx context.Context) error {
+		return fakeRunTask(ctx)
+	})
+	if err := g.Wait(); err != nil {
+		// handle err
+	}
+}
+
+func ExampleGroup_maxproc() {
+	g := Group{}
+	// set max concurrency
+	g.GOMAXPROCS(2)
+	g.Go(func(ctx context.Context) error {
+		return fakeRunTask(context.Background())
+	})
+	g.Go(func(ctx context.Context) error {
+		return fakeRunTask(context.Background())
+	})
+	if err := g.Wait(); err != nil {
+		// handle err
+	}
+}