Initial commit

This commit is contained in:
Donny
2019-04-22 20:46:32 +08:00
commit 49ab8aadd1
25441 changed files with 4055000 additions and 0 deletions

View File

@@ -0,0 +1,33 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"error.go",
"priority_queue.go",
"queue.go",
"ring.go",
],
importpath = "go-common/app/service/main/account/model/queue",
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,32 @@
/*
Copyright 2014 Workiva, LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package queue
import "errors"
var (
// ErrDisposed is returned when an operation is performed on a disposed
// queue.
ErrDisposed = errors.New(`queue: disposed`)
// ErrTimeout is returned when an applicable queue operation times out.
ErrTimeout = errors.New(`queue: poll timed out`)
// ErrEmptyQueue is returned when an non-applicable queue operation was called
// due to the queue's empty item state
ErrEmptyQueue = errors.New(`queue: empty queue`)
)

View File

@@ -0,0 +1,267 @@
/*
Copyright 2014 Workiva, LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
/*
The priority queue is almost a spitting image of the logic
used for a regular queue. In order to keep the logic fast,
this code is repeated instead of using casts to cast to interface{}
back and forth. If Go had inheritance and generics, this problem
would be easier to solve.
*/
package queue
import "sync"
// Item is an item that can be added to the priority queue.
type Item interface {
// Compare returns a bool that can be used to determine
// ordering in the priority queue. Assuming the queue
// is in ascending order, this should return > logic.
// Return 1 to indicate this object is greater than the
// the other logic, 0 to indicate equality, and -1 to indicate
// less than other.
Compare(other Item) int
HashCode() int64
}
type priorityItems []Item
func (items *priorityItems) swap(i, j int) {
(*items)[i], (*items)[j] = (*items)[j], (*items)[i]
}
func (items *priorityItems) pop() Item {
size := len(*items)
// Move last leaf to root, and 'pop' the last item.
items.swap(size-1, 0)
item := (*items)[size-1] // Item to return.
(*items)[size-1], *items = nil, (*items)[:size-1]
// 'Bubble down' to restore heap property.
index := 0
childL, childR := 2*index+1, 2*index+2
for len(*items) > childL {
child := childL
if len(*items) > childR && (*items)[childR].Compare((*items)[childL]) < 0 {
child = childR
}
if (*items)[child].Compare((*items)[index]) < 0 {
items.swap(index, child)
index = child
childL, childR = 2*index+1, 2*index+2
} else {
break
}
}
return item
}
func (items *priorityItems) get(number int) []Item {
returnItems := make([]Item, 0, number)
for i := 0; i < number; i++ {
if len(*items) == 0 {
break
}
returnItems = append(returnItems, items.pop())
}
return returnItems
}
func (items *priorityItems) push(item Item) {
// Stick the item as the end of the last level.
*items = append(*items, item)
// 'Bubble up' to restore heap property.
index := len(*items) - 1
parent := int((index - 1) / 2)
for parent >= 0 && (*items)[parent].Compare(item) > 0 {
items.swap(index, parent)
index = parent
parent = int((index - 1) / 2)
}
}
// PriorityQueue is similar to queue except that it takes
// items that implement the Item interface and adds them
// to the queue in priority order.
type PriorityQueue struct {
waiters waiters
items priorityItems
itemMap map[int64]struct{}
lock sync.Mutex
disposeLock sync.Mutex
disposed bool
allowDuplicates bool
}
// Put adds items to the queue.
func (pq *PriorityQueue) Put(items ...Item) error {
if len(items) == 0 {
return nil
}
pq.lock.Lock()
defer pq.lock.Unlock()
if pq.disposed {
return ErrDisposed
}
for _, item := range items {
if pq.allowDuplicates {
pq.items.push(item)
} else if _, ok := pq.itemMap[item.HashCode()]; !ok {
pq.itemMap[item.HashCode()] = struct{}{}
pq.items.push(item)
}
}
for {
sema := pq.waiters.get()
if sema == nil {
break
}
sema.response.Add(1)
sema.ready <- true
sema.response.Wait()
if len(pq.items) == 0 {
break
}
}
return nil
}
// Get retrieves items from the queue. If the queue is empty,
// this call blocks until the next item is added to the queue. This
// will attempt to retrieve number of items.
func (pq *PriorityQueue) Get(number int) ([]Item, error) {
if number < 1 {
return nil, nil
}
pq.lock.Lock()
if pq.disposed {
pq.lock.Unlock()
return nil, ErrDisposed
}
var items []Item
// Remove references to popped items.
deleteItems := func(items []Item) {
for _, item := range items {
delete(pq.itemMap, item.HashCode())
}
}
if len(pq.items) == 0 {
sema := newSema()
pq.waiters.put(sema)
pq.lock.Unlock()
<-sema.ready
if pq.Disposed() {
return nil, ErrDisposed
}
items = pq.items.get(number)
if !pq.allowDuplicates {
deleteItems(items)
}
sema.response.Done()
return items, nil
}
items = pq.items.get(number)
deleteItems(items)
pq.lock.Unlock()
return items, nil
}
// Peek will look at the next item without removing it from the queue.
func (pq *PriorityQueue) Peek() Item {
pq.lock.Lock()
defer pq.lock.Unlock()
if len(pq.items) > 0 {
return pq.items[0]
}
return nil
}
// Empty returns a bool indicating if there are any items left
// in the queue.
func (pq *PriorityQueue) Empty() bool {
pq.lock.Lock()
defer pq.lock.Unlock()
return len(pq.items) == 0
}
// Len returns a number indicating how many items are in the queue.
func (pq *PriorityQueue) Len() int {
pq.lock.Lock()
defer pq.lock.Unlock()
return len(pq.items)
}
// Disposed returns a bool indicating if this queue has been disposed.
func (pq *PriorityQueue) Disposed() bool {
pq.disposeLock.Lock()
defer pq.disposeLock.Unlock()
return pq.disposed
}
// Dispose will prevent any further reads/writes to this queue
// and frees available resources.
func (pq *PriorityQueue) Dispose() {
pq.lock.Lock()
defer pq.lock.Unlock()
pq.disposeLock.Lock()
defer pq.disposeLock.Unlock()
pq.disposed = true
for _, waiter := range pq.waiters {
waiter.response.Add(1)
waiter.ready <- true
}
pq.items = nil
pq.waiters = nil
}
// NewPriorityQueue is the constructor for a priority queue.
func NewPriorityQueue(hint int, allowDuplicates bool) *PriorityQueue {
return &PriorityQueue{
items: make(priorityItems, 0, hint),
itemMap: make(map[int64]struct{}, hint),
allowDuplicates: allowDuplicates,
}
}

View File

@@ -0,0 +1,411 @@
/*
Copyright 2014 Workiva, LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
/*
Package queue includes a regular queue and a priority queue.
These queues rely on waitgroups to pause listening threads
on empty queues until a message is received. If any thread
calls Dispose on the queue, any listeners are immediately returned
with an error. Any subsequent put to the queue will return an error
as opposed to panicking as with channels. Queues will grow with unbounded
behavior as opposed to channels which can be buffered but will pause
while a thread attempts to put to a full channel.
Recently added is a lockless ring buffer using the same basic C design as
found here:
http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue
Modified for use with Go with the addition of some dispose semantics providing
the capability to release blocked threads. This works for both puts
and gets, either will return an error if they are blocked and the buffer
is disposed. This could serve as a signal to kill a goroutine. All threadsafety
is acheived using CAS operations, making this buffer pretty quick.
Benchmarks:
BenchmarkPriorityQueue-8 2000000 782 ns/op
BenchmarkQueue-8 2000000 671 ns/op
BenchmarkChannel-8 1000000 2083 ns/op
BenchmarkQueuePut-8 20000 84299 ns/op
BenchmarkQueueGet-8 20000 80753 ns/op
BenchmarkExecuteInParallel-8 20000 68891 ns/op
BenchmarkRBLifeCycle-8 10000000 177 ns/op
BenchmarkRBPut-8 30000000 58.1 ns/op
BenchmarkRBGet-8 50000000 26.8 ns/op
TODO: We really need a Fibonacci heap for the priority queue.
TODO: Unify the types of queue to the same interface.
*/
package queue
import (
"runtime"
"sync"
"sync/atomic"
"time"
)
type waiters []*sema
func (w *waiters) get() *sema {
if len(*w) == 0 {
return nil
}
sema := (*w)[0]
copy((*w)[0:], (*w)[1:])
(*w)[len(*w)-1] = nil // or the zero value of T
*w = (*w)[:len(*w)-1]
return sema
}
func (w *waiters) put(sema *sema) {
*w = append(*w, sema)
}
func (w *waiters) remove(sema *sema) {
if len(*w) == 0 {
return
}
// build new slice, copy all except sema
ws := *w
newWs := make(waiters, 0, len(*w))
for i := range ws {
if ws[i] != sema {
newWs = append(newWs, ws[i])
}
}
*w = newWs
}
type items []interface{}
func (items *items) get(number int64) []interface{} {
returnItems := make([]interface{}, 0, number)
index := int64(0)
for i := int64(0); i < number; i++ {
if i >= int64(len(*items)) {
break
}
returnItems = append(returnItems, (*items)[i])
(*items)[i] = nil
index++
}
*items = (*items)[index:]
return returnItems
}
func (items *items) peek() (interface{}, bool) {
length := len(*items)
if length == 0 {
return nil, false
}
return (*items)[0], true
}
func (items *items) getUntil(checker func(item interface{}) bool) []interface{} {
length := len(*items)
if len(*items) == 0 {
// returning nil here actually wraps that nil in a list
// of interfaces... thanks go
return []interface{}{}
}
returnItems := make([]interface{}, 0, length)
index := -1
for i, item := range *items {
if !checker(item) {
break
}
returnItems = append(returnItems, item)
index = i
(*items)[i] = nil // prevent memory leak
}
*items = (*items)[index+1:]
return returnItems
}
type sema struct {
ready chan bool
response *sync.WaitGroup
}
func newSema() *sema {
return &sema{
ready: make(chan bool, 1),
response: &sync.WaitGroup{},
}
}
// Queue is the struct responsible for tracking the state
// of the queue.
type Queue struct {
waiters waiters
items items
lock sync.Mutex
disposed bool
}
// Put will add the specified items to the queue.
func (q *Queue) Put(items ...interface{}) error {
if len(items) == 0 {
return nil
}
q.lock.Lock()
if q.disposed {
q.lock.Unlock()
return ErrDisposed
}
q.items = append(q.items, items...)
for {
sema := q.waiters.get()
if sema == nil {
break
}
sema.response.Add(1)
select {
case sema.ready <- true:
sema.response.Wait()
default:
// This semaphore timed out.
}
if len(q.items) == 0 {
break
}
}
q.lock.Unlock()
return nil
}
// Get retrieves items from the queue. If there are some items in the
// queue, get will return a number UP TO the number passed in as a
// parameter. If no items are in the queue, this method will pause
// until items are added to the queue.
func (q *Queue) Get(number int64) ([]interface{}, error) {
return q.Poll(number, 0)
}
// Poll retrieves items from the queue. If there are some items in the queue,
// Poll will return a number UP TO the number passed in as a parameter. If no
// items are in the queue, this method will pause until items are added to the
// queue or the provided timeout is reached. A non-positive timeout will block
// until items are added. If a timeout occurs, ErrTimeout is returned.
func (q *Queue) Poll(number int64, timeout time.Duration) ([]interface{}, error) {
if number < 1 {
// thanks again go
return []interface{}{}, nil
}
q.lock.Lock()
if q.disposed {
q.lock.Unlock()
return nil, ErrDisposed
}
var items []interface{}
if len(q.items) == 0 {
sema := newSema()
q.waiters.put(sema)
q.lock.Unlock()
var timeoutC <-chan time.Time
if timeout > 0 {
timeoutC = time.After(timeout)
}
select {
case <-sema.ready:
// we are now inside the put's lock
if q.disposed {
return nil, ErrDisposed
}
items = q.items.get(number)
sema.response.Done()
return items, nil
case <-timeoutC:
// cleanup the sema that was added to waiters
select {
case sema.ready <- true:
// we called this before Put() could
// Remove sema from waiters.
q.lock.Lock()
q.waiters.remove(sema)
q.lock.Unlock()
default:
// Put() got it already, we need to call Done() so Put() can move on
sema.response.Done()
}
return nil, ErrTimeout
}
}
items = q.items.get(number)
q.lock.Unlock()
return items, nil
}
// Peek returns a the first item in the queue by value
// without modifying the queue.
func (q *Queue) Peek() (interface{}, error) {
q.lock.Lock()
defer q.lock.Unlock()
if q.disposed {
return nil, ErrDisposed
}
peekItem, ok := q.items.peek()
if !ok {
return nil, ErrEmptyQueue
}
return peekItem, nil
}
// TakeUntil takes a function and returns a list of items that
// match the checker until the checker returns false. This does not
// wait if there are no items in the queue.
func (q *Queue) TakeUntil(checker func(item interface{}) bool) ([]interface{}, error) {
if checker == nil {
return nil, nil
}
q.lock.Lock()
if q.disposed {
q.lock.Unlock()
return nil, ErrDisposed
}
result := q.items.getUntil(checker)
q.lock.Unlock()
return result, nil
}
// Empty returns a bool indicating if this bool is empty.
func (q *Queue) Empty() bool {
q.lock.Lock()
defer q.lock.Unlock()
return len(q.items) == 0
}
// Len returns the number of items in this queue.
func (q *Queue) Len() int64 {
q.lock.Lock()
defer q.lock.Unlock()
return int64(len(q.items))
}
// Disposed returns a bool indicating if this queue
// has had disposed called on it.
func (q *Queue) Disposed() bool {
q.lock.Lock()
defer q.lock.Unlock()
return q.disposed
}
// Dispose will dispose of this queue and returns
// the items disposed. Any subsequent calls to Get
// or Put will return an error.
func (q *Queue) Dispose() []interface{} {
q.lock.Lock()
defer q.lock.Unlock()
q.disposed = true
for _, waiter := range q.waiters {
waiter.response.Add(1)
select {
case waiter.ready <- true:
// release Poll immediately
default:
// ignore if it's a timeout or in the get
}
}
disposedItems := q.items
q.items = nil
q.waiters = nil
return disposedItems
}
// New is a constructor for a new threadsafe queue.
func New(hint int64) *Queue {
return &Queue{
items: make([]interface{}, 0, hint),
}
}
// ExecuteInParallel will (in parallel) call the provided function
// with each item in the queue until the queue is exhausted. When the queue
// is exhausted execution is complete and all goroutines will be killed.
// This means that the queue will be disposed so cannot be used again.
func ExecuteInParallel(q *Queue, fn func(interface{})) {
if q == nil {
return
}
q.lock.Lock() // so no one touches anything in the middle
// of this process
todo, done := uint64(len(q.items)), int64(-1)
// this is important or we might face an infinite loop
if todo == 0 {
return
}
numCPU := 1
if runtime.NumCPU() > 1 {
numCPU = runtime.NumCPU() - 1
}
var wg sync.WaitGroup
wg.Add(numCPU)
items := q.items
for i := 0; i < numCPU; i++ {
go func() {
for {
index := atomic.AddInt64(&done, 1)
if index >= int64(todo) {
wg.Done()
break
}
fn(items[index])
items[index] = 0
}
}()
}
wg.Wait()
q.lock.Unlock()
q.Dispose()
}

View File

@@ -0,0 +1,203 @@
/*
Copyright 2014 Workiva, LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package queue
import (
"runtime"
"sync/atomic"
"time"
)
// roundUp takes a uint64 greater than 0 and rounds it up to the next
// power of 2.
func roundUp(v uint64) uint64 {
v--
v |= v >> 1
v |= v >> 2
v |= v >> 4
v |= v >> 8
v |= v >> 16
v |= v >> 32
v++
return v
}
type node struct {
position uint64
data interface{}
}
type nodes []*node
// RingBuffer is a MPMC buffer that achieves threadsafety with CAS operations
// only. A put on full or get on empty call will block until an item
// is put or retrieved. Calling Dispose on the RingBuffer will unblock
// any blocked threads with an error. This buffer is similar to the buffer
// described here: http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue
// with some minor additions.
type RingBuffer struct {
_padding0 [8]uint64
queue uint64
_padding1 [8]uint64
dequeue uint64
_padding2 [8]uint64
mask, disposed uint64
_padding3 [8]uint64
nodes nodes
}
func (rb *RingBuffer) init(size uint64) {
size = roundUp(size)
rb.nodes = make(nodes, size)
for i := uint64(0); i < size; i++ {
rb.nodes[i] = &node{position: i}
}
rb.mask = size - 1 // so we don't have to do this with every put/get operation
}
// Put adds the provided item to the queue. If the queue is full, this
// call will block until an item is added to the queue or Dispose is called
// on the queue. An error will be returned if the queue is disposed.
func (rb *RingBuffer) Put(item interface{}) error {
_, err := rb.put(item, false)
return err
}
// Offer adds the provided item to the queue if there is space. If the queue
// is full, this call will return false. An error will be returned if the
// queue is disposed.
func (rb *RingBuffer) Offer(item interface{}) (bool, error) {
return rb.put(item, true)
}
func (rb *RingBuffer) put(item interface{}, offer bool) (bool, error) {
var n *node
pos := atomic.LoadUint64(&rb.queue)
L:
for {
if atomic.LoadUint64(&rb.disposed) == 1 {
return false, ErrDisposed
}
n = rb.nodes[pos&rb.mask]
seq := atomic.LoadUint64(&n.position)
switch dif := seq - pos; {
case dif == 0:
if atomic.CompareAndSwapUint64(&rb.queue, pos, pos+1) {
break L
}
// case dif < 0:
// panic(`Ring buffer in a compromised state during a put operation.`)
default:
pos = atomic.LoadUint64(&rb.queue)
}
if offer {
return false, nil
}
runtime.Gosched() // free up the cpu before the next iteration
}
n.data = item
atomic.StoreUint64(&n.position, pos+1)
return true, nil
}
// Get will return the next item in the queue. This call will block
// if the queue is empty. This call will unblock when an item is added
// to the queue or Dispose is called on the queue. An error will be returned
// if the queue is disposed.
func (rb *RingBuffer) Get() (interface{}, error) {
return rb.Poll(0)
}
// Poll will return the next item in the queue. This call will block
// if the queue is empty. This call will unblock when an item is added
// to the queue, Dispose is called on the queue, or the timeout is reached. An
// error will be returned if the queue is disposed or a timeout occurs. A
// non-positive timeout will block indefinitely.
func (rb *RingBuffer) Poll(timeout time.Duration) (interface{}, error) {
var (
n *node
pos = atomic.LoadUint64(&rb.dequeue)
start time.Time
)
if timeout > 0 {
start = time.Now()
}
L:
for {
if atomic.LoadUint64(&rb.disposed) == 1 {
return nil, ErrDisposed
}
n = rb.nodes[pos&rb.mask]
seq := atomic.LoadUint64(&n.position)
switch dif := seq - (pos + 1); {
case dif == 0:
if atomic.CompareAndSwapUint64(&rb.dequeue, pos, pos+1) {
break L
}
// case dif < 0:
// panic(`Ring buffer in compromised state during a get operation.`)
default:
pos = atomic.LoadUint64(&rb.dequeue)
}
if timeout > 0 && time.Since(start) >= timeout {
return nil, ErrTimeout
}
runtime.Gosched() // free up the cpu before the next iteration
}
data := n.data
n.data = nil
atomic.StoreUint64(&n.position, pos+rb.mask+1)
return data, nil
}
// Len returns the number of items in the queue.
func (rb *RingBuffer) Len() uint64 {
return atomic.LoadUint64(&rb.queue) - atomic.LoadUint64(&rb.dequeue)
}
// Cap returns the capacity of this ring buffer.
func (rb *RingBuffer) Cap() uint64 {
return uint64(len(rb.nodes))
}
// Dispose will dispose of this queue and free any blocked threads
// in the Put and/or Get methods. Calling those methods on a disposed
// queue will return an error.
func (rb *RingBuffer) Dispose() {
atomic.CompareAndSwapUint64(&rb.disposed, 0, 1)
}
// IsDisposed will return a bool indicating if this queue has been
// disposed.
func (rb *RingBuffer) IsDisposed() bool {
return atomic.LoadUint64(&rb.disposed) == 1
}
// NewRingBuffer will allocate, initialize, and return a ring buffer
// with the specified size.
func NewRingBuffer(size uint64) *RingBuffer {
rb := &RingBuffer{}
rb.init(size)
return rb
}