Initial commit

vendor/github.com/wvanbergen/kafka/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Willem van Bergen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/wvanbergen/kafka/consumergroup/BUILD.bazel

@@ -0,0 +1,31 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "consumer_group.go",
+        "offset_manager.go",
+        "utils.go",
+    ],
+    importmap = "go-common/vendor/github.com/wvanbergen/kafka/consumergroup",
+    importpath = "github.com/wvanbergen/kafka/consumergroup",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/Shopify/sarama:go_default_library",
+        "//vendor/github.com/wvanbergen/kazoo-go:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/github.com/wvanbergen/kafka/consumergroup/consumer_group.go

@@ -0,0 +1,514 @@
+package consumergroup
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/Shopify/sarama"
+	"github.com/wvanbergen/kazoo-go"
+)
+
+var (
+	AlreadyClosing = errors.New("The consumer group is already shutting down.")
+)
+
+type Config struct {
+	*sarama.Config
+
+	Zookeeper *kazoo.Config
+
+	Offsets struct {
+		Initial           int64         // The initial offset method to use if the consumer has no previously stored offset. Must be either sarama.OffsetOldest (default) or sarama.OffsetNewest.
+		ProcessingTimeout time.Duration // Time to wait for all the offsets for a partition to be processed after stopping to consume from it. Defaults to 1 minute.
+		CommitInterval    time.Duration // The interval between which the processed offsets are commited.
+		ResetOffsets      bool          // Resets the offsets for the consumergroup so that it won't resume from where it left off previously.
+	}
+}
+
+func NewConfig() *Config {
+	config := &Config{}
+	config.Config = sarama.NewConfig()
+	config.Zookeeper = kazoo.NewConfig()
+	config.Offsets.Initial = sarama.OffsetOldest
+	config.Offsets.ProcessingTimeout = 60 * time.Second
+	config.Offsets.CommitInterval = 10 * time.Second
+
+	return config
+}
+
+func (cgc *Config) Validate() error {
+	if cgc.Zookeeper.Timeout <= 0 {
+		return sarama.ConfigurationError("ZookeeperTimeout should have a duration > 0")
+	}
+
+	if cgc.Offsets.CommitInterval < 0 {
+		return sarama.ConfigurationError("CommitInterval should have a duration >= 0")
+	}
+
+	if cgc.Offsets.Initial != sarama.OffsetOldest && cgc.Offsets.Initial != sarama.OffsetNewest {
+		return errors.New("Offsets.Initial should be sarama.OffsetOldest or sarama.OffsetNewest.")
+	}
+
+	if cgc.Config != nil {
+		if err := cgc.Config.Validate(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// The ConsumerGroup type holds all the information for a consumer that is part
+// of a consumer group. Call JoinConsumerGroup to start a consumer.
+type ConsumerGroup struct {
+	config *Config
+
+	consumer sarama.Consumer
+	kazoo    *kazoo.Kazoo
+	group    *kazoo.Consumergroup
+	instance *kazoo.ConsumergroupInstance
+
+	wg             sync.WaitGroup
+	singleShutdown sync.Once
+
+	messages chan *sarama.ConsumerMessage
+	errors   chan error
+	stopper  chan struct{}
+
+	consumers kazoo.ConsumergroupInstanceList
+
+	offsetManager OffsetManager
+}
+
+// Connects to a consumer group, using Zookeeper for auto-discovery
+func JoinConsumerGroup(name string, topics []string, zookeeper []string, config *Config) (cg *ConsumerGroup, err error) {
+
+	if name == "" {
+		return nil, sarama.ConfigurationError("Empty consumergroup name")
+	}
+
+	if len(topics) == 0 {
+		return nil, sarama.ConfigurationError("No topics provided")
+	}
+
+	if len(zookeeper) == 0 {
+		return nil, errors.New("You need to provide at least one zookeeper node address!")
+	}
+
+	if config == nil {
+		config = NewConfig()
+	}
+	config.ClientID = name
+
+	// Validate configuration
+	if err = config.Validate(); err != nil {
+		return
+	}
+
+	var kz *kazoo.Kazoo
+	if kz, err = kazoo.NewKazoo(zookeeper, config.Zookeeper); err != nil {
+		return
+	}
+
+	brokers, err := kz.BrokerList()
+	if err != nil {
+		kz.Close()
+		return
+	}
+
+	group := kz.Consumergroup(name)
+
+	if config.Offsets.ResetOffsets {
+		err = group.ResetOffsets()
+		if err != nil {
+			kz.Close()
+			return
+		}
+	}
+
+	instance := group.NewInstance()
+
+	var consumer sarama.Consumer
+	if consumer, err = sarama.NewConsumer(brokers, config.Config); err != nil {
+		kz.Close()
+		return
+	}
+
+	cg = &ConsumerGroup{
+		config:   config,
+		consumer: consumer,
+
+		kazoo:    kz,
+		group:    group,
+		instance: instance,
+
+		messages: make(chan *sarama.ConsumerMessage, config.ChannelBufferSize),
+		errors:   make(chan error, config.ChannelBufferSize),
+		stopper:  make(chan struct{}),
+	}
+
+	// Register consumer group
+	if exists, err := cg.group.Exists(); err != nil {
+		cg.Logf("FAILED to check for existence of consumergroup: %s!\n", err)
+		_ = consumer.Close()
+		_ = kz.Close()
+		return nil, err
+	} else if !exists {
+		cg.Logf("Consumergroup `%s` does not yet exists, creating...\n", cg.group.Name)
+		if err := cg.group.Create(); err != nil {
+			cg.Logf("FAILED to create consumergroup in Zookeeper: %s!\n", err)
+			_ = consumer.Close()
+			_ = kz.Close()
+			return nil, err
+		}
+	}
+
+	// Register itself with zookeeper
+	if err := cg.instance.Register(topics); err != nil {
+		cg.Logf("FAILED to register consumer instance: %s!\n", err)
+		return nil, err
+	} else {
+		cg.Logf("Consumer instance registered (%s).", cg.instance.ID)
+	}
+
+	offsetConfig := OffsetManagerConfig{CommitInterval: config.Offsets.CommitInterval}
+	cg.offsetManager = NewZookeeperOffsetManager(cg, &offsetConfig)
+
+	go cg.topicListConsumer(topics)
+
+	return
+}
+
+// Returns a channel that you can read to obtain events from Kafka to process.
+func (cg *ConsumerGroup) Messages() <-chan *sarama.ConsumerMessage {
+	return cg.messages
+}
+
+// Returns a channel that you can read to obtain events from Kafka to process.
+func (cg *ConsumerGroup) Errors() <-chan error {
+	return cg.errors
+}
+
+func (cg *ConsumerGroup) Closed() bool {
+	return cg.instance == nil
+}
+
+func (cg *ConsumerGroup) Close() error {
+	shutdownError := AlreadyClosing
+	cg.singleShutdown.Do(func() {
+		defer cg.kazoo.Close()
+
+		shutdownError = nil
+
+		close(cg.stopper)
+		cg.wg.Wait()
+
+		if err := cg.offsetManager.Close(); err != nil {
+			cg.Logf("FAILED closing the offset manager: %s!\n", err)
+		}
+
+		if shutdownError = cg.instance.Deregister(); shutdownError != nil {
+			cg.Logf("FAILED deregistering consumer instance: %s!\n", shutdownError)
+		} else {
+			cg.Logf("Deregistered consumer instance %s.\n", cg.instance.ID)
+		}
+
+		if shutdownError = cg.consumer.Close(); shutdownError != nil {
+			cg.Logf("FAILED closing the Sarama client: %s\n", shutdownError)
+		}
+
+		close(cg.messages)
+		close(cg.errors)
+		cg.instance = nil
+	})
+
+	return shutdownError
+}
+
+func (cg *ConsumerGroup) Logf(format string, args ...interface{}) {
+	var identifier string
+	if cg.instance == nil {
+		identifier = "(defunct)"
+	} else {
+		identifier = cg.instance.ID[len(cg.instance.ID)-12:]
+	}
+	sarama.Logger.Printf("[%s/%s] %s", cg.group.Name, identifier, fmt.Sprintf(format, args...))
+}
+
+func (cg *ConsumerGroup) InstanceRegistered() (bool, error) {
+	return cg.instance.Registered()
+}
+
+func (cg *ConsumerGroup) CommitUpto(message *sarama.ConsumerMessage) error {
+	cg.offsetManager.MarkAsProcessed(message.Topic, message.Partition, message.Offset)
+	return nil
+}
+
+func (cg *ConsumerGroup) FlushOffsets() error {
+	return cg.offsetManager.Flush()
+}
+
+func (cg *ConsumerGroup) topicListConsumer(topics []string) {
+	for {
+		select {
+		case <-cg.stopper:
+			return
+		default:
+		}
+
+		consumers, consumerChanges, err := cg.group.WatchInstances()
+		if err != nil {
+			cg.Logf("FAILED to get list of registered consumer instances: %s\n", err)
+			return
+		}
+
+		cg.consumers = consumers
+		cg.Logf("Currently registered consumers: %d\n", len(cg.consumers))
+
+		stopper := make(chan struct{})
+
+		for _, topic := range topics {
+			cg.wg.Add(1)
+			go cg.topicConsumer(topic, cg.messages, cg.errors, stopper)
+		}
+
+		select {
+		case <-cg.stopper:
+			close(stopper)
+			return
+
+		case <-consumerChanges:
+			registered, err := cg.instance.Registered()
+			if err != nil {
+				cg.Logf("FAILED to get register status: %s\n", err)
+			} else if !registered {
+				err = cg.instance.Register(topics)
+				if err != nil {
+					cg.Logf("FAILED to register consumer instance: %s!\n", err)
+				} else {
+					cg.Logf("Consumer instance registered (%s).", cg.instance.ID)
+				}
+			}
+
+			cg.Logf("Triggering rebalance due to consumer list change\n")
+			close(stopper)
+			cg.wg.Wait()
+		}
+	}
+}
+
+func (cg *ConsumerGroup) topicConsumer(topic string, messages chan<- *sarama.ConsumerMessage, errors chan<- error, stopper <-chan struct{}) {
+	defer cg.wg.Done()
+
+	select {
+	case <-stopper:
+		return
+	default:
+	}
+
+	cg.Logf("%s :: Started topic consumer\n", topic)
+
+	// Fetch a list of partition IDs
+	partitions, err := cg.kazoo.Topic(topic).Partitions()
+	if err != nil {
+		cg.Logf("%s :: FAILED to get list of partitions: %s\n", topic, err)
+		cg.errors <- &sarama.ConsumerError{
+			Topic:     topic,
+			Partition: -1,
+			Err:       err,
+		}
+		return
+	}
+
+	partitionLeaders, err := retrievePartitionLeaders(partitions)
+	if err != nil {
+		cg.Logf("%s :: FAILED to get leaders of partitions: %s\n", topic, err)
+		cg.errors <- &sarama.ConsumerError{
+			Topic:     topic,
+			Partition: -1,
+			Err:       err,
+		}
+		return
+	}
+
+	dividedPartitions := dividePartitionsBetweenConsumers(cg.consumers, partitionLeaders)
+	myPartitions := dividedPartitions[cg.instance.ID]
+	cg.Logf("%s :: Claiming %d of %d partitions", topic, len(myPartitions), len(partitionLeaders))
+
+	// Consume all the assigned partitions
+	var wg sync.WaitGroup
+	for _, pid := range myPartitions {
+
+		wg.Add(1)
+		go cg.partitionConsumer(topic, pid.ID, messages, errors, &wg, stopper)
+	}
+
+	wg.Wait()
+	cg.Logf("%s :: Stopped topic consumer\n", topic)
+}
+
+func (cg *ConsumerGroup) consumePartition(topic string, partition int32, nextOffset int64) (sarama.PartitionConsumer, error) {
+	consumer, err := cg.consumer.ConsumePartition(topic, partition, nextOffset)
+	if err == sarama.ErrOffsetOutOfRange {
+		cg.Logf("%s/%d :: Partition consumer offset out of Range.\n", topic, partition)
+		// if the offset is out of range, simplistically decide whether to use OffsetNewest or OffsetOldest
+		// if the configuration specified offsetOldest, then switch to the oldest available offset, else
+		// switch to the newest available offset.
+		if cg.config.Offsets.Initial == sarama.OffsetOldest {
+			nextOffset = sarama.OffsetOldest
+			cg.Logf("%s/%d :: Partition consumer offset reset to oldest available offset.\n", topic, partition)
+		} else {
+			nextOffset = sarama.OffsetNewest
+			cg.Logf("%s/%d :: Partition consumer offset reset to newest available offset.\n", topic, partition)
+		}
+		// retry the consumePartition with the adjusted offset
+		consumer, err = cg.consumer.ConsumePartition(topic, partition, nextOffset)
+	}
+	if err != nil {
+		cg.Logf("%s/%d :: FAILED to start partition consumer: %s\n", topic, partition, err)
+		return nil, err
+	}
+	return consumer, err
+}
+
+// Consumes a partition
+func (cg *ConsumerGroup) partitionConsumer(topic string, partition int32, messages chan<- *sarama.ConsumerMessage, errors chan<- error, wg *sync.WaitGroup, stopper <-chan struct{}) {
+	defer wg.Done()
+
+	select {
+	case <-stopper:
+		return
+	default:
+	}
+
+	// Since ProcessingTimeout is the amount of time we'll wait for the final batch
+	// of messages to be processed before releasing a partition, we need to wait slightly
+	// longer than that before timing out here to ensure that another consumer has had
+	// enough time to release the partition. Hence, +2 seconds.
+	maxRetries := int(cg.config.Offsets.ProcessingTimeout/time.Second) + 2
+	for tries := 0; tries < maxRetries; tries++ {
+		if err := cg.instance.ClaimPartition(topic, partition); err == nil {
+			break
+		} else if tries+1 < maxRetries {
+			if err == kazoo.ErrPartitionClaimedByOther {
+				// Another consumer still owns this partition. We should wait longer for it to release it.
+				time.Sleep(1 * time.Second)
+			} else {
+				// An unexpected error occurred. Log it and continue trying until we hit the timeout.
+				cg.Logf("%s/%d :: FAILED to claim partition on attempt %v of %v; retrying in 1 second. Error: %v", topic, partition, tries+1, maxRetries, err)
+				time.Sleep(1 * time.Second)
+			}
+		} else {
+			cg.Logf("%s/%d :: FAILED to claim the partition: %s\n", topic, partition, err)
+			cg.errors <- &sarama.ConsumerError{
+				Topic:     topic,
+				Partition: partition,
+				Err:       err,
+			}
+			return
+		}
+	}
+
+	defer func() {
+		err := cg.instance.ReleasePartition(topic, partition)
+		if err != nil {
+			cg.Logf("%s/%d :: FAILED to release partition: %s\n", topic, partition, err)
+			cg.errors <- &sarama.ConsumerError{
+				Topic:     topic,
+				Partition: partition,
+				Err:       err,
+			}
+		}
+	}()
+
+	nextOffset, err := cg.offsetManager.InitializePartition(topic, partition)
+	if err != nil {
+		cg.Logf("%s/%d :: FAILED to determine initial offset: %s\n", topic, partition, err)
+		return
+	}
+
+	if nextOffset >= 0 {
+		cg.Logf("%s/%d :: Partition consumer starting at offset %d.\n", topic, partition, nextOffset)
+	} else {
+		nextOffset = cg.config.Offsets.Initial
+		if nextOffset == sarama.OffsetOldest {
+			cg.Logf("%s/%d :: Partition consumer starting at the oldest available offset.\n", topic, partition)
+		} else if nextOffset == sarama.OffsetNewest {
+			cg.Logf("%s/%d :: Partition consumer listening for new messages only.\n", topic, partition)
+		}
+	}
+
+	consumer, err := cg.consumePartition(topic, partition, nextOffset)
+
+	if err != nil {
+		cg.Logf("%s/%d :: FAILED to start partition consumer: %s\n", topic, partition, err)
+		return
+	}
+
+	defer consumer.Close()
+
+	err = nil
+	var lastOffset int64 = -1 // aka unknown
+partitionConsumerLoop:
+	for {
+		select {
+		case <-stopper:
+			break partitionConsumerLoop
+
+		case err := <-consumer.Errors():
+			if err == nil {
+				cg.Logf("%s/%d :: Consumer encountered an invalid state: re-establishing consumption of partition.\n", topic, partition)
+
+				// Errors encountered (if any) are logged in the consumerPartition function
+				var cErr error
+				consumer, cErr = cg.consumePartition(topic, partition, lastOffset)
+				if cErr != nil {
+					break partitionConsumerLoop
+				}
+				continue partitionConsumerLoop
+			}
+
+			for {
+				select {
+				case errors <- err:
+					continue partitionConsumerLoop
+
+				case <-stopper:
+					break partitionConsumerLoop
+				}
+			}
+
+		case message := <-consumer.Messages():
+			if message == nil {
+				cg.Logf("%s/%d :: Consumer encountered an invalid state: re-establishing consumption of partition.\n", topic, partition)
+
+				// Errors encountered (if any) are logged in the consumerPartition function
+				var cErr error
+				consumer, cErr = cg.consumePartition(topic, partition, lastOffset)
+				if cErr != nil {
+					break partitionConsumerLoop
+				}
+				continue partitionConsumerLoop
+
+			}
+
+			for {
+				select {
+				case <-stopper:
+					break partitionConsumerLoop
+
+				case messages <- message:
+					lastOffset = message.Offset
+					continue partitionConsumerLoop
+				}
+			}
+		}
+	}
+
+	cg.Logf("%s/%d :: Stopping partition consumer at offset %d\n", topic, partition, lastOffset)
+	if err := cg.offsetManager.FinalizePartition(topic, partition, lastOffset, cg.config.Offsets.ProcessingTimeout); err != nil {
+		cg.Logf("%s/%d :: %s\n", topic, partition, err)
+	}
+}

vendor/github.com/wvanbergen/kafka/consumergroup/offset_manager.go

@@ -0,0 +1,298 @@
+package consumergroup
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+)
+
+// OffsetManager is the main interface consumergroup requires to manage offsets of the consumergroup.
+type OffsetManager interface {
+
+	// InitializePartition is called when the consumergroup is starting to consume a
+	// partition. It should return the last processed offset for this partition. Note:
+	// the same partition can be initialized multiple times during a single run of a
+	// consumer group due to other consumer instances coming online and offline.
+	InitializePartition(topic string, partition int32) (int64, error)
+
+	// MarkAsProcessed tells the offset manager than a certain message has been successfully
+	// processed by the consumer, and should be committed. The implementation does not have
+	// to store this offset right away, but should return true if it intends to do this at
+	// some point.
+	//
+	// Offsets should generally be increasing if the consumer
+	// processes events serially, but this cannot be guaranteed if the consumer does any
+	// asynchronous processing. This can be handled in various ways, e.g. by only accepting
+	// offsets that are higehr than the offsets seen before for the same partition.
+	MarkAsProcessed(topic string, partition int32, offset int64) bool
+
+	// Flush tells the offset manager to immediately commit offsets synchronously and to
+	// return any errors that may have occured during the process.
+	Flush() error
+
+	// FinalizePartition is called when the consumergroup is done consuming a
+	// partition. In this method, the offset manager can flush any remaining offsets to its
+	// backend store. It should return an error if it was not able to commit the offset.
+	// Note: it's possible that the consumergroup instance will start to consume the same
+	// partition again after this function is called.
+	FinalizePartition(topic string, partition int32, lastOffset int64, timeout time.Duration) error
+
+	// Close is called when the consumergroup is shutting down. In normal circumstances, all
+	// offsets are committed because FinalizePartition is called for all the running partition
+	// consumers. You may want to check for this to be true, and try to commit any outstanding
+	// offsets. If this doesn't succeed, it should return an error.
+	Close() error
+}
+
+var (
+	UncleanClose = errors.New("Not all offsets were committed before shutdown was completed")
+)
+
+// OffsetManagerConfig holds configuration setting son how the offset manager should behave.
+type OffsetManagerConfig struct {
+	CommitInterval time.Duration // Interval between offset flushes to the backend store.
+	VerboseLogging bool          // Whether to enable verbose logging.
+}
+
+// NewOffsetManagerConfig returns a new OffsetManagerConfig with sane defaults.
+func NewOffsetManagerConfig() *OffsetManagerConfig {
+	return &OffsetManagerConfig{
+		CommitInterval: 10 * time.Second,
+	}
+}
+
+type (
+	topicOffsets    map[int32]*partitionOffsetTracker
+	offsetsMap      map[string]topicOffsets
+	offsetCommitter func(int64) error
+)
+
+type partitionOffsetTracker struct {
+	l                      sync.Mutex
+	waitingForOffset       int64
+	highestProcessedOffset int64
+	lastCommittedOffset    int64
+	done                   chan struct{}
+}
+
+type zookeeperOffsetManager struct {
+	config  *OffsetManagerConfig
+	l       sync.RWMutex
+	offsets offsetsMap
+	cg      *ConsumerGroup
+
+	closing, closed, flush chan struct{}
+	flushErr               chan error
+}
+
+// NewZookeeperOffsetManager returns an offset manager that uses Zookeeper
+// to store offsets.
+func NewZookeeperOffsetManager(cg *ConsumerGroup, config *OffsetManagerConfig) OffsetManager {
+	if config == nil {
+		config = NewOffsetManagerConfig()
+	}
+
+	zom := &zookeeperOffsetManager{
+		config:   config,
+		cg:       cg,
+		offsets:  make(offsetsMap),
+		closing:  make(chan struct{}),
+		closed:   make(chan struct{}),
+		flush:    make(chan struct{}),
+		flushErr: make(chan error),
+	}
+
+	go zom.offsetCommitter()
+
+	return zom
+}
+
+func (zom *zookeeperOffsetManager) InitializePartition(topic string, partition int32) (int64, error) {
+	zom.l.Lock()
+	defer zom.l.Unlock()
+
+	if zom.offsets[topic] == nil {
+		zom.offsets[topic] = make(topicOffsets)
+	}
+
+	nextOffset, err := zom.cg.group.FetchOffset(topic, partition)
+	if err != nil {
+		return 0, err
+	}
+
+	zom.offsets[topic][partition] = &partitionOffsetTracker{
+		highestProcessedOffset: nextOffset - 1,
+		lastCommittedOffset:    nextOffset - 1,
+		done:                   make(chan struct{}),
+	}
+
+	return nextOffset, nil
+}
+
+func (zom *zookeeperOffsetManager) FinalizePartition(topic string, partition int32, lastOffset int64, timeout time.Duration) error {
+	zom.l.RLock()
+	tracker := zom.offsets[topic][partition]
+	zom.l.RUnlock()
+
+	if lastOffset >= 0 {
+		if lastOffset-tracker.highestProcessedOffset > 0 {
+			zom.cg.Logf("%s/%d :: Last processed offset: %d. Waiting up to %ds for another %d messages to process...", topic, partition, tracker.highestProcessedOffset, timeout/time.Second, lastOffset-tracker.highestProcessedOffset)
+			if !tracker.waitForOffset(lastOffset, timeout) {
+				return fmt.Errorf("TIMEOUT waiting for offset %d. Last committed offset: %d", lastOffset, tracker.lastCommittedOffset)
+			}
+		}
+
+		if err := zom.commitOffset(topic, partition, tracker); err != nil {
+			return fmt.Errorf("FAILED to commit offset %d to Zookeeper. Last committed offset: %d", tracker.highestProcessedOffset, tracker.lastCommittedOffset)
+		}
+	}
+
+	zom.l.Lock()
+	delete(zom.offsets[topic], partition)
+	zom.l.Unlock()
+
+	return nil
+}
+
+func (zom *zookeeperOffsetManager) MarkAsProcessed(topic string, partition int32, offset int64) bool {
+	zom.l.RLock()
+	defer zom.l.RUnlock()
+	if p, ok := zom.offsets[topic][partition]; ok {
+		return p.markAsProcessed(offset)
+	} else {
+		return false
+	}
+}
+
+func (zom *zookeeperOffsetManager) Flush() error {
+	zom.flush <- struct{}{}
+	return <-zom.flushErr
+}
+
+func (zom *zookeeperOffsetManager) Close() error {
+	close(zom.closing)
+	<-zom.closed
+
+	zom.l.Lock()
+	defer zom.l.Unlock()
+
+	var closeError error
+	for _, partitionOffsets := range zom.offsets {
+		if len(partitionOffsets) > 0 {
+			closeError = UncleanClose
+		}
+	}
+
+	return closeError
+}
+
+func (zom *zookeeperOffsetManager) offsetCommitter() {
+	var tickerChan <-chan time.Time
+	if zom.config.CommitInterval != 0 {
+		commitTicker := time.NewTicker(zom.config.CommitInterval)
+		tickerChan = commitTicker.C
+		defer commitTicker.Stop()
+	}
+
+	for {
+		select {
+		case <-zom.closing:
+			close(zom.closed)
+			return
+		case <-tickerChan:
+			if err := zom.commitOffsets(); err != nil {
+				zom.cg.errors <- err
+			}
+		case <-zom.flush:
+			zom.flushErr <- zom.commitOffsets()
+		}
+	}
+}
+
+func (zom *zookeeperOffsetManager) commitOffsets() error {
+	zom.l.RLock()
+	defer zom.l.RUnlock()
+
+	var returnErr error
+	for topic, partitionOffsets := range zom.offsets {
+		for partition, offsetTracker := range partitionOffsets {
+			err := zom.commitOffset(topic, partition, offsetTracker)
+			switch err {
+			case nil:
+				// noop
+			default:
+				returnErr = err
+			}
+		}
+	}
+	return returnErr
+}
+
+func (zom *zookeeperOffsetManager) commitOffset(topic string, partition int32, tracker *partitionOffsetTracker) error {
+	err := tracker.commit(func(offset int64) error {
+		if offset >= 0 {
+			return zom.cg.group.CommitOffset(topic, partition, offset+1)
+		} else {
+			return nil
+		}
+	})
+
+	if err != nil {
+		zom.cg.Logf("FAILED to commit offset %d for %s/%d!", tracker.highestProcessedOffset, topic, partition)
+	} else if zom.config.VerboseLogging {
+		zom.cg.Logf("Committed offset %d for %s/%d!", tracker.lastCommittedOffset, topic, partition)
+	}
+
+	return err
+}
+
+// MarkAsProcessed marks the provided offset as highest processed offset if
+// it's higher than any previous offset it has received.
+func (pot *partitionOffsetTracker) markAsProcessed(offset int64) bool {
+	pot.l.Lock()
+	defer pot.l.Unlock()
+	if offset > pot.highestProcessedOffset {
+		pot.highestProcessedOffset = offset
+		if pot.waitingForOffset == pot.highestProcessedOffset {
+			close(pot.done)
+		}
+		return true
+	} else {
+		return false
+	}
+}
+
+// Commit calls a committer function if the highest processed offset is out
+// of sync with the last committed offset.
+func (pot *partitionOffsetTracker) commit(committer offsetCommitter) error {
+	pot.l.Lock()
+	defer pot.l.Unlock()
+
+	if pot.highestProcessedOffset > pot.lastCommittedOffset {
+		if err := committer(pot.highestProcessedOffset); err != nil {
+			return err
+		}
+		pot.lastCommittedOffset = pot.highestProcessedOffset
+		return nil
+	} else {
+		return nil
+	}
+}
+
+func (pot *partitionOffsetTracker) waitForOffset(offset int64, timeout time.Duration) bool {
+	pot.l.Lock()
+	if offset > pot.highestProcessedOffset {
+		pot.waitingForOffset = offset
+		pot.l.Unlock()
+		select {
+		case <-pot.done:
+			return true
+		case <-time.After(timeout):
+			return false
+		}
+	} else {
+		pot.l.Unlock()
+		return true
+	}
+}

vendor/github.com/wvanbergen/kafka/consumergroup/utils.go

@@ -0,0 +1,113 @@
+package consumergroup
+
+import (
+	"crypto/rand"
+	"fmt"
+	"io"
+	"os"
+	"sort"
+
+	"github.com/wvanbergen/kazoo-go"
+)
+
+func retrievePartitionLeaders(partitions kazoo.PartitionList) (partitionLeaders, error) {
+
+	pls := make(partitionLeaders, 0, len(partitions))
+	for _, partition := range partitions {
+		leader, err := partition.Leader()
+		if err != nil {
+			return nil, err
+		}
+
+		pl := partitionLeader{id: partition.ID, leader: leader, partition: partition}
+		pls = append(pls, pl)
+	}
+
+	return pls, nil
+}
+
+// Divides a set of partitions between a set of consumers.
+func dividePartitionsBetweenConsumers(consumers kazoo.ConsumergroupInstanceList, partitions partitionLeaders) map[string][]*kazoo.Partition {
+	result := make(map[string][]*kazoo.Partition)
+
+	plen := len(partitions)
+	clen := len(consumers)
+	if clen == 0 {
+		return result
+	}
+
+	sort.Sort(partitions)
+	sort.Sort(consumers)
+
+	n := plen / clen
+	m := plen % clen
+	p := 0
+	for i, consumer := range consumers {
+		first := p
+		last := first + n
+		if m > 0 && i < m {
+			last++
+		}
+		if last > plen {
+			last = plen
+		}
+
+		for _, pl := range partitions[first:last] {
+			result[consumer.ID] = append(result[consumer.ID], pl.partition)
+		}
+		p = last
+	}
+
+	return result
+}
+
+type partitionLeader struct {
+	id        int32
+	leader    int32
+	partition *kazoo.Partition
+}
+
+// A sortable slice of PartitionLeader structs
+type partitionLeaders []partitionLeader
+
+func (pls partitionLeaders) Len() int {
+	return len(pls)
+}
+
+func (pls partitionLeaders) Less(i, j int) bool {
+	return pls[i].leader < pls[j].leader || (pls[i].leader == pls[j].leader && pls[i].id < pls[j].id)
+}
+
+func (s partitionLeaders) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func generateUUID() (string, error) {
+	uuid := make([]byte, 16)
+	n, err := io.ReadFull(rand.Reader, uuid)
+	if n != len(uuid) || err != nil {
+		return "", err
+	}
+	// variant bits; see section 4.1.1
+	uuid[8] = uuid[8]&^0xc0 | 0x80
+	// version 4 (pseudo-random); see section 4.1.3
+	uuid[6] = uuid[6]&^0xf0 | 0x40
+	return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil
+}
+
+func generateConsumerID() (consumerID string, err error) {
+	var uuid, hostname string
+
+	uuid, err = generateUUID()
+	if err != nil {
+		return
+	}
+
+	hostname, err = os.Hostname()
+	if err != nil {
+		return
+	}
+
+	consumerID = fmt.Sprintf("%s:%s", hostname, uuid)
+	return
+}

vendor/github.com/wvanbergen/kazoo-go/BUILD.bazel

@@ -0,0 +1,28 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "consumergroup.go",
+        "kazoo.go",
+        "topic_metadata.go",
+    ],
+    importmap = "go-common/vendor/github.com/wvanbergen/kazoo-go",
+    importpath = "github.com/wvanbergen/kazoo-go",
+    visibility = ["//visibility:public"],
+    deps = ["//vendor/github.com/samuel/go-zookeeper/zk:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/github.com/wvanbergen/kazoo-go/MIT-LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Willem van Bergen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/wvanbergen/kazoo-go/Makefile

@@ -0,0 +1,83 @@
+.PHONY: confluent/kafka/* confluent/zookeeper/* confluent/registry/* confluent/start confluent/stop fmt vet errcheck test test/create_kafka_topics dependencies dependencies/*
+
+
+default: fmt vet errcheck test
+
+
+# Confluent platform tasks
+
+confluent/start: confluent/rest/start
+
+confluent/stop: confluent/rest/stop confluent/registry/stop confluent/kafka/stop confluent/zookeeper/stop
+
+# Download & extract tasks
+
+confluent/confluent.tgz:
+	mkdir -p confluent && wget http://packages.confluent.io/archive/1.0/confluent-1.0-2.10.4.tar.gz -O confluent/confluent.tgz
+
+confluent/EXTRACTED: confluent/confluent.tgz
+	tar xzf confluent/confluent.tgz -C confluent --strip-components 1 && mkdir confluent/logs && touch confluent/EXTRACTED
+
+# Zookeeper tasks
+
+confluent/zookeeper/start: confluent/EXTRACTED
+	nohup confluent/bin/zookeeper-server-start confluent/etc/kafka/zookeeper.properties 2> confluent/logs/zookeeper.err > confluent/logs/zookeeper.out < /dev/null &
+	while ! nc localhost 2181 </dev/null; do echo "Waiting for zookeeper..."; sleep 1; done
+
+confluent/zookeeper/stop: confluent/EXTRACTED
+	confluent/bin/zookeeper-server-stop
+
+# Kafka tasks
+
+confluent/kafka/start: confluent/zookeeper/start confluent/EXTRACTED
+	nohup confluent/bin/kafka-server-start confluent/etc/kafka/server.properties 2> confluent/logs/kafka.err > confluent/logs/kafka.out < /dev/null &
+	while ! nc localhost 9092 </dev/null; do echo "Waiting for Kafka..."; sleep 1; done
+
+confluent/kafka/stop: confluent/EXTRACTED
+	confluent/bin/kafka-server-stop
+
+# schema-registry tasks
+
+confluent/registry/start: confluent/kafka/start confluent/EXTRACTED
+	nohup confluent/bin/schema-registry-start confluent/etc/schema-registry/schema-registry.properties 2> confluent/logs/schema-registry.err > confluent/logs/schema-registry.out < /dev/null &
+	while ! nc localhost 8081 </dev/null; do echo "Waiting for schema registry..."; sleep 1; done
+
+confluent/registry/stop: confluent/EXTRACTED
+	confluent/bin/kafka-server-stop
+
+# REST proxy tasks
+
+confluent/rest/start: confluent/registry/start confluent/EXTRACTED
+	nohup confluent/bin/kafka-rest-start confluent/etc/kafka-rest/kafka-rest.properties 2> confluent/logs/kafka-rest.err > confluent/logs/kafka-rest.out < /dev/null &
+	while ! nc localhost 8082 </dev/null; do echo "Waiting for REST proxy..."; sleep 1; done
+
+confluent/rest/stop: confluent/EXTRACTED
+	confluent/bin/kafka-rest-stop
+
+
+# CI tasks
+
+test:
+	go test -v -race ./...
+
+vet:
+	go vet ./...
+
+errcheck:
+	errcheck ./...
+
+fmt:
+	@if [ -n "$$(go fmt ./...)" ]; then echo 'Please run go fmt on your code.' && exit 1; fi
+
+dependencies: dependencies/errcheck dependencies/get
+
+dependencies/errcheck:
+	go get github.com/kisielk/errcheck
+
+dependencies/get:
+	go get -t ./...
+
+test/create_kafka_topics: confluent/kafka/start
+	confluent/bin/kafka-topics --create --partitions  1 --replication-factor 1 --topic test.1  --zookeeper localhost:2181
+	confluent/bin/kafka-topics --create --partitions  4 --replication-factor 1 --topic test.4  --zookeeper localhost:2181 --config retention.ms=604800000
+	confluent/bin/kafka-topics --create --partitions 64 --replication-factor 1 --topic test.64 --zookeeper localhost:2181

vendor/github.com/wvanbergen/kazoo-go/README.md

@@ -0,0 +1,6 @@
+# Kazoo
+
+Kazoo is a library to interact with the Kafka metadata that lives in Zookeeper.
+It provides discovery of the cluster's brokers, topic metadata, and consumer groups.
+
+API documentation can be found on [godoc.org](http://godoc.org/github.com/wvanbergen/kazoo-go).

vendor/github.com/wvanbergen/kazoo-go/consumergroup.go

@@ -0,0 +1,514 @@
+package kazoo
+
+import (
+	"crypto/rand"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/samuel/go-zookeeper/zk"
+)
+
+var (
+	ErrRunningInstances          = errors.New("Cannot deregister a consumergroup with running instances")
+	ErrInstanceAlreadyRegistered = errors.New("Cannot register consumer instance because it already is registered")
+	ErrInstanceNotRegistered     = errors.New("Cannot deregister consumer instance because it not registered")
+	ErrPartitionClaimedByOther   = errors.New("Cannot claim partition: it is already claimed by another instance")
+	ErrPartitionNotClaimed       = errors.New("Cannot release partition: it is not claimed by this instance")
+)
+
+// Consumergroup represents a high-level consumer that is registered in Zookeeper,
+type Consumergroup struct {
+	kz   *Kazoo
+	Name string
+}
+
+// ConsumergroupInstance represents an instance of a Consumergroup.
+type ConsumergroupInstance struct {
+	cg *Consumergroup
+	ID string
+}
+
+// ConsumergroupList implements the sortable interface on top of a consumer group list
+type ConsumergroupList []*Consumergroup
+
+// ConsumergroupInstanceList implements the sortable interface on top of a consumer instance list
+type ConsumergroupInstanceList []*ConsumergroupInstance
+
+type Registration struct {
+	Pattern      RegPattern     `json:"pattern"`
+	Subscription map[string]int `json:"subscription"`
+	Timestamp    int64          `json:"timestamp"`
+	Version      RegVersion     `json:"version"`
+}
+
+type RegPattern string
+
+const (
+	RegPatternStatic    RegPattern = "static"
+	RegPatternWhiteList RegPattern = "white_list"
+	RegPatternBlackList RegPattern = "black_list"
+)
+
+type RegVersion int
+
+const (
+	RegDefaultVersion RegVersion = 1
+)
+
+// Consumergroups returns all the registered consumergroups
+func (kz *Kazoo) Consumergroups() (ConsumergroupList, error) {
+	root := fmt.Sprintf("%s/consumers", kz.conf.Chroot)
+	cgs, _, err := kz.conn.Children(root)
+	if err != nil {
+		return nil, err
+	}
+
+	result := make(ConsumergroupList, 0, len(cgs))
+	for _, cg := range cgs {
+		result = append(result, kz.Consumergroup(cg))
+	}
+	return result, nil
+}
+
+// Consumergroup instantiates a new consumergroup.
+func (kz *Kazoo) Consumergroup(name string) *Consumergroup {
+	return &Consumergroup{Name: name, kz: kz}
+}
+
+// Exists checks whether the consumergroup has been registered in Zookeeper
+func (cg *Consumergroup) Exists() (bool, error) {
+	return cg.kz.exists(fmt.Sprintf("%s/consumers/%s", cg.kz.conf.Chroot, cg.Name))
+}
+
+// Create registers the consumergroup in zookeeper
+func (cg *Consumergroup) Create() error {
+	return cg.kz.mkdirRecursive(fmt.Sprintf("%s/consumers/%s", cg.kz.conf.Chroot, cg.Name))
+}
+
+// Delete removes the consumergroup from zookeeper
+func (cg *Consumergroup) Delete() error {
+	if instances, err := cg.Instances(); err != nil {
+		return err
+	} else if len(instances) > 0 {
+		return ErrRunningInstances
+	}
+
+	return cg.kz.deleteRecursive(fmt.Sprintf("%s/consumers/%s", cg.kz.conf.Chroot, cg.Name))
+}
+
+// Instances returns a map of all running instances inside this consumergroup.
+func (cg *Consumergroup) Instances() (ConsumergroupInstanceList, error) {
+	root := fmt.Sprintf("%s/consumers/%s/ids", cg.kz.conf.Chroot, cg.Name)
+	if exists, err := cg.kz.exists(root); err != nil {
+		return nil, err
+	} else if exists {
+		cgis, _, err := cg.kz.conn.Children(root)
+		if err != nil {
+			return nil, err
+		}
+
+		result := make(ConsumergroupInstanceList, 0, len(cgis))
+		for _, cgi := range cgis {
+			result = append(result, cg.Instance(cgi))
+		}
+		return result, nil
+	} else {
+		result := make(ConsumergroupInstanceList, 0)
+		return result, nil
+	}
+}
+
+// WatchInstances returns a ConsumergroupInstanceList, and a channel that will be closed
+// as soon the instance list changes.
+func (cg *Consumergroup) WatchInstances() (ConsumergroupInstanceList, <-chan zk.Event, error) {
+	node := fmt.Sprintf("%s/consumers/%s/ids", cg.kz.conf.Chroot, cg.Name)
+	if exists, err := cg.kz.exists(node); err != nil {
+		return nil, nil, err
+	} else if !exists {
+		if err := cg.kz.mkdirRecursive(node); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	cgis, _, c, err := cg.kz.conn.ChildrenW(node)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	result := make(ConsumergroupInstanceList, 0, len(cgis))
+	for _, cgi := range cgis {
+		result = append(result, cg.Instance(cgi))
+	}
+
+	return result, c, nil
+}
+
+// NewInstance instantiates a new ConsumergroupInstance inside this consumer group,
+// using a newly generated ID.
+func (cg *Consumergroup) NewInstance() *ConsumergroupInstance {
+	id, err := generateConsumerInstanceID()
+	if err != nil {
+		panic(err)
+	}
+	return cg.Instance(id)
+}
+
+// Instance instantiates a new ConsumergroupInstance inside this consumer group,
+// using an existing ID.
+func (cg *Consumergroup) Instance(id string) *ConsumergroupInstance {
+	return &ConsumergroupInstance{cg: cg, ID: id}
+}
+
+// PartitionOwner returns the ConsumergroupInstance that has claimed the given partition.
+// This can be nil if nobody has claimed it yet.
+func (cg *Consumergroup) PartitionOwner(topic string, partition int32) (*ConsumergroupInstance, error) {
+	node := fmt.Sprintf("%s/consumers/%s/owners/%s/%d", cg.kz.conf.Chroot, cg.Name, topic, partition)
+	val, _, err := cg.kz.conn.Get(node)
+
+	// If the node does not exists, nobody has claimed it.
+	switch err {
+	case nil:
+		return &ConsumergroupInstance{cg: cg, ID: string(val)}, nil
+	case zk.ErrNoNode:
+		return nil, nil
+	default:
+		return nil, err
+	}
+}
+
+// WatchPartitionOwner retrieves what instance is currently owning the partition, and sets a
+// Zookeeper watch to be notified of changes. If the partition currently does not have an owner,
+// the function returns nil for every return value. In this case is should be safe to claim
+// the partition for an instance.
+func (cg *Consumergroup) WatchPartitionOwner(topic string, partition int32) (*ConsumergroupInstance, <-chan zk.Event, error) {
+	node := fmt.Sprintf("%s/consumers/%s/owners/%s/%d", cg.kz.conf.Chroot, cg.Name, topic, partition)
+	instanceID, _, changed, err := cg.kz.conn.GetW(node)
+
+	switch err {
+	case nil:
+		return &ConsumergroupInstance{cg: cg, ID: string(instanceID)}, changed, nil
+
+	case zk.ErrNoNode:
+		return nil, nil, nil
+
+	default:
+		return nil, nil, err
+	}
+}
+
+// Registered checks whether the consumergroup instance is registered in Zookeeper.
+func (cgi *ConsumergroupInstance) Registered() (bool, error) {
+	node := fmt.Sprintf("%s/consumers/%s/ids/%s", cgi.cg.kz.conf.Chroot, cgi.cg.Name, cgi.ID)
+	return cgi.cg.kz.exists(node)
+}
+
+// Registered returns current registration of the consumer group instance.
+func (cgi *ConsumergroupInstance) Registration() (*Registration, error) {
+	node := fmt.Sprintf("%s/consumers/%s/ids/%s", cgi.cg.kz.conf.Chroot, cgi.cg.Name, cgi.ID)
+	val, _, err := cgi.cg.kz.conn.Get(node)
+	if err != nil {
+		return nil, err
+	}
+
+	reg := &Registration{}
+	if err := json.Unmarshal(val, reg); err != nil {
+		return nil, err
+	}
+	return reg, nil
+}
+
+// RegisterSubscription registers the consumer instance in Zookeeper, with its subscription.
+func (cgi *ConsumergroupInstance) RegisterWithSubscription(subscriptionJSON []byte) error {
+	if exists, err := cgi.Registered(); err != nil {
+		return err
+	} else if exists {
+		return ErrInstanceAlreadyRegistered
+	}
+
+	// Create an ephemeral node for the the consumergroup instance.
+	node := fmt.Sprintf("%s/consumers/%s/ids/%s", cgi.cg.kz.conf.Chroot, cgi.cg.Name, cgi.ID)
+	return cgi.cg.kz.create(node, subscriptionJSON, true)
+}
+
+// Register registers the consumergroup instance in Zookeeper.
+func (cgi *ConsumergroupInstance) Register(topics []string) error {
+	subscription := make(map[string]int)
+	for _, topic := range topics {
+		subscription[topic] = 1
+	}
+
+	data, err := json.Marshal(&Registration{
+		Pattern:      RegPatternStatic,
+		Subscription: subscription,
+		Timestamp:    time.Now().Unix(),
+		Version:      RegDefaultVersion,
+	})
+	if err != nil {
+		return err
+	}
+
+	return cgi.RegisterWithSubscription(data)
+}
+
+// Deregister removes the registration of the instance from zookeeper.
+func (cgi *ConsumergroupInstance) Deregister() error {
+	node := fmt.Sprintf("%s/consumers/%s/ids/%s", cgi.cg.kz.conf.Chroot, cgi.cg.Name, cgi.ID)
+	exists, stat, err := cgi.cg.kz.conn.Exists(node)
+	if err != nil {
+		return err
+	} else if !exists {
+		return ErrInstanceNotRegistered
+	}
+
+	return cgi.cg.kz.conn.Delete(node, stat.Version)
+}
+
+// Claim claims a topic/partition ownership for a consumer ID within a group. If the
+// partition is already claimed by another running instance, it will return ErrAlreadyClaimed.
+func (cgi *ConsumergroupInstance) ClaimPartition(topic string, partition int32) error {
+	root := fmt.Sprintf("%s/consumers/%s/owners/%s", cgi.cg.kz.conf.Chroot, cgi.cg.Name, topic)
+	if err := cgi.cg.kz.mkdirRecursive(root); err != nil {
+		return err
+	}
+
+	// Create an ephemeral node for the partition to claim the partition for this instance
+	node := fmt.Sprintf("%s/%d", root, partition)
+	err := cgi.cg.kz.create(node, []byte(cgi.ID), true)
+	switch err {
+	case zk.ErrNodeExists:
+		data, _, err := cgi.cg.kz.conn.Get(node)
+		if err != nil {
+			return err
+		}
+		if string(data) != cgi.ID {
+			// Return a separate error for this, to allow for implementing a retry mechanism.
+			return ErrPartitionClaimedByOther
+		}
+		return nil
+	default:
+		return err
+	}
+}
+
+// ReleasePartition releases a claim to a partition.
+func (cgi *ConsumergroupInstance) ReleasePartition(topic string, partition int32) error {
+	owner, err := cgi.cg.PartitionOwner(topic, partition)
+	if err != nil {
+		return err
+	}
+	if owner == nil || owner.ID != cgi.ID {
+		return ErrPartitionNotClaimed
+	}
+
+	node := fmt.Sprintf("%s/consumers/%s/owners/%s/%d", cgi.cg.kz.conf.Chroot, cgi.cg.Name, topic, partition)
+	return cgi.cg.kz.conn.Delete(node, 0)
+}
+
+// Topics retrieves the list of topics the consumergroup has claimed ownership of at some point.
+func (cg *Consumergroup) Topics() (TopicList, error) {
+	root := fmt.Sprintf("%s/consumers/%s/owners", cg.kz.conf.Chroot, cg.Name)
+	children, _, err := cg.kz.conn.Children(root)
+	if err != nil {
+		return nil, err
+	}
+
+	result := make(TopicList, 0, len(children))
+	for _, name := range children {
+		result = append(result, cg.kz.Topic(name))
+	}
+	return result, nil
+}
+
+// CommitOffset commits an offset to a group/topic/partition
+func (cg *Consumergroup) CommitOffset(topic string, partition int32, offset int64) error {
+	node := fmt.Sprintf("%s/consumers/%s/offsets/%s/%d", cg.kz.conf.Chroot, cg.Name, topic, partition)
+	data := []byte(fmt.Sprintf("%d", offset))
+
+	_, stat, err := cg.kz.conn.Get(node)
+	switch err {
+	case zk.ErrNoNode: // Create a new node
+		return cg.kz.create(node, data, false)
+
+	case nil: // Update the existing node
+		_, err := cg.kz.conn.Set(node, data, stat.Version)
+		return err
+
+	default:
+		return err
+	}
+}
+
+// FetchOffset retrieves an offset to a group/topic/partition
+func (cg *Consumergroup) FetchOffset(topic string, partition int32) (int64, error) {
+	node := fmt.Sprintf("%s/consumers/%s/offsets/%s/%d", cg.kz.conf.Chroot, cg.Name, topic, partition)
+	val, _, err := cg.kz.conn.Get(node)
+	if err == zk.ErrNoNode {
+		return -1, nil
+	} else if err != nil {
+		return -1, err
+	}
+	return strconv.ParseInt(string(val), 10, 64)
+}
+
+// FetchOffset retrieves all the commmitted offsets for a group
+func (cg *Consumergroup) FetchAllOffsets() (map[string]map[int32]int64, error) {
+	result := make(map[string]map[int32]int64)
+
+	offsetsNode := fmt.Sprintf("%s/consumers/%s/offsets", cg.kz.conf.Chroot, cg.Name)
+	topics, _, err := cg.kz.conn.Children(offsetsNode)
+	if err == zk.ErrNoNode {
+		return result, nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	for _, topic := range topics {
+		result[topic] = make(map[int32]int64)
+		topicNode := fmt.Sprintf("%s/consumers/%s/offsets/%s", cg.kz.conf.Chroot, cg.Name, topic)
+		partitions, _, err := cg.kz.conn.Children(topicNode)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, partition := range partitions {
+			partitionNode := fmt.Sprintf("%s/consumers/%s/offsets/%s/%s", cg.kz.conf.Chroot, cg.Name, topic, partition)
+			val, _, err := cg.kz.conn.Get(partitionNode)
+			if err != nil {
+				return nil, err
+			}
+
+			partition, err := strconv.ParseInt(partition, 10, 32)
+			if err != nil {
+				return nil, err
+			}
+
+			offset, err := strconv.ParseInt(string(val), 10, 64)
+			if err != nil {
+				return nil, err
+			}
+
+			result[topic][int32(partition)] = offset
+		}
+	}
+
+	return result, nil
+}
+
+func (cg *Consumergroup) ResetOffsets() error {
+	offsetsNode := fmt.Sprintf("%s/consumers/%s/offsets", cg.kz.conf.Chroot, cg.Name)
+	topics, _, err := cg.kz.conn.Children(offsetsNode)
+	if err == zk.ErrNoNode {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	for _, topic := range topics {
+		topicNode := fmt.Sprintf("%s/consumers/%s/offsets/%s", cg.kz.conf.Chroot, cg.Name, topic)
+		partitions, stat, err := cg.kz.conn.Children(topicNode)
+		if err != nil {
+			return err
+		}
+
+		for _, partition := range partitions {
+			partitionNode := fmt.Sprintf("%s/consumers/%s/offsets/%s/%s", cg.kz.conf.Chroot, cg.Name, topic, partition)
+			exists, stat, err := cg.kz.conn.Exists(partitionNode)
+			if exists {
+				if err = cg.kz.conn.Delete(partitionNode, stat.Version); err != nil {
+					if err != zk.ErrNoNode {
+						return err
+					}
+				}
+			}
+		}
+
+		if err := cg.kz.conn.Delete(topicNode, stat.Version); err != nil {
+			if err != zk.ErrNoNode {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// generateUUID Generates a UUIDv4.
+func generateUUID() (string, error) {
+	uuid := make([]byte, 16)
+	n, err := io.ReadFull(rand.Reader, uuid)
+	if n != len(uuid) || err != nil {
+		return "", err
+	}
+	// variant bits; see section 4.1.1
+	uuid[8] = uuid[8]&^0xc0 | 0x80
+	// version 4 (pseudo-random); see section 4.1.3
+	uuid[6] = uuid[6]&^0xf0 | 0x40
+	return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil
+}
+
+// generateConsumerInstanceID generates a consumergroup Instance ID
+// that is almost certain to be unique.
+func generateConsumerInstanceID() (string, error) {
+	uuid, err := generateUUID()
+	if err != nil {
+		return "", err
+	}
+
+	hostname, err := os.Hostname()
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%s:%s", hostname, uuid), nil
+}
+
+// Find returns the consumergroup with the given name if it exists in the list.
+// Otherwise it will return `nil`.
+func (cgl ConsumergroupList) Find(name string) *Consumergroup {
+	for _, cg := range cgl {
+		if cg.Name == name {
+			return cg
+		}
+	}
+	return nil
+}
+
+func (cgl ConsumergroupList) Len() int {
+	return len(cgl)
+}
+
+func (cgl ConsumergroupList) Less(i, j int) bool {
+	return cgl[i].Name < cgl[j].Name
+}
+
+func (cgl ConsumergroupList) Swap(i, j int) {
+	cgl[i], cgl[j] = cgl[j], cgl[i]
+}
+
+// Find returns the consumergroup instance with the given ID if it exists in the list.
+// Otherwise it will return `nil`.
+func (cgil ConsumergroupInstanceList) Find(id string) *ConsumergroupInstance {
+	for _, cgi := range cgil {
+		if cgi.ID == id {
+			return cgi
+		}
+	}
+	return nil
+}
+
+func (cgil ConsumergroupInstanceList) Len() int {
+	return len(cgil)
+}
+
+func (cgil ConsumergroupInstanceList) Less(i, j int) bool {
+	return cgil[i].ID < cgil[j].ID
+}
+
+func (cgil ConsumergroupInstanceList) Swap(i, j int) {
+	cgil[i], cgil[j] = cgil[j], cgil[i]
+}

vendor/github.com/wvanbergen/kazoo-go/kazoo.go

@@ -0,0 +1,224 @@
+package kazoo
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/samuel/go-zookeeper/zk"
+)
+
+var (
+	FailedToClaimPartition = errors.New("Failed to claim partition for this consumer instance. Do you have a rogue consumer running?")
+)
+
+// ParseConnectionString parses a zookeeper connection string in the form of
+// host1:2181,host2:2181/chroot and returns the list of servers, and the chroot.
+func ParseConnectionString(zookeeper string) (nodes []string, chroot string) {
+	nodesAndChroot := strings.SplitN(zookeeper, "/", 2)
+	if len(nodesAndChroot) == 2 {
+		chroot = fmt.Sprintf("/%s", nodesAndChroot[1])
+	}
+	nodes = strings.Split(nodesAndChroot[0], ",")
+	return
+}
+
+// BuildConnectionString builds a Zookeeper connection string for a list of nodes.
+// Returns a string like "zk1:2181,zk2:2181,zk3:2181"
+func BuildConnectionString(nodes []string) string {
+	return strings.Join(nodes, ",")
+}
+
+// ConnectionStringWithChroot builds a Zookeeper connection string for a list
+// of nodes and a chroot. The chroot should start with "/".
+// Returns a string like "zk1:2181,zk2:2181,zk3:2181/chroot"
+func BuildConnectionStringWithChroot(nodes []string, chroot string) string {
+	return fmt.Sprintf("%s%s", strings.Join(nodes, ","), chroot)
+}
+
+// Kazoo interacts with the Kafka metadata in Zookeeper
+type Kazoo struct {
+	conn *zk.Conn
+	conf *Config
+}
+
+// Config holds configuration values f.
+type Config struct {
+	// The chroot the Kafka installation is registerde under. Defaults to "".
+	Chroot string
+
+	// The amount of time the Zookeeper client can be disconnected from the Zookeeper cluster
+	// before the cluster will get rid of watches and ephemeral nodes. Defaults to 1 second.
+	Timeout time.Duration
+}
+
+// NewConfig instantiates a new Config struct with sane defaults.
+func NewConfig() *Config {
+	return &Config{Timeout: 1 * time.Second}
+}
+
+// NewKazoo creates a new connection instance
+func NewKazoo(servers []string, conf *Config) (*Kazoo, error) {
+	if conf == nil {
+		conf = NewConfig()
+	}
+
+	conn, _, err := zk.Connect(servers, conf.Timeout)
+	if err != nil {
+		return nil, err
+	}
+	return &Kazoo{conn, conf}, nil
+}
+
+// NewKazooFromConnectionString creates a new connection instance
+// based on a zookeeer connection string that can include a chroot.
+func NewKazooFromConnectionString(connectionString string, conf *Config) (*Kazoo, error) {
+	if conf == nil {
+		conf = NewConfig()
+	}
+
+	nodes, chroot := ParseConnectionString(connectionString)
+	conf.Chroot = chroot
+	return NewKazoo(nodes, conf)
+}
+
+// Brokers returns a map of all the brokers that make part of the
+// Kafka cluster that is registered in Zookeeper.
+func (kz *Kazoo) Brokers() (map[int32]string, error) {
+	root := fmt.Sprintf("%s/brokers/ids", kz.conf.Chroot)
+	children, _, err := kz.conn.Children(root)
+	if err != nil {
+		return nil, err
+	}
+
+	type brokerEntry struct {
+		Host string `json:"host"`
+		Port int    `json:"port"`
+	}
+
+	result := make(map[int32]string)
+	for _, child := range children {
+		brokerID, err := strconv.ParseInt(child, 10, 32)
+		if err != nil {
+			return nil, err
+		}
+
+		value, _, err := kz.conn.Get(path.Join(root, child))
+		if err != nil {
+			return nil, err
+		}
+
+		var brokerNode brokerEntry
+		if err := json.Unmarshal(value, &brokerNode); err != nil {
+			return nil, err
+		}
+
+		result[int32(brokerID)] = fmt.Sprintf("%s:%d", brokerNode.Host, brokerNode.Port)
+	}
+
+	return result, nil
+}
+
+// BrokerList returns a slice of broker addresses that can be used to connect to
+// the Kafka cluster, e.g. using `sarama.NewAsyncProducer()`.
+func (kz *Kazoo) BrokerList() ([]string, error) {
+	brokers, err := kz.Brokers()
+	if err != nil {
+		return nil, err
+	}
+
+	result := make([]string, 0, len(brokers))
+	for _, broker := range brokers {
+		result = append(result, broker)
+	}
+
+	return result, nil
+}
+
+// Controller returns what broker is currently acting as controller of the Kafka cluster
+func (kz *Kazoo) Controller() (int32, error) {
+	type controllerEntry struct {
+		BrokerID int32 `json:"brokerid"`
+	}
+
+	node := fmt.Sprintf("%s/controller", kz.conf.Chroot)
+	data, _, err := kz.conn.Get(node)
+	if err != nil {
+		return -1, err
+	}
+
+	var controllerNode controllerEntry
+	if err := json.Unmarshal(data, &controllerNode); err != nil {
+		return -1, err
+	}
+
+	return controllerNode.BrokerID, nil
+}
+
+// Close closes the connection with the Zookeeper cluster
+func (kz *Kazoo) Close() error {
+	kz.conn.Close()
+	return nil
+}
+
+////////////////////////////////////////////////////////////////////////
+// Util methods
+////////////////////////////////////////////////////////////////////////
+
+// Exists checks existence of a node
+func (kz *Kazoo) exists(node string) (ok bool, err error) {
+	ok, _, err = kz.conn.Exists(node)
+	return
+}
+
+// DeleteAll deletes a node recursively
+func (kz *Kazoo) deleteRecursive(node string) (err error) {
+	children, stat, err := kz.conn.Children(node)
+	if err == zk.ErrNoNode {
+		return nil
+	} else if err != nil {
+		return
+	}
+
+	for _, child := range children {
+		if err = kz.deleteRecursive(path.Join(node, child)); err != nil {
+			return
+		}
+	}
+
+	return kz.conn.Delete(node, stat.Version)
+}
+
+// MkdirAll creates a directory recursively
+func (kz *Kazoo) mkdirRecursive(node string) (err error) {
+	parent := path.Dir(node)
+	if parent != "/" {
+		if err = kz.mkdirRecursive(parent); err != nil {
+			return
+		}
+	}
+
+	_, err = kz.conn.Create(node, nil, 0, zk.WorldACL(zk.PermAll))
+	if err == zk.ErrNodeExists {
+		err = nil
+	}
+	return
+}
+
+// Create stores a new value at node. Fails if already set.
+func (kz *Kazoo) create(node string, value []byte, ephemeral bool) (err error) {
+	if err = kz.mkdirRecursive(path.Dir(node)); err != nil {
+		return
+	}
+
+	flags := int32(0)
+	if ephemeral {
+		flags = zk.FlagEphemeral
+	}
+	_, err = kz.conn.Create(node, value, flags, zk.WorldACL(zk.PermAll))
+	return
+}

vendor/github.com/wvanbergen/kazoo-go/topic_metadata.go

@@ -0,0 +1,255 @@
+package kazoo
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"github.com/samuel/go-zookeeper/zk"
+)
+
+// Topic interacts with Kafka's topic metadata in Zookeeper.
+type Topic struct {
+	Name string
+	kz   *Kazoo
+}
+
+// TopicList is a type that implements the sortable interface for a list of Topic instances.
+type TopicList []*Topic
+
+// Partition interacts with Kafka's partition metadata in Zookeeper.
+type Partition struct {
+	topic    *Topic
+	ID       int32
+	Replicas []int32
+}
+
+// PartitionList is a type that implements the sortable interface for a list of Partition instances
+type PartitionList []*Partition
+
+// Topics returns a list of all registered Kafka topics.
+func (kz *Kazoo) Topics() (TopicList, error) {
+	root := fmt.Sprintf("%s/brokers/topics", kz.conf.Chroot)
+	children, _, err := kz.conn.Children(root)
+	if err != nil {
+		return nil, err
+	}
+
+	result := make(TopicList, 0, len(children))
+	for _, name := range children {
+		result = append(result, kz.Topic(name))
+	}
+	return result, nil
+}
+
+// WatchTopics returns a list of all registered Kafka topics, and
+// watches that list for changes.
+func (kz *Kazoo) WatchTopics() (TopicList, <-chan zk.Event, error) {
+	root := fmt.Sprintf("%s/brokers/topics", kz.conf.Chroot)
+	children, _, c, err := kz.conn.ChildrenW(root)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	result := make(TopicList, 0, len(children))
+	for _, name := range children {
+		result = append(result, kz.Topic(name))
+	}
+	return result, c, nil
+}
+
+// Topic returns a Topic instance for a given topic name
+func (kz *Kazoo) Topic(topic string) *Topic {
+	return &Topic{Name: topic, kz: kz}
+}
+
+// Exists returns true if the topic exists on the Kafka cluster.
+func (t *Topic) Exists() (bool, error) {
+	return t.kz.exists(fmt.Sprintf("%s/brokers/topics/%s", t.kz.conf.Chroot, t.Name))
+}
+
+// Partitions returns a list of all partitions for the topic.
+func (t *Topic) Partitions() (PartitionList, error) {
+	node := fmt.Sprintf("%s/brokers/topics/%s", t.kz.conf.Chroot, t.Name)
+	value, _, err := t.kz.conn.Get(node)
+	if err != nil {
+		return nil, err
+	}
+
+	return t.parsePartitions(value)
+}
+
+// WatchPartitions returns a list of all partitions for the topic, and watches the topic for changes.
+func (t *Topic) WatchPartitions() (PartitionList, <-chan zk.Event, error) {
+	node := fmt.Sprintf("%s/brokers/topics/%s", t.kz.conf.Chroot, t.Name)
+	value, _, c, err := t.kz.conn.GetW(node)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	list, err := t.parsePartitions(value)
+	return list, c, err
+}
+
+// parsePartitions pases the JSON representation of the partitions
+// that is stored as data on the topic node in Zookeeper.
+func (t *Topic) parsePartitions(value []byte) (PartitionList, error) {
+	type topicMetadata struct {
+		Partitions map[string][]int32 `json:"partitions"`
+	}
+
+	var tm topicMetadata
+	if err := json.Unmarshal(value, &tm); err != nil {
+		return nil, err
+	}
+
+	result := make(PartitionList, len(tm.Partitions))
+	for partitionNumber, replicas := range tm.Partitions {
+		partitionID, err := strconv.ParseInt(partitionNumber, 10, 32)
+		if err != nil {
+			return nil, err
+		}
+
+		replicaIDs := make([]int32, 0, len(replicas))
+		for _, r := range replicas {
+			replicaIDs = append(replicaIDs, int32(r))
+		}
+		result[partitionID] = t.Partition(int32(partitionID), replicaIDs)
+	}
+
+	return result, nil
+}
+
+// Partition returns a Partition instance for the topic.
+func (t *Topic) Partition(id int32, replicas []int32) *Partition {
+	return &Partition{ID: id, Replicas: replicas, topic: t}
+}
+
+// Config returns topic-level configuration settings as a map.
+func (t *Topic) Config() (map[string]string, error) {
+	value, _, err := t.kz.conn.Get(fmt.Sprintf("%s/config/topics/%s", t.kz.conf.Chroot, t.Name))
+	if err != nil {
+		return nil, err
+	}
+
+	var topicConfig struct {
+		ConfigMap map[string]string `json:"config"`
+	}
+
+	if err := json.Unmarshal(value, &topicConfig); err != nil {
+		return nil, err
+	}
+
+	return topicConfig.ConfigMap, nil
+}
+
+// Topic returns the Topic of this partition.
+func (p *Partition) Topic() *Topic {
+	return p.topic
+}
+
+// Key returns a unique identifier for the partition, using the form "topic/partition".
+func (p *Partition) Key() string {
+	return fmt.Sprintf("%s/%d", p.topic.Name, p.ID)
+}
+
+// PreferredReplica returns the preferred replica for this partition.
+func (p *Partition) PreferredReplica() int32 {
+	if len(p.Replicas) > 0 {
+		return p.Replicas[0]
+	} else {
+		return -1
+	}
+}
+
+// Leader returns the broker ID of the broker that is currently the leader for the partition.
+func (p *Partition) Leader() (int32, error) {
+	if state, err := p.state(); err != nil {
+		return -1, err
+	} else {
+		return state.Leader, nil
+	}
+}
+
+// ISR returns the broker IDs of the current in-sync replica set for the partition
+func (p *Partition) ISR() ([]int32, error) {
+	if state, err := p.state(); err != nil {
+		return nil, err
+	} else {
+		return state.ISR, nil
+	}
+}
+
+func (p *Partition) UnderReplicated() (bool, error) {
+	if state, err := p.state(); err != nil {
+		return false, err
+	} else {
+		return len(state.ISR) < len(p.Replicas), nil
+	}
+}
+
+func (p *Partition) UsesPreferredReplica() (bool, error) {
+	if state, err := p.state(); err != nil {
+		return false, err
+	} else {
+		return len(state.ISR) > 0 && state.ISR[0] == p.Replicas[0], nil
+	}
+}
+
+// partitionState represents the partition state as it is stored as JSON
+// in Zookeeper on the partition's state node.
+type partitionState struct {
+	Leader int32   `json:"leader"`
+	ISR    []int32 `json:"isr"`
+}
+
+// state retrieves and parses the partition State
+func (p *Partition) state() (partitionState, error) {
+	var state partitionState
+	node := fmt.Sprintf("%s/brokers/topics/%s/partitions/%d/state", p.topic.kz.conf.Chroot, p.topic.Name, p.ID)
+	value, _, err := p.topic.kz.conn.Get(node)
+	if err != nil {
+		return state, err
+	}
+
+	if err := json.Unmarshal(value, &state); err != nil {
+		return state, err
+	}
+
+	return state, nil
+}
+
+// Find returns the topic with the given name if it exists in the topic list,
+// and will return `nil` otherwise.
+func (tl TopicList) Find(name string) *Topic {
+	for _, topic := range tl {
+		if topic.Name == name {
+			return topic
+		}
+	}
+	return nil
+}
+
+func (tl TopicList) Len() int {
+	return len(tl)
+}
+
+func (tl TopicList) Less(i, j int) bool {
+	return tl[i].Name < tl[j].Name
+}
+
+func (tl TopicList) Swap(i, j int) {
+	tl[i], tl[j] = tl[j], tl[i]
+}
+
+func (pl PartitionList) Len() int {
+	return len(pl)
+}
+
+func (pl PartitionList) Less(i, j int) bool {
+	return pl[i].topic.Name < pl[j].topic.Name || (pl[i].topic.Name == pl[j].topic.Name && pl[i].ID < pl[j].ID)
+}
+
+func (pl PartitionList) Swap(i, j int) {
+	pl[i], pl[j] = pl[j], pl[i]
+}