bingyi/gf
1
0
Fork 0
mirror of https://gitee.com/johng/gf synced 2026-06-06 16:21:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

thirdparty package kafka updated to date

Browse Source
This commit is contained in:
John
2019-01-02 11:02:03 +08:00
parent d6aa2b2512
commit adf06a2b0d
98 changed files with 4670 additions and 1816 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
g/database/gdb/gdb.go
View File

@ -4,8 +4,7 @@
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.
// Package gdb provides ORM features for popular relationship databases.
// 数据库ORM.
// Package gdb provides ORM features for popular relationship databases/数据库ORM.
// 默认内置支持MySQL, 其他数据库需要手动import对应的数据库引擎第三方包.
package gdb

2
g/database/gkafka/gkafka.go
View File

@ -4,7 +4,7 @@
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.
// Kafka Client.
// Package gkafka provides producer and consumer client for kafka server/Kafka客户端.
package gkafka
import (

1
third/github.com/Shopify/sarama/.gitignore vendored
View File

@ -24,3 +24,4 @@ _testmain.go
*.exe
coverage.txt
profile.out

11
third/github.com/Shopify/sarama/.travis.yml
View File

@ -1,8 +1,7 @@
language: go
go:
- 1.8.x
- 1.9.x
- 1.10.x
- 1.11.x
env:
global:
@ -12,9 +11,9 @@ env:
- KAFKA_HOSTNAME=localhost
- DEBUG=true
matrix:
- KAFKA_VERSION=0.11.0.2
- KAFKA_VERSION=1.0.0
- KAFKA_VERSION=1.1.0
- KAFKA_VERSION=1.1.1
- KAFKA_VERSION=2.0.1
- KAFKA_VERSION=2.1.0
before_install:
- export REPOSITORY_ROOT=${TRAVIS_BUILD_DIR}
@ -28,7 +27,7 @@ script:
- make test
- make vet
- make errcheck
- make fmt
- if [[ "$TRAVIS_GO_VERSION" == 1.11* ]]; then make fmt; fi
after_success:
- bash <(curl -s https://codecov.io/bash)

82
third/github.com/Shopify/sarama/CHANGELOG.md
View File

@ -1,5 +1,87 @@
# Changelog
#### Version 1.20.0 (2018-12-10)
New Features:
- Add support for zstd compression
([#1170](https://github.com/Shopify/sarama/pull/1170)).
- Add support for Idempotent Producer
([#1152](https://github.com/Shopify/sarama/pull/1152)).
- Add support support for Kafka 2.1.0
([#1229](https://github.com/Shopify/sarama/pull/1229)).
- Add support support for OffsetCommit request/response pairs versions v1 to v5
([#1201](https://github.com/Shopify/sarama/pull/1201)).
- Add support support for OffsetFetch request/response pair up to version v5
([#1198](https://github.com/Shopify/sarama/pull/1198)).
Improvements:
- Export broker's Rack setting
([#1173](https://github.com/Shopify/sarama/pull/1173)).
- Always use latest patch version of Go on CI
([#1202](https://github.com/Shopify/sarama/pull/1202)).
- Add error codes 61 to 72
([#1195](https://github.com/Shopify/sarama/pull/1195)).
Bug Fixes:
- Fix build without cgo
([#1182](https://github.com/Shopify/sarama/pull/1182)).
- Fix go vet suggestion in consumer group file
([#1209](https://github.com/Shopify/sarama/pull/1209)).
- Fix typos in code and comments
([#1228](https://github.com/Shopify/sarama/pull/1228)).
#### Version 1.19.0 (2018-09-27)
New Features:
- Implement a higher-level consumer group
([#1099](https://github.com/Shopify/sarama/pull/1099)).
Improvements:
- Add support for Go 1.11
([#1176](https://github.com/Shopify/sarama/pull/1176)).
Bug Fixes:
- Fix encoding of `MetadataResponse` with version 2 and higher
([#1174](https://github.com/Shopify/sarama/pull/1174)).
- Fix race condition in mock async producer
([#1174](https://github.com/Shopify/sarama/pull/1174)).
#### Version 1.18.0 (2018-09-07)
New Features:
- Make `Partitioner.RequiresConsistency` vary per-message
([#1112](https://github.com/Shopify/sarama/pull/1112)).
- Add customizable partitioner
([#1118](https://github.com/Shopify/sarama/pull/1118)).
- Add `ClusterAdmin` support for `CreateTopic`, `DeleteTopic`, `CreatePartitions`,
`DeleteRecords`, `DescribeConfig`, `AlterConfig`, `CreateACL`, `ListAcls`, `DeleteACL`
([#1055](https://github.com/Shopify/sarama/pull/1055)).
Improvements:
- Add support for Kafka 2.0.0
([#1149](https://github.com/Shopify/sarama/pull/1149)).
- Allow setting `LocalAddr` when dialing an address to support multi-homed hosts
([#1123](https://github.com/Shopify/sarama/pull/1123)).
- Simpler offset management
([#1127](https://github.com/Shopify/sarama/pull/1127)).
Bug Fixes:
- Fix mutation of `ProducerMessage.MetaData` when producing to Kafka
([#1110](https://github.com/Shopify/sarama/pull/1110)).
- Fix consumer block when response did not contain all the
expected topic/partition blocks
([#1086](https://github.com/Shopify/sarama/pull/1086)).
- Fix consumer block when response contains only constrol messages
([#1115](https://github.com/Shopify/sarama/pull/1115)).
- Add timeout config for ClusterAdmin requests
([#1142](https://github.com/Shopify/sarama/pull/1142)).
- Add version check when producing message with headers
([#1117](https://github.com/Shopify/sarama/pull/1117)).
- Fix `MetadataRequest` for empty list of topics
([#1132](https://github.com/Shopify/sarama/pull/1132)).
- Fix producer topic metadata on-demand fetch when topic error happens in metadata response
([#1125](https://github.com/Shopify/sarama/pull/1125)).
#### Version 1.17.0 (2018-05-30)
New Features:

2
third/github.com/Shopify/sarama/Makefile
View File

@ -4,7 +4,7 @@ default: fmt vet errcheck test
test:
echo "" > coverage.txt
for d in `go list ./... | grep -v vendor`; do \
go test -p 1 -v -timeout 90s -race -coverprofile=profile.out -covermode=atomic $$d || exit 1; \
go test -p 1 -v -timeout 240s -race -coverprofile=profile.out -covermode=atomic $$d || exit 1; \
if [ -f profile.out ]; then \
cat profile.out >> coverage.txt; \
rm profile.out; \

4
third/github.com/Shopify/sarama/README.md
View File

@ -1,7 +1,7 @@
sarama
======
[![GoDoc](https://godoc.org/github.com/Shopify/sarama?status.png)](https://godoc.org/github.com/Shopify/sarama)
[![GoDoc](https://godoc.org/github.com/Shopify/sarama?status.svg)](https://godoc.org/github.com/Shopify/sarama)
[![Build Status](https://travis-ci.org/Shopify/sarama.svg?branch=master)](https://travis-ci.org/Shopify/sarama)
[![Coverage](https://codecov.io/gh/Shopify/sarama/branch/master/graph/badge.svg)](https://codecov.io/gh/Shopify/sarama)
@ -21,7 +21,7 @@ You might also want to look at the [Frequently Asked Questions](https://github.c
Sarama provides a "2 releases + 2 months" compatibility guarantee: we support
the two latest stable releases of Kafka and Go, and we provide a two month
grace period for older releases. This means we currently officially support
Go 1.8 through 1.10, and Kafka 0.11 through 1.1, although older releases are
Go 1.8 through 1.11, and Kafka 1.0 through 2.0, although older releases are
still likely to work.
Sarama follows semantic versioning and provides API stability via the gopkg.in service.

11
third/github.com/Shopify/sarama/admin.go
View File

@ -118,6 +118,7 @@ func (ca *clusterAdmin) CreateTopic(topic string, detail *TopicDetail, validateO
request := &CreateTopicsRequest{
TopicDetails: topicDetails,
ValidateOnly: validateOnly,
Timeout: ca.conf.Admin.Timeout,
}
if ca.conf.Version.IsAtLeast(V0_11_0_0) {
@ -155,7 +156,10 @@ func (ca *clusterAdmin) DeleteTopic(topic string) error {
return ErrInvalidTopic
}
request := &DeleteTopicsRequest{Topics: []string{topic}}
request := &DeleteTopicsRequest{
Topics: []string{topic},
Timeout: ca.conf.Admin.Timeout,
}
if ca.conf.Version.IsAtLeast(V0_11_0_0) {
request.Version = 1
@ -192,6 +196,7 @@ func (ca *clusterAdmin) CreatePartitions(topic string, count int32, assignment [
request := &CreatePartitionsRequest{
TopicPartitions: topicPartitions,
Timeout: ca.conf.Admin.Timeout,
}
b, err := ca.Controller()
@ -225,7 +230,9 @@ func (ca *clusterAdmin) DeleteRecords(topic string, partitionOffsets map[int32]i
topics := make(map[string]*DeleteRecordsRequestTopic)
topics[topic] = &DeleteRecordsRequestTopic{PartitionOffsets: partitionOffsets}
request := &DeleteRecordsRequest{
Topics: topics}
Topics: topics,
Timeout: ca.conf.Admin.Timeout,
}
b, err := ca.Controller()
if err != nil {

214
third/github.com/Shopify/sarama/async_producer.go
View File

@ -47,6 +47,50 @@ type AsyncProducer interface {
Errors() <-chan *ProducerError
}
// transactionManager keeps the state necessary to ensure idempotent production
type transactionManager struct {
producerID int64
producerEpoch int16
sequenceNumbers map[string]int32
mutex sync.Mutex
}
const (
noProducerID = -1
noProducerEpoch = -1
)
func (t *transactionManager) getAndIncrementSequenceNumber(topic string, partition int32) int32 {
key := fmt.Sprintf("%s-%d", topic, partition)
t.mutex.Lock()
defer t.mutex.Unlock()
sequence := t.sequenceNumbers[key]
t.sequenceNumbers[key] = sequence + 1
return sequence
}
func newTransactionManager(conf *Config, client Client) (*transactionManager, error) {
txnmgr := &transactionManager{
producerID: noProducerID,
producerEpoch: noProducerEpoch,
}
if conf.Producer.Idempotent {
initProducerIDResponse, err := client.InitProducerID()
if err != nil {
return nil, err
}
txnmgr.producerID = initProducerIDResponse.ProducerID
txnmgr.producerEpoch = initProducerIDResponse.ProducerEpoch
txnmgr.sequenceNumbers = make(map[string]int32)
txnmgr.mutex = sync.Mutex{}
Logger.Printf("Obtained a ProducerId: %d and ProducerEpoch: %d\n", txnmgr.producerID, txnmgr.producerEpoch)
}
return txnmgr, nil
}
type asyncProducer struct {
client Client
conf *Config
@ -56,9 +100,11 @@ type asyncProducer struct {
input, successes, retries chan *ProducerMessage
inFlight sync.WaitGroup
brokers map[*Broker]chan<- *ProducerMessage
brokerRefs map[chan<- *ProducerMessage]int
brokers map[*Broker]*brokerProducer
brokerRefs map[*brokerProducer]int
brokerLock sync.Mutex
txnmgr *transactionManager
}
// NewAsyncProducer creates a new AsyncProducer using the given broker addresses and configuration.
@ -84,6 +130,11 @@ func NewAsyncProducerFromClient(client Client) (AsyncProducer, error) {
return nil, ErrClosedClient
}
txnmgr, err := newTransactionManager(client.Config(), client)
if err != nil {
return nil, err
}
p := &asyncProducer{
client: client,
conf: client.Config(),
@ -91,8 +142,9 @@ func NewAsyncProducerFromClient(client Client) (AsyncProducer, error) {
input: make(chan *ProducerMessage),
successes: make(chan *ProducerMessage),
retries: make(chan *ProducerMessage),
brokers: make(map[*Broker]chan<- *ProducerMessage),
brokerRefs: make(map[chan<- *ProducerMessage]int),
brokers: make(map[*Broker]*brokerProducer),
brokerRefs: make(map[*brokerProducer]int),
txnmgr: txnmgr,
}
// launch our singleton dispatchers
@ -145,9 +197,10 @@ type ProducerMessage struct {
// least version 0.10.0.
Timestamp time.Time
retries int
flags flagSet
expectation chan *ProducerError
retries int
flags flagSet
expectation chan *ProducerError
sequenceNumber int32
}
const producerMessageOverhead = 26 // the metadata overhead of CRC, flags, etc.
@ -328,6 +381,10 @@ func (tp *topicProducer) dispatch() {
continue
}
}
// All messages being retried (sent or not) have already had their retry count updated
if tp.parent.conf.Producer.Idempotent && msg.retries == 0 {
msg.sequenceNumber = tp.parent.txnmgr.getAndIncrementSequenceNumber(msg.Topic, msg.Partition)
}
handler := tp.handlers[msg.Partition]
if handler == nil {
@ -394,9 +451,9 @@ type partitionProducer struct {
partition int32
input <-chan *ProducerMessage
leader *Broker
breaker *breaker.Breaker
output chan<- *ProducerMessage
leader *Broker
breaker *breaker.Breaker
brokerProducer *brokerProducer
// highWatermark tracks the "current" retry level, which is the only one where we actually let messages through,
// all other messages get buffered in retryState[msg.retries].buf to preserve ordering
@ -431,9 +488,9 @@ func (pp *partitionProducer) dispatch() {
// on the first message
pp.leader, _ = pp.parent.client.Leader(pp.topic, pp.partition)
if pp.leader != nil {
pp.output = pp.parent.getBrokerProducer(pp.leader)
pp.brokerProducer = pp.parent.getBrokerProducer(pp.leader)
pp.parent.inFlight.Add(1) // we're generating a syn message; track it so we don't shut down while it's still inflight
pp.output <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: syn}
pp.brokerProducer.input <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: syn}
}
for msg := range pp.input {
@ -465,7 +522,7 @@ func (pp *partitionProducer) dispatch() {
// if we made it this far then the current msg contains real data, and can be sent to the next goroutine
// without breaking any of our ordering guarantees
if pp.output == nil {
if pp.brokerProducer == nil {
if err := pp.updateLeader(); err != nil {
pp.parent.returnError(msg, err)
time.Sleep(pp.parent.conf.Producer.Retry.Backoff)
@ -474,11 +531,11 @@ func (pp *partitionProducer) dispatch() {
Logger.Printf("producer/leader/%s/%d selected broker %d\n", pp.topic, pp.partition, pp.leader.ID())
}
pp.output <- msg
pp.brokerProducer.input <- msg
}
if pp.output != nil {
pp.parent.unrefBrokerProducer(pp.leader, pp.output)
if pp.brokerProducer != nil {
pp.parent.unrefBrokerProducer(pp.leader, pp.brokerProducer)
}
}
@ -490,12 +547,12 @@ func (pp *partitionProducer) newHighWatermark(hwm int) {
// back to us and we can safely flush the backlog (otherwise we risk re-ordering messages)
pp.retryState[pp.highWatermark].expectChaser = true
pp.parent.inFlight.Add(1) // we're generating a fin message; track it so we don't shut down while it's still inflight
pp.output <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: fin, retries: pp.highWatermark - 1}
pp.brokerProducer.input <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: fin, retries: pp.highWatermark - 1}
// a new HWM means that our current broker selection is out of date
Logger.Printf("producer/leader/%s/%d abandoning broker %d\n", pp.topic, pp.partition, pp.leader.ID())
pp.parent.unrefBrokerProducer(pp.leader, pp.output)
pp.output = nil
pp.parent.unrefBrokerProducer(pp.leader, pp.brokerProducer)
pp.brokerProducer = nil
}
func (pp *partitionProducer) flushRetryBuffers() {
@ -503,7 +560,7 @@ func (pp *partitionProducer) flushRetryBuffers() {
for {
pp.highWatermark--
if pp.output == nil {
if pp.brokerProducer == nil {
if err := pp.updateLeader(); err != nil {
pp.parent.returnErrors(pp.retryState[pp.highWatermark].buf, err)
goto flushDone
@ -512,7 +569,7 @@ func (pp *partitionProducer) flushRetryBuffers() {
}
for _, msg := range pp.retryState[pp.highWatermark].buf {
pp.output <- msg
pp.brokerProducer.input <- msg
}
flushDone:
@ -537,16 +594,16 @@ func (pp *partitionProducer) updateLeader() error {
return err
}
pp.output = pp.parent.getBrokerProducer(pp.leader)
pp.brokerProducer = pp.parent.getBrokerProducer(pp.leader)
pp.parent.inFlight.Add(1) // we're generating a syn message; track it so we don't shut down while it's still inflight
pp.output <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: syn}
pp.brokerProducer.input <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: syn}
return nil
})
}
// one per broker; also constructs an associated flusher
func (p *asyncProducer) newBrokerProducer(broker *Broker) chan<- *ProducerMessage {
func (p *asyncProducer) newBrokerProducer(broker *Broker) *brokerProducer {
var (
input = make(chan *ProducerMessage)
bridge = make(chan *produceSet)
@ -580,7 +637,7 @@ func (p *asyncProducer) newBrokerProducer(broker *Broker) chan<- *ProducerMessag
close(responses)
})
return input
return bp
}
type brokerProducerResponse struct {
@ -595,7 +652,7 @@ type brokerProducer struct {
parent *asyncProducer
broker *Broker
input <-chan *ProducerMessage
input chan *ProducerMessage
output chan<- *produceSet
responses <-chan *brokerProducerResponse
@ -740,16 +797,17 @@ func (bp *brokerProducer) handleResponse(response *brokerProducerResponse) {
func (bp *brokerProducer) handleSuccess(sent *produceSet, response *ProduceResponse) {
// we iterate through the blocks in the request set, not the response, so that we notice
// if the response is missing a block completely
sent.eachPartition(func(topic string, partition int32, msgs []*ProducerMessage) {
var retryTopics []string
sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) {
if response == nil {
// this only happens when RequiredAcks is NoResponse, so we have to assume success
bp.parent.returnSuccesses(msgs)
bp.parent.returnSuccesses(pSet.msgs)
return
}
block := response.GetBlock(topic, partition)
if block == nil {
bp.parent.returnErrors(msgs, ErrIncompleteResponse)
bp.parent.returnErrors(pSet.msgs, ErrIncompleteResponse)
return
}
@ -757,45 +815,107 @@ func (bp *brokerProducer) handleSuccess(sent *produceSet, response *ProduceRespo
// Success
case ErrNoError:
if bp.parent.conf.Version.IsAtLeast(V0_10_0_0) && !block.Timestamp.IsZero() {
for _, msg := range msgs {
for _, msg := range pSet.msgs {
msg.Timestamp = block.Timestamp
}
}
for i, msg := range msgs {
for i, msg := range pSet.msgs {
msg.Offset = block.Offset + int64(i)
}
bp.parent.returnSuccesses(msgs)
bp.parent.returnSuccesses(pSet.msgs)
// Duplicate
case ErrDuplicateSequenceNumber:
bp.parent.returnSuccesses(pSet.msgs)
// Retriable errors
case ErrInvalidMessage, ErrUnknownTopicOrPartition, ErrLeaderNotAvailable, ErrNotLeaderForPartition,
ErrRequestTimedOut, ErrNotEnoughReplicas, ErrNotEnoughReplicasAfterAppend:
Logger.Printf("producer/broker/%d state change to [retrying] on %s/%d because %v\n",
bp.broker.ID(), topic, partition, block.Err)
bp.currentRetries[topic][partition] = block.Err
bp.parent.retryMessages(msgs, block.Err)
bp.parent.retryMessages(bp.buffer.dropPartition(topic, partition), block.Err)
retryTopics = append(retryTopics, topic)
// Other non-retriable errors
default:
bp.parent.returnErrors(msgs, block.Err)
bp.parent.returnErrors(pSet.msgs, block.Err)
}
})
if len(retryTopics) > 0 {
if bp.parent.conf.Producer.Idempotent {
err := bp.parent.client.RefreshMetadata(retryTopics...)
if err != nil {
Logger.Printf("Failed refreshing metadata because of %v\n", err)
}
}
sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) {
block := response.GetBlock(topic, partition)
if block == nil {
// handled in the previous "eachPartition" loop
return
}
switch block.Err {
case ErrInvalidMessage, ErrUnknownTopicOrPartition, ErrLeaderNotAvailable, ErrNotLeaderForPartition,
ErrRequestTimedOut, ErrNotEnoughReplicas, ErrNotEnoughReplicasAfterAppend:
Logger.Printf("producer/broker/%d state change to [retrying] on %s/%d because %v\n",
bp.broker.ID(), topic, partition, block.Err)
if bp.currentRetries[topic] == nil {
bp.currentRetries[topic] = make(map[int32]error)
}
bp.currentRetries[topic][partition] = block.Err
if bp.parent.conf.Producer.Idempotent {
go bp.parent.retryBatch(topic, partition, pSet, block.Err)
} else {
bp.parent.retryMessages(pSet.msgs, block.Err)
}
// dropping the following messages has the side effect of incrementing their retry count
bp.parent.retryMessages(bp.buffer.dropPartition(topic, partition), block.Err)
}
})
}
}
func (p *asyncProducer) retryBatch(topic string, partition int32, pSet *partitionSet, kerr KError) {
Logger.Printf("Retrying batch for %v-%d because of %s\n", topic, partition, kerr)
produceSet := newProduceSet(p)
produceSet.msgs[topic] = make(map[int32]*partitionSet)
produceSet.msgs[topic][partition] = pSet
produceSet.bufferBytes += pSet.bufferBytes
produceSet.bufferCount += len(pSet.msgs)
for _, msg := range pSet.msgs {
if msg.retries >= p.conf.Producer.Retry.Max {
p.returnError(msg, kerr)
return
}
msg.retries++
}
// it's expected that a metadata refresh has been requested prior to calling retryBatch
leader, err := p.client.Leader(topic, partition)
if err != nil {
Logger.Printf("Failed retrying batch for %v-%d because of %v while looking up for new leader\n", topic, partition, err)
for _, msg := range pSet.msgs {
p.returnError(msg, kerr)
}
return
}
bp := p.getBrokerProducer(leader)
bp.output <- produceSet
}
func (bp *brokerProducer) handleError(sent *produceSet, err error) {
switch err.(type) {
case PacketEncodingError:
sent.eachPartition(func(topic string, partition int32, msgs []*ProducerMessage) {
bp.parent.returnErrors(msgs, err)
sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) {
bp.parent.returnErrors(pSet.msgs, err)
})
default:
Logger.Printf("producer/broker/%d state change to [closing] because %s\n", bp.broker.ID(), err)
bp.parent.abandonBrokerConnection(bp.broker)
_ = bp.broker.Close()
bp.closing = err
sent.eachPartition(func(topic string, partition int32, msgs []*ProducerMessage) {
bp.parent.retryMessages(msgs, err)
sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) {
bp.parent.retryMessages(pSet.msgs, err)
})
bp.buffer.eachPartition(func(topic string, partition int32, msgs []*ProducerMessage) {
bp.parent.retryMessages(msgs, err)
bp.buffer.eachPartition(func(topic string, partition int32, pSet *partitionSet) {
bp.parent.retryMessages(pSet.msgs, err)
})
bp.rollOver()
}
@ -892,7 +1012,7 @@ func (p *asyncProducer) retryMessages(batch []*ProducerMessage, err error) {
}
}
func (p *asyncProducer) getBrokerProducer(broker *Broker) chan<- *ProducerMessage {
func (p *asyncProducer) getBrokerProducer(broker *Broker) *brokerProducer {
p.brokerLock.Lock()
defer p.brokerLock.Unlock()
@ -909,13 +1029,13 @@ func (p *asyncProducer) getBrokerProducer(broker *Broker) chan<- *ProducerMessag
return bp
}
func (p *asyncProducer) unrefBrokerProducer(broker *Broker, bp chan<- *ProducerMessage) {
func (p *asyncProducer) unrefBrokerProducer(broker *Broker, bp *brokerProducer) {
p.brokerLock.Lock()
defer p.brokerLock.Unlock()
p.brokerRefs[bp]--
if p.brokerRefs[bp] == 0 {
close(bp)
close(bp.input)
delete(p.brokerRefs, bp)
if p.brokers[broker] == bp {

250
third/github.com/Shopify/sarama/async_producer_test.go
View File

@ -459,6 +459,7 @@ func TestAsyncProducerMultipleRetries(t *testing.T) {
metadataLeader2 := new(MetadataResponse)
metadataLeader2.AddBroker(leader2.Addr(), leader2.BrokerID())
metadataLeader2.AddTopicPartition("my_topic", 0, leader2.BrokerID(), nil, nil, ErrNoError)
seedBroker.Returns(metadataLeader2)
leader2.Returns(prodNotLeader)
seedBroker.Returns(metadataLeader1)
@ -753,6 +754,255 @@ func TestAsyncProducerNoReturns(t *testing.T) {
leader.Close()
}
func TestAsyncProducerIdempotentGoldenPath(t *testing.T) {
broker := NewMockBroker(t, 1)
metadataResponse := &MetadataResponse{
Version: 1,
ControllerID: 1,
}
metadataResponse.AddBroker(broker.Addr(), broker.BrokerID())
metadataResponse.AddTopicPartition("my_topic", 0, broker.BrokerID(), nil, nil, ErrNoError)
broker.Returns(metadataResponse)
initProducerID := &InitProducerIDResponse{
ThrottleTime: 0,
ProducerID: 1000,
ProducerEpoch: 1,
}
broker.Returns(initProducerID)
config := NewConfig()
config.Producer.Flush.Messages = 10
config.Producer.Return.Successes = true
config.Producer.Retry.Max = 4
config.Producer.RequiredAcks = WaitForAll
config.Producer.Retry.Backoff = 0
config.Producer.Idempotent = true
config.Net.MaxOpenRequests = 1
config.Version = V0_11_0_0
producer, err := NewAsyncProducer([]string{broker.Addr()}, config)
if err != nil {
t.Fatal(err)
}
for i := 0; i < 10; i++ {
producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
}
prodSuccess := &ProduceResponse{
Version: 3,
ThrottleTime: 0,
}
prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError)
broker.Returns(prodSuccess)
expectResults(t, producer, 10, 0)
broker.Close()
closeProducer(t, producer)
}
func TestAsyncProducerIdempotentRetryCheckBatch(t *testing.T) {
//Logger = log.New(os.Stderr, "", log.LstdFlags)
tests := []struct {
name string
failAfterWrite bool
}{
{"FailAfterWrite", true},
{"FailBeforeWrite", false},
}
for _, test := range tests {
broker := NewMockBroker(t, 1)
metadataResponse := &MetadataResponse{
Version: 1,
ControllerID: 1,
}
metadataResponse.AddBroker(broker.Addr(), broker.BrokerID())
metadataResponse.AddTopicPartition("my_topic", 0, broker.BrokerID(), nil, nil, ErrNoError)
initProducerIDResponse := &InitProducerIDResponse{
ThrottleTime: 0,
ProducerID: 1000,
ProducerEpoch: 1,
}
prodNotLeaderResponse := &ProduceResponse{
Version: 3,
ThrottleTime: 0,
}
prodNotLeaderResponse.AddTopicPartition("my_topic", 0, ErrNotEnoughReplicas)
prodDuplicate := &ProduceResponse{
Version: 3,
ThrottleTime: 0,
}
prodDuplicate.AddTopicPartition("my_topic", 0, ErrDuplicateSequenceNumber)
prodOutOfSeq := &ProduceResponse{
Version: 3,
ThrottleTime: 0,
}
prodOutOfSeq.AddTopicPartition("my_topic", 0, ErrOutOfOrderSequenceNumber)
prodSuccessResponse := &ProduceResponse{
Version: 3,
ThrottleTime: 0,
}
prodSuccessResponse.AddTopicPartition("my_topic", 0, ErrNoError)
prodCounter := 0
lastBatchFirstSeq := -1
lastBatchSize := -1
lastSequenceWrittenToDisk := -1
handlerFailBeforeWrite := func(req *request) (res encoder) {
switch req.body.key() {
case 3:
return metadataResponse
case 22:
return initProducerIDResponse
case 0:
prodCounter++
preq := req.body.(*ProduceRequest)
batch := preq.records["my_topic"][0].RecordBatch
batchFirstSeq := int(batch.FirstSequence)
batchSize := len(batch.Records)
if lastSequenceWrittenToDisk == batchFirstSeq-1 { //in sequence append
if lastBatchFirstSeq == batchFirstSeq { //is a batch retry
if lastBatchSize == batchSize { //good retry
// mock write to disk
lastSequenceWrittenToDisk = batchFirstSeq + batchSize - 1
return prodSuccessResponse
}
t.Errorf("[%s] Retried Batch firstSeq=%d with different size old=%d new=%d", test.name, batchFirstSeq, lastBatchSize, batchSize)
return prodOutOfSeq
} else { // not a retry
// save batch just received for future check
lastBatchFirstSeq = batchFirstSeq
lastBatchSize = batchSize
if prodCounter%2 == 1 {
if test.failAfterWrite {
// mock write to disk
lastSequenceWrittenToDisk = batchFirstSeq + batchSize - 1
}
return prodNotLeaderResponse
}
// mock write to disk
lastSequenceWrittenToDisk = batchFirstSeq + batchSize - 1
return prodSuccessResponse
}
} else {
if lastBatchFirstSeq == batchFirstSeq && lastBatchSize == batchSize { // is a good batch retry
if lastSequenceWrittenToDisk == (batchFirstSeq + batchSize - 1) { // we already have the messages
return prodDuplicate
}
// mock write to disk
lastSequenceWrittenToDisk = batchFirstSeq + batchSize - 1
return prodSuccessResponse
} else { //out of sequence / bad retried batch
if lastBatchFirstSeq == batchFirstSeq && lastBatchSize != batchSize {
t.Errorf("[%s] Retried Batch firstSeq=%d with different size old=%d new=%d", test.name, batchFirstSeq, lastBatchSize, batchSize)
} else if lastSequenceWrittenToDisk+1 != batchFirstSeq {
t.Errorf("[%s] Out of sequence message lastSequence=%d new batch starts at=%d", test.name, lastSequenceWrittenToDisk, batchFirstSeq)
} else {
t.Errorf("[%s] Unexpected error", test.name)
}
return prodOutOfSeq
}
}
}
return nil
}
config := NewConfig()
config.Version = V0_11_0_0
config.Producer.Idempotent = true
config.Net.MaxOpenRequests = 1
config.Producer.RequiredAcks = WaitForAll
config.Producer.Return.Successes = true
config.Producer.Flush.Frequency = 50 * time.Millisecond
config.Producer.Retry.Backoff = 100 * time.Millisecond
broker.setHandler(handlerFailBeforeWrite)
producer, err := NewAsyncProducer([]string{broker.Addr()}, config)
if err != nil {
t.Fatal(err)
}
for i := 0; i < 3; i++ {
producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
}
go func() {
for i := 0; i < 7; i++ {
producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder("goroutine")}
time.Sleep(100 * time.Millisecond)
}
}()
expectResults(t, producer, 10, 0)
broker.Close()
closeProducer(t, producer)
}
}
func TestAsyncProducerIdempotentErrorOnOutOfSeq(t *testing.T) {
broker := NewMockBroker(t, 1)
metadataResponse := &MetadataResponse{
Version: 1,
ControllerID: 1,
}
metadataResponse.AddBroker(broker.Addr(), broker.BrokerID())
metadataResponse.AddTopicPartition("my_topic", 0, broker.BrokerID(), nil, nil, ErrNoError)
broker.Returns(metadataResponse)
initProducerID := &InitProducerIDResponse{
ThrottleTime: 0,
ProducerID: 1000,
ProducerEpoch: 1,
}
broker.Returns(initProducerID)
config := NewConfig()
config.Producer.Flush.Messages = 10
config.Producer.Return.Successes = true
config.Producer.Retry.Max = 400000
config.Producer.RequiredAcks = WaitForAll
config.Producer.Retry.Backoff = 0
config.Producer.Idempotent = true
config.Net.MaxOpenRequests = 1
config.Version = V0_11_0_0
producer, err := NewAsyncProducer([]string{broker.Addr()}, config)
if err != nil {
t.Fatal(err)
}
for i := 0; i < 10; i++ {
producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
}
prodOutOfSeq := &ProduceResponse{
Version: 3,
ThrottleTime: 0,
}
prodOutOfSeq.AddTopicPartition("my_topic", 0, ErrOutOfOrderSequenceNumber)
broker.Returns(prodOutOfSeq)
expectResults(t, producer, 0, 10)
broker.Close()
closeProducer(t, producer)
}
// This example shows how to use the producer while simultaneously
// reading the Errors channel to know about any failures.
func ExampleAsyncProducer_select() {

129
third/github.com/Shopify/sarama/balance_strategy.go Normal file
View File

@ -0,0 +1,129 @@
package sarama
import (
"math"
"sort"
)
// BalanceStrategyPlan is the results of any BalanceStrategy.Plan attempt.
// It contains an allocation of topic/partitions by memberID in the form of
// a `memberID -> topic -> partitions` map.
type BalanceStrategyPlan map[string]map[string][]int32
// Add assigns a topic with a number partitions to a member.
func (p BalanceStrategyPlan) Add(memberID, topic string, partitions ...int32) {
if len(partitions) == 0 {
return
}
if _, ok := p[memberID]; !ok {
p[memberID] = make(map[string][]int32, 1)
}
p[memberID][topic] = append(p[memberID][topic], partitions...)
}
// --------------------------------------------------------------------
// BalanceStrategy is used to balance topics and partitions
// across memebers of a consumer group
type BalanceStrategy interface {
// Name uniquely identifies the strategy.
Name() string
// Plan accepts a map of `memberID -> metadata` and a map of `topic -> partitions`
// and returns a distribution plan.
Plan(members map[string]ConsumerGroupMemberMetadata, topics map[string][]int32) (BalanceStrategyPlan, error)
}
// --------------------------------------------------------------------
// BalanceStrategyRange is the default and assigns partitions as ranges to consumer group members.
// Example with one topic T with six partitions (0..5) and two members (M1, M2):
// M1: {T: [0, 1, 2]}
// M2: {T: [3, 4, 5]}
var BalanceStrategyRange = &balanceStrategy{
name: "range",
coreFn: func(plan BalanceStrategyPlan, memberIDs []string, topic string, partitions []int32) {
step := float64(len(partitions)) / float64(len(memberIDs))
for i, memberID := range memberIDs {
pos := float64(i)
min := int(math.Floor(pos*step + 0.5))
max := int(math.Floor((pos+1)*step + 0.5))
plan.Add(memberID, topic, partitions[min:max]...)
}
},
}
// BalanceStrategyRoundRobin assigns partitions to members in alternating order.
// Example with topic T with six partitions (0..5) and two members (M1, M2):
// M1: {T: [0, 2, 4]}
// M2: {T: [1, 3, 5]}
var BalanceStrategyRoundRobin = &balanceStrategy{
name: "roundrobin",
coreFn: func(plan BalanceStrategyPlan, memberIDs []string, topic string, partitions []int32) {
for i, part := range partitions {
memberID := memberIDs[i%len(memberIDs)]
plan.Add(memberID, topic, part)
}
},
}
// --------------------------------------------------------------------
type balanceStrategy struct {
name string
coreFn func(plan BalanceStrategyPlan, memberIDs []string, topic string, partitions []int32)
}
// Name implements BalanceStrategy.
func (s *balanceStrategy) Name() string { return s.name }
// Balance implements BalanceStrategy.
func (s *balanceStrategy) Plan(members map[string]ConsumerGroupMemberMetadata, topics map[string][]int32) (BalanceStrategyPlan, error) {
// Build members by topic map
mbt := make(map[string][]string)
for memberID, meta := range members {
for _, topic := range meta.Topics {
mbt[topic] = append(mbt[topic], memberID)
}
}
// Sort members for each topic
for topic, memberIDs := range mbt {
sort.Sort(&balanceStrategySortable{
topic: topic,
memberIDs: memberIDs,
})
}
// Assemble plan
plan := make(BalanceStrategyPlan, len(members))
for topic, memberIDs := range mbt {
s.coreFn(plan, memberIDs, topic, topics[topic])
}
return plan, nil
}
type balanceStrategySortable struct {
topic string
memberIDs []string
}
func (p balanceStrategySortable) Len() int { return len(p.memberIDs) }
func (p balanceStrategySortable) Swap(i, j int) {
p.memberIDs[i], p.memberIDs[j] = p.memberIDs[j], p.memberIDs[i]
}
func (p balanceStrategySortable) Less(i, j int) bool {
return balanceStrategyHashValue(p.topic, p.memberIDs[i]) < balanceStrategyHashValue(p.topic, p.memberIDs[j])
}
func balanceStrategyHashValue(vv ...string) uint32 {
h := uint32(2166136261)
for _, s := range vv {
for _, c := range s {
h ^= uint32(c)
h *= 16777619
}
}
return h
}

102
third/github.com/Shopify/sarama/balance_strategy_test.go Normal file
View File

@ -0,0 +1,102 @@
package sarama
import (
"reflect"
"testing"
)
func TestBalanceStrategyRange(t *testing.T) {
tests := []struct {
members map[string][]string
topics map[string][]int32
expected BalanceStrategyPlan
}{
{
members: map[string][]string{"M1": {"T1", "T2"}, "M2": {"T1", "T2"}},
topics: map[string][]int32{"T1": {0, 1, 2, 3}, "T2": {0, 1, 2, 3}},
expected: BalanceStrategyPlan{
"M1": map[string][]int32{"T1": {0, 1}, "T2": {2, 3}},
"M2": map[string][]int32{"T1": {2, 3}, "T2": {0, 1}},
},
},
{
members: map[string][]string{"M1": {"T1", "T2"}, "M2": {"T1", "T2"}},
topics: map[string][]int32{"T1": {0, 1, 2}, "T2": {0, 1, 2}},
expected: BalanceStrategyPlan{
"M1": map[string][]int32{"T1": {0, 1}, "T2": {2}},
"M2": map[string][]int32{"T1": {2}, "T2": {0, 1}},
},
},
{
members: map[string][]string{"M1": {"T1"}, "M2": {"T1", "T2"}},
topics: map[string][]int32{"T1": {0, 1}, "T2": {0, 1}},
expected: BalanceStrategyPlan{
"M1": map[string][]int32{"T1": {0}},
"M2": map[string][]int32{"T1": {1}, "T2": {0, 1}},
},
},
}
strategy := BalanceStrategyRange
if strategy.Name() != "range" {
t.Errorf("Unexpected stategy name\nexpected: range\nactual: %v", strategy.Name())
}
for _, test := range tests {
members := make(map[string]ConsumerGroupMemberMetadata)
for memberID, topics := range test.members {
members[memberID] = ConsumerGroupMemberMetadata{Topics: topics}
}
actual, err := strategy.Plan(members, test.topics)
if err != nil {
t.Errorf("Unexpected error %v", err)
} else if !reflect.DeepEqual(actual, test.expected) {
t.Errorf("Plan does not match expectation\nexpected: %#v\nactual: %#v", test.expected, actual)
}
}
}
func TestBalanceStrategyRoundRobin(t *testing.T) {
tests := []struct {
members map[string][]string
topics map[string][]int32
expected BalanceStrategyPlan
}{
{
members: map[string][]string{"M1": {"T1", "T2"}, "M2": {"T1", "T2"}},
topics: map[string][]int32{"T1": {0, 1, 2, 3}, "T2": {0, 1, 2, 3}},
expected: BalanceStrategyPlan{
"M1": map[string][]int32{"T1": {0, 2}, "T2": {1, 3}},
"M2": map[string][]int32{"T1": {1, 3}, "T2": {0, 2}},
},
},
{
members: map[string][]string{"M1": {"T1", "T2"}, "M2": {"T1", "T2"}},
topics: map[string][]int32{"T1": {0, 1, 2}, "T2": {0, 1, 2}},
expected: BalanceStrategyPlan{
"M1": map[string][]int32{"T1": {0, 2}, "T2": {1}},
"M2": map[string][]int32{"T1": {1}, "T2": {0, 2}},
},
},
}
strategy := BalanceStrategyRoundRobin
if strategy.Name() != "roundrobin" {
t.Errorf("Unexpected stategy name\nexpected: range\nactual: %v", strategy.Name())
}
for _, test := range tests {
members := make(map[string]ConsumerGroupMemberMetadata)
for memberID, topics := range test.members {
members[memberID] = ConsumerGroupMemberMetadata{Topics: topics}
}
actual, err := strategy.Plan(members, test.topics)
if err != nil {
t.Errorf("Unexpected error %v", err)
} else if !reflect.DeepEqual(actual, test.expected) {
t.Errorf("Plan does not match expectation\nexpected: %#v\nactual: %#v", test.expected, actual)
}
}
}

12
third/github.com/Shopify/sarama/broker.go
View File

@ -86,6 +86,7 @@ func (b *Broker) Open(conf *Config) error {
dialer := net.Dialer{
Timeout: conf.Net.DialTimeout,
KeepAlive: conf.Net.KeepAlive,
LocalAddr: conf.Net.LocalAddr,
}
if conf.Net.TLS.Enable {
@ -207,6 +208,17 @@ func (b *Broker) Addr() string {
return b.addr
}
// Rack returns the broker's rack as retrieved from Kafka's metadata or the
// empty string if it is not known. The returned value corresponds to the
// broker's broker.rack configuration setting. Requires protocol version to be
// at least v0.10.0.0.
func (b *Broker) Rack() string {
if b.rack == nil {
return ""
}
return *b.rack
}
func (b *Broker) GetMetadata(request *MetadataRequest) (*MetadataResponse, error) {
response := new(MetadataResponse)

10
third/github.com/Shopify/sarama/broker_test.go
View File

@ -51,10 +51,20 @@ func TestBrokerAccessors(t *testing.T) {
t.Error("New broker didn't have the correct address")
}
if broker.Rack() != "" {
t.Error("New broker didn't have an unknown rack.")
}
broker.id = 34
if broker.ID() != 34 {
t.Error("Manually setting broker ID did not take effect.")
}
rack := "dc1"
broker.rack = &rack
if broker.Rack() != rack {
t.Error("Manually setting broker rack did not take effect.")
}
}
func TestSimpleBrokerCommunication(t *testing.T) {

69
third/github.com/Shopify/sarama/client.go
View File

@ -17,7 +17,7 @@ type Client interface {
// altered after it has been created.
Config() *Config
// Controller returns the cluster controller broker.
// Controller returns the cluster controller broker. Requires Kafka 0.10 or higher.
Controller() (*Broker, error)
// Brokers returns the current set of active brokers as retrieved from cluster metadata.
@ -67,6 +67,9 @@ type Client interface {
// in local cache. This function only works on Kafka 0.8.2 and higher.
RefreshCoordinator(consumerGroup string) error
// InitProducerID retrieves information required for Idempotent Producer
InitProducerID() (*InitProducerIDResponse, error)
// Close shuts down all broker connections managed by this client. It is required
// to call this function before a client object passes out of scope, as it will
// otherwise leak memory. You must close any Producers or Consumers using a client
@ -100,10 +103,11 @@ type client struct {
seedBrokers []*Broker
deadSeeds []*Broker
controllerID int32 // cluster controller broker id
brokers map[int32]*Broker // maps broker ids to brokers
metadata map[string]map[int32]*PartitionMetadata // maps topics to partition ids to metadata
coordinators map[string]int32 // Maps consumer group names to coordinating broker IDs
controllerID int32 // cluster controller broker id
brokers map[int32]*Broker // maps broker ids to brokers
metadata map[string]map[int32]*PartitionMetadata // maps topics to partition ids to metadata
metadataTopics map[string]none // topics that need to collect metadata
coordinators map[string]int32 // Maps consumer group names to coordinating broker IDs
// If the number of partitions is large, we can get some churn calling cachedPartitions,
// so the result is cached. It is important to update this value whenever metadata is changed
@ -136,6 +140,7 @@ func NewClient(addrs []string, conf *Config) (Client, error) {
closed: make(chan none),
brokers: make(map[int32]*Broker),
metadata: make(map[string]map[int32]*PartitionMetadata),
metadataTopics: make(map[string]none),
cachedPartitionsResults: make(map[string][maxPartitionIndex][]int32),
coordinators: make(map[string]int32),
}
@ -174,13 +179,33 @@ func (client *client) Config() *Config {
func (client *client) Brokers() []*Broker {
client.lock.RLock()
defer client.lock.RUnlock()
brokers := make([]*Broker, 0)
brokers := make([]*Broker, 0, len(client.brokers))
for _, broker := range client.brokers {
brokers = append(brokers, broker)
}
return brokers
}
func (client *client) InitProducerID() (*InitProducerIDResponse, error) {
var err error
for broker := client.any(); broker != nil; broker = client.any() {
req := &InitProducerIDRequest{}
response, err := broker.InitProducerID(req)
switch err.(type) {
case nil:
return response, nil
default:
// some error, remove that broker and try again
Logger.Printf("Client got error from broker %d when issuing InitProducerID : %v\n", broker.ID(), err)
_ = broker.Close()
client.deregisterBroker(broker)
}
}
return nil, err
}
func (client *client) Close() error {
if client.Closed() {
// Chances are this is being called from a defer() and the error will go unobserved
@ -207,6 +232,7 @@ func (client *client) Close() error {
client.brokers = nil
client.metadata = nil
client.metadataTopics = nil
return nil
}
@ -231,6 +257,22 @@ func (client *client) Topics() ([]string, error) {
return ret, nil
}
func (client *client) MetadataTopics() ([]string, error) {
if client.Closed() {
return nil, ErrClosedClient
}
client.lock.RLock()
defer client.lock.RUnlock()
ret := make([]string, 0, len(client.metadataTopics))
for topic := range client.metadataTopics {
ret = append(ret, topic)
}
return ret, nil
}
func (client *client) Partitions(topic string) ([]int32, error) {
if client.Closed() {
return nil, ErrClosedClient
@ -388,6 +430,10 @@ func (client *client) Controller() (*Broker, error) {
return nil, ErrClosedClient
}
if !client.conf.Version.IsAtLeast(V0_10_0_0) {
return nil, ErrUnsupportedVersion
}
controller := client.cachedController()
if controller == nil {
if err := client.refreshMetadata(); err != nil {
@ -645,7 +691,7 @@ func (client *client) refreshMetadata() error {
topics := []string{}
if !client.conf.Metadata.Full {
if specificTopics, err := client.Topics(); err != nil {
if specificTopics, err := client.MetadataTopics(); err != nil {
return err
} else if len(specificTopics) == 0 {
return ErrNoTopicsToUpdateMetadata
@ -700,7 +746,7 @@ func (client *client) tryRefreshMetadata(topics []string, attemptsRemaining int)
return err
default:
// some other error, remove that broker and try again
Logger.Println("client/metadata got error from broker while fetching metadata:", err)
Logger.Printf("client/metadata got error from broker %d while fetching metadata: %v\n", broker.ID(), err)
_ = broker.Close()
client.deregisterBroker(broker)
}
@ -728,9 +774,16 @@ func (client *client) updateMetadata(data *MetadataResponse, allKnownMetaData bo
if allKnownMetaData {
client.metadata = make(map[string]map[int32]*PartitionMetadata)
client.metadataTopics = make(map[string]none)
client.cachedPartitionsResults = make(map[string][maxPartitionIndex][]int32)
}
for _, topic := range data.Topics {
// topics must be added firstly to `metadataTopics` to guarantee that all
// requested topics must be recorded to keep them trackable for periodically
// metadata refresh.
if _, exists := client.metadataTopics[topic.Name]; !exists {
client.metadataTopics[topic.Name] = none{}
}
delete(client.metadata, topic.Name)
delete(client.cachedPartitionsResults, topic.Name)

5
third/github.com/Shopify/sarama/client_test.go
View File

@ -481,10 +481,11 @@ func TestClientController(t *testing.T) {
t.Fatal(err)
}
defer safeClose(t, client2)
if _, err = client2.Controller(); err != ErrControllerNotAvailable {
t.Errorf("Expected Contoller() to return %s, found %s", ErrControllerNotAvailable, err)
if _, err = client2.Controller(); err != ErrUnsupportedVersion {
t.Errorf("Expected Contoller() to return %s, found %s", ErrUnsupportedVersion, err)
}
}
func TestClientCoordinatorWithConsumerOffsetsTopic(t *testing.T) {
seedBroker := NewMockBroker(t, 1)
staleCoordinator := NewMockBroker(t, 2)

12
third/github.com/Shopify/sarama/client_tls_test.go
View File

@ -33,12 +33,12 @@ func TestTLS(t *testing.T) {
nva := time.Now().Add(1 * time.Hour)
caTemplate := &x509.Certificate{
Subject: pkix.Name{CommonName: "ca"},
Issuer: pkix.Name{CommonName: "ca"},
SerialNumber: big.NewInt(0),
NotAfter: nva,
NotBefore: nvb,
IsCA: true,
Subject: pkix.Name{CommonName: "ca"},
Issuer: pkix.Name{CommonName: "ca"},
SerialNumber: big.NewInt(0),
NotAfter: nva,
NotBefore: nvb,
IsCA: true,
BasicConstraintsValid: true,
KeyUsage: x509.KeyUsageCertSign,
}

75
third/github.com/Shopify/sarama/compress.go Normal file
View File

@ -0,0 +1,75 @@
package sarama
import (
"bytes"
"compress/gzip"
"fmt"
"sync"
"gitee.com/johng/gf/third/github.com/eapache/go-xerial-snappy"
"gitee.com/johng/gf/third/github.com/pierrec/lz4"
)
var (
lz4WriterPool = sync.Pool{
New: func() interface{} {
return lz4.NewWriter(nil)
},
}
gzipWriterPool = sync.Pool{
New: func() interface{} {
return gzip.NewWriter(nil)
},
}
)
func compress(cc CompressionCodec, level int, data []byte) ([]byte, error) {
switch cc {
case CompressionNone:
return data, nil
case CompressionGZIP:
var (
err error
buf bytes.Buffer
writer *gzip.Writer
)
if level != CompressionLevelDefault {
writer, err = gzip.NewWriterLevel(&buf, level)
if err != nil {
return nil, err
}
} else {
writer = gzipWriterPool.Get().(*gzip.Writer)
defer gzipWriterPool.Put(writer)
writer.Reset(&buf)
}
if _, err := writer.Write(data); err != nil {
return nil, err
}
if err := writer.Close(); err != nil {
return nil, err
}
return buf.Bytes(), nil
case CompressionSnappy:
return snappy.Encode(data), nil
case CompressionLZ4:
writer := lz4WriterPool.Get().(*lz4.Writer)
defer lz4WriterPool.Put(writer)
var buf bytes.Buffer
writer.Reset(&buf)
if _, err := writer.Write(data); err != nil {
return nil, err
}
if err := writer.Close(); err != nil {
return nil, err
}
return buf.Bytes(), nil
case CompressionZSTD:
return zstdCompressLevel(nil, data, level)
default:
return nil, PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", cc)}
}
}

137
third/github.com/Shopify/sarama/config.go
View File

@ -5,6 +5,7 @@ import (
"crypto/tls"
"fmt"
"io/ioutil"
"net"
"regexp"
"time"
@ -17,6 +18,13 @@ var validID = regexp.MustCompile(`\A[A-Za-z0-9._-]+\z`)
// Config is used to pass multiple configuration options to Sarama's constructors.
type Config struct {
// Admin is the namespace for ClusterAdmin properties used by the administrative Kafka client.
Admin struct {
// The maximum duration the administrative Kafka client will wait for ClusterAdmin operations,
// including topics, brokers, configurations and ACLs (defaults to 3 seconds).
Timeout time.Duration
}
// Net is the namespace for network-level properties used by the Broker, and
// shared by the Client/Producer/Consumer.
Net struct {
@ -58,6 +66,12 @@ type Config struct {
// KeepAlive specifies the keep-alive period for an active network connection.
// If zero, keep-alives are disabled. (default is 0: disabled).
KeepAlive time.Duration
// LocalAddr is the local address to use when dialing an
// address. The address must be of a compatible type for the
// network being dialed.
// If nil, a local address is automatically chosen.
LocalAddr net.Addr
}
// Metadata is the namespace for metadata management properties used by the
@ -110,6 +124,9 @@ type Config struct {
// (defaults to hashing the message key). Similar to the `partitioner.class`
// setting for the JVM producer.
Partitioner PartitionerConstructor
// If enabled, the producer will ensure that exactly one copy of each message is
// written.
Idempotent bool
// Return specifies what channels will be populated. If they are set to true,
// you must read from the respective channels to prevent deadlock. If,
@ -159,14 +176,55 @@ type Config struct {
// Consumer is the namespace for configuration related to consuming messages,
// used by the Consumer.
//
// Note that Sarama's Consumer type does not currently support automatic
// consumer-group rebalancing and offset tracking. For Zookeeper-based
// tracking (Kafka 0.8.2 and earlier), the https://github.com/wvanbergen/kafka
// library builds on Sarama to add this support. For Kafka-based tracking
// (Kafka 0.9 and later), the https://github.com/bsm/sarama-cluster library
// builds on Sarama to add this support.
Consumer struct {
// Group is the namespace for configuring consumer group.
Group struct {
Session struct {
// The timeout used to detect consumer failures when using Kafka's group management facility.
// The consumer sends periodic heartbeats to indicate its liveness to the broker.
// If no heartbeats are received by the broker before the expiration of this session timeout,
// then the broker will remove this consumer from the group and initiate a rebalance.
// Note that the value must be in the allowable range as configured in the broker configuration
// by `group.min.session.timeout.ms` and `group.max.session.timeout.ms` (default 10s)
Timeout time.Duration
}
Heartbeat struct {
// The expected time between heartbeats to the consumer coordinator when using Kafka's group
// management facilities. Heartbeats are used to ensure that the consumer's session stays active and
// to facilitate rebalancing when new consumers join or leave the group.
// The value must be set lower than Consumer.Group.Session.Timeout, but typically should be set no
// higher than 1/3 of that value.
// It can be adjusted even lower to control the expected time for normal rebalances (default 3s)
Interval time.Duration
}
Rebalance struct {
// Strategy for allocating topic partitions to members (default BalanceStrategyRange)
Strategy BalanceStrategy
// The maximum allowed time for each worker to join the group once a rebalance has begun.
// This is basically a limit on the amount of time needed for all tasks to flush any pending
// data and commit offsets. If the timeout is exceeded, then the worker will be removed from
// the group, which will cause offset commit failures (default 60s).
Timeout time.Duration
Retry struct {
// When a new consumer joins a consumer group the set of consumers attempt to "rebalance"
// the load to assign partitions to each consumer. If the set of consumers changes while
// this assignment is taking place the rebalance will fail and retry. This setting controls
// the maximum number of attempts before giving up (default 4).
Max int
// Backoff time between retries during rebalance (default 2s)
Backoff time.Duration
}
}
Member struct {
// Custom metadata to include when joining the group. The user data for all joined members
// can be retrieved by sending a DescribeGroupRequest to the broker that is the
// coordinator for the group.
UserData []byte
}
}
Retry struct {
// How long to wait after a failing to read from a partition before
// trying again (default 2s).
@ -248,6 +306,12 @@ type Config struct {
// broker version 0.9.0 or later.
// (default is 0: disabled).
Retention time.Duration
Retry struct {
// The total number of times to retry failing commit
// requests during OffsetManager shutdown (default 3).
Max int
}
}
}
@ -279,6 +343,8 @@ type Config struct {
func NewConfig() *Config {
c := &Config{}
c.Admin.Timeout = 3 * time.Second
c.Net.MaxOpenRequests = 5
c.Net.DialTimeout = 30 * time.Second
c.Net.ReadTimeout = 30 * time.Second
@ -307,6 +373,14 @@ func NewConfig() *Config {
c.Consumer.Return.Errors = false
c.Consumer.Offsets.CommitInterval = 1 * time.Second
c.Consumer.Offsets.Initial = OffsetNewest
c.Consumer.Offsets.Retry.Max = 3
c.Consumer.Group.Session.Timeout = 10 * time.Second
c.Consumer.Group.Heartbeat.Interval = 3 * time.Second
c.Consumer.Group.Rebalance.Strategy = BalanceStrategyRange
c.Consumer.Group.Rebalance.Timeout = 60 * time.Second
c.Consumer.Group.Rebalance.Retry.Max = 4
c.Consumer.Group.Rebalance.Retry.Backoff = 2 * time.Second
c.ClientID = defaultClientID
c.ChannelBufferSize = 256
@ -355,6 +429,15 @@ func (c *Config) Validate() error {
if c.Consumer.Offsets.Retention%time.Millisecond != 0 {
Logger.Println("Consumer.Offsets.Retention only supports millisecond precision; nanoseconds will be truncated.")
}
if c.Consumer.Group.Session.Timeout%time.Millisecond != 0 {
Logger.Println("Consumer.Group.Session.Timeout only supports millisecond precision; nanoseconds will be truncated.")
}
if c.Consumer.Group.Heartbeat.Interval%time.Millisecond != 0 {
Logger.Println("Consumer.Group.Heartbeat.Interval only supports millisecond precision; nanoseconds will be truncated.")
}
if c.Consumer.Group.Rebalance.Timeout%time.Millisecond != 0 {
Logger.Println("Consumer.Group.Rebalance.Timeout only supports millisecond precision; nanoseconds will be truncated.")
}
if c.ClientID == defaultClientID {
Logger.Println("ClientID is the default of 'sarama', you should consider setting it to something application-specific.")
}
@ -377,6 +460,12 @@ func (c *Config) Validate() error {
return ConfigurationError("Net.SASL.Password must not be empty when SASL is enabled")
}
// validate the Admin values
switch {
case c.Admin.Timeout <= 0:
return ConfigurationError("Admin.Timeout must be > 0")
}
// validate the Metadata values
switch {
case c.Metadata.Retry.Max < 0:
@ -425,6 +514,21 @@ func (c *Config) Validate() error {
}
}
if c.Producer.Idempotent {
if !c.Version.IsAtLeast(V0_11_0_0) {
return ConfigurationError("Idempotent producer requires Version >= V0_11_0_0")
}
if c.Producer.Retry.Max == 0 {
return ConfigurationError("Idempotent producer requires Producer.Retry.Max >= 1")
}
if c.Producer.RequiredAcks != WaitForAll {
return ConfigurationError("Idempotent producer requires Producer.RequiredAcks to be WaitForAll")
}
if c.Net.MaxOpenRequests > 1 {
return ConfigurationError("Idempotent producer requires Net.MaxOpenRequests to be 1")
}
}
// validate the Consumer values
switch {
case c.Consumer.Fetch.Min <= 0:
@ -443,7 +547,26 @@ func (c *Config) Validate() error {
return ConfigurationError("Consumer.Offsets.CommitInterval must be > 0")
case c.Consumer.Offsets.Initial != OffsetOldest && c.Consumer.Offsets.Initial != OffsetNewest:
return ConfigurationError("Consumer.Offsets.Initial must be OffsetOldest or OffsetNewest")
case c.Consumer.Offsets.Retry.Max < 0:
return ConfigurationError("Consumer.Offsets.Retry.Max must be >= 0")
}
// validate the Consumer Group values
switch {
case c.Consumer.Group.Session.Timeout <= 2*time.Millisecond:
return ConfigurationError("Consumer.Group.Session.Timeout must be >= 2ms")
case c.Consumer.Group.Heartbeat.Interval < 1*time.Millisecond:
return ConfigurationError("Consumer.Group.Heartbeat.Interval must be >= 1ms")
case c.Consumer.Group.Heartbeat.Interval >= c.Consumer.Group.Session.Timeout:
return ConfigurationError("Consumer.Group.Heartbeat.Interval must be < Consumer.Group.Session.Timeout")
case c.Consumer.Group.Rebalance.Strategy == nil:
return ConfigurationError("Consumer.Group.Rebalance.Strategy must not be empty")
case c.Consumer.Group.Rebalance.Timeout <= time.Millisecond:
return ConfigurationError("Consumer.Group.Rebalance.Timeout must be >= 1ms")
case c.Consumer.Group.Rebalance.Retry.Max < 0:
return ConfigurationError("Consumer.Group.Rebalance.Retry.Max must be >= 0")
case c.Consumer.Group.Rebalance.Retry.Backoff < 0:
return ConfigurationError("Consumer.Group.Rebalance.Retry.Backoff must be >= 0")
}
// validate misc shared values

50
third/github.com/Shopify/sarama/config_test.go
View File

@ -122,6 +122,28 @@ func TestMetadataConfigValidates(t *testing.T) {
}
}
func TestAdminConfigValidates(t *testing.T) {
tests := []struct {
name string
cfg func(*Config) // resorting to using a function as a param because of internal composite structs
err string
}{
{"Timeout",
func(cfg *Config) {
cfg.Admin.Timeout = 0
},
"Admin.Timeout must be > 0"},
}
for i, test := range tests {
c := NewConfig()
test.cfg(c)
if err := c.Validate(); string(err.(ConfigurationError)) != test.err {
t.Errorf("[%d]:[%s] Expected %s, Got %s\n", i, test.name, test.err, err)
}
}
}
func TestProducerConfigValidates(t *testing.T) {
tests := []struct {
name string
@ -185,6 +207,32 @@ func TestProducerConfigValidates(t *testing.T) {
cfg.Producer.Retry.Backoff = -1
},
"Producer.Retry.Backoff must be >= 0"},
{"Idempotent Version",
func(cfg *Config) {
cfg.Producer.Idempotent = true
cfg.Version = V0_10_0_0
},
"Idempotent producer requires Version >= V0_11_0_0"},
{"Idempotent with Producer.Retry.Max",
func(cfg *Config) {
cfg.Version = V0_11_0_0
cfg.Producer.Idempotent = true
cfg.Producer.Retry.Max = 0
},
"Idempotent producer requires Producer.Retry.Max >= 1"},
{"Idempotent with Producer.RequiredAcks",
func(cfg *Config) {
cfg.Version = V0_11_0_0
cfg.Producer.Idempotent = true
},
"Idempotent producer requires Producer.RequiredAcks to be WaitForAll"},
{"Idempotent with Net.MaxOpenRequests",
func(cfg *Config) {
cfg.Version = V0_11_0_0
cfg.Producer.Idempotent = true
cfg.Producer.RequiredAcks = WaitForAll
},
"Idempotent producer requires Net.MaxOpenRequests to be 1"},
}
for i, test := range tests {
@ -200,7 +248,7 @@ func TestLZ4ConfigValidation(t *testing.T) {
config := NewConfig()
config.Producer.Compression = CompressionLZ4
if err := config.Validate(); string(err.(ConfigurationError)) != "lz4 compression requires Version >= V0_10_0_0" {
t.Error("Expected invalid lz4/kakfa version error, got ", err)
t.Error("Expected invalid lz4/kafka version error, got ", err)
}
config.Version = V0_10_0_0
if err := config.Validate(); err != nil {

774
third/github.com/Shopify/sarama/consumer_group.go Normal file
View File

@ -0,0 +1,774 @@
package sarama
import (
"context"
"errors"
"fmt"
"sort"
"sync"
"time"
)
// ErrClosedConsumerGroup is the error returned when a method is called on a consumer group that has been closed.
var ErrClosedConsumerGroup = errors.New("kafka: tried to use a consumer group that was closed")
// ConsumerGroup is responsible for dividing up processing of topics and partitions
// over a collection of processes (the members of the consumer group).
type ConsumerGroup interface {
// Consume joins a cluster of consumers for a given list of topics and
// starts a blocking ConsumerGroupSession through the ConsumerGroupHandler.
//
// The life-cycle of a session is represented by the following steps:
//
// 1. The consumers join the group (as explained in https://kafka.apache.org/documentation/#intro_consumers)
// and is assigned their "fair share" of partitions, aka 'claims'.
// 2. Before processing starts, the handler's Setup() hook is called to notify the user
// of the claims and allow any necessary preparation or alteration of state.
// 3. For each of the assigned claims the handler's ConsumeClaim() function is then called
// in a separate goroutine which requires it to be thread-safe. Any state must be carefully protected
// from concurrent reads/writes.
// 4. The session will persist until one of the ConsumeClaim() functions exits. This can be either when the
// parent context is cancelled or when a server-side rebalance cycle is initiated.
// 5. Once all the ConsumeClaim() loops have exited, the handler's Cleanup() hook is called
// to allow the user to perform any final tasks before a rebalance.
// 6. Finally, marked offsets are committed one last time before claims are released.
//
// Please note, that once a rebalance is triggered, sessions must be completed within
// Config.Consumer.Group.Rebalance.Timeout. This means that ConsumeClaim() functions must exit
// as quickly as possible to allow time for Cleanup() and the final offset commit. If the timeout
// is exceeded, the consumer will be removed from the group by Kafka, which will cause offset
// commit failures.
Consume(ctx context.Context, topics []string, handler ConsumerGroupHandler) error
// Errors returns a read channel of errors that occurred during the consumer life-cycle.
// By default, errors are logged and not returned over this channel.
// If you want to implement any custom error handling, set your config's
// Consumer.Return.Errors setting to true, and read from this channel.
Errors() <-chan error
// Close stops the ConsumerGroup and detaches any running sessions. It is required to call
// this function before the object passes out of scope, as it will otherwise leak memory.
Close() error
}
type consumerGroup struct {
client Client
ownClient bool
config *Config
consumer Consumer
groupID string
memberID string
errors chan error
lock sync.Mutex
closed chan none
closeOnce sync.Once
}
// NewConsumerGroup creates a new consumer group the given broker addresses and configuration.
func NewConsumerGroup(addrs []string, groupID string, config *Config) (ConsumerGroup, error) {
client, err := NewClient(addrs, config)
if err != nil {
return nil, err
}
c, err := NewConsumerGroupFromClient(groupID, client)
if err != nil {
_ = client.Close()
return nil, err
}
c.(*consumerGroup).ownClient = true
return c, nil
}
// NewConsumerGroupFromClient creates a new consumer group using the given client. It is still
// necessary to call Close() on the underlying client when shutting down this consumer.
// PLEASE NOTE: consumer groups can only re-use but not share clients.
func NewConsumerGroupFromClient(groupID string, client Client) (ConsumerGroup, error) {
config := client.Config()
if !config.Version.IsAtLeast(V0_10_2_0) {
return nil, ConfigurationError("consumer groups require Version to be >= V0_10_2_0")
}
consumer, err := NewConsumerFromClient(client)
if err != nil {
return nil, err
}
return &consumerGroup{
client: client,
consumer: consumer,
config: config,
groupID: groupID,
errors: make(chan error, config.ChannelBufferSize),
closed: make(chan none),
}, nil
}
// Errors implements ConsumerGroup.
func (c *consumerGroup) Errors() <-chan error { return c.errors }
// Close implements ConsumerGroup.
func (c *consumerGroup) Close() (err error) {
c.closeOnce.Do(func() {
close(c.closed)
c.lock.Lock()
defer c.lock.Unlock()
// leave group
if e := c.leave(); e != nil {
err = e
}
// drain errors
go func() {
close(c.errors)
}()
for e := range c.errors {
err = e
}
if c.ownClient {
if e := c.client.Close(); e != nil {
err = e
}
}
})
return
}
// Consume implements ConsumerGroup.
func (c *consumerGroup) Consume(ctx context.Context, topics []string, handler ConsumerGroupHandler) error {
// Ensure group is not closed
select {
case <-c.closed:
return ErrClosedConsumerGroup
default:
}
c.lock.Lock()
defer c.lock.Unlock()
// Quick exit when no topics are provided
if len(topics) == 0 {
return fmt.Errorf("no topics provided")
}
// Refresh metadata for requested topics
if err := c.client.RefreshMetadata(topics...); err != nil {
return err
}
// Get coordinator
coordinator, err := c.client.Coordinator(c.groupID)
if err != nil {
return err
}
// Init session
sess, err := c.newSession(ctx, coordinator, topics, handler, c.config.Consumer.Group.Rebalance.Retry.Max)
if err == ErrClosedClient {
return ErrClosedConsumerGroup
} else if err != nil {
return err
}
// Wait for session exit signal
<-sess.ctx.Done()
// Gracefully release session claims
return sess.release(true)
}
func (c *consumerGroup) newSession(ctx context.Context, coordinator *Broker, topics []string, handler ConsumerGroupHandler, retries int) (*consumerGroupSession, error) {
// Join consumer group
join, err := c.joinGroupRequest(coordinator, topics)
if err != nil {
_ = coordinator.Close()
return nil, err
}
switch join.Err {
case ErrNoError:
c.memberID = join.MemberId
case ErrUnknownMemberId, ErrIllegalGeneration: // reset member ID and retry immediately
c.memberID = ""
return c.newSession(ctx, coordinator, topics, handler, retries)
case ErrRebalanceInProgress: // retry after backoff
if retries <= 0 {
return nil, join.Err
}
select {
case <-c.closed:
return nil, ErrClosedConsumerGroup
case <-time.After(c.config.Consumer.Group.Rebalance.Retry.Backoff):
}
return c.newSession(ctx, coordinator, topics, handler, retries-1)
default:
return nil, join.Err
}
// Prepare distribution plan if we joined as the leader
var plan BalanceStrategyPlan
if join.LeaderId == join.MemberId {
members, err := join.GetMembers()
if err != nil {
return nil, err
}
plan, err = c.balance(members)
if err != nil {
return nil, err
}
}
// Sync consumer group
sync, err := c.syncGroupRequest(coordinator, plan, join.GenerationId)
if err != nil {
_ = coordinator.Close()
return nil, err
}
switch sync.Err {
case ErrNoError:
case ErrUnknownMemberId, ErrIllegalGeneration: // reset member ID and retry immediately
c.memberID = ""
return c.newSession(ctx, coordinator, topics, handler, retries)
case ErrRebalanceInProgress: // retry after backoff
if retries <= 0 {
return nil, sync.Err
}
select {
case <-c.closed:
return nil, ErrClosedConsumerGroup
case <-time.After(c.config.Consumer.Group.Rebalance.Retry.Backoff):
}
return c.newSession(ctx, coordinator, topics, handler, retries-1)
default:
return nil, sync.Err
}
// Retrieve and sort claims
var claims map[string][]int32
if len(sync.MemberAssignment) > 0 {
members, err := sync.GetMemberAssignment()
if err != nil {
return nil, err
}
claims = members.Topics
for _, partitions := range claims {
sort.Sort(int32Slice(partitions))
}
}
return newConsumerGroupSession(ctx, c, claims, join.MemberId, join.GenerationId, handler)
}
func (c *consumerGroup) joinGroupRequest(coordinator *Broker, topics []string) (*JoinGroupResponse, error) {
req := &JoinGroupRequest{
GroupId: c.groupID,
MemberId: c.memberID,
SessionTimeout: int32(c.config.Consumer.Group.Session.Timeout / time.Millisecond),
ProtocolType: "consumer",
}
if c.config.Version.IsAtLeast(V0_10_1_0) {
req.Version = 1
req.RebalanceTimeout = int32(c.config.Consumer.Group.Rebalance.Timeout / time.Millisecond)
}
meta := &ConsumerGroupMemberMetadata{
Topics: topics,
UserData: c.config.Consumer.Group.Member.UserData,
}
strategy := c.config.Consumer.Group.Rebalance.Strategy
if err := req.AddGroupProtocolMetadata(strategy.Name(), meta); err != nil {
return nil, err
}
return coordinator.JoinGroup(req)
}
func (c *consumerGroup) syncGroupRequest(coordinator *Broker, plan BalanceStrategyPlan, generationID int32) (*SyncGroupResponse, error) {
req := &SyncGroupRequest{
GroupId: c.groupID,
MemberId: c.memberID,
GenerationId: generationID,
}
for memberID, topics := range plan {
err := req.AddGroupAssignmentMember(memberID, &ConsumerGroupMemberAssignment{
Topics: topics,
})
if err != nil {
return nil, err
}
}
return coordinator.SyncGroup(req)
}
func (c *consumerGroup) heartbeatRequest(coordinator *Broker, memberID string, generationID int32) (*HeartbeatResponse, error) {
req := &HeartbeatRequest{
GroupId: c.groupID,
MemberId: memberID,
GenerationId: generationID,
}
return coordinator.Heartbeat(req)
}
func (c *consumerGroup) balance(members map[string]ConsumerGroupMemberMetadata) (BalanceStrategyPlan, error) {
topics := make(map[string][]int32)
for _, meta := range members {
for _, topic := range meta.Topics {
topics[topic] = nil
}
}
for topic := range topics {
partitions, err := c.client.Partitions(topic)
if err != nil {
return nil, err
}
topics[topic] = partitions
}
strategy := c.config.Consumer.Group.Rebalance.Strategy
return strategy.Plan(members, topics)
}
// Leaves the cluster, called by Close, protected by lock.
func (c *consumerGroup) leave() error {
if c.memberID == "" {
return nil
}
coordinator, err := c.client.Coordinator(c.groupID)
if err != nil {
return err
}
resp, err := coordinator.LeaveGroup(&LeaveGroupRequest{
GroupId: c.groupID,
MemberId: c.memberID,
})
if err != nil {
_ = coordinator.Close()
return err
}
// Unset memberID
c.memberID = ""
// Check response
switch resp.Err {
case ErrRebalanceInProgress, ErrUnknownMemberId, ErrNoError:
return nil
default:
return resp.Err
}
}
func (c *consumerGroup) handleError(err error, topic string, partition int32) {
select {
case <-c.closed:
return
default:
}
if _, ok := err.(*ConsumerError); !ok && topic != "" && partition > -1 {
err = &ConsumerError{
Topic: topic,
Partition: partition,
Err: err,
}
}
if c.config.Consumer.Return.Errors {
select {
case c.errors <- err:
default:
}
} else {
Logger.Println(err)
}
}
// --------------------------------------------------------------------
// ConsumerGroupSession represents a consumer group member session.
type ConsumerGroupSession interface {
// Claims returns information about the claimed partitions by topic.
Claims() map[string][]int32
// MemberID returns the cluster member ID.
MemberID() string
// GenerationID returns the current generation ID.
GenerationID() int32
// MarkOffset marks the provided offset, alongside a metadata string
// that represents the state of the partition consumer at that point in time. The
// metadata string can be used by another consumer to restore that state, so it
// can resume consumption.
//
// To follow upstream conventions, you are expected to mark the offset of the
// next message to read, not the last message read. Thus, when calling `MarkOffset`
// you should typically add one to the offset of the last consumed message.
//
// Note: calling MarkOffset does not necessarily commit the offset to the backend
// store immediately for efficiency reasons, and it may never be committed if
// your application crashes. This means that you may end up processing the same
// message twice, and your processing should ideally be idempotent.
MarkOffset(topic string, partition int32, offset int64, metadata string)
// ResetOffset resets to the provided offset, alongside a metadata string that
// represents the state of the partition consumer at that point in time. Reset
// acts as a counterpart to MarkOffset, the difference being that it allows to
// reset an offset to an earlier or smaller value, where MarkOffset only
// allows incrementing the offset. cf MarkOffset for more details.
ResetOffset(topic string, partition int32, offset int64, metadata string)
// MarkMessage marks a message as consumed.
MarkMessage(msg *ConsumerMessage, metadata string)
// Context returns the session context.
Context() context.Context
}
type consumerGroupSession struct {
parent *consumerGroup
memberID string
generationID int32
handler ConsumerGroupHandler
claims map[string][]int32
offsets *offsetManager
ctx context.Context
cancel func()
waitGroup sync.WaitGroup
releaseOnce sync.Once
hbDying, hbDead chan none
}
func newConsumerGroupSession(ctx context.Context, parent *consumerGroup, claims map[string][]int32, memberID string, generationID int32, handler ConsumerGroupHandler) (*consumerGroupSession, error) {
// init offset manager
offsets, err := newOffsetManagerFromClient(parent.groupID, memberID, generationID, parent.client)
if err != nil {
return nil, err
}
// init context
ctx, cancel := context.WithCancel(ctx)
// init session
sess := &consumerGroupSession{
parent: parent,
memberID: memberID,
generationID: generationID,
handler: handler,
offsets: offsets,
claims: claims,
ctx: ctx,
cancel: cancel,
hbDying: make(chan none),
hbDead: make(chan none),
}
// start heartbeat loop
go sess.heartbeatLoop()
// create a POM for each claim
for topic, partitions := range claims {
for _, partition := range partitions {
pom, err := offsets.ManagePartition(topic, partition)
if err != nil {
_ = sess.release(false)
return nil, err
}
// handle POM errors
go func(topic string, partition int32) {
for err := range pom.Errors() {
sess.parent.handleError(err, topic, partition)
}
}(topic, partition)
}
}
// perform setup
if err := handler.Setup(sess); err != nil {
_ = sess.release(true)
return nil, err
}
// start consuming
for topic, partitions := range claims {
for _, partition := range partitions {
sess.waitGroup.Add(1)
go func(topic string, partition int32) {
defer sess.waitGroup.Done()
// cancel the as session as soon as the first
// goroutine exits
defer sess.cancel()
// consume a single topic/partition, blocking
sess.consume(topic, partition)
}(topic, partition)
}
}
return sess, nil
}
func (s *consumerGroupSession) Claims() map[string][]int32 { return s.claims }
func (s *consumerGroupSession) MemberID() string { return s.memberID }
func (s *consumerGroupSession) GenerationID() int32 { return s.generationID }
func (s *consumerGroupSession) MarkOffset(topic string, partition int32, offset int64, metadata string) {
if pom := s.offsets.findPOM(topic, partition); pom != nil {
pom.MarkOffset(offset, metadata)
}
}
func (s *consumerGroupSession) ResetOffset(topic string, partition int32, offset int64, metadata string) {
if pom := s.offsets.findPOM(topic, partition); pom != nil {
pom.ResetOffset(offset, metadata)
}
}
func (s *consumerGroupSession) MarkMessage(msg *ConsumerMessage, metadata string) {
s.MarkOffset(msg.Topic, msg.Partition, msg.Offset+1, metadata)
}
func (s *consumerGroupSession) Context() context.Context {
return s.ctx
}
func (s *consumerGroupSession) consume(topic string, partition int32) {
// quick exit if rebalance is due
select {
case <-s.ctx.Done():
return
case <-s.parent.closed:
return
default:
}
// get next offset
offset := s.parent.config.Consumer.Offsets.Initial
if pom := s.offsets.findPOM(topic, partition); pom != nil {
offset, _ = pom.NextOffset()
}
// create new claim
claim, err := newConsumerGroupClaim(s, topic, partition, offset)
if err != nil {
s.parent.handleError(err, topic, partition)
return
}
// handle errors
go func() {
for err := range claim.Errors() {
s.parent.handleError(err, topic, partition)
}
}()
// trigger close when session is done
go func() {
select {
case <-s.ctx.Done():
case <-s.parent.closed:
}
claim.AsyncClose()
}()
// start processing
if err := s.handler.ConsumeClaim(s, claim); err != nil {
s.parent.handleError(err, topic, partition)
}
// ensure consumer is closed & drained
claim.AsyncClose()
for _, err := range claim.waitClosed() {
s.parent.handleError(err, topic, partition)
}
}
func (s *consumerGroupSession) release(withCleanup bool) (err error) {
// signal release, stop heartbeat
s.cancel()
// wait for consumers to exit
s.waitGroup.Wait()
// perform release
s.releaseOnce.Do(func() {
if withCleanup {
if e := s.handler.Cleanup(s); e != nil {
s.parent.handleError(err, "", -1)
err = e
}
}
if e := s.offsets.Close(); e != nil {
err = e
}
close(s.hbDying)
<-s.hbDead
})
return
}
func (s *consumerGroupSession) heartbeatLoop() {
defer close(s.hbDead)
defer s.cancel() // trigger the end of the session on exit
pause := time.NewTicker(s.parent.config.Consumer.Group.Heartbeat.Interval)
defer pause.Stop()
retries := s.parent.config.Metadata.Retry.Max
for {
coordinator, err := s.parent.client.Coordinator(s.parent.groupID)
if err != nil {
if retries <= 0 {
s.parent.handleError(err, "", -1)
return
}
select {
case <-s.hbDying:
return
case <-time.After(s.parent.config.Metadata.Retry.Backoff):
retries--
}
continue
}
resp, err := s.parent.heartbeatRequest(coordinator, s.memberID, s.generationID)
if err != nil {
_ = coordinator.Close()
retries--
continue
}
switch resp.Err {
case ErrNoError:
retries = s.parent.config.Metadata.Retry.Max
case ErrRebalanceInProgress, ErrUnknownMemberId, ErrIllegalGeneration:
return
default:
s.parent.handleError(err, "", -1)
return
}
select {
case <-pause.C:
case <-s.hbDying:
return
}
}
}
// --------------------------------------------------------------------
// ConsumerGroupHandler instances are used to handle individual topic/partition claims.
// It also provides hooks for your consumer group session life-cycle and allow you to
// trigger logic before or after the consume loop(s).
//
// PLEASE NOTE that handlers are likely be called from several goroutines concurrently,
// ensure that all state is safely protected against race conditions.
type ConsumerGroupHandler interface {
// Setup is run at the beginning of a new session, before ConsumeClaim.
Setup(ConsumerGroupSession) error
// Cleanup is run at the end of a session, once all ConsumeClaim goroutines have exited
// but before the offsets are committed for the very last time.
Cleanup(ConsumerGroupSession) error
// ConsumeClaim must start a consumer loop of ConsumerGroupClaim's Messages().
// Once the Messages() channel is closed, the Handler must finish its processing
// loop and exit.
ConsumeClaim(ConsumerGroupSession, ConsumerGroupClaim) error
}
// ConsumerGroupClaim processes Kafka messages from a given topic and partition within a consumer group.
type ConsumerGroupClaim interface {
// Topic returns the consumed topic name.
Topic() string
// Partition returns the consumed partition.
Partition() int32
// InitialOffset returns the initial offset that was used as a starting point for this claim.
InitialOffset() int64
// HighWaterMarkOffset returns the high water mark offset of the partition,
// i.e. the offset that will be used for the next message that will be produced.
// You can use this to determine how far behind the processing is.
HighWaterMarkOffset() int64
// Messages returns the read channel for the messages that are returned by
// the broker. The messages channel will be closed when a new rebalance cycle
// is due. You must finish processing and mark offsets within
// Config.Consumer.Group.Session.Timeout before the topic/partition is eventually
// re-assigned to another group member.
Messages() <-chan *ConsumerMessage
}
type consumerGroupClaim struct {
topic string
partition int32
offset int64
PartitionConsumer
}
func newConsumerGroupClaim(sess *consumerGroupSession, topic string, partition int32, offset int64) (*consumerGroupClaim, error) {
pcm, err := sess.parent.consumer.ConsumePartition(topic, partition, offset)
if err == ErrOffsetOutOfRange {
offset = sess.parent.config.Consumer.Offsets.Initial
pcm, err = sess.parent.consumer.ConsumePartition(topic, partition, offset)
}
if err != nil {
return nil, err
}
go func() {
for err := range pcm.Errors() {
sess.parent.handleError(err, topic, partition)
}
}()
return &consumerGroupClaim{
topic: topic,
partition: partition,
offset: offset,
PartitionConsumer: pcm,
}, nil
}
func (c *consumerGroupClaim) Topic() string { return c.topic }
func (c *consumerGroupClaim) Partition() int32 { return c.partition }
func (c *consumerGroupClaim) InitialOffset() int64 { return c.offset }
// Drains messages and errors, ensures the claim is fully closed.
func (c *consumerGroupClaim) waitClosed() (errs ConsumerErrors) {
go func() {
for range c.Messages() {
}
}()
for err := range c.Errors() {
errs = append(errs, err)
}
return
}

2
third/github.com/Shopify/sarama/consumer_group_members.go
View File

@ -1,5 +1,6 @@
package sarama
//ConsumerGroupMemberMetadata holds the metadata for consumer group
type ConsumerGroupMemberMetadata struct {
Version int16
Topics []string
@ -36,6 +37,7 @@ func (m *ConsumerGroupMemberMetadata) decode(pd packetDecoder) (err error) {
return nil
}
//ConsumerGroupMemberAssignment holds the member assignment for a consume group
type ConsumerGroupMemberAssignment struct {
Version int16
Topics map[string][]int32

58
third/github.com/Shopify/sarama/consumer_group_test.go Normal file
View File

@ -0,0 +1,58 @@
package sarama
import (
"context"
"fmt"
)
type exampleConsumerGroupHandler struct{}
func (exampleConsumerGroupHandler) Setup(_ ConsumerGroupSession) error { return nil }
func (exampleConsumerGroupHandler) Cleanup(_ ConsumerGroupSession) error { return nil }
func (h exampleConsumerGroupHandler) ConsumeClaim(sess ConsumerGroupSession, claim ConsumerGroupClaim) error {
for msg := range claim.Messages() {
fmt.Printf("Message topic:%q partition:%d offset:%d\n", msg.Topic, msg.Partition, msg.Offset)
sess.MarkMessage(msg, "")
}
return nil
}
func ExampleConsumerGroup() {
// Init config, specify appropriate version
config := NewConfig()
config.Version = V1_0_0_0
config.Consumer.Return.Errors = true
// Start with a client
client, err := NewClient([]string{"localhost:9092"}, config)
if err != nil {
panic(err)
}
defer func() { _ = client.Close() }()
// Start a new consumer group
group, err := NewConsumerGroupFromClient("my-group", client)
if err != nil {
panic(err)
}
defer func() { _ = group.Close() }()
// Track errors
go func() {
for err := range group.Errors() {
fmt.Println("ERROR", err)
}
}()
// Iterate over consumer sessions.
ctx := context.Background()
for {
topics := []string{"my-topic"}
handler := exampleConsumerGroupHandler{}
err := group.Consume(ctx, topics, handler)
if err != nil {
panic(err)
}
}
}

1
third/github.com/Shopify/sarama/consumer_metadata_request.go
View File

@ -1,5 +1,6 @@
package sarama
//ConsumerMetadataRequest is used for metadata requests
type ConsumerMetadataRequest struct {
ConsumerGroup string
}

1
third/github.com/Shopify/sarama/consumer_metadata_response.go
View File

@ -5,6 +5,7 @@ import (
"strconv"
)
//ConsumerMetadataResponse holds the reponse for a consumer gorup meta data request
type ConsumerMetadataResponse struct {
Err KError
Coordinator *Broker

63
third/github.com/Shopify/sarama/decompress.go Normal file
View File

@ -0,0 +1,63 @@
package sarama
import (
"bytes"
"compress/gzip"
"fmt"
"io/ioutil"
"sync"
"gitee.com/johng/gf/third/github.com/eapache/go-xerial-snappy"
"gitee.com/johng/gf/third/github.com/pierrec/lz4"
)
var (
lz4ReaderPool = sync.Pool{
New: func() interface{} {
return lz4.NewReader(nil)
},
}
gzipReaderPool sync.Pool
)
func decompress(cc CompressionCodec, data []byte) ([]byte, error) {
switch cc {
case CompressionNone:
return data, nil
case CompressionGZIP:
var (
err error
reader *gzip.Reader
readerIntf = gzipReaderPool.Get()
)
if readerIntf != nil {
reader = readerIntf.(*gzip.Reader)
} else {
reader, err = gzip.NewReader(bytes.NewReader(data))
if err != nil {
return nil, err
}
}
defer gzipReaderPool.Put(reader)
if err := reader.Reset(bytes.NewReader(data)); err != nil {
return nil, err
}
return ioutil.ReadAll(reader)
case CompressionSnappy:
return snappy.Decode(data)
case CompressionLZ4:
reader := lz4ReaderPool.Get().(*lz4.Reader)
defer lz4ReaderPool.Put(reader)
reader.Reset(bytes.NewReader(data))
return ioutil.ReadAll(reader)
case CompressionZSTD:
return zstdDecompress(nil, data)
default:
return nil, PacketDecodingError{fmt.Sprintf("invalid compression specified (%d)", cc)}
}
}

33
third/github.com/Shopify/sarama/describe_configs_request.go
View File

@ -1,15 +1,17 @@
package sarama
type DescribeConfigsRequest struct {
Version int16
Resources []*ConfigResource
IncludeSynonyms bool
}
type ConfigResource struct {
Type ConfigResourceType
Name string
ConfigNames []string
}
type DescribeConfigsRequest struct {
Resources []*ConfigResource
}
func (r *DescribeConfigsRequest) encode(pe packetEncoder) error {
if err := pe.putArrayLength(len(r.Resources)); err != nil {
return err
@ -30,6 +32,10 @@ func (r *DescribeConfigsRequest) encode(pe packetEncoder) error {
}
}
if r.Version >= 1 {
pe.putBool(r.IncludeSynonyms)
}
return nil
}
@ -74,6 +80,14 @@ func (r *DescribeConfigsRequest) decode(pd packetDecoder, version int16) (err er
}
r.Resources[i].ConfigNames = cfnames
}
r.Version = version
if r.Version >= 1 {
b, err := pd.getBool()
if err != nil {
return err
}
r.IncludeSynonyms = b
}
return nil
}
@ -83,9 +97,16 @@ func (r *DescribeConfigsRequest) key() int16 {
}
func (r *DescribeConfigsRequest) version() int16 {
return 0
return r.Version
}
func (r *DescribeConfigsRequest) requiredVersion() KafkaVersion {
return V0_11_0_0
switch r.Version {
case 1:
return V1_0_0_0
case 2:
return V2_0_0_0
default:
return V0_11_0_0
}
}

31
third/github.com/Shopify/sarama/describe_configs_request_test.go
View File

@ -33,23 +33,33 @@ var (
}
singleDescribeConfigsRequestAllConfigs = []byte{
0, 0, 0, 1, // 1 config
2, // a topic
0, 3, 'f', 'o', 'o', // topic name: foo
255, 255, 255, 255, // all configs
}
singleDescribeConfigsRequestAllConfigsv1 = []byte{
0, 0, 0, 1, // 1 config
2, // a topic
0, 3, 'f', 'o', 'o', // topic name: foo
255, 255, 255, 255, // no configs
1, //synoms
}
)
func TestDescribeConfigsRequest(t *testing.T) {
func TestDescribeConfigsRequestv0(t *testing.T) {
var request *DescribeConfigsRequest
request = &DescribeConfigsRequest{
Version: 0,
Resources: []*ConfigResource{},
}
testRequest(t, "no requests", request, emptyDescribeConfigsRequest)
configs := []string{"segment.ms"}
request = &DescribeConfigsRequest{
Version: 0,
Resources: []*ConfigResource{
&ConfigResource{
Type: TopicResource,
@ -62,6 +72,7 @@ func TestDescribeConfigsRequest(t *testing.T) {
testRequest(t, "one config", request, singleDescribeConfigsRequest)
request = &DescribeConfigsRequest{
Version: 0,
Resources: []*ConfigResource{
&ConfigResource{
Type: TopicResource,
@ -78,6 +89,7 @@ func TestDescribeConfigsRequest(t *testing.T) {
testRequest(t, "two configs", request, doubleDescribeConfigsRequest)
request = &DescribeConfigsRequest{
Version: 0,
Resources: []*ConfigResource{
&ConfigResource{
Type: TopicResource,
@ -88,3 +100,20 @@ func TestDescribeConfigsRequest(t *testing.T) {
testRequest(t, "one topic, all configs", request, singleDescribeConfigsRequestAllConfigs)
}
func TestDescribeConfigsRequestv1(t *testing.T) {
var request *DescribeConfigsRequest
request = &DescribeConfigsRequest{
Version: 1,
Resources: []*ConfigResource{
{
Type: TopicResource,
Name: "foo",
},
},
IncludeSynonyms: true,
}
testRequest(t, "one topic, all configs", request, singleDescribeConfigsRequestAllConfigsv1)
}

158
third/github.com/Shopify/sarama/describe_configs_response.go
View File

@ -1,8 +1,41 @@
package sarama
import "time"
import (
"fmt"
"time"
)
type ConfigSource int8
func (s ConfigSource) String() string {
switch s {
case SourceUnknown:
return "Unknown"
case SourceTopic:
return "Topic"
case SourceDynamicBroker:
return "DynamicBroker"
case SourceDynamicDefaultBroker:
return "DynamicDefaultBroker"
case SourceStaticBroker:
return "StaticBroker"
case SourceDefault:
return "Default"
}
return fmt.Sprintf("Source Invalid: %d", int(s))
}
const (
SourceUnknown ConfigSource = 0
SourceTopic ConfigSource = 1
SourceDynamicBroker ConfigSource = 2
SourceDynamicDefaultBroker ConfigSource = 3
SourceStaticBroker ConfigSource = 4
SourceDefault ConfigSource = 5
)
type DescribeConfigsResponse struct {
Version int16
ThrottleTime time.Duration
Resources []*ResourceResponse
}
@ -20,7 +53,15 @@ type ConfigEntry struct {
Value string
ReadOnly bool
Default bool
Source ConfigSource
Sensitive bool
Synonyms []*ConfigSynonym
}
type ConfigSynonym struct {
ConfigName string
ConfigValue string
Source ConfigSource
}
func (r *DescribeConfigsResponse) encode(pe packetEncoder) (err error) {
@ -30,14 +71,16 @@ func (r *DescribeConfigsResponse) encode(pe packetEncoder) (err error) {
}
for _, c := range r.Resources {
if err = c.encode(pe); err != nil {
if err = c.encode(pe, r.Version); err != nil {
return err
}
}
return nil
}
func (r *DescribeConfigsResponse) decode(pd packetDecoder, version int16) (err error) {
r.Version = version
throttleTime, err := pd.getInt32()
if err != nil {
return err
@ -66,14 +109,21 @@ func (r *DescribeConfigsResponse) key() int16 {
}
func (r *DescribeConfigsResponse) version() int16 {
return 0
return r.Version
}
func (r *DescribeConfigsResponse) requiredVersion() KafkaVersion {
return V0_11_0_0
switch r.Version {
case 1:
return V1_0_0_0
case 2:
return V2_0_0_0
default:
return V0_11_0_0
}
}
func (r *ResourceResponse) encode(pe packetEncoder) (err error) {
func (r *ResourceResponse) encode(pe packetEncoder, version int16) (err error) {
pe.putInt16(r.ErrorCode)
if err = pe.putString(r.ErrorMsg); err != nil {
@ -91,7 +141,7 @@ func (r *ResourceResponse) encode(pe packetEncoder) (err error) {
}
for _, c := range r.Configs {
if err = c.encode(pe); err != nil {
if err = c.encode(pe, version); err != nil {
return err
}
}
@ -139,7 +189,7 @@ func (r *ResourceResponse) decode(pd packetDecoder, version int16) (err error) {
return nil
}
func (r *ConfigEntry) encode(pe packetEncoder) (err error) {
func (r *ConfigEntry) encode(pe packetEncoder, version int16) (err error) {
if err = pe.putString(r.Name); err != nil {
return err
}
@ -149,12 +199,32 @@ func (r *ConfigEntry) encode(pe packetEncoder) (err error) {
}
pe.putBool(r.ReadOnly)
pe.putBool(r.Default)
pe.putBool(r.Sensitive)
if version <= 0 {
pe.putBool(r.Default)
pe.putBool(r.Sensitive)
} else {
pe.putInt8(int8(r.Source))
pe.putBool(r.Sensitive)
if err := pe.putArrayLength(len(r.Synonyms)); err != nil {
return err
}
for _, c := range r.Synonyms {
if err = c.encode(pe, version); err != nil {
return err
}
}
}
return nil
}
//https://cwiki.apache.org/confluence/display/KAFKA/KIP-226+-+Dynamic+Broker+Configuration
func (r *ConfigEntry) decode(pd packetDecoder, version int16) (err error) {
if version == 0 {
r.Source = SourceUnknown
}
name, err := pd.getString()
if err != nil {
return err
@ -173,16 +243,78 @@ func (r *ConfigEntry) decode(pd packetDecoder, version int16) (err error) {
}
r.ReadOnly = read
de, err := pd.getBool()
if err != nil {
return err
if version == 0 {
defaultB, err := pd.getBool()
if err != nil {
return err
}
r.Default = defaultB
} else {
source, err := pd.getInt8()
if err != nil {
return err
}
r.Source = ConfigSource(source)
}
r.Default = de
sensitive, err := pd.getBool()
if err != nil {
return err
}
r.Sensitive = sensitive
if version > 0 {
n, err := pd.getArrayLength()
if err != nil {
return err
}
r.Synonyms = make([]*ConfigSynonym, n)
for i := 0; i < n; i++ {
s := &ConfigSynonym{}
if err := s.decode(pd, version); err != nil {
return err
}
r.Synonyms[i] = s
}
}
return nil
}
func (c *ConfigSynonym) encode(pe packetEncoder, version int16) (err error) {
err = pe.putString(c.ConfigName)
if err != nil {
return err
}
err = pe.putString(c.ConfigValue)
if err != nil {
return err
}
pe.putInt8(int8(c.Source))
return nil
}
func (c *ConfigSynonym) decode(pd packetDecoder, version int16) error {
name, err := pd.getString()
if err != nil {
return nil
}
c.ConfigName = name
value, err := pd.getString()
if err != nil {
return nil
}
c.ConfigValue = value
source, err := pd.getInt8()
if err != nil {
return nil
}
c.Source = ConfigSource(source)
return nil
}

119
third/github.com/Shopify/sarama/describe_configs_response_test.go
View File

@ -10,7 +10,7 @@ var (
0, 0, 0, 0, // no configs
}
describeConfigsResponsePopulated = []byte{
describeConfigsResponsePopulatedv0 = []byte{
0, 0, 0, 0, //throttle
0, 0, 0, 1, // response
0, 0, //errorcode
@ -24,9 +24,44 @@ var (
0, // Default
0, // Sensitive
}
describeConfigsResponsePopulatedv1 = []byte{
0, 0, 0, 0, //throttle
0, 0, 0, 1, // response
0, 0, //errorcode
0, 0, //string
2, // topic
0, 3, 'f', 'o', 'o',
0, 0, 0, 1, //configs
0, 10, 's', 'e', 'g', 'm', 'e', 'n', 't', '.', 'm', 's',
0, 4, '1', '0', '0', '0',
0, // ReadOnly
4, // Source
0, // Sensitive
0, 0, 0, 0, // No Synonym
}
describeConfigsResponseWithSynonymv1 = []byte{
0, 0, 0, 0, //throttle
0, 0, 0, 1, // response
0, 0, //errorcode
0, 0, //string
2, // topic
0, 3, 'f', 'o', 'o',
0, 0, 0, 1, //configs
0, 10, 's', 'e', 'g', 'm', 'e', 'n', 't', '.', 'm', 's',
0, 4, '1', '0', '0', '0',
0, // ReadOnly
4, // Source
0, // Sensitive
0, 0, 0, 1, // 1 Synonym
0, 14, 'l', 'o', 'g', '.', 's', 'e', 'g', 'm', 'e', 'n', 't', '.', 'm', 's',
0, 4, '1', '0', '0', '0',
4, // Source
}
)
func TestDescribeConfigsResponse(t *testing.T) {
func TestDescribeConfigsResponsev0(t *testing.T) {
var response *DescribeConfigsResponse
response = &DescribeConfigsResponse{
@ -38,7 +73,7 @@ func TestDescribeConfigsResponse(t *testing.T) {
}
response = &DescribeConfigsResponse{
Resources: []*ResourceResponse{
Version: 0, Resources: []*ResourceResponse{
&ResourceResponse{
ErrorCode: 0,
ErrorMsg: "",
@ -56,5 +91,81 @@ func TestDescribeConfigsResponse(t *testing.T) {
},
},
}
testResponse(t, "response with error", response, describeConfigsResponsePopulated)
testResponse(t, "response with error", response, describeConfigsResponsePopulatedv0)
}
func TestDescribeConfigsResponsev1(t *testing.T) {
var response *DescribeConfigsResponse
response = &DescribeConfigsResponse{
Resources: []*ResourceResponse{},
}
testVersionDecodable(t, "empty", response, describeConfigsResponseEmpty, 0)
if len(response.Resources) != 0 {
t.Error("Expected no groups")
}
response = &DescribeConfigsResponse{
Version: 1,
Resources: []*ResourceResponse{
&ResourceResponse{
ErrorCode: 0,
ErrorMsg: "",
Type: TopicResource,
Name: "foo",
Configs: []*ConfigEntry{
&ConfigEntry{
Name: "segment.ms",
Value: "1000",
ReadOnly: false,
Source: SourceStaticBroker,
Sensitive: false,
Synonyms: []*ConfigSynonym{},
},
},
},
},
}
testResponse(t, "response with error", response, describeConfigsResponsePopulatedv1)
}
func TestDescribeConfigsResponseWithSynonym(t *testing.T) {
var response *DescribeConfigsResponse
response = &DescribeConfigsResponse{
Resources: []*ResourceResponse{},
}
testVersionDecodable(t, "empty", response, describeConfigsResponseEmpty, 0)
if len(response.Resources) != 0 {
t.Error("Expected no groups")
}
response = &DescribeConfigsResponse{
Version: 1,
Resources: []*ResourceResponse{
&ResourceResponse{
ErrorCode: 0,
ErrorMsg: "",
Type: TopicResource,
Name: "foo",
Configs: []*ConfigEntry{
&ConfigEntry{
Name: "segment.ms",
Value: "1000",
ReadOnly: false,
Source: SourceStaticBroker,
Sensitive: false,
Synonyms: []*ConfigSynonym{
{
ConfigName: "log.segment.ms",
ConfigValue: "1000",
Source: SourceStaticBroker,
},
},
},
},
},
},
}
testResponse(t, "response with error", response, describeConfigsResponseWithSynonymv1)
}

2
third/github.com/Shopify/sarama/dev.yml
View File

@ -2,7 +2,7 @@ name: sarama
up:
- go:
version: '1.9'
version: '1.11'
commands:
test:

36
third/github.com/Shopify/sarama/errors.go
View File

@ -145,6 +145,18 @@ const (
ErrSASLAuthenticationFailed KError = 58
ErrUnknownProducerID KError = 59
ErrReassignmentInProgress KError = 60
ErrDelegationTokenAuthDisabled KError = 61
ErrDelegationTokenNotFound KError = 62
ErrDelegationTokenOwnerMismatch KError = 63
ErrDelegationTokenRequestNotAllowed KError = 64
ErrDelegationTokenAuthorizationFailed KError = 65
ErrDelegationTokenExpired KError = 66
ErrInvalidPrincipalType KError = 67
ErrNonEmptyGroup KError = 68
ErrGroupIDNotFound KError = 69
ErrFetchSessionIDNotFound KError = 70
ErrInvalidFetchSessionEpoch KError = 71
ErrListenerNotFound KError = 72
)
func (err KError) Error() string {
@ -275,6 +287,30 @@ func (err KError) Error() string {
return "kafka server: The broker could not locate the producer metadata associated with the Producer ID."
case ErrReassignmentInProgress:
return "kafka server: A partition reassignment is in progress."
case ErrDelegationTokenAuthDisabled:
return "kafka server: Delegation Token feature is not enabled."
case ErrDelegationTokenNotFound:
return "kafka server: Delegation Token is not found on server."
case ErrDelegationTokenOwnerMismatch:
return "kafka server: Specified Principal is not valid Owner/Renewer."
case ErrDelegationTokenRequestNotAllowed:
return "kafka server: Delegation Token requests are not allowed on PLAINTEXT/1-way SSL channels and on delegation token authenticated channels."
case ErrDelegationTokenAuthorizationFailed:
return "kafka server: Delegation Token authorization failed."
case ErrDelegationTokenExpired:
return "kafka server: Delegation Token is expired."
case ErrInvalidPrincipalType:
return "kafka server: Supplied principalType is not supported."
case ErrNonEmptyGroup:
return "kafka server: The group is not empty."
case ErrGroupIDNotFound:
return "kafka server: The group id does not exist."
case ErrFetchSessionIDNotFound:
return "kafka server: The fetch session ID was not found."
case ErrInvalidFetchSessionEpoch:
return "kafka server: The fetch session epoch is invalid."
case ErrListenerNotFound:
return "kafka server: There is no listener on the leader broker that matches the listener on which metadata request was processed."
}
return fmt.Sprintf("Unknown error, how did this happen? Error code = %d", err)

25
third/github.com/Shopify/sarama/fetch_response.go
View File

@ -33,7 +33,7 @@ type FetchResponseBlock struct {
HighWaterMarkOffset int64
LastStableOffset int64
AbortedTransactions []*AbortedTransaction
Records *Records // deprecated: use FetchResponseBlock.Records
Records *Records // deprecated: use FetchResponseBlock.RecordsSet
RecordsSet []*Records
Partial bool
}
@ -104,15 +104,26 @@ func (b *FetchResponseBlock) decode(pd packetDecoder, version int16) (err error)
return err
}
// If we have at least one full records, we skip incomplete ones
if partial && len(b.RecordsSet) > 0 {
break
n, err := records.numRecords()
if err != nil {
return err
}
b.RecordsSet = append(b.RecordsSet, records)
if n > 0 || (partial && len(b.RecordsSet) == 0) {
b.RecordsSet = append(b.RecordsSet, records)
if b.Records == nil {
b.Records = records
if b.Records == nil {
b.Records = records
}
}
overflow, err := records.isOverflow()
if err != nil {
return err
}
if partial || overflow {
break
}
}

83
third/github.com/Shopify/sarama/fetch_response_test.go
View File

@ -27,6 +27,29 @@ var (
0xFF, 0xFF, 0xFF, 0xFF,
0x00, 0x00, 0x00, 0x02, 0x00, 0xEE}
overflowMessageFetchResponse = []byte{
0x00, 0x00, 0x00, 0x01,
0x00, 0x05, 't', 'o', 'p', 'i', 'c',
0x00, 0x00, 0x00, 0x01,
0x00, 0x00, 0x00, 0x05,
0x00, 0x01,
0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x10, 0x10,
0x00, 0x00, 0x00, 0x30,
// messageSet
0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00,
0x00, 0x00, 0x00, 0x10,
// message
0x23, 0x96, 0x4a, 0xf7, // CRC
0x00,
0x00,
0xFF, 0xFF, 0xFF, 0xFF,
0x00, 0x00, 0x00, 0x02, 0x00, 0xEE,
// overflow messageSet
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0x00, 0x00, 0x00, 0xFF,
// overflow bytes
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
oneRecordFetchResponse = []byte{
0x00, 0x00, 0x00, 0x00, // ThrottleTime
0x00, 0x00, 0x00, 0x01, // Number of Topics
@ -148,6 +171,66 @@ func TestOneMessageFetchResponse(t *testing.T) {
}
}
func TestOverflowMessageFetchResponse(t *testing.T) {
response := FetchResponse{}
testVersionDecodable(t, "overflow message", &response, overflowMessageFetchResponse, 0)
if len(response.Blocks) != 1 {
t.Fatal("Decoding produced incorrect number of topic blocks.")
}
if len(response.Blocks["topic"]) != 1 {
t.Fatal("Decoding produced incorrect number of partition blocks for topic.")
}
block := response.GetBlock("topic", 5)
if block == nil {
t.Fatal("GetBlock didn't return block.")
}
if block.Err != ErrOffsetOutOfRange {
t.Error("Decoding didn't produce correct error code.")
}
if block.HighWaterMarkOffset != 0x10101010 {
t.Error("Decoding didn't produce correct high water mark offset.")
}
partial, err := block.Records.isPartial()
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if partial {
t.Error("Decoding detected a partial trailing message where there wasn't one.")
}
overflow, err := block.Records.isOverflow()
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if !overflow {
t.Error("Decoding detected a partial trailing message where there wasn't one.")
}
n, err := block.Records.numRecords()
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if n != 1 {
t.Fatal("Decoding produced incorrect number of messages.")
}
msgBlock := block.Records.MsgSet.Messages[0]
if msgBlock.Offset != 0x550000 {
t.Error("Decoding produced incorrect message offset.")
}
msg := msgBlock.Msg
if msg.Codec != CompressionNone {
t.Error("Decoding produced incorrect message compression.")
}
if msg.Key != nil {
t.Error("Decoding produced message key where there was none.")
}
if !bytes.Equal(msg.Value, []byte{0x00, 0xEE}) {
t.Error("Decoding produced incorrect message value.")
}
}
func TestOneRecordFetchResponse(t *testing.T) {
response := FetchResponse{}
testVersionDecodable(t, "one record", &response, oneRecordFetchResponse, 4)

418
third/github.com/Shopify/sarama/functional_consumer_group_test.go Normal file
View File

@ -0,0 +1,418 @@
// +build go1.9
package sarama
import (
"context"
"fmt"
"log"
"reflect"
"sync"
"sync/atomic"
"testing"
"time"
)
func TestFuncConsumerGroupPartitioning(t *testing.T) {
checkKafkaVersion(t, "0.10.2")
setupFunctionalTest(t)
defer teardownFunctionalTest(t)
groupID := testFuncConsumerGroupID(t)
// start M1
m1 := runTestFuncConsumerGroupMember(t, groupID, "M1", 0, nil)
defer m1.Stop()
m1.WaitForState(2)
m1.WaitForClaims(map[string]int{"test.4": 4})
m1.WaitForHandlers(4)
// start M2
m2 := runTestFuncConsumerGroupMember(t, groupID, "M2", 0, nil, "test.1", "test.4")
defer m2.Stop()
m2.WaitForState(2)
// assert that claims are shared among both members
m1.WaitForClaims(map[string]int{"test.4": 2})
m1.WaitForHandlers(2)
m2.WaitForClaims(map[string]int{"test.1": 1, "test.4": 2})
m2.WaitForHandlers(3)
// shutdown M1, wait for M2 to take over
m1.AssertCleanShutdown()
m2.WaitForClaims(map[string]int{"test.1": 1, "test.4": 4})
m2.WaitForHandlers(5)
// shutdown M2
m2.AssertCleanShutdown()
}
func TestFuncConsumerGroupExcessConsumers(t *testing.T) {
checkKafkaVersion(t, "0.10.2")
setupFunctionalTest(t)
defer teardownFunctionalTest(t)
groupID := testFuncConsumerGroupID(t)
// start members
m1 := runTestFuncConsumerGroupMember(t, groupID, "M1", 0, nil)
defer m1.Stop()
m2 := runTestFuncConsumerGroupMember(t, groupID, "M2", 0, nil)
defer m2.Stop()
m3 := runTestFuncConsumerGroupMember(t, groupID, "M3", 0, nil)
defer m3.Stop()
m4 := runTestFuncConsumerGroupMember(t, groupID, "M4", 0, nil)
defer m4.Stop()
m1.WaitForClaims(map[string]int{"test.4": 1})
m2.WaitForClaims(map[string]int{"test.4": 1})
m3.WaitForClaims(map[string]int{"test.4": 1})
m4.WaitForClaims(map[string]int{"test.4": 1})
// start M5
m5 := runTestFuncConsumerGroupMember(t, groupID, "M5", 0, nil)
defer m5.Stop()
m5.WaitForState(1)
m5.AssertNoErrs()
// assert that claims are shared among both members
m4.AssertCleanShutdown()
m5.WaitForState(2)
m5.WaitForClaims(map[string]int{"test.4": 1})
// shutdown everything
m1.AssertCleanShutdown()
m2.AssertCleanShutdown()
m3.AssertCleanShutdown()
m5.AssertCleanShutdown()
}
func TestFuncConsumerGroupFuzzy(t *testing.T) {
checkKafkaVersion(t, "0.10.2")
setupFunctionalTest(t)
defer teardownFunctionalTest(t)
if err := testFuncConsumerGroupFuzzySeed("test.4"); err != nil {
t.Fatal(err)
}
groupID := testFuncConsumerGroupID(t)
sink := &testFuncConsumerGroupSink{msgs: make(chan testFuncConsumerGroupMessage, 20000)}
waitForMessages := func(t *testing.T, n int) {
t.Helper()
for i := 0; i < 600; i++ {
if sink.Len() >= n {
break
}
time.Sleep(100 * time.Millisecond)
}
if sz := sink.Len(); sz < n {
log.Fatalf("expected to consume %d messages, but consumed %d", n, sz)
}
}
defer runTestFuncConsumerGroupMember(t, groupID, "M1", 1500, sink).Stop()
defer runTestFuncConsumerGroupMember(t, groupID, "M2", 3000, sink).Stop()
defer runTestFuncConsumerGroupMember(t, groupID, "M3", 1500, sink).Stop()
defer runTestFuncConsumerGroupMember(t, groupID, "M4", 200, sink).Stop()
defer runTestFuncConsumerGroupMember(t, groupID, "M5", 100, sink).Stop()
waitForMessages(t, 3000)
defer runTestFuncConsumerGroupMember(t, groupID, "M6", 300, sink).Stop()
defer runTestFuncConsumerGroupMember(t, groupID, "M7", 400, sink).Stop()
defer runTestFuncConsumerGroupMember(t, groupID, "M8", 500, sink).Stop()
defer runTestFuncConsumerGroupMember(t, groupID, "M9", 2000, sink).Stop()
waitForMessages(t, 8000)
defer runTestFuncConsumerGroupMember(t, groupID, "M10", 1000, sink).Stop()
waitForMessages(t, 10000)
defer runTestFuncConsumerGroupMember(t, groupID, "M11", 1000, sink).Stop()
defer runTestFuncConsumerGroupMember(t, groupID, "M12", 2500, sink).Stop()
waitForMessages(t, 12000)
defer runTestFuncConsumerGroupMember(t, groupID, "M13", 1000, sink).Stop()
waitForMessages(t, 15000)
if umap := sink.Close(); len(umap) != 15000 {
dupes := make(map[string][]string)
for k, v := range umap {
if len(v) > 1 {
dupes[k] = v
}
}
t.Fatalf("expected %d unique messages to be consumed but got %d, including %d duplicates:\n%v", 15000, len(umap), len(dupes), dupes)
}
}
// --------------------------------------------------------------------
func testFuncConsumerGroupID(t *testing.T) string {
return fmt.Sprintf("sarama.%s%d", t.Name(), time.Now().UnixNano())
}
func testFuncConsumerGroupFuzzySeed(topic string) error {
client, err := NewClient(kafkaBrokers, nil)
if err != nil {
return err
}
defer func() { _ = client.Close() }()
total := int64(0)
for pn := int32(0); pn < 4; pn++ {
newest, err := client.GetOffset(topic, pn, OffsetNewest)
if err != nil {
return err
}
oldest, err := client.GetOffset(topic, pn, OffsetOldest)
if err != nil {
return err
}
total = total + newest - oldest
}
if total >= 21000 {
return nil
}
producer, err := NewAsyncProducerFromClient(client)
if err != nil {
return err
}
for i := total; i < 21000; i++ {
producer.Input() <- &ProducerMessage{Topic: topic, Value: ByteEncoder([]byte("testdata"))}
}
return producer.Close()
}
type testFuncConsumerGroupMessage struct {
ClientID string
*ConsumerMessage
}
type testFuncConsumerGroupSink struct {
msgs chan testFuncConsumerGroupMessage
count int32
}
func (s *testFuncConsumerGroupSink) Len() int {
if s == nil {
return -1
}
return int(atomic.LoadInt32(&s.count))
}
func (s *testFuncConsumerGroupSink) Push(clientID string, m *ConsumerMessage) {
if s != nil {
s.msgs <- testFuncConsumerGroupMessage{ClientID: clientID, ConsumerMessage: m}
atomic.AddInt32(&s.count, 1)
}
}
func (s *testFuncConsumerGroupSink) Close() map[string][]string {
close(s.msgs)
res := make(map[string][]string)
for msg := range s.msgs {
key := fmt.Sprintf("%s-%d:%d", msg.Topic, msg.Partition, msg.Offset)
res[key] = append(res[key], msg.ClientID)
}
return res
}
type testFuncConsumerGroupMember struct {
ConsumerGroup
clientID string
claims map[string]int
state int32
handlers int32
errs []error
maxMessages int32
isCapped bool
sink *testFuncConsumerGroupSink
t *testing.T
mu sync.RWMutex
}
func runTestFuncConsumerGroupMember(t *testing.T, groupID, clientID string, maxMessages int32, sink *testFuncConsumerGroupSink, topics ...string) *testFuncConsumerGroupMember {
t.Helper()
config := NewConfig()
config.ClientID = clientID
config.Version = V0_10_2_0
config.Consumer.Return.Errors = true
config.Consumer.Offsets.Initial = OffsetOldest
config.Consumer.Group.Rebalance.Timeout = 10 * time.Second
group, err := NewConsumerGroup(kafkaBrokers, groupID, config)
if err != nil {
t.Fatal(err)
return nil
}
if len(topics) == 0 {
topics = []string{"test.4"}
}
member := &testFuncConsumerGroupMember{
ConsumerGroup: group,
clientID: clientID,
claims: make(map[string]int),
maxMessages: maxMessages,
isCapped: maxMessages != 0,
sink: sink,
t: t,
}
go member.loop(topics)
return member
}
func (m *testFuncConsumerGroupMember) AssertCleanShutdown() {
m.t.Helper()
if err := m.Close(); err != nil {
m.t.Fatalf("unexpected error on Close(): %v", err)
}
m.WaitForState(4)
m.WaitForHandlers(0)
m.AssertNoErrs()
}
func (m *testFuncConsumerGroupMember) AssertNoErrs() {
m.t.Helper()
var errs []error
m.mu.RLock()
errs = append(errs, m.errs...)
m.mu.RUnlock()
if len(errs) != 0 {
m.t.Fatalf("unexpected consumer errors: %v", errs)
}
}
func (m *testFuncConsumerGroupMember) WaitForState(expected int32) {
m.t.Helper()
m.waitFor("state", expected, func() (interface{}, error) {
return atomic.LoadInt32(&m.state), nil
})
}
func (m *testFuncConsumerGroupMember) WaitForHandlers(expected int) {
m.t.Helper()
m.waitFor("handlers", expected, func() (interface{}, error) {
return int(atomic.LoadInt32(&m.handlers)), nil
})
}
func (m *testFuncConsumerGroupMember) WaitForClaims(expected map[string]int) {
m.t.Helper()
m.waitFor("claims", expected, func() (interface{}, error) {
m.mu.RLock()
claims := m.claims
m.mu.RUnlock()
return claims, nil
})
}
func (m *testFuncConsumerGroupMember) Stop() { _ = m.Close() }
func (m *testFuncConsumerGroupMember) Setup(s ConsumerGroupSession) error {
// store claims
claims := make(map[string]int)
for topic, partitions := range s.Claims() {
claims[topic] = len(partitions)
}
m.mu.Lock()
m.claims = claims
m.mu.Unlock()
// enter post-setup state
atomic.StoreInt32(&m.state, 2)
return nil
}
func (m *testFuncConsumerGroupMember) Cleanup(s ConsumerGroupSession) error {
// enter post-cleanup state
atomic.StoreInt32(&m.state, 3)
return nil
}
func (m *testFuncConsumerGroupMember) ConsumeClaim(s ConsumerGroupSession, c ConsumerGroupClaim) error {
atomic.AddInt32(&m.handlers, 1)
defer atomic.AddInt32(&m.handlers, -1)
for msg := range c.Messages() {
if n := atomic.AddInt32(&m.maxMessages, -1); m.isCapped && n < 0 {
break
}
s.MarkMessage(msg, "")
m.sink.Push(m.clientID, msg)
}
return nil
}
func (m *testFuncConsumerGroupMember) waitFor(kind string, expected interface{}, factory func() (interface{}, error)) {
m.t.Helper()
deadline := time.NewTimer(60 * time.Second)
defer deadline.Stop()
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
var actual interface{}
for {
var err error
if actual, err = factory(); err != nil {
m.t.Errorf("failed retrieve value, expected %s %#v but received error %v", kind, expected, err)
}
if reflect.DeepEqual(expected, actual) {
return
}
select {
case <-deadline.C:
m.t.Fatalf("ttl exceeded, expected %s %#v but got %#v", kind, expected, actual)
return
case <-ticker.C:
}
}
}
func (m *testFuncConsumerGroupMember) loop(topics []string) {
defer atomic.StoreInt32(&m.state, 4)
go func() {
for err := range m.Errors() {
_ = m.Close()
m.mu.Lock()
m.errs = append(m.errs, err)
m.mu.Unlock()
}
}()
ctx := context.Background()
for {
// set state to pre-consume
atomic.StoreInt32(&m.state, 1)
if err := m.Consume(ctx, topics, m); err == ErrClosedConsumerGroup {
return
} else if err != nil {
m.mu.Lock()
m.errs = append(m.errs, err)
m.mu.Unlock()
return
}
// return if capped
if n := atomic.LoadInt32(&m.maxMessages); m.isCapped && n < 0 {
return
}
}
}

24
third/github.com/Shopify/sarama/functional_consumer_test.go
View File

@ -81,7 +81,7 @@ func TestVersionMatrix(t *testing.T) {
// protocol versions and compressions for the except of LZ4.
testVersions := versionRange(V0_8_2_0)
allCodecsButLZ4 := []CompressionCodec{CompressionNone, CompressionGZIP, CompressionSnappy}
producedMessages := produceMsgs(t, testVersions, allCodecsButLZ4, 17, 100)
producedMessages := produceMsgs(t, testVersions, allCodecsButLZ4, 17, 100, false)
// When/Then
consumeMsgs(t, testVersions, producedMessages)
@ -98,7 +98,20 @@ func TestVersionMatrixLZ4(t *testing.T) {
// and all possible compressions.
testVersions := versionRange(V0_10_0_0)
allCodecs := []CompressionCodec{CompressionNone, CompressionGZIP, CompressionSnappy, CompressionLZ4}
producedMessages := produceMsgs(t, testVersions, allCodecs, 17, 100)
producedMessages := produceMsgs(t, testVersions, allCodecs, 17, 100, false)
// When/Then
consumeMsgs(t, testVersions, producedMessages)
}
func TestVersionMatrixIdempotent(t *testing.T) {
setupFunctionalTest(t)
defer teardownFunctionalTest(t)
// Produce lot's of message with all possible combinations of supported
// protocol versions starting with v0.11 (first where idempotent was supported)
testVersions := versionRange(V0_11_0_0)
producedMessages := produceMsgs(t, testVersions, []CompressionCodec{CompressionNone}, 17, 100, true)
// When/Then
consumeMsgs(t, testVersions, producedMessages)
@ -133,7 +146,7 @@ func versionRange(lower KafkaVersion) []KafkaVersion {
return versions
}
func produceMsgs(t *testing.T, clientVersions []KafkaVersion, codecs []CompressionCodec, flush int, countPerVerCodec int) []*ProducerMessage {
func produceMsgs(t *testing.T, clientVersions []KafkaVersion, codecs []CompressionCodec, flush int, countPerVerCodec int, idempotent bool) []*ProducerMessage {
var wg sync.WaitGroup
var producedMessagesMu sync.Mutex
var producedMessages []*ProducerMessage
@ -145,6 +158,11 @@ func produceMsgs(t *testing.T, clientVersions []KafkaVersion, codecs []Compressi
prodCfg.Producer.Return.Errors = true
prodCfg.Producer.Flush.MaxMessages = flush
prodCfg.Producer.Compression = codec
prodCfg.Producer.Idempotent = idempotent
if idempotent {
prodCfg.Producer.RequiredAcks = WaitForAll
prodCfg.Net.MaxOpenRequests = 1
}
p, err := NewSyncProducer(kafkaBrokers, prodCfg)
if err != nil {

15
third/github.com/Shopify/sarama/length_field.go
View File

@ -5,6 +5,19 @@ import "encoding/binary"
// LengthField implements the PushEncoder and PushDecoder interfaces for calculating 4-byte lengths.
type lengthField struct {
startOffset int
length int32
}
func (l *lengthField) decode(pd packetDecoder) error {
var err error
l.length, err = pd.getInt32()
if err != nil {
return err
}
if l.length > int32(pd.remaining()) {
return ErrInsufficientData
}
return nil
}
func (l *lengthField) saveOffset(in int) {
@ -21,7 +34,7 @@ func (l *lengthField) run(curOffset int, buf []byte) error {
}
func (l *lengthField) check(curOffset int, buf []byte) error {
if uint32(curOffset-l.startOffset-4) != binary.BigEndian.Uint32(buf[l.startOffset:]) {
if int32(curOffset-l.startOffset-4) != l.length {
return PacketDecodingError{"length field invalid"}
}

86
third/github.com/Shopify/sarama/message.go
View File

@ -1,27 +1,22 @@
package sarama
import (
"bytes"
"compress/gzip"
"fmt"
"io/ioutil"
"time"
"gitee.com/johng/gf/third/github.com/eapache/go-xerial-snappy"
"gitee.com/johng/gf/third/github.com/pierrec/lz4"
)
// CompressionCodec represents the various compression codecs recognized by Kafka in messages.
type CompressionCodec int8
// only the last two bits are really used
const compressionCodecMask int8 = 0x03
// The lowest 3 bits contain the compression codec used for the message
const compressionCodecMask int8 = 0x07
const (
CompressionNone CompressionCodec = 0
CompressionGZIP CompressionCodec = 1
CompressionSnappy CompressionCodec = 2
CompressionLZ4 CompressionCodec = 3
CompressionZSTD CompressionCodec = 4
)
func (cc CompressionCodec) String() string {
@ -76,47 +71,12 @@ func (m *Message) encode(pe packetEncoder) error {
payload = m.compressedCache
m.compressedCache = nil
} else if m.Value != nil {
switch m.Codec {
case CompressionNone:
payload = m.Value
case CompressionGZIP:
var buf bytes.Buffer
var writer *gzip.Writer
if m.CompressionLevel != CompressionLevelDefault {
writer, err = gzip.NewWriterLevel(&buf, m.CompressionLevel)
if err != nil {
return err
}
} else {
writer = gzip.NewWriter(&buf)
}
if _, err = writer.Write(m.Value); err != nil {
return err
}
if err = writer.Close(); err != nil {
return err
}
m.compressedCache = buf.Bytes()
payload = m.compressedCache
case CompressionSnappy:
tmp := snappy.Encode(m.Value)
m.compressedCache = tmp
payload = m.compressedCache
case CompressionLZ4:
var buf bytes.Buffer
writer := lz4.NewWriter(&buf)
if _, err = writer.Write(m.Value); err != nil {
return err
}
if err = writer.Close(); err != nil {
return err
}
m.compressedCache = buf.Bytes()
payload = m.compressedCache
default:
return PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", m.Codec)}
payload, err = compress(m.Codec, m.CompressionLevel, m.Value)
if err != nil {
return err
}
m.compressedCache = payload
// Keep in mind the compressed payload size for metric gathering
m.compressedSize = len(payload)
}
@ -172,44 +132,18 @@ func (m *Message) decode(pd packetDecoder) (err error) {
switch m.Codec {
case CompressionNone:
// nothing to do
case CompressionGZIP:
default:
if m.Value == nil {
break
}
reader, err := gzip.NewReader(bytes.NewReader(m.Value))
m.Value, err = decompress(m.Codec, m.Value)
if err != nil {
return err
}
if m.Value, err = ioutil.ReadAll(reader); err != nil {
return err
}
if err := m.decodeSet(); err != nil {
return err
}
case CompressionSnappy:
if m.Value == nil {
break
}
if m.Value, err = snappy.Decode(m.Value); err != nil {
return err
}
if err := m.decodeSet(); err != nil {
return err
}
case CompressionLZ4:
if m.Value == nil {
break
}
reader := lz4.NewReader(bytes.NewReader(m.Value))
if m.Value, err = ioutil.ReadAll(reader); err != nil {
return err
}
if err := m.decodeSet(); err != nil {
return err
}
default:
return PacketDecodingError{fmt.Sprintf("invalid compression specified (%d)", m.Codec)}
}
return pd.pop()

8
third/github.com/Shopify/sarama/message_set.go
View File

@ -47,6 +47,7 @@ func (msb *MessageBlock) decode(pd packetDecoder) (err error) {
type MessageSet struct {
PartialTrailingMessage bool // whether the set on the wire contained an incomplete trailing MessageBlock
OverflowMessage bool // whether the set on the wire contained an overflow message
Messages []*MessageBlock
}
@ -85,7 +86,12 @@ func (ms *MessageSet) decode(pd packetDecoder) (err error) {
case ErrInsufficientData:
// As an optimization the server is allowed to return a partial message at the
// end of the message set. Clients should handle this case. So we just ignore such things.
ms.PartialTrailingMessage = true
if msb.Offset == -1 {
// This is an overflow message caused by chunked down conversion
ms.OverflowMessage = true
} else {
ms.PartialTrailingMessage = true
}
return nil
default:
return err

44
third/github.com/Shopify/sarama/message_test.go
View File

@ -52,6 +52,17 @@ var (
5, 93, 204, 2, // LZ4 checksum
}
emptyZSTDMessage = []byte{
180, 172, 84, 179, // CRC
0x01, // version byte
0x04, // attribute flags: zstd
0, 0, 1, 88, 141, 205, 89, 56, // timestamp
0xFF, 0xFF, 0xFF, 0xFF, // key
0x00, 0x00, 0x00, 0x09, // len
// ZSTD data
0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x00, 0x01, 0x00, 0x00,
}
emptyBulkSnappyMessage = []byte{
180, 47, 53, 209, //CRC
0x00, // magic version byte
@ -86,6 +97,17 @@ var (
112, 185, 52, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 121, 87, 72, 224, 0, 0, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 14, 121, 87, 72, 224, 0, 0, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0,
71, 129, 23, 111, // LZ4 checksum
}
emptyBulkZSTDMessage = []byte{
203, 151, 133, 28, // CRC
0x01, // Version
0x04, // attribute flags (ZSTD)
255, 255, 249, 209, 212, 181, 73, 201, // timestamp
0xFF, 0xFF, 0xFF, 0xFF, // key
0x00, 0x00, 0x00, 0x26, // len
// ZSTD data
0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x34, 0xcd, 0x0, 0x0, 0x78, 0x0, 0x0, 0xe, 0x79, 0x57, 0x48, 0xe0, 0x0, 0x0, 0xff, 0xff, 0xff, 0xff, 0x0, 0x1, 0x3, 0x0, 0x3d, 0xbd, 0x0, 0x3b, 0x15, 0x0, 0xb, 0xd2, 0x34, 0xc1, 0x78,
}
)
func TestMessageEncoding(t *testing.T) {
@ -101,6 +123,12 @@ func TestMessageEncoding(t *testing.T) {
message.Timestamp = time.Unix(1479847795, 0)
message.Version = 1
testEncodable(t, "empty lz4", &message, emptyLZ4Message)
message.Value = []byte{}
message.Codec = CompressionZSTD
message.Timestamp = time.Unix(1479847795, 0)
message.Version = 1
testEncodable(t, "empty zstd", &message, emptyZSTDMessage)
}
func TestMessageDecoding(t *testing.T) {
@ -179,6 +207,22 @@ func TestMessageDecodingBulkLZ4(t *testing.T) {
}
}
func TestMessageDecodingBulkZSTD(t *testing.T) {
message := Message{}
testDecodable(t, "bulk zstd", &message, emptyBulkZSTDMessage)
if message.Codec != CompressionZSTD {
t.Errorf("Decoding produced codec %d, but expected %d.", message.Codec, CompressionZSTD)
}
if message.Key != nil {
t.Errorf("Decoding produced key %+v, but none was expected.", message.Key)
}
if message.Set == nil {
t.Error("Decoding produced no set, but one was expected.")
} else if len(message.Set.Messages) != 2 {
t.Errorf("Decoding produced a set with %d messages, but 2 were expected.", len(message.Set.Messages))
}
}
func TestMessageDecodingVersion1(t *testing.T) {
message := Message{Version: 1}
testDecodable(t, "decoding empty v1 message", &message, emptyV1Message)

2
third/github.com/Shopify/sarama/metadata_request.go
View File

@ -10,7 +10,7 @@ func (r *MetadataRequest) encode(pe packetEncoder) error {
if r.Version < 0 || r.Version > 5 {
return PacketEncodingError{"invalid or unsupported MetadataRequest version field"}
}
if r.Version == 0 || r.Topics != nil || len(r.Topics) > 0 {
if r.Version == 0 || len(r.Topics) > 0 {
err := pe.putArrayLength(len(r.Topics))
if err != nil {
return err

11
third/github.com/Shopify/sarama/metadata_response.go
View File

@ -207,6 +207,10 @@ func (r *MetadataResponse) decode(pd packetDecoder, version int16) (err error) {
}
func (r *MetadataResponse) encode(pe packetEncoder) error {
if r.Version >= 3 {
pe.putInt32(r.ThrottleTimeMs)
}
err := pe.putArrayLength(len(r.Brokers))
if err != nil {
return err
@ -218,6 +222,13 @@ func (r *MetadataResponse) encode(pe packetEncoder) error {
}
}
if r.Version >= 2 {
err := pe.putNullableString(r.ClusterID)
if err != nil {
return err
}
}
if r.Version >= 1 {
pe.putInt32(r.ControllerID)
}

2
third/github.com/Shopify/sarama/mockresponses.go
View File

@ -523,7 +523,7 @@ func (mr *MockOffsetFetchResponse) SetOffset(group, topic string, partition int3
partitions = make(map[int32]*OffsetFetchResponseBlock)
topics[topic] = partitions
}
partitions[partition] = &OffsetFetchResponseBlock{offset, metadata, kerror}
partitions[partition] = &OffsetFetchResponseBlock{offset, 0, metadata, kerror}
return mr
}

3
third/github.com/Shopify/sarama/mocks/async_producer.go
View File

@ -44,6 +44,7 @@ func NewAsyncProducer(t ErrorReporter, config *sarama.Config) *AsyncProducer {
defer func() {
close(mp.successes)
close(mp.errors)
close(mp.closed)
}()
for msg := range mp.input {
@ -86,8 +87,6 @@ func NewAsyncProducer(t ErrorReporter, config *sarama.Config) *AsyncProducer {
mp.t.Errorf("Expected to exhaust all expectations, but %d are left.", len(mp.expectations))
}
mp.l.Unlock()
close(mp.closed)
}()
return mp

8
third/github.com/Shopify/sarama/offset_commit_request.go
View File

@ -52,12 +52,14 @@ type OffsetCommitRequest struct {
// - 0 (kafka 0.8.1 and later)
// - 1 (kafka 0.8.2 and later)
// - 2 (kafka 0.9.0 and later)
// - 3 (kafka 0.11.0 and later)
// - 4 (kafka 2.0.0 and later)
Version int16
blocks map[string]map[int32]*offsetCommitRequestBlock
}
func (r *OffsetCommitRequest) encode(pe packetEncoder) error {
if r.Version < 0 || r.Version > 2 {
if r.Version < 0 || r.Version > 4 {
return PacketEncodingError{"invalid or unsupported OffsetCommitRequest version field"}
}
@ -174,6 +176,10 @@ func (r *OffsetCommitRequest) requiredVersion() KafkaVersion {
return V0_8_2_0
case 2:
return V0_9_0_0
case 3:
return V0_11_0_0
case 4:
return V2_0_0_0
default:
return MinVersion
}

27
third/github.com/Shopify/sarama/offset_commit_request_test.go
View File

@ -1,6 +1,9 @@
package sarama
import "testing"
import (
"fmt"
"testing"
)
var (
offsetCommitRequestNoBlocksV0 = []byte{
@ -76,15 +79,17 @@ func TestOffsetCommitRequestV1(t *testing.T) {
testRequest(t, "one block v1", request, offsetCommitRequestOneBlockV1)
}
func TestOffsetCommitRequestV2(t *testing.T) {
request := new(OffsetCommitRequest)
request.ConsumerGroup = "foobar"
request.ConsumerID = "cons"
request.ConsumerGroupGeneration = 0x1122
request.RetentionTime = 0x4433
request.Version = 2
testRequest(t, "no blocks v2", request, offsetCommitRequestNoBlocksV2)
func TestOffsetCommitRequestV2ToV4(t *testing.T) {
for version := 2; version <= 4; version++ {
request := new(OffsetCommitRequest)
request.ConsumerGroup = "foobar"
request.ConsumerID = "cons"
request.ConsumerGroupGeneration = 0x1122
request.RetentionTime = 0x4433
request.Version = int16(version)
testRequest(t, fmt.Sprintf("no blocks v%d", version), request, offsetCommitRequestNoBlocksV2)
request.AddBlock("topic", 0x5221, 0xDEADBEEF, 0, "metadata")
testRequest(t, "one block v2", request, offsetCommitRequestOneBlockV2)
request.AddBlock("topic", 0x5221, 0xDEADBEEF, 0, "metadata")
testRequest(t, fmt.Sprintf("one block v%d", version), request, offsetCommitRequestOneBlockV2)
}
}

31
third/github.com/Shopify/sarama/offset_commit_response.go
View File

@ -1,7 +1,9 @@
package sarama
type OffsetCommitResponse struct {
Errors map[string]map[int32]KError
Version int16
ThrottleTimeMs int32
Errors map[string]map[int32]KError
}
func (r *OffsetCommitResponse) AddError(topic string, partition int32, kerror KError) {
@ -17,6 +19,9 @@ func (r *OffsetCommitResponse) AddError(topic string, partition int32, kerror KE
}
func (r *OffsetCommitResponse) encode(pe packetEncoder) error {
if r.Version >= 3 {
pe.putInt32(r.ThrottleTimeMs)
}
if err := pe.putArrayLength(len(r.Errors)); err != nil {
return err
}
@ -36,6 +41,15 @@ func (r *OffsetCommitResponse) encode(pe packetEncoder) error {
}
func (r *OffsetCommitResponse) decode(pd packetDecoder, version int16) (err error) {
r.Version = version
if version >= 3 {
r.ThrottleTimeMs, err = pd.getInt32()
if err != nil {
return err
}
}
numTopics, err := pd.getArrayLength()
if err != nil || numTopics == 0 {
return err
@ -77,9 +91,20 @@ func (r *OffsetCommitResponse) key() int16 {
}
func (r *OffsetCommitResponse) version() int16 {
return 0
return r.Version
}
func (r *OffsetCommitResponse) requiredVersion() KafkaVersion {
return MinVersion
switch r.Version {
case 1:
return V0_8_2_0
case 2:
return V0_9_0_0
case 3:
return V0_11_0_0
case 4:
return V2_0_0_0
default:
return MinVersion
}
}

15
third/github.com/Shopify/sarama/offset_commit_response_test.go
View File

@ -1,6 +1,7 @@
package sarama
import (
"fmt"
"testing"
)
@ -22,3 +23,17 @@ func TestNormalOffsetCommitResponse(t *testing.T) {
// unpredictable map traversal order.
testResponse(t, "normal", &response, nil)
}
func TestOffsetCommitResponseWithThrottleTime(t *testing.T) {
for version := 3; version <= 4; version++ {
response := OffsetCommitResponse{
Version: int16(version),
ThrottleTimeMs: 123,
}
response.AddError("t", 0, ErrNotLeaderForPartition)
response.Errors["m"] = make(map[int32]KError)
// The response encoded form cannot be checked for it varies due to
// unpredictable map traversal order.
testResponse(t, fmt.Sprintf("v%d with throttle time", version), &response, nil)
}
}

39
third/github.com/Shopify/sarama/offset_fetch_request.go
View File

@ -1,28 +1,33 @@
package sarama
type OffsetFetchRequest struct {
ConsumerGroup string
Version int16
ConsumerGroup string
partitions map[string][]int32
}
func (r *OffsetFetchRequest) encode(pe packetEncoder) (err error) {
if r.Version < 0 || r.Version > 1 {
if r.Version < 0 || r.Version > 5 {
return PacketEncodingError{"invalid or unsupported OffsetFetchRequest version field"}
}
if err = pe.putString(r.ConsumerGroup); err != nil {
return err
}
if err = pe.putArrayLength(len(r.partitions)); err != nil {
return err
}
for topic, partitions := range r.partitions {
if err = pe.putString(topic); err != nil {
if r.Version >= 2 && r.partitions == nil {
pe.putInt32(-1)
} else {
if err = pe.putArrayLength(len(r.partitions)); err != nil {
return err
}
if err = pe.putInt32Array(partitions); err != nil {
return err
for topic, partitions := range r.partitions {
if err = pe.putString(topic); err != nil {
return err
}
if err = pe.putInt32Array(partitions); err != nil {
return err
}
}
}
return nil
@ -37,7 +42,7 @@ func (r *OffsetFetchRequest) decode(pd packetDecoder, version int16) (err error)
if err != nil {
return err
}
if partitionCount == 0 {
if (partitionCount == 0 && version < 2) || partitionCount < 0 {
return nil
}
r.partitions = make(map[string][]int32)
@ -67,11 +72,25 @@ func (r *OffsetFetchRequest) requiredVersion() KafkaVersion {
switch r.Version {
case 1:
return V0_8_2_0
case 2:
return V0_10_2_0
case 3:
return V0_11_0_0
case 4:
return V2_0_0_0
case 5:
return V2_1_0_0
default:
return MinVersion
}
}
func (r *OffsetFetchRequest) ZeroPartitions() {
if r.partitions == nil && r.Version >= 2 {
r.partitions = make(map[string][]int32)
}
}
func (r *OffsetFetchRequest) AddPartition(topic string, partitionID int32) {
if r.partitions == nil {
r.partitions = make(map[string][]int32)

42
third/github.com/Shopify/sarama/offset_fetch_request_test.go
View File

@ -1,6 +1,9 @@
package sarama
import "testing"
import (
"fmt"
"testing"
)
var (
offsetFetchRequestNoGroupNoPartitions = []byte{
@ -17,15 +20,36 @@ var (
0x00, 0x0D, 't', 'o', 'p', 'i', 'c', 'T', 'h', 'e', 'F', 'i', 'r', 's', 't',
0x00, 0x00, 0x00, 0x01,
0x4F, 0x4F, 0x4F, 0x4F}
offsetFetchRequestAllPartitions = []byte{
0x00, 0x04, 'b', 'l', 'a', 'h',
0xff, 0xff, 0xff, 0xff}
)
func TestOffsetFetchRequest(t *testing.T) {
request := new(OffsetFetchRequest)
testRequest(t, "no group, no partitions", request, offsetFetchRequestNoGroupNoPartitions)
func TestOffsetFetchRequestNoPartitions(t *testing.T) {
for version := 0; version <= 5; version++ {
request := new(OffsetFetchRequest)
request.Version = int16(version)
request.ZeroPartitions()
testRequest(t, fmt.Sprintf("no group, no partitions %d", version), request, offsetFetchRequestNoGroupNoPartitions)
request.ConsumerGroup = "blah"
testRequest(t, "no partitions", request, offsetFetchRequestNoPartitions)
request.AddPartition("topicTheFirst", 0x4F4F4F4F)
testRequest(t, "one partition", request, offsetFetchRequestOnePartition)
request.ConsumerGroup = "blah"
testRequest(t, fmt.Sprintf("no partitions %d", version), request, offsetFetchRequestNoPartitions)
}
}
func TestOffsetFetchRequest(t *testing.T) {
for version := 0; version <= 5; version++ {
request := new(OffsetFetchRequest)
request.Version = int16(version)
request.ConsumerGroup = "blah"
request.AddPartition("topicTheFirst", 0x4F4F4F4F)
testRequest(t, fmt.Sprintf("one partition %d", version), request, offsetFetchRequestOnePartition)
}
}
func TestOffsetFetchRequestAllPartitions(t *testing.T) {
for version := 2; version <= 5; version++ {
request := &OffsetFetchRequest{Version: int16(version), ConsumerGroup: "blah"}
testRequest(t, fmt.Sprintf("all partitions %d", version), request, offsetFetchRequestAllPartitions)
}
}

120
third/github.com/Shopify/sarama/offset_fetch_response.go
View File

@ -1,17 +1,25 @@
package sarama
type OffsetFetchResponseBlock struct {
Offset int64
Metadata string
Err KError
Offset int64
LeaderEpoch int32
Metadata string
Err KError
}
func (b *OffsetFetchResponseBlock) decode(pd packetDecoder) (err error) {
func (b *OffsetFetchResponseBlock) decode(pd packetDecoder, version int16) (err error) {
b.Offset, err = pd.getInt64()
if err != nil {
return err
}
if version >= 5 {
b.LeaderEpoch, err = pd.getInt32()
if err != nil {
return err
}
}
b.Metadata, err = pd.getString()
if err != nil {
return err
@ -26,9 +34,13 @@ func (b *OffsetFetchResponseBlock) decode(pd packetDecoder) (err error) {
return nil
}
func (b *OffsetFetchResponseBlock) encode(pe packetEncoder) (err error) {
func (b *OffsetFetchResponseBlock) encode(pe packetEncoder, version int16) (err error) {
pe.putInt64(b.Offset)
if version >= 5 {
pe.putInt32(b.LeaderEpoch)
}
err = pe.putString(b.Metadata)
if err != nil {
return err
@ -40,10 +52,17 @@ func (b *OffsetFetchResponseBlock) encode(pe packetEncoder) (err error) {
}
type OffsetFetchResponse struct {
Blocks map[string]map[int32]*OffsetFetchResponseBlock
Version int16
ThrottleTimeMs int32
Blocks map[string]map[int32]*OffsetFetchResponseBlock
Err KError
}
func (r *OffsetFetchResponse) encode(pe packetEncoder) error {
if r.Version >= 3 {
pe.putInt32(r.ThrottleTimeMs)
}
if err := pe.putArrayLength(len(r.Blocks)); err != nil {
return err
}
@ -56,53 +75,75 @@ func (r *OffsetFetchResponse) encode(pe packetEncoder) error {
}
for partition, block := range partitions {
pe.putInt32(partition)
if err := block.encode(pe); err != nil {
if err := block.encode(pe, r.Version); err != nil {
return err
}
}
}
if r.Version >= 2 {
pe.putInt16(int16(r.Err))
}
return nil
}
func (r *OffsetFetchResponse) decode(pd packetDecoder, version int16) (err error) {
r.Version = version
if version >= 3 {
r.ThrottleTimeMs, err = pd.getInt32()
if err != nil {
return err
}
}
numTopics, err := pd.getArrayLength()
if err != nil || numTopics == 0 {
if err != nil {
return err
}
r.Blocks = make(map[string]map[int32]*OffsetFetchResponseBlock, numTopics)
for i := 0; i < numTopics; i++ {
name, err := pd.getString()
if err != nil {
return err
}
numBlocks, err := pd.getArrayLength()
if err != nil {
return err
}
if numBlocks == 0 {
r.Blocks[name] = nil
continue
}
r.Blocks[name] = make(map[int32]*OffsetFetchResponseBlock, numBlocks)
for j := 0; j < numBlocks; j++ {
id, err := pd.getInt32()
if numTopics > 0 {
r.Blocks = make(map[string]map[int32]*OffsetFetchResponseBlock, numTopics)
for i := 0; i < numTopics; i++ {
name, err := pd.getString()
if err != nil {
return err
}
block := new(OffsetFetchResponseBlock)
err = block.decode(pd)
numBlocks, err := pd.getArrayLength()
if err != nil {
return err
}
r.Blocks[name][id] = block
if numBlocks == 0 {
r.Blocks[name] = nil
continue
}
r.Blocks[name] = make(map[int32]*OffsetFetchResponseBlock, numBlocks)
for j := 0; j < numBlocks; j++ {
id, err := pd.getInt32()
if err != nil {
return err
}
block := new(OffsetFetchResponseBlock)
err = block.decode(pd, version)
if err != nil {
return err
}
r.Blocks[name][id] = block
}
}
}
if version >= 2 {
kerr, err := pd.getInt16()
if err != nil {
return err
}
r.Err = KError(kerr)
}
return nil
}
@ -111,11 +152,24 @@ func (r *OffsetFetchResponse) key() int16 {
}
func (r *OffsetFetchResponse) version() int16 {
return 0
return r.Version
}
func (r *OffsetFetchResponse) requiredVersion() KafkaVersion {
return MinVersion
switch r.Version {
case 1:
return V0_8_2_0
case 2:
return V0_10_2_0
case 3:
return V0_11_0_0
case 4:
return V2_0_0_0
case 5:
return V2_1_0_0
default:
return MinVersion
}
}
func (r *OffsetFetchResponse) GetBlock(topic string, partition int32) *OffsetFetchResponseBlock {

57
third/github.com/Shopify/sarama/offset_fetch_response_test.go
View File

@ -1,22 +1,65 @@
package sarama
import "testing"
import (
"fmt"
"testing"
)
var (
emptyOffsetFetchResponse = []byte{
0x00, 0x00, 0x00, 0x00}
emptyOffsetFetchResponseV2 = []byte{
0x00, 0x00, 0x00, 0x00,
0x00, 0x2A}
emptyOffsetFetchResponseV3 = []byte{
0x00, 0x00, 0x00, 0x09,
0x00, 0x00, 0x00, 0x00,
0x00, 0x2A}
)
func TestEmptyOffsetFetchResponse(t *testing.T) {
response := OffsetFetchResponse{}
testResponse(t, "empty", &response, emptyOffsetFetchResponse)
for version := 0; version <= 1; version++ {
response := OffsetFetchResponse{Version: int16(version)}
testResponse(t, fmt.Sprintf("empty v%d", version), &response, emptyOffsetFetchResponse)
}
responseV2 := OffsetFetchResponse{Version: 2, Err: ErrInvalidRequest}
testResponse(t, "empty V2", &responseV2, emptyOffsetFetchResponseV2)
for version := 3; version <= 5; version++ {
responseV3 := OffsetFetchResponse{Version: int16(version), Err: ErrInvalidRequest, ThrottleTimeMs: 9}
testResponse(t, fmt.Sprintf("empty v%d", version), &responseV3, emptyOffsetFetchResponseV3)
}
}
func TestNormalOffsetFetchResponse(t *testing.T) {
response := OffsetFetchResponse{}
response.AddBlock("t", 0, &OffsetFetchResponseBlock{0, "md", ErrRequestTimedOut})
response.Blocks["m"] = nil
// The response encoded form cannot be checked for it varies due to
// unpredictable map traversal order.
testResponse(t, "normal", &response, nil)
// Hence the 'nil' as byte[] parameter in the 'testResponse(..)' calls
for version := 0; version <= 1; version++ {
response := OffsetFetchResponse{Version: int16(version)}
response.AddBlock("t", 0, &OffsetFetchResponseBlock{0, 0, "md", ErrRequestTimedOut})
response.Blocks["m"] = nil
testResponse(t, fmt.Sprintf("Normal v%d", version), &response, nil)
}
responseV2 := OffsetFetchResponse{Version: 2, Err: ErrInvalidRequest}
responseV2.AddBlock("t", 0, &OffsetFetchResponseBlock{0, 0, "md", ErrRequestTimedOut})
responseV2.Blocks["m"] = nil
testResponse(t, "normal V2", &responseV2, nil)
for version := 3; version <= 4; version++ {
responseV3 := OffsetFetchResponse{Version: int16(version), Err: ErrInvalidRequest, ThrottleTimeMs: 9}
responseV3.AddBlock("t", 0, &OffsetFetchResponseBlock{0, 0, "md", ErrRequestTimedOut})
responseV3.Blocks["m"] = nil
testResponse(t, fmt.Sprintf("Normal v%d", version), &responseV3, nil)
}
responseV5 := OffsetFetchResponse{Version: 5, Err: ErrInvalidRequest, ThrottleTimeMs: 9}
responseV5.AddBlock("t", 0, &OffsetFetchResponseBlock{Offset: 10, LeaderEpoch: 100, Metadata: "md", Err: ErrRequestTimedOut})
responseV5.Blocks["m"] = nil
testResponse(t, "normal V5", &responseV5, nil)
}

664
third/github.com/Shopify/sarama/offset_manager.go
View File

@ -25,27 +25,49 @@ type offsetManager struct {
client Client
conf *Config
group string
ticker *time.Ticker
lock sync.Mutex
poms map[string]map[int32]*partitionOffsetManager
boms map[*Broker]*brokerOffsetManager
memberID string
generation int32
broker *Broker
brokerLock sync.RWMutex
poms map[string]map[int32]*partitionOffsetManager
pomsLock sync.RWMutex
closeOnce sync.Once
closing chan none
closed chan none
}
// NewOffsetManagerFromClient creates a new OffsetManager from the given client.
// It is still necessary to call Close() on the underlying client when finished with the partition manager.
func NewOffsetManagerFromClient(group string, client Client) (OffsetManager, error) {
return newOffsetManagerFromClient(group, "", GroupGenerationUndefined, client)
}
func newOffsetManagerFromClient(group, memberID string, generation int32, client Client) (*offsetManager, error) {
// Check that we are not dealing with a closed Client before processing any other arguments
if client.Closed() {
return nil, ErrClosedClient
}
conf := client.Config()
om := &offsetManager{
client: client,
conf: client.Config(),
conf: conf,
group: group,
ticker: time.NewTicker(conf.Consumer.Offsets.CommitInterval),
poms: make(map[string]map[int32]*partitionOffsetManager),
boms: make(map[*Broker]*brokerOffsetManager),
memberID: memberID,
generation: generation,
closing: make(chan none),
closed: make(chan none),
}
go withRecover(om.mainLoop)
return om, nil
}
@ -56,8 +78,8 @@ func (om *offsetManager) ManagePartition(topic string, partition int32) (Partiti
return nil, err
}
om.lock.Lock()
defer om.lock.Unlock()
om.pomsLock.Lock()
defer om.pomsLock.Unlock()
topicManagers := om.poms[topic]
if topicManagers == nil {
@ -74,53 +96,307 @@ func (om *offsetManager) ManagePartition(topic string, partition int32) (Partiti
}
func (om *offsetManager) Close() error {
om.closeOnce.Do(func() {
// exit the mainLoop
close(om.closing)
<-om.closed
// mark all POMs as closed
om.asyncClosePOMs()
// flush one last time
for attempt := 0; attempt <= om.conf.Consumer.Offsets.Retry.Max; attempt++ {
om.flushToBroker()
if om.releasePOMs(false) == 0 {
break
}
}
om.releasePOMs(true)
om.brokerLock.Lock()
om.broker = nil
om.brokerLock.Unlock()
})
return nil
}
func (om *offsetManager) refBrokerOffsetManager(broker *Broker) *brokerOffsetManager {
om.lock.Lock()
defer om.lock.Unlock()
bom := om.boms[broker]
if bom == nil {
bom = om.newBrokerOffsetManager(broker)
om.boms[broker] = bom
func (om *offsetManager) fetchInitialOffset(topic string, partition int32, retries int) (int64, string, error) {
broker, err := om.coordinator()
if err != nil {
if retries <= 0 {
return 0, "", err
}
return om.fetchInitialOffset(topic, partition, retries-1)
}
bom.refs++
req := new(OffsetFetchRequest)
req.Version = 1
req.ConsumerGroup = om.group
req.AddPartition(topic, partition)
return bom
resp, err := broker.FetchOffset(req)
if err != nil {
if retries <= 0 {
return 0, "", err
}
om.releaseCoordinator(broker)
return om.fetchInitialOffset(topic, partition, retries-1)
}
block := resp.GetBlock(topic, partition)
if block == nil {
return 0, "", ErrIncompleteResponse
}
switch block.Err {
case ErrNoError:
return block.Offset, block.Metadata, nil
case ErrNotCoordinatorForConsumer:
if retries <= 0 {
return 0, "", block.Err
}
om.releaseCoordinator(broker)
return om.fetchInitialOffset(topic, partition, retries-1)
case ErrOffsetsLoadInProgress:
if retries <= 0 {
return 0, "", block.Err
}
select {
case <-om.closing:
return 0, "", block.Err
case <-time.After(om.conf.Metadata.Retry.Backoff):
}
return om.fetchInitialOffset(topic, partition, retries-1)
default:
return 0, "", block.Err
}
}
func (om *offsetManager) unrefBrokerOffsetManager(bom *brokerOffsetManager) {
om.lock.Lock()
defer om.lock.Unlock()
func (om *offsetManager) coordinator() (*Broker, error) {
om.brokerLock.RLock()
broker := om.broker
om.brokerLock.RUnlock()
bom.refs--
if broker != nil {
return broker, nil
}
if bom.refs == 0 {
close(bom.updateSubscriptions)
if om.boms[bom.broker] == bom {
delete(om.boms, bom.broker)
om.brokerLock.Lock()
defer om.brokerLock.Unlock()
if broker := om.broker; broker != nil {
return broker, nil
}
if err := om.client.RefreshCoordinator(om.group); err != nil {
return nil, err
}
broker, err := om.client.Coordinator(om.group)
if err != nil {
return nil, err
}
om.broker = broker
return broker, nil
}
func (om *offsetManager) releaseCoordinator(b *Broker) {
om.brokerLock.Lock()
if om.broker == b {
om.broker = nil
}
om.brokerLock.Unlock()
}
func (om *offsetManager) mainLoop() {
defer om.ticker.Stop()
defer close(om.closed)
for {
select {
case <-om.ticker.C:
om.flushToBroker()
om.releasePOMs(false)
case <-om.closing:
return
}
}
}
func (om *offsetManager) abandonBroker(bom *brokerOffsetManager) {
om.lock.Lock()
defer om.lock.Unlock()
func (om *offsetManager) flushToBroker() {
req := om.constructRequest()
if req == nil {
return
}
delete(om.boms, bom.broker)
broker, err := om.coordinator()
if err != nil {
om.handleError(err)
return
}
resp, err := broker.CommitOffset(req)
if err != nil {
om.handleError(err)
om.releaseCoordinator(broker)
_ = broker.Close()
return
}
om.handleResponse(broker, req, resp)
}
func (om *offsetManager) abandonPartitionOffsetManager(pom *partitionOffsetManager) {
om.lock.Lock()
defer om.lock.Unlock()
func (om *offsetManager) constructRequest() *OffsetCommitRequest {
var r *OffsetCommitRequest
var perPartitionTimestamp int64
if om.conf.Consumer.Offsets.Retention == 0 {
perPartitionTimestamp = ReceiveTime
r = &OffsetCommitRequest{
Version: 1,
ConsumerGroup: om.group,
ConsumerID: om.memberID,
ConsumerGroupGeneration: om.generation,
}
} else {
r = &OffsetCommitRequest{
Version: 2,
RetentionTime: int64(om.conf.Consumer.Offsets.Retention / time.Millisecond),
ConsumerGroup: om.group,
ConsumerID: om.memberID,
ConsumerGroupGeneration: om.generation,
}
delete(om.poms[pom.topic], pom.partition)
if len(om.poms[pom.topic]) == 0 {
delete(om.poms, pom.topic)
}
om.pomsLock.RLock()
defer om.pomsLock.RUnlock()
for _, topicManagers := range om.poms {
for _, pom := range topicManagers {
pom.lock.Lock()
if pom.dirty {
r.AddBlock(pom.topic, pom.partition, pom.offset, perPartitionTimestamp, pom.metadata)
}
pom.lock.Unlock()
}
}
if len(r.blocks) > 0 {
return r
}
return nil
}
func (om *offsetManager) handleResponse(broker *Broker, req *OffsetCommitRequest, resp *OffsetCommitResponse) {
om.pomsLock.RLock()
defer om.pomsLock.RUnlock()
for _, topicManagers := range om.poms {
for _, pom := range topicManagers {
if req.blocks[pom.topic] == nil || req.blocks[pom.topic][pom.partition] == nil {
continue
}
var err KError
var ok bool
if resp.Errors[pom.topic] == nil {
pom.handleError(ErrIncompleteResponse)
continue
}
if err, ok = resp.Errors[pom.topic][pom.partition]; !ok {
pom.handleError(ErrIncompleteResponse)
continue
}
switch err {
case ErrNoError:
block := req.blocks[pom.topic][pom.partition]
pom.updateCommitted(block.offset, block.metadata)
case ErrNotLeaderForPartition, ErrLeaderNotAvailable,
ErrConsumerCoordinatorNotAvailable, ErrNotCoordinatorForConsumer:
// not a critical error, we just need to redispatch
om.releaseCoordinator(broker)
case ErrOffsetMetadataTooLarge, ErrInvalidCommitOffsetSize:
// nothing we can do about this, just tell the user and carry on
pom.handleError(err)
case ErrOffsetsLoadInProgress:
// nothing wrong but we didn't commit, we'll get it next time round
break
case ErrUnknownTopicOrPartition:
// let the user know *and* try redispatching - if topic-auto-create is
// enabled, redispatching should trigger a metadata req and create the
// topic; if not then re-dispatching won't help, but we've let the user
// know and it shouldn't hurt either (see https://github.com/Shopify/sarama/issues/706)
fallthrough
default:
// dunno, tell the user and try redispatching
pom.handleError(err)
om.releaseCoordinator(broker)
}
}
}
}
func (om *offsetManager) handleError(err error) {
om.pomsLock.RLock()
defer om.pomsLock.RUnlock()
for _, topicManagers := range om.poms {
for _, pom := range topicManagers {
pom.handleError(err)
}
}
}
func (om *offsetManager) asyncClosePOMs() {
om.pomsLock.RLock()
defer om.pomsLock.RUnlock()
for _, topicManagers := range om.poms {
for _, pom := range topicManagers {
pom.AsyncClose()
}
}
}
// Releases/removes closed POMs once they are clean (or when forced)
func (om *offsetManager) releasePOMs(force bool) (remaining int) {
om.pomsLock.Lock()
defer om.pomsLock.Unlock()
for topic, topicManagers := range om.poms {
for partition, pom := range topicManagers {
pom.lock.Lock()
releaseDue := pom.done && (force || !pom.dirty)
pom.lock.Unlock()
if releaseDue {
pom.release()
delete(om.poms[topic], partition)
if len(om.poms[topic]) == 0 {
delete(om.poms, topic)
}
}
}
remaining += len(om.poms[topic])
}
return
}
func (om *offsetManager) findPOM(topic string, partition int32) *partitionOffsetManager {
om.pomsLock.RLock()
defer om.pomsLock.RUnlock()
if partitions, ok := om.poms[topic]; ok {
if pom, ok := partitions[partition]; ok {
return pom
}
}
return nil
}
// Partition Offset Manager
@ -187,138 +463,26 @@ type partitionOffsetManager struct {
offset int64
metadata string
dirty bool
clean sync.Cond
broker *brokerOffsetManager
done bool
errors chan *ConsumerError
rebalance chan none
dying chan none
releaseOnce sync.Once
errors chan *ConsumerError
}
func (om *offsetManager) newPartitionOffsetManager(topic string, partition int32) (*partitionOffsetManager, error) {
pom := &partitionOffsetManager{
offset, metadata, err := om.fetchInitialOffset(topic, partition, om.conf.Metadata.Retry.Max)
if err != nil {
return nil, err
}
return &partitionOffsetManager{
parent: om,
topic: topic,
partition: partition,
errors: make(chan *ConsumerError, om.conf.ChannelBufferSize),
rebalance: make(chan none, 1),
dying: make(chan none),
}
pom.clean.L = &pom.lock
if err := pom.selectBroker(); err != nil {
return nil, err
}
if err := pom.fetchInitialOffset(om.conf.Metadata.Retry.Max); err != nil {
return nil, err
}
pom.broker.updateSubscriptions <- pom
go withRecover(pom.mainLoop)
return pom, nil
}
func (pom *partitionOffsetManager) mainLoop() {
for {
select {
case <-pom.rebalance:
if err := pom.selectBroker(); err != nil {
pom.handleError(err)
pom.rebalance <- none{}
} else {
pom.broker.updateSubscriptions <- pom
}
case <-pom.dying:
if pom.broker != nil {
select {
case <-pom.rebalance:
case pom.broker.updateSubscriptions <- pom:
}
pom.parent.unrefBrokerOffsetManager(pom.broker)
}
pom.parent.abandonPartitionOffsetManager(pom)
close(pom.errors)
return
}
}
}
func (pom *partitionOffsetManager) selectBroker() error {
if pom.broker != nil {
pom.parent.unrefBrokerOffsetManager(pom.broker)
pom.broker = nil
}
var broker *Broker
var err error
if err = pom.parent.client.RefreshCoordinator(pom.parent.group); err != nil {
return err
}
if broker, err = pom.parent.client.Coordinator(pom.parent.group); err != nil {
return err
}
pom.broker = pom.parent.refBrokerOffsetManager(broker)
return nil
}
func (pom *partitionOffsetManager) fetchInitialOffset(retries int) error {
request := new(OffsetFetchRequest)
request.Version = 1
request.ConsumerGroup = pom.parent.group
request.AddPartition(pom.topic, pom.partition)
response, err := pom.broker.broker.FetchOffset(request)
if err != nil {
return err
}
block := response.GetBlock(pom.topic, pom.partition)
if block == nil {
return ErrIncompleteResponse
}
switch block.Err {
case ErrNoError:
pom.offset = block.Offset
pom.metadata = block.Metadata
return nil
case ErrNotCoordinatorForConsumer:
if retries <= 0 {
return block.Err
}
if err := pom.selectBroker(); err != nil {
return err
}
return pom.fetchInitialOffset(retries - 1)
case ErrOffsetsLoadInProgress:
if retries <= 0 {
return block.Err
}
time.Sleep(pom.parent.conf.Metadata.Retry.Backoff)
return pom.fetchInitialOffset(retries - 1)
default:
return block.Err
}
}
func (pom *partitionOffsetManager) handleError(err error) {
cErr := &ConsumerError{
Topic: pom.topic,
Partition: pom.partition,
Err: err,
}
if pom.parent.conf.Consumer.Return.Errors {
pom.errors <- cErr
} else {
Logger.Println(cErr)
}
offset: offset,
metadata: metadata,
}, nil
}
func (pom *partitionOffsetManager) Errors() <-chan *ConsumerError {
@ -353,7 +517,6 @@ func (pom *partitionOffsetManager) updateCommitted(offset int64, metadata string
if pom.offset == offset && pom.metadata == metadata {
pom.dirty = false
pom.clean.Signal()
}
}
@ -369,16 +532,9 @@ func (pom *partitionOffsetManager) NextOffset() (int64, string) {
}
func (pom *partitionOffsetManager) AsyncClose() {
go func() {
pom.lock.Lock()
defer pom.lock.Unlock()
for pom.dirty {
pom.clean.Wait()
}
close(pom.dying)
}()
pom.lock.Lock()
pom.done = true
pom.lock.Unlock()
}
func (pom *partitionOffsetManager) Close() error {
@ -395,166 +551,22 @@ func (pom *partitionOffsetManager) Close() error {
return nil
}
// Broker Offset Manager
type brokerOffsetManager struct {
parent *offsetManager
broker *Broker
timer *time.Ticker
updateSubscriptions chan *partitionOffsetManager
subscriptions map[*partitionOffsetManager]none
refs int
}
func (om *offsetManager) newBrokerOffsetManager(broker *Broker) *brokerOffsetManager {
bom := &brokerOffsetManager{
parent: om,
broker: broker,
timer: time.NewTicker(om.conf.Consumer.Offsets.CommitInterval),
updateSubscriptions: make(chan *partitionOffsetManager),
subscriptions: make(map[*partitionOffsetManager]none),
func (pom *partitionOffsetManager) handleError(err error) {
cErr := &ConsumerError{
Topic: pom.topic,
Partition: pom.partition,
Err: err,
}
go withRecover(bom.mainLoop)
return bom
}
func (bom *brokerOffsetManager) mainLoop() {
for {
select {
case <-bom.timer.C:
if len(bom.subscriptions) > 0 {
bom.flushToBroker()
}
case s, ok := <-bom.updateSubscriptions:
if !ok {
bom.timer.Stop()
return
}
if _, ok := bom.subscriptions[s]; ok {
delete(bom.subscriptions, s)
} else {
bom.subscriptions[s] = none{}
}
}
}
}
func (bom *brokerOffsetManager) flushToBroker() {
request := bom.constructRequest()
if request == nil {
return
}
response, err := bom.broker.CommitOffset(request)
if err != nil {
bom.abort(err)
return
}
for s := range bom.subscriptions {
if request.blocks[s.topic] == nil || request.blocks[s.topic][s.partition] == nil {
continue
}
var err KError
var ok bool
if response.Errors[s.topic] == nil {
s.handleError(ErrIncompleteResponse)
delete(bom.subscriptions, s)
s.rebalance <- none{}
continue
}
if err, ok = response.Errors[s.topic][s.partition]; !ok {
s.handleError(ErrIncompleteResponse)
delete(bom.subscriptions, s)
s.rebalance <- none{}
continue
}
switch err {
case ErrNoError:
block := request.blocks[s.topic][s.partition]
s.updateCommitted(block.offset, block.metadata)
case ErrNotLeaderForPartition, ErrLeaderNotAvailable,
ErrConsumerCoordinatorNotAvailable, ErrNotCoordinatorForConsumer:
// not a critical error, we just need to redispatch
delete(bom.subscriptions, s)
s.rebalance <- none{}
case ErrOffsetMetadataTooLarge, ErrInvalidCommitOffsetSize:
// nothing we can do about this, just tell the user and carry on
s.handleError(err)
case ErrOffsetsLoadInProgress:
// nothing wrong but we didn't commit, we'll get it next time round
break
case ErrUnknownTopicOrPartition:
// let the user know *and* try redispatching - if topic-auto-create is
// enabled, redispatching should trigger a metadata request and create the
// topic; if not then re-dispatching won't help, but we've let the user
// know and it shouldn't hurt either (see https://github.com/Shopify/sarama/issues/706)
fallthrough
default:
// dunno, tell the user and try redispatching
s.handleError(err)
delete(bom.subscriptions, s)
s.rebalance <- none{}
}
}
}
func (bom *brokerOffsetManager) constructRequest() *OffsetCommitRequest {
var r *OffsetCommitRequest
var perPartitionTimestamp int64
if bom.parent.conf.Consumer.Offsets.Retention == 0 {
perPartitionTimestamp = ReceiveTime
r = &OffsetCommitRequest{
Version: 1,
ConsumerGroup: bom.parent.group,
ConsumerGroupGeneration: GroupGenerationUndefined,
}
if pom.parent.conf.Consumer.Return.Errors {
pom.errors <- cErr
} else {
r = &OffsetCommitRequest{
Version: 2,
RetentionTime: int64(bom.parent.conf.Consumer.Offsets.Retention / time.Millisecond),
ConsumerGroup: bom.parent.group,
ConsumerGroupGeneration: GroupGenerationUndefined,
}
Logger.Println(cErr)
}
for s := range bom.subscriptions {
s.lock.Lock()
if s.dirty {
r.AddBlock(s.topic, s.partition, s.offset, perPartitionTimestamp, s.metadata)
}
s.lock.Unlock()
}
if len(r.blocks) > 0 {
return r
}
return nil
}
func (bom *brokerOffsetManager) abort(err error) {
_ = bom.broker.Close() // we don't care about the error this might return, we already have one
bom.parent.abandonBroker(bom)
for pom := range bom.subscriptions {
pom.handleError(err)
pom.rebalance <- none{}
}
for s := range bom.updateSubscriptions {
if _, ok := bom.subscriptions[s]; !ok {
s.handleError(err)
s.rebalance <- none{}
}
}
bom.subscriptions = make(map[*partitionOffsetManager]none)
func (pom *partitionOffsetManager) release() {
pom.releaseOnce.Do(func() {
go close(pom.errors)
})
}

50
third/github.com/Shopify/sarama/offset_manager_test.go
View File

@ -5,13 +5,16 @@ import (
"time"
)
func initOffsetManager(t *testing.T) (om OffsetManager,
func initOffsetManager(t *testing.T, retention time.Duration) (om OffsetManager,
testClient Client, broker, coordinator *MockBroker) {
config := NewConfig()
config.Metadata.Retry.Max = 1
config.Consumer.Offsets.CommitInterval = 1 * time.Millisecond
config.Version = V0_9_0_0
if retention > 0 {
config.Consumer.Offsets.Retention = retention
}
broker = NewMockBroker(t, 1)
coordinator = NewMockBroker(t, 2)
@ -64,31 +67,30 @@ func initPartitionOffsetManager(t *testing.T, om OffsetManager,
func TestNewOffsetManager(t *testing.T) {
seedBroker := NewMockBroker(t, 1)
seedBroker.Returns(new(MetadataResponse))
defer seedBroker.Close()
testClient, err := NewClient([]string{seedBroker.Addr()}, nil)
if err != nil {
t.Fatal(err)
}
_, err = NewOffsetManagerFromClient("group", testClient)
om, err := NewOffsetManagerFromClient("group", testClient)
if err != nil {
t.Error(err)
}
safeClose(t, om)
safeClose(t, testClient)
_, err = NewOffsetManagerFromClient("group", testClient)
if err != ErrClosedClient {
t.Errorf("Error expected for closed client; actual value: %v", err)
}
seedBroker.Close()
}
// Test recovery from ErrNotCoordinatorForConsumer
// on first fetchInitialOffset call
func TestOffsetManagerFetchInitialFail(t *testing.T) {
om, testClient, broker, coordinator := initOffsetManager(t)
om, testClient, broker, coordinator := initOffsetManager(t, 0)
// Error on first fetchInitialOffset call
responseBlock := OffsetFetchResponseBlock{
@ -131,7 +133,7 @@ func TestOffsetManagerFetchInitialFail(t *testing.T) {
// Test fetchInitialOffset retry on ErrOffsetsLoadInProgress
func TestOffsetManagerFetchInitialLoadInProgress(t *testing.T) {
om, testClient, broker, coordinator := initOffsetManager(t)
om, testClient, broker, coordinator := initOffsetManager(t, 0)
// Error on first fetchInitialOffset call
responseBlock := OffsetFetchResponseBlock{
@ -164,7 +166,7 @@ func TestOffsetManagerFetchInitialLoadInProgress(t *testing.T) {
}
func TestPartitionOffsetManagerInitialOffset(t *testing.T) {
om, testClient, broker, coordinator := initOffsetManager(t)
om, testClient, broker, coordinator := initOffsetManager(t, 0)
testClient.Config().Consumer.Offsets.Initial = OffsetOldest
// Kafka returns -1 if no offset has been stored for this partition yet.
@ -186,7 +188,7 @@ func TestPartitionOffsetManagerInitialOffset(t *testing.T) {
}
func TestPartitionOffsetManagerNextOffset(t *testing.T) {
om, testClient, broker, coordinator := initOffsetManager(t)
om, testClient, broker, coordinator := initOffsetManager(t, 0)
pom := initPartitionOffsetManager(t, om, coordinator, 5, "test_meta")
offset, meta := pom.NextOffset()
@ -205,7 +207,7 @@ func TestPartitionOffsetManagerNextOffset(t *testing.T) {
}
func TestPartitionOffsetManagerResetOffset(t *testing.T) {
om, testClient, broker, coordinator := initOffsetManager(t)
om, testClient, broker, coordinator := initOffsetManager(t, 0)
pom := initPartitionOffsetManager(t, om, coordinator, 5, "original_meta")
ocResponse := new(OffsetCommitResponse)
@ -231,9 +233,7 @@ func TestPartitionOffsetManagerResetOffset(t *testing.T) {
}
func TestPartitionOffsetManagerResetOffsetWithRetention(t *testing.T) {
om, testClient, broker, coordinator := initOffsetManager(t)
testClient.Config().Consumer.Offsets.Retention = time.Hour
om, testClient, broker, coordinator := initOffsetManager(t, time.Hour)
pom := initPartitionOffsetManager(t, om, coordinator, 5, "original_meta")
ocResponse := new(OffsetCommitResponse)
@ -269,7 +269,7 @@ func TestPartitionOffsetManagerResetOffsetWithRetention(t *testing.T) {
}
func TestPartitionOffsetManagerMarkOffset(t *testing.T) {
om, testClient, broker, coordinator := initOffsetManager(t)
om, testClient, broker, coordinator := initOffsetManager(t, 0)
pom := initPartitionOffsetManager(t, om, coordinator, 5, "original_meta")
ocResponse := new(OffsetCommitResponse)
@ -294,9 +294,7 @@ func TestPartitionOffsetManagerMarkOffset(t *testing.T) {
}
func TestPartitionOffsetManagerMarkOffsetWithRetention(t *testing.T) {
om, testClient, broker, coordinator := initOffsetManager(t)
testClient.Config().Consumer.Offsets.Retention = time.Hour
om, testClient, broker, coordinator := initOffsetManager(t, time.Hour)
pom := initPartitionOffsetManager(t, om, coordinator, 5, "original_meta")
ocResponse := new(OffsetCommitResponse)
@ -331,7 +329,7 @@ func TestPartitionOffsetManagerMarkOffsetWithRetention(t *testing.T) {
}
func TestPartitionOffsetManagerCommitErr(t *testing.T) {
om, testClient, broker, coordinator := initOffsetManager(t)
om, testClient, broker, coordinator := initOffsetManager(t, 0)
pom := initPartitionOffsetManager(t, om, coordinator, 5, "meta")
// Error on one partition
@ -353,24 +351,14 @@ func TestPartitionOffsetManagerCommitErr(t *testing.T) {
ocResponse2 := new(OffsetCommitResponse)
newCoordinator.Returns(ocResponse2)
// For RefreshCoordinator()
broker.Returns(&ConsumerMetadataResponse{
CoordinatorID: newCoordinator.BrokerID(),
CoordinatorHost: "127.0.0.1",
CoordinatorPort: newCoordinator.Port(),
})
// No error, no need to refresh coordinator
// Error on the wrong partition for this pom
ocResponse3 := new(OffsetCommitResponse)
ocResponse3.AddError("my_topic", 1, ErrNoError)
newCoordinator.Returns(ocResponse3)
// For RefreshCoordinator()
broker.Returns(&ConsumerMetadataResponse{
CoordinatorID: newCoordinator.BrokerID(),
CoordinatorHost: "127.0.0.1",
CoordinatorPort: newCoordinator.Port(),
})
// No error, no need to refresh coordinator
// ErrUnknownTopicOrPartition/ErrNotLeaderForPartition/ErrLeaderNotAvailable block
ocResponse4 := new(OffsetCommitResponse)
@ -405,7 +393,7 @@ func TestPartitionOffsetManagerCommitErr(t *testing.T) {
// Test of recovery from abort
func TestAbortPartitionOffsetManager(t *testing.T) {
om, testClient, broker, coordinator := initOffsetManager(t)
om, testClient, broker, coordinator := initOffsetManager(t, 0)
pom := initPartitionOffsetManager(t, om, coordinator, 5, "meta")
// this triggers an error in the CommitOffset request,

34
third/github.com/Shopify/sarama/offset_request.go
View File

@ -27,12 +27,20 @@ func (b *offsetRequestBlock) decode(pd packetDecoder, version int16) (err error)
}
type OffsetRequest struct {
Version int16
blocks map[string]map[int32]*offsetRequestBlock
Version int16
replicaID int32
isReplicaIDSet bool
blocks map[string]map[int32]*offsetRequestBlock
}
func (r *OffsetRequest) encode(pe packetEncoder) error {
pe.putInt32(-1) // replica ID is always -1 for clients
if r.isReplicaIDSet {
pe.putInt32(r.replicaID)
} else {
// default replica ID is always -1 for clients
pe.putInt32(-1)
}
err := pe.putArrayLength(len(r.blocks))
if err != nil {
return err
@ -59,10 +67,14 @@ func (r *OffsetRequest) encode(pe packetEncoder) error {
func (r *OffsetRequest) decode(pd packetDecoder, version int16) error {
r.Version = version
// Ignore replica ID
if _, err := pd.getInt32(); err != nil {
replicaID, err := pd.getInt32()
if err != nil {
return err
}
if replicaID >= 0 {
r.SetReplicaID(replicaID)
}
blockCount, err := pd.getArrayLength()
if err != nil {
return err
@ -113,6 +125,18 @@ func (r *OffsetRequest) requiredVersion() KafkaVersion {
}
}
func (r *OffsetRequest) SetReplicaID(id int32) {
r.replicaID = id
r.isReplicaIDSet = true
}
func (r *OffsetRequest) ReplicaID() int32 {
if r.isReplicaIDSet {
return r.replicaID
}
return -1
}
func (r *OffsetRequest) AddBlock(topic string, partitionID int32, time int64, maxOffsets int32) {
if r.blocks == nil {
r.blocks = make(map[string]map[int32]*offsetRequestBlock)

16
third/github.com/Shopify/sarama/offset_request_test.go
View File

@ -23,6 +23,10 @@ var (
0x00, 0x00, 0x00, 0x01,
0x00, 0x00, 0x00, 0x04,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}
offsetRequestReplicaID = []byte{
0x00, 0x00, 0x00, 0x2a,
0x00, 0x00, 0x00, 0x00}
)
func TestOffsetRequest(t *testing.T) {
@ -41,3 +45,15 @@ func TestOffsetRequestV1(t *testing.T) {
request.AddBlock("bar", 4, 1, 2) // Last argument is ignored for V1
testRequest(t, "one block", request, offsetRequestOneBlockV1)
}
func TestOffsetRequestReplicaID(t *testing.T) {
request := new(OffsetRequest)
replicaID := int32(42)
request.SetReplicaID(replicaID)
if found := request.ReplicaID(); found != replicaID {
t.Errorf("replicaID: expected %v, found %v", replicaID, found)
}
testRequest(t, "with replica ID", request, offsetRequestReplicaID)
}

8
third/github.com/Shopify/sarama/produce_response.go
View File

@ -179,5 +179,11 @@ func (r *ProduceResponse) AddTopicPartition(topic string, partition int32, err K
byTopic = make(map[int32]*ProduceResponseBlock)
r.Blocks[topic] = byTopic
}
byTopic[partition] = &ProduceResponseBlock{Err: err}
block := &ProduceResponseBlock{
Err: err,
}
if r.Version >= 2 {
block.Timestamp = time.Now()
}
byTopic[partition] = block
}

21
third/github.com/Shopify/sarama/produce_set.go
View File

@ -2,6 +2,7 @@ package sarama
import (
"encoding/binary"
"errors"
"time"
)
@ -61,9 +62,13 @@ func (ps *produceSet) add(msg *ProducerMessage) error {
batch := &RecordBatch{
FirstTimestamp: timestamp,
Version: 2,
ProducerID: -1, /* No producer id */
Codec: ps.parent.conf.Producer.Compression,
CompressionLevel: ps.parent.conf.Producer.CompressionLevel,
ProducerID: ps.parent.txnmgr.producerID,
ProducerEpoch: ps.parent.txnmgr.producerEpoch,
}
if ps.parent.conf.Producer.Idempotent {
batch.FirstSequence = msg.sequenceNumber
}
set = &partitionSet{recordsToSend: newDefaultRecords(batch)}
size = recordBatchOverhead
@ -72,9 +77,12 @@ func (ps *produceSet) add(msg *ProducerMessage) error {
}
partitions[msg.Partition] = set
}
set.msgs = append(set.msgs, msg)
if ps.parent.conf.Version.IsAtLeast(V0_11_0_0) {
if ps.parent.conf.Producer.Idempotent && msg.sequenceNumber < set.recordsToSend.RecordBatch.FirstSequence {
return errors.New("Assertion failed: Message out of sequence added to a batch")
}
// We are being conservative here to avoid having to prep encode the record
size += maximumRecordOverhead
rec := &Record{
@ -120,8 +128,8 @@ func (ps *produceSet) buildRequest() *ProduceRequest {
req.Version = 3
}
for topic, partitionSet := range ps.msgs {
for partition, set := range partitionSet {
for topic, partitionSets := range ps.msgs {
for partition, set := range partitionSets {
if req.Version >= 3 {
// If the API version we're hitting is 3 or greater, we need to calculate
// offsets for each record in the batch relative to FirstOffset.
@ -137,7 +145,6 @@ func (ps *produceSet) buildRequest() *ProduceRequest {
record.OffsetDelta = int64(i)
}
}
req.AddBatch(topic, partition, rb)
continue
}
@ -183,10 +190,10 @@ func (ps *produceSet) buildRequest() *ProduceRequest {
return req
}
func (ps *produceSet) eachPartition(cb func(topic string, partition int32, msgs []*ProducerMessage)) {
func (ps *produceSet) eachPartition(cb func(topic string, partition int32, pSet *partitionSet)) {
for topic, partitionSet := range ps.msgs {
for partition, set := range partitionSet {
cb(topic, partition, set.msgs)
cb(topic, partition, set)
}
}
}

97
third/github.com/Shopify/sarama/produce_set_test.go
View File

@ -7,8 +7,11 @@ import (
)
func makeProduceSet() (*asyncProducer, *produceSet) {
conf := NewConfig()
txnmgr, _ := newTransactionManager(conf, nil)
parent := &asyncProducer{
conf: NewConfig(),
conf: conf,
txnmgr: txnmgr,
}
return parent, newProduceSet(parent)
}
@ -72,8 +75,8 @@ func TestProduceSetPartitionTracking(t *testing.T) {
seenT1P1 := false
seenT2P0 := false
ps.eachPartition(func(topic string, partition int32, msgs []*ProducerMessage) {
if len(msgs) != 1 {
ps.eachPartition(func(topic string, partition int32, pSet *partitionSet) {
if len(pSet.msgs) != 1 {
t.Error("Wrong message count")
}
@ -253,3 +256,91 @@ func TestProduceSetV3RequestBuilding(t *testing.T) {
}
}
}
func TestProduceSetIdempotentRequestBuilding(t *testing.T) {
const pID = 1000
const pEpoch = 1234
config := NewConfig()
config.Producer.RequiredAcks = WaitForAll
config.Producer.Idempotent = true
config.Version = V0_11_0_0
parent := &asyncProducer{
conf: config,
txnmgr: &transactionManager{
producerID: pID,
producerEpoch: pEpoch,
},
}
ps := newProduceSet(parent)
now := time.Now()
msg := &ProducerMessage{
Topic: "t1",
Partition: 0,
Key: StringEncoder(TestMessage),
Value: StringEncoder(TestMessage),
Headers: []RecordHeader{
RecordHeader{
Key: []byte("header-1"),
Value: []byte("value-1"),
},
RecordHeader{
Key: []byte("header-2"),
Value: []byte("value-2"),
},
RecordHeader{
Key: []byte("header-3"),
Value: []byte("value-3"),
},
},
Timestamp: now,
sequenceNumber: 123,
}
for i := 0; i < 10; i++ {
safeAddMessage(t, ps, msg)
msg.Timestamp = msg.Timestamp.Add(time.Second)
}
req := ps.buildRequest()
if req.Version != 3 {
t.Error("Wrong request version")
}
batch := req.records["t1"][0].RecordBatch
if batch.FirstTimestamp != now {
t.Errorf("Wrong first timestamp: %v", batch.FirstTimestamp)
}
if batch.ProducerID != pID {
t.Errorf("Wrong producerID: %v", batch.ProducerID)
}
if batch.ProducerEpoch != pEpoch {
t.Errorf("Wrong producerEpoch: %v", batch.ProducerEpoch)
}
if batch.FirstSequence != 123 {
t.Errorf("Wrong first sequence: %v", batch.FirstSequence)
}
for i := 0; i < 10; i++ {
rec := batch.Records[i]
if rec.TimestampDelta != time.Duration(i)*time.Second {
t.Errorf("Wrong timestamp delta: %v", rec.TimestampDelta)
}
if rec.OffsetDelta != int64(i) {
t.Errorf("Wrong relative inner offset, expected %d, got %d", i, rec.OffsetDelta)
}
for j, h := range batch.Records[i].Headers {
exp := fmt.Sprintf("header-%d", j+1)
if string(h.Key) != exp {
t.Errorf("Wrong header key, expected %v, got %v", exp, h.Key)
}
exp = fmt.Sprintf("value-%d", j+1)
if string(h.Value) != exp {
t.Errorf("Wrong header value, expected %v, got %v", exp, h.Value)
}
}
}
}

70
third/github.com/Shopify/sarama/record_batch.go
View File

@ -1,14 +1,8 @@
package sarama
import (
"bytes"
"compress/gzip"
"fmt"
"io/ioutil"
"time"
"gitee.com/johng/gf/third/github.com/eapache/go-xerial-snappy"
"gitee.com/johng/gf/third/github.com/pierrec/lz4"
)
const recordBatchOverhead = 49
@ -174,27 +168,9 @@ func (b *RecordBatch) decode(pd packetDecoder) (err error) {
return err
}
switch b.Codec {
case CompressionNone:
case CompressionGZIP:
reader, err := gzip.NewReader(bytes.NewReader(recBuffer))
if err != nil {
return err
}
if recBuffer, err = ioutil.ReadAll(reader); err != nil {
return err
}
case CompressionSnappy:
if recBuffer, err = snappy.Decode(recBuffer); err != nil {
return err
}
case CompressionLZ4:
reader := lz4.NewReader(bytes.NewReader(recBuffer))
if recBuffer, err = ioutil.ReadAll(reader); err != nil {
return err
}
default:
return PacketDecodingError{fmt.Sprintf("invalid compression specified (%d)", b.Codec)}
recBuffer, err = decompress(b.Codec, recBuffer)
if err != nil {
return err
}
b.recordsLen = len(recBuffer)
@ -215,44 +191,8 @@ func (b *RecordBatch) encodeRecords(pe packetEncoder) error {
}
b.recordsLen = len(raw)
switch b.Codec {
case CompressionNone:
b.compressedRecords = raw
case CompressionGZIP:
var buf bytes.Buffer
var writer *gzip.Writer
if b.CompressionLevel != CompressionLevelDefault {
writer, err = gzip.NewWriterLevel(&buf, b.CompressionLevel)
if err != nil {
return err
}
} else {
writer = gzip.NewWriter(&buf)
}
if _, err := writer.Write(raw); err != nil {
return err
}
if err := writer.Close(); err != nil {
return err
}
b.compressedRecords = buf.Bytes()
case CompressionSnappy:
b.compressedRecords = snappy.Encode(raw)
case CompressionLZ4:
var buf bytes.Buffer
writer := lz4.NewWriter(&buf)
if _, err := writer.Write(raw); err != nil {
return err
}
if err := writer.Close(); err != nil {
return err
}
b.compressedRecords = buf.Bytes()
default:
return PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", b.Codec)}
}
return nil
b.compressedRecords, err = compress(b.Codec, b.CompressionLevel, raw)
return err
}
func (b *RecordBatch) computeAttributes() int16 {

21
third/github.com/Shopify/sarama/records.go
View File

@ -163,6 +163,27 @@ func (r *Records) isControl() (bool, error) {
return false, fmt.Errorf("unknown records type: %v", r.recordsType)
}
func (r *Records) isOverflow() (bool, error) {
if r.recordsType == unknownRecords {
if empty, err := r.setTypeFromFields(); err != nil || empty {
return false, err
}
}
switch r.recordsType {
case unknownRecords:
return false, nil
case legacyRecords:
if r.MsgSet == nil {
return false, nil
}
return r.MsgSet.OverflowMessage, nil
case defaultRecords:
return false, nil
}
return false, fmt.Errorf("unknown records type: %v", r.recordsType)
}
func magicValue(pd packetDecoder) (int8, error) {
dec, err := pd.peek(magicOffset, magicLength)
if err != nil {

10
third/github.com/Shopify/sarama/tools/README.md Normal file
View File

@ -0,0 +1,10 @@
# Sarama tools
This folder contains applications that are useful for exploration of your Kafka cluster, or instrumentation.
Some of these tools mirror tools that ship with Kafka, but these tools won't require installing the JVM to function.
- [kafka-console-producer](./kafka-console-producer): a command line tool to produce a single message to your Kafka custer.
- [kafka-console-partitionconsumer](./kafka-console-partitionconsumer): (deprecated) a command line tool to consume a single partition of a topic on your Kafka cluster.
- [kafka-console-consumer](./kafka-console-consumer): a command line tool to consume arbitrary partitions of a topic on your Kafka cluster.
To install all tools, run `go get github.com/Shopify/sarama/tools/...`

2
third/github.com/Shopify/sarama/tools/kafka-console-consumer/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
kafka-console-consumer
kafka-console-consumer.test

29
third/github.com/Shopify/sarama/tools/kafka-console-consumer/README.md Normal file
View File

@ -0,0 +1,29 @@
# kafka-console-consumer
A simple command line tool to consume partitions of a topic and print the
messages on the standard output.
### Installation
go get github.com/Shopify/sarama/tools/kafka-console-consumer
### Usage
# Minimum invocation
kafka-console-consumer -topic=test -brokers=kafka1:9092
# It will pick up a KAFKA_PEERS environment variable
export KAFKA_PEERS=kafka1:9092,kafka2:9092,kafka3:9092
kafka-console-consumer -topic=test
# You can specify the offset you want to start at. It can be either
# `oldest`, `newest`. The default is `newest`.
kafka-console-consumer -topic=test -offset=oldest
kafka-console-consumer -topic=test -offset=newest
# You can specify the partition(s) you want to consume as a comma-separated
# list. The default is `all`.
kafka-console-consumer -topic=test -partitions=1,2,3
# Display all command line options
kafka-console-consumer -help

145
third/github.com/Shopify/sarama/tools/kafka-console-consumer/kafka-console-consumer.go Normal file
View File

@ -0,0 +1,145 @@
package main
import (
"flag"
"fmt"
"log"
"os"
"os/signal"
"strconv"
"strings"
"sync"
"gitee.com/johng/gf/third/github.com/Shopify/sarama"
)
var (
brokerList = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The comma separated list of brokers in the Kafka cluster")
topic = flag.String("topic", "", "REQUIRED: the topic to consume")
partitions = flag.String("partitions", "all", "The partitions to consume, can be 'all' or comma-separated numbers")
offset = flag.String("offset", "newest", "The offset to start with. Can be `oldest`, `newest`")
verbose = flag.Bool("verbose", false, "Whether to turn on sarama logging")
bufferSize = flag.Int("buffer-size", 256, "The buffer size of the message channel.")
logger = log.New(os.Stderr, "", log.LstdFlags)
)
func main() {
flag.Parse()
if *brokerList == "" {
printUsageErrorAndExit("You have to provide -brokers as a comma-separated list, or set the KAFKA_PEERS environment variable.")
}
if *topic == "" {
printUsageErrorAndExit("-topic is required")
}
if *verbose {
sarama.Logger = logger
}
var initialOffset int64
switch *offset {
case "oldest":
initialOffset = sarama.OffsetOldest
case "newest":
initialOffset = sarama.OffsetNewest
default:
printUsageErrorAndExit("-offset should be `oldest` or `newest`")
}
c, err := sarama.NewConsumer(strings.Split(*brokerList, ","), nil)
if err != nil {
printErrorAndExit(69, "Failed to start consumer: %s", err)
}
partitionList, err := getPartitions(c)
if err != nil {
printErrorAndExit(69, "Failed to get the list of partitions: %s", err)
}
var (
messages = make(chan *sarama.ConsumerMessage, *bufferSize)
closing = make(chan struct{})
wg sync.WaitGroup
)
go func() {
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Kill, os.Interrupt)
<-signals
logger.Println("Initiating shutdown of consumer...")
close(closing)
}()
for _, partition := range partitionList {
pc, err := c.ConsumePartition(*topic, partition, initialOffset)
if err != nil {
printErrorAndExit(69, "Failed to start consumer for partition %d: %s", partition, err)
}
go func(pc sarama.PartitionConsumer) {
<-closing
pc.AsyncClose()
}(pc)
wg.Add(1)
go func(pc sarama.PartitionConsumer) {
defer wg.Done()
for message := range pc.Messages() {
messages <- message
}
}(pc)
}
go func() {
for msg := range messages {
fmt.Printf("Partition:\t%d\n", msg.Partition)
fmt.Printf("Offset:\t%d\n", msg.Offset)
fmt.Printf("Key:\t%s\n", string(msg.Key))
fmt.Printf("Value:\t%s\n", string(msg.Value))
fmt.Println()
}
}()
wg.Wait()
logger.Println("Done consuming topic", *topic)
close(messages)
if err := c.Close(); err != nil {
logger.Println("Failed to close consumer: ", err)
}
}
func getPartitions(c sarama.Consumer) ([]int32, error) {
if *partitions == "all" {
return c.Partitions(*topic)
}
tmp := strings.Split(*partitions, ",")
var pList []int32
for i := range tmp {
val, err := strconv.ParseInt(tmp[i], 10, 32)
if err != nil {
return nil, err
}
pList = append(pList, int32(val))
}
return pList, nil
}
func printErrorAndExit(code int, format string, values ...interface{}) {
fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
fmt.Fprintln(os.Stderr)
os.Exit(code)
}
func printUsageErrorAndExit(format string, values ...interface{}) {
fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
fmt.Fprintln(os.Stderr)
fmt.Fprintln(os.Stderr, "Available command line options:")
flag.PrintDefaults()
os.Exit(64)
}

2
third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
kafka-console-partitionconsumer
kafka-console-partitionconsumer.test

28
third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/README.md Normal file
View File

@ -0,0 +1,28 @@
# kafka-console-partitionconsumer
NOTE: this tool is deprecated in favour of the more general and more powerful
`kafka-console-consumer`.
A simple command line tool to consume a partition of a topic and print the messages
on the standard output.
### Installation
go get github.com/Shopify/sarama/tools/kafka-console-partitionconsumer
### Usage
# Minimum invocation
kafka-console-partitionconsumer -topic=test -partition=4 -brokers=kafka1:9092
# It will pick up a KAFKA_PEERS environment variable
export KAFKA_PEERS=kafka1:9092,kafka2:9092,kafka3:9092
kafka-console-partitionconsumer -topic=test -partition=4
# You can specify the offset you want to start at. It can be either
# `oldest`, `newest`, or a specific offset number
kafka-console-partitionconsumer -topic=test -partition=3 -offset=oldest
kafka-console-partitionconsumer -topic=test -partition=2 -offset=1337
# Display all command line options
kafka-console-partitionconsumer -help

102
third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/kafka-console-partitionconsumer.go Normal file
View File

@ -0,0 +1,102 @@
package main
import (
"flag"
"fmt"
"log"
"os"
"os/signal"
"strconv"
"strings"
"gitee.com/johng/gf/third/github.com/Shopify/sarama"
)
var (
brokerList = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The comma separated list of brokers in the Kafka cluster")
topic = flag.String("topic", "", "REQUIRED: the topic to consume")
partition = flag.Int("partition", -1, "REQUIRED: the partition to consume")
offset = flag.String("offset", "newest", "The offset to start with. Can be `oldest`, `newest`, or an actual offset")
verbose = flag.Bool("verbose", false, "Whether to turn on sarama logging")
logger = log.New(os.Stderr, "", log.LstdFlags)
)
func main() {
flag.Parse()
if *brokerList == "" {
printUsageErrorAndExit("You have to provide -brokers as a comma-separated list, or set the KAFKA_PEERS environment variable.")
}
if *topic == "" {
printUsageErrorAndExit("-topic is required")
}
if *partition == -1 {
printUsageErrorAndExit("-partition is required")
}
if *verbose {
sarama.Logger = logger
}
var (
initialOffset int64
offsetError error
)
switch *offset {
case "oldest":
initialOffset = sarama.OffsetOldest
case "newest":
initialOffset = sarama.OffsetNewest
default:
initialOffset, offsetError = strconv.ParseInt(*offset, 10, 64)
}
if offsetError != nil {
printUsageErrorAndExit("Invalid initial offset: %s", *offset)
}
c, err := sarama.NewConsumer(strings.Split(*brokerList, ","), nil)
if err != nil {
printErrorAndExit(69, "Failed to start consumer: %s", err)
}
pc, err := c.ConsumePartition(*topic, int32(*partition), initialOffset)
if err != nil {
printErrorAndExit(69, "Failed to start partition consumer: %s", err)
}
go func() {
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Kill, os.Interrupt)
<-signals
pc.AsyncClose()
}()
for msg := range pc.Messages() {
fmt.Printf("Offset:\t%d\n", msg.Offset)
fmt.Printf("Key:\t%s\n", string(msg.Key))
fmt.Printf("Value:\t%s\n", string(msg.Value))
fmt.Println()
}
if err := c.Close(); err != nil {
logger.Println("Failed to close consumer: ", err)
}
}
func printErrorAndExit(code int, format string, values ...interface{}) {
fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
fmt.Fprintln(os.Stderr)
os.Exit(code)
}
func printUsageErrorAndExit(format string, values ...interface{}) {
fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
fmt.Fprintln(os.Stderr)
fmt.Fprintln(os.Stderr, "Available command line options:")
flag.PrintDefaults()
os.Exit(64)
}

2
third/github.com/Shopify/sarama/tools/kafka-console-producer/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
kafka-console-producer
kafka-console-producer.test

34
third/github.com/Shopify/sarama/tools/kafka-console-producer/README.md Normal file
View File

@ -0,0 +1,34 @@
# kafka-console-producer
A simple command line tool to produce a single message to Kafka.
### Installation
go get github.com/Shopify/sarama/tools/kafka-console-producer
### Usage
# Minimum invocation
kafka-console-producer -topic=test -value=value -brokers=kafka1:9092
# It will pick up a KAFKA_PEERS environment variable
export KAFKA_PEERS=kafka1:9092,kafka2:9092,kafka3:9092
kafka-console-producer -topic=test -value=value
# It will read the value from stdin by using pipes
echo "hello world" | kafka-console-producer -topic=test
# Specify a key:
echo "hello world" | kafka-console-producer -topic=test -key=key
# Partitioning: by default, kafka-console-producer will partition as follows:
# - manual partitioning if a -partition is provided
# - hash partitioning by key if a -key is provided
# - random partioning otherwise.
#
# You can override this using the -partitioner argument:
echo "hello world" | kafka-console-producer -topic=test -key=key -partitioner=random
# Display all command line options
kafka-console-producer -help

124
third/github.com/Shopify/sarama/tools/kafka-console-producer/kafka-console-producer.go Normal file
View File

@ -0,0 +1,124 @@
package main
import (
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"strings"
"gitee.com/johng/gf/third/github.com/Shopify/sarama"
"gitee.com/johng/gf/third/github.com/rcrowley/go-metrics"
)
var (
brokerList = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The comma separated list of brokers in the Kafka cluster. You can also set the KAFKA_PEERS environment variable")
topic = flag.String("topic", "", "REQUIRED: the topic to produce to")
key = flag.String("key", "", "The key of the message to produce. Can be empty.")
value = flag.String("value", "", "REQUIRED: the value of the message to produce. You can also provide the value on stdin.")
partitioner = flag.String("partitioner", "", "The partitioning scheme to use. Can be `hash`, `manual`, or `random`")
partition = flag.Int("partition", -1, "The partition to produce to.")
verbose = flag.Bool("verbose", false, "Turn on sarama logging to stderr")
showMetrics = flag.Bool("metrics", false, "Output metrics on successful publish to stderr")
silent = flag.Bool("silent", false, "Turn off printing the message's topic, partition, and offset to stdout")
logger = log.New(os.Stderr, "", log.LstdFlags)
)
func main() {
flag.Parse()
if *brokerList == "" {
printUsageErrorAndExit("no -brokers specified. Alternatively, set the KAFKA_PEERS environment variable")
}
if *topic == "" {
printUsageErrorAndExit("no -topic specified")
}
if *verbose {
sarama.Logger = logger
}
config := sarama.NewConfig()
config.Producer.RequiredAcks = sarama.WaitForAll
config.Producer.Return.Successes = true
switch *partitioner {
case "":
if *partition >= 0 {
config.Producer.Partitioner = sarama.NewManualPartitioner
} else {
config.Producer.Partitioner = sarama.NewHashPartitioner
}
case "hash":
config.Producer.Partitioner = sarama.NewHashPartitioner
case "random":
config.Producer.Partitioner = sarama.NewRandomPartitioner
case "manual":
config.Producer.Partitioner = sarama.NewManualPartitioner
if *partition == -1 {
printUsageErrorAndExit("-partition is required when partitioning manually")
}
default:
printUsageErrorAndExit(fmt.Sprintf("Partitioner %s not supported.", *partitioner))
}
message := &sarama.ProducerMessage{Topic: *topic, Partition: int32(*partition)}
if *key != "" {
message.Key = sarama.StringEncoder(*key)
}
if *value != "" {
message.Value = sarama.StringEncoder(*value)
} else if stdinAvailable() {
bytes, err := ioutil.ReadAll(os.Stdin)
if err != nil {
printErrorAndExit(66, "Failed to read data from the standard input: %s", err)
}
message.Value = sarama.ByteEncoder(bytes)
} else {
printUsageErrorAndExit("-value is required, or you have to provide the value on stdin")
}
producer, err := sarama.NewSyncProducer(strings.Split(*brokerList, ","), config)
if err != nil {
printErrorAndExit(69, "Failed to open Kafka producer: %s", err)
}
defer func() {
if err := producer.Close(); err != nil {
logger.Println("Failed to close Kafka producer cleanly:", err)
}
}()
partition, offset, err := producer.SendMessage(message)
if err != nil {
printErrorAndExit(69, "Failed to produce message: %s", err)
} else if !*silent {
fmt.Printf("topic=%s\tpartition=%d\toffset=%d\n", *topic, partition, offset)
}
if *showMetrics {
metrics.WriteOnce(config.MetricRegistry, os.Stderr)
}
}
func printErrorAndExit(code int, format string, values ...interface{}) {
fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
fmt.Fprintln(os.Stderr)
os.Exit(code)
}
func printUsageErrorAndExit(message string) {
fmt.Fprintln(os.Stderr, "ERROR:", message)
fmt.Fprintln(os.Stderr)
fmt.Fprintln(os.Stderr, "Available command line options:")
flag.PrintDefaults()
os.Exit(64)
}
func stdinAvailable() bool {
stat, _ := os.Stdin.Stat()
return (stat.Mode() & os.ModeCharDevice) == 0
}

14
third/github.com/Shopify/sarama/utils.go
View File

@ -155,6 +155,10 @@ var (
V0_11_0_2 = newKafkaVersion(0, 11, 0, 2)
V1_0_0_0 = newKafkaVersion(1, 0, 0, 0)
V1_1_0_0 = newKafkaVersion(1, 1, 0, 0)
V1_1_1_0 = newKafkaVersion(1, 1, 1, 0)
V2_0_0_0 = newKafkaVersion(2, 0, 0, 0)
V2_0_1_0 = newKafkaVersion(2, 0, 1, 0)
V2_1_0_0 = newKafkaVersion(2, 1, 0, 0)
SupportedVersions = []KafkaVersion{
V0_8_2_0,
@ -173,9 +177,13 @@ var (
V0_11_0_2,
V1_0_0_0,
V1_1_0_0,
V1_1_1_0,
V2_0_0_0,
V2_0_1_0,
V2_1_0_0,
}
MinVersion = V0_8_2_0
MaxVersion = V1_1_0_0
MaxVersion = V2_1_0_0
)
func ParseKafkaVersion(s string) (KafkaVersion, error) {
@ -206,7 +214,7 @@ func scanKafkaVersion(s string, pattern string, format string, v [3]*uint) error
func (v KafkaVersion) String() string {
if v.version[0] == 0 {
return fmt.Sprintf("0.%d.%d.%d", v.version[1], v.version[2], v.version[3])
} else {
return fmt.Sprintf("%d.%d.%d", v.version[0], v.version[1], v.version[2])
}
return fmt.Sprintf("%d.%d.%d", v.version[0], v.version[1], v.version[2])
}

22
third/github.com/Shopify/sarama/vagrant/boot_cluster.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/sh
set -ex
# Launch and wait for toxiproxy
${REPOSITORY_ROOT}/vagrant/run_toxiproxy.sh &
while ! nc -q 1 localhost 2181 </dev/null; do echo "Waiting"; sleep 1; done
while ! nc -q 1 localhost 9092 </dev/null; do echo "Waiting"; sleep 1; done
# Launch and wait for Zookeeper
for i in 1 2 3 4 5; do
KAFKA_PORT=`expr $i + 9090`
cd ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT} && bin/zookeeper-server-start.sh -daemon config/zookeeper.properties
done
while ! nc -q 1 localhost 21805 </dev/null; do echo "Waiting"; sleep 1; done
# Launch and wait for Kafka
for i in 1 2 3 4 5; do
KAFKA_PORT=`expr $i + 9090`
cd ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT} && bin/kafka-server-start.sh -daemon config/server.properties
done
while ! nc -q 1 localhost 29095 </dev/null; do echo "Waiting"; sleep 1; done

8
third/github.com/Shopify/sarama/vagrant/create_topics.sh Executable file
View File

@ -0,0 +1,8 @@
#!/bin/sh
set -ex
cd ${KAFKA_INSTALL_ROOT}/kafka-9092
bin/kafka-topics.sh --create --partitions 1 --replication-factor 3 --topic test.1 --zookeeper localhost:2181
bin/kafka-topics.sh --create --partitions 4 --replication-factor 3 --topic test.4 --zookeeper localhost:2181
bin/kafka-topics.sh --create --partitions 64 --replication-factor 3 --topic test.64 --zookeeper localhost:2181

15
third/github.com/Shopify/sarama/vagrant/halt_cluster.sh Executable file
View File

@ -0,0 +1,15 @@
#!/bin/sh
set -ex
for i in 1 2 3 4 5; do
KAFKA_PORT=`expr $i + 9090`
cd ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT} && bin/kafka-server-stop.sh
done
for i in 1 2 3 4 5; do
KAFKA_PORT=`expr $i + 9090`
cd ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT} && bin/zookeeper-server-stop.sh
done
killall toxiproxy

49
third/github.com/Shopify/sarama/vagrant/install_cluster.sh Executable file
View File

@ -0,0 +1,49 @@
#!/bin/sh
set -ex
TOXIPROXY_VERSION=2.1.3
mkdir -p ${KAFKA_INSTALL_ROOT}
if [ ! -f ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_VERSION}.tgz ]; then
wget --quiet https://www-us.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz -O ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_VERSION}.tgz
fi
if [ ! -f ${KAFKA_INSTALL_ROOT}/toxiproxy-${TOXIPROXY_VERSION} ]; then
wget --quiet https://github.com/Shopify/toxiproxy/releases/download/v${TOXIPROXY_VERSION}/toxiproxy-server-linux-amd64 -O ${KAFKA_INSTALL_ROOT}/toxiproxy-${TOXIPROXY_VERSION}
chmod +x ${KAFKA_INSTALL_ROOT}/toxiproxy-${TOXIPROXY_VERSION}
fi
rm -f ${KAFKA_INSTALL_ROOT}/toxiproxy
ln -s ${KAFKA_INSTALL_ROOT}/toxiproxy-${TOXIPROXY_VERSION} ${KAFKA_INSTALL_ROOT}/toxiproxy
for i in 1 2 3 4 5; do
ZK_PORT=`expr $i + 2180`
ZK_PORT_REAL=`expr $i + 21800`
KAFKA_PORT=`expr $i + 9090`
KAFKA_PORT_REAL=`expr $i + 29090`
# unpack kafka
mkdir -p ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}
tar xzf ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_VERSION}.tgz -C ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT} --strip-components 1
# broker configuration
cp ${REPOSITORY_ROOT}/vagrant/server.properties ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/
sed -i s/KAFKAID/${KAFKA_PORT}/g ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/server.properties
sed -i s/KAFKAPORT/${KAFKA_PORT_REAL}/g ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/server.properties
sed -i s/KAFKA_HOSTNAME/${KAFKA_HOSTNAME}/g ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/server.properties
sed -i s/ZK_PORT/${ZK_PORT}/g ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/server.properties
KAFKA_DATADIR="${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/data"
mkdir -p ${KAFKA_DATADIR}
sed -i s#KAFKA_DATADIR#${KAFKA_DATADIR}#g ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/server.properties
# zookeeper configuration
cp ${REPOSITORY_ROOT}/vagrant/zookeeper.properties ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/
sed -i s/KAFKAID/${KAFKA_PORT}/g ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/zookeeper.properties
sed -i s/ZK_PORT/${ZK_PORT_REAL}/g ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/zookeeper.properties
ZK_DATADIR="${KAFKA_INSTALL_ROOT}/zookeeper-${ZK_PORT}"
mkdir -p ${ZK_DATADIR}
sed -i s#ZK_DATADIR#${ZK_DATADIR}#g ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/zookeeper.properties
echo $i > ${KAFKA_INSTALL_ROOT}/zookeeper-${ZK_PORT}/myid
done

9
third/github.com/Shopify/sarama/vagrant/kafka.conf Normal file
View File

@ -0,0 +1,9 @@
start on started zookeeper-ZK_PORT
stop on stopping zookeeper-ZK_PORT
# Use a script instead of exec (using env stanza leaks KAFKA_HEAP_OPTS from zookeeper)
script
sleep 2
export KAFKA_HEAP_OPTS="-Xmx320m"
exec /opt/kafka-KAFKAID/bin/kafka-server-start.sh /opt/kafka-KAFKAID/config/server.properties
end script

15
third/github.com/Shopify/sarama/vagrant/provision.sh Executable file
View File

@ -0,0 +1,15 @@
#!/bin/sh
set -ex
apt-get update
yes | apt-get install default-jre
export KAFKA_INSTALL_ROOT=/opt
export KAFKA_HOSTNAME=192.168.100.67
export KAFKA_VERSION=1.0.0
export REPOSITORY_ROOT=/vagrant
sh /vagrant/vagrant/install_cluster.sh
sh /vagrant/vagrant/setup_services.sh
sh /vagrant/vagrant/create_topics.sh

22
third/github.com/Shopify/sarama/vagrant/run_toxiproxy.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/sh
set -ex
${KAFKA_INSTALL_ROOT}/toxiproxy -port 8474 -host 0.0.0.0 &
PID=$!
while ! nc -q 1 localhost 8474 </dev/null; do echo "Waiting"; sleep 1; done
wget -O/dev/null -S --post-data='{"name":"zk1", "upstream":"localhost:21801", "listen":"0.0.0.0:2181"}' localhost:8474/proxies
wget -O/dev/null -S --post-data='{"name":"zk2", "upstream":"localhost:21802", "listen":"0.0.0.0:2182"}' localhost:8474/proxies
wget -O/dev/null -S --post-data='{"name":"zk3", "upstream":"localhost:21803", "listen":"0.0.0.0:2183"}' localhost:8474/proxies
wget -O/dev/null -S --post-data='{"name":"zk4", "upstream":"localhost:21804", "listen":"0.0.0.0:2184"}' localhost:8474/proxies
wget -O/dev/null -S --post-data='{"name":"zk5", "upstream":"localhost:21805", "listen":"0.0.0.0:2185"}' localhost:8474/proxies
wget -O/dev/null -S --post-data='{"name":"kafka1", "upstream":"localhost:29091", "listen":"0.0.0.0:9091"}' localhost:8474/proxies
wget -O/dev/null -S --post-data='{"name":"kafka2", "upstream":"localhost:29092", "listen":"0.0.0.0:9092"}' localhost:8474/proxies
wget -O/dev/null -S --post-data='{"name":"kafka3", "upstream":"localhost:29093", "listen":"0.0.0.0:9093"}' localhost:8474/proxies
wget -O/dev/null -S --post-data='{"name":"kafka4", "upstream":"localhost:29094", "listen":"0.0.0.0:9094"}' localhost:8474/proxies
wget -O/dev/null -S --post-data='{"name":"kafka5", "upstream":"localhost:29095", "listen":"0.0.0.0:9095"}' localhost:8474/proxies
wait $PID

127
third/github.com/Shopify/sarama/vagrant/server.properties Normal file
View File

@ -0,0 +1,127 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# see kafka.server.KafkaConfig for additional details and defaults
############################# Server Basics #############################
# The id of the broker. This must be set to a unique integer for each broker.
broker.id=KAFKAID
reserved.broker.max.id=10000
############################# Socket Server Settings #############################
# The port the socket server listens on
port=KAFKAPORT
# Hostname the broker will bind to. If not set, the server will bind to all interfaces
host.name=localhost
# Hostname the broker will advertise to producers and consumers. If not set, it uses the
# value for "host.name" if configured. Otherwise, it will use the value returned from
# java.net.InetAddress.getCanonicalHostName().
advertised.host.name=KAFKA_HOSTNAME
advertised.port=KAFKAID
# The port to publish to ZooKeeper for clients to use. If this is not set,
# it will publish the same port that the broker binds to.
# advertised.port=<port accessible by clients>
# The number of threads handling network requests
num.network.threads=2
# The number of threads doing disk I/O
num.io.threads=8
# The send buffer (SO_SNDBUF) used by the socket server
socket.send.buffer.bytes=1048576
# The receive buffer (SO_RCVBUF) used by the socket server
socket.receive.buffer.bytes=1048576
# The maximum size of a request that the socket server will accept (protection against OOM)
socket.request.max.bytes=104857600
############################# Log Basics #############################
# A comma seperated list of directories under which to store log files
log.dirs=KAFKA_DATADIR
# The default number of log partitions per topic. More partitions allow greater
# parallelism for consumption, but this will also result in more files across
# the brokers.
num.partitions=2
# Create new topics with a replication factor of 2 so failover can be tested
# more easily.
default.replication.factor=2
auto.create.topics.enable=false
delete.topic.enable=true
############################# Log Flush Policy #############################
# Messages are immediately written to the filesystem but by default we only fsync() to sync
# the OS cache lazily. The following configurations control the flush of data to disk.
# There are a few important trade-offs here:
# 1. Durability: Unflushed data may be lost if you are not using replication.
# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks.
# The settings below allow one to configure the flush policy to flush data after a period of time or
# every N messages (or both). This can be done globally and overridden on a per-topic basis.
# The number of messages to accept before forcing a flush of data to disk
#log.flush.interval.messages=10000
# The maximum amount of time a message can sit in a log before we force a flush
#log.flush.interval.ms=1000
############################# Log Retention Policy #############################
# The following configurations control the disposal of log segments. The policy can
# be set to delete segments after a period of time, or after a given size has accumulated.
# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
# from the end of the log.
# The minimum age of a log file to be eligible for deletion
log.retention.hours=168
# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining
# segments don't drop below log.retention.bytes.
log.retention.bytes=268435456
# The maximum size of a log segment file. When this size is reached a new log segment will be created.
log.segment.bytes=268435456
# The interval at which log segments are checked to see if they can be deleted according
# to the retention policies
log.retention.check.interval.ms=60000
# By default the log cleaner is disabled and the log retention policy will default to just delete segments after their retention expires.
# If log.cleaner.enable=true is set the cleaner will be enabled and individual logs can then be marked for log compaction.
log.cleaner.enable=false
############################# Zookeeper #############################
# Zookeeper connection string (see zookeeper docs for details).
# This is a comma separated host:port pairs, each corresponding to a zk
# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
# You can also append an optional chroot string to the urls to specify the
# root directory for all kafka znodes.
zookeeper.connect=localhost:ZK_PORT
# Timeout in ms for connecting to zookeeper
zookeeper.session.timeout.ms=3000
zookeeper.connection.timeout.ms=3000

29
third/github.com/Shopify/sarama/vagrant/setup_services.sh Executable file
View File

@ -0,0 +1,29 @@
#!/bin/sh
set -ex
stop toxiproxy || true
cp ${REPOSITORY_ROOT}/vagrant/toxiproxy.conf /etc/init/toxiproxy.conf
cp ${REPOSITORY_ROOT}/vagrant/run_toxiproxy.sh ${KAFKA_INSTALL_ROOT}/
start toxiproxy
for i in 1 2 3 4 5; do
ZK_PORT=`expr $i + 2180`
KAFKA_PORT=`expr $i + 9090`
stop zookeeper-${ZK_PORT} || true
# set up zk service
cp ${REPOSITORY_ROOT}/vagrant/zookeeper.conf /etc/init/zookeeper-${ZK_PORT}.conf
sed -i s/KAFKAID/${KAFKA_PORT}/g /etc/init/zookeeper-${ZK_PORT}.conf
# set up kafka service
cp ${REPOSITORY_ROOT}/vagrant/kafka.conf /etc/init/kafka-${KAFKA_PORT}.conf
sed -i s/KAFKAID/${KAFKA_PORT}/g /etc/init/kafka-${KAFKA_PORT}.conf
sed -i s/ZK_PORT/${ZK_PORT}/g /etc/init/kafka-${KAFKA_PORT}.conf
start zookeeper-${ZK_PORT}
done
# Wait for the last kafka node to finish booting
while ! nc -q 1 localhost 29095 </dev/null; do echo "Waiting"; sleep 1; done

6
third/github.com/Shopify/sarama/vagrant/toxiproxy.conf Normal file
View File

@ -0,0 +1,6 @@
start on started networking
stop on shutdown
env KAFKA_INSTALL_ROOT=/opt
exec /opt/run_toxiproxy.sh

7
third/github.com/Shopify/sarama/vagrant/zookeeper.conf Normal file
View File

@ -0,0 +1,7 @@
start on started toxiproxy
stop on stopping toxiproxy
script
export KAFKA_HEAP_OPTS="-Xmx192m"
exec /opt/kafka-KAFKAID/bin/zookeeper-server-start.sh /opt/kafka-KAFKAID/config/zookeeper.properties
end script

36
third/github.com/Shopify/sarama/vagrant/zookeeper.properties Normal file
View File

@ -0,0 +1,36 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# the directory where the snapshot is stored.
dataDir=ZK_DATADIR
# the port at which the clients will connect
clientPort=ZK_PORT
# disable the per-ip limit on the number of connections since this is a non-production config
maxClientCnxns=0
# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial synchronization phase can take
initLimit=10
# The number of ticks that can pass between
# sending a request and getting an acknowledgement
syncLimit=5
server.1=localhost:2281:2381
server.2=localhost:2282:2382
server.3=localhost:2283:2383
server.4=localhost:2284:2384
server.5=localhost:2285:2385

13
third/github.com/Shopify/sarama/zstd_cgo.go Normal file
View File

@ -0,0 +1,13 @@
// +build cgo
package sarama
import "gitee.com/johng/gf/third/github.com/DataDog/zstd"
func zstdDecompress(dst, src []byte) ([]byte, error) {
return zstd.Decompress(dst, src)
}
func zstdCompressLevel(dst, src []byte, level int) ([]byte, error) {
return zstd.CompressLevel(dst, src, level)
}

17
third/github.com/Shopify/sarama/zstd_fallback.go Normal file
View File

@ -0,0 +1,17 @@
// +build !cgo
package sarama
import (
"errors"
)
var errZstdCgo = errors.New("zstd compression requires building with cgo enabled")
func zstdDecompress(dst, src []byte) ([]byte, error) {
return nil, errZstdCgo
}
func zstdCompressLevel(dst, src []byte, level int) ([]byte, error) {
return nil, errZstdCgo
}

4
third/github.com/go-sql-driver/mysql/README.md
View File

@ -58,7 +58,7 @@ _Go MySQL Driver_ is an implementation of Go's `database/sql/driver` interface.
Use `mysql` as `driverName` and a valid [DSN](#dsn-data-source-name) as `dataSourceName`:
```go
import "database/sql"
import _ "github.com/go-sql-driver/mysql"
import _ "gitee.com/johng/gf/third/github.com/go-sql-driver/mysql"
db, err := sql.Open("mysql", "user:password@/dbname")
```
@ -431,7 +431,7 @@ See [context support in the database/sql package](https://golang.org/doc/go1.8#d
### `LOAD DATA LOCAL INFILE` support
For this feature you need direct access to the package. Therefore you must change the import path (no `_`):
```go
import "github.com/go-sql-driver/mysql"
import "gitee.com/johng/gf/third/github.com/go-sql-driver/mysql"
```
Files must be whitelisted by registering them with `mysql.RegisterLocalFile(filepath)` (recommended) or the Whitelist check must be deactivated by using the DSN parameter `allowAllFiles=true` ([*Might be insecure!*](http://dev.mysql.com/doc/refman/5.7/en/load-data-local.html)).

2
third/github.com/go-sql-driver/mysql/driver.go
View File

@ -9,7 +9,7 @@
// The driver should be used via the database/sql package:
//
// import "database/sql"
// import _ "github.com/go-sql-driver/mysql"
// import _ "gitee.com/johng/gf/third/github.com/go-sql-driver/mysql"
//
// db, err := sql.Open("mysql", "user:password@/dbname")
//

16
third/github.com/johng-cn/sarama-cluster/balancer.go
View File

@ -116,12 +116,13 @@ func (info topicInfo) RoundRobin() map[string][]int32 {
// --------------------------------------------------------------------
type balancer struct {
client sarama.Client
topics map[string]topicInfo
client sarama.Client
topics map[string]topicInfo
strategy Strategy
}
func newBalancerFromMeta(client sarama.Client, members map[string]sarama.ConsumerGroupMemberMetadata) (*balancer, error) {
balancer := newBalancer(client)
func newBalancerFromMeta(client sarama.Client, strategy Strategy, members map[string]sarama.ConsumerGroupMemberMetadata) (*balancer, error) {
balancer := newBalancer(client, strategy)
for memberID, meta := range members {
for _, topic := range meta.Topics {
if err := balancer.Topic(topic, memberID); err != nil {
@ -132,10 +133,11 @@ func newBalancerFromMeta(client sarama.Client, members map[string]sarama.Consume
return balancer, nil
}
func newBalancer(client sarama.Client) *balancer {
func newBalancer(client sarama.Client, strategy Strategy) *balancer {
return &balancer{
client: client,
topics: make(map[string]topicInfo),
strategy: strategy,
}
}
@ -156,10 +158,10 @@ func (r *balancer) Topic(name string, memberID string) error {
return nil
}
func (r *balancer) Perform(s Strategy) map[string]map[string][]int32 {
func (r *balancer) Perform() map[string]map[string][]int32 {
res := make(map[string]map[string][]int32, 1)
for topic, info := range r.topics {
for memberID, partitions := range info.Perform(s) {
for memberID, partitions := range info.Perform(r.strategy) {
if _, ok := res[memberID]; !ok {
res[memberID] = make(map[string][]int32, 1)
}

130
third/github.com/johng-cn/sarama-cluster/balancer_test.go
View File

@ -1,130 +0,0 @@
package cluster
import (
"gitee.com/johng/gf/third/github.com/Shopify/sarama"
. "gitee.com/johng/gf/third/github.com/onsi/ginkgo"
. "gitee.com/johng/gf/third/github.com/onsi/ginkgo/extensions/table"
. "gitee.com/johng/gf/third/github.com/onsi/gomega"
)
var _ = Describe("Notification", func() {
It("should init and convert", func() {
n := newNotification(map[string][]int32{
"a": {1, 2, 3},
"b": {4, 5},
"c": {1, 2},
})
Expect(n).To(Equal(&Notification{
Type: RebalanceStart,
Current: map[string][]int32{"a": {1, 2, 3}, "b": {4, 5}, "c": {1, 2}},
}))
o := n.success(map[string][]int32{
"a": {3, 4},
"b": {1, 2, 3, 4},
"d": {3, 4},
})
Expect(o).To(Equal(&Notification{
Type: RebalanceOK,
Claimed: map[string][]int32{"a": {4}, "b": {1, 2, 3}, "d": {3, 4}},
Released: map[string][]int32{"a": {1, 2}, "b": {5}, "c": {1, 2}},
Current: map[string][]int32{"a": {3, 4}, "b": {1, 2, 3, 4}, "d": {3, 4}},
}))
})
})
var _ = Describe("balancer", func() {
var subject *balancer
BeforeEach(func() {
client := &mockClient{
topics: map[string][]int32{
"one": {0, 1, 2, 3},
"two": {0, 1, 2},
"three": {0, 1},
},
}
var err error
subject, err = newBalancerFromMeta(client, map[string]sarama.ConsumerGroupMemberMetadata{
"b": {Topics: []string{"three", "one"}},
"a": {Topics: []string{"one", "two"}},
})
Expect(err).NotTo(HaveOccurred())
})
It("should parse from meta data", func() {
Expect(subject.topics).To(HaveLen(3))
})
It("should perform", func() {
Expect(subject.Perform(StrategyRange)).To(Equal(map[string]map[string][]int32{
"a": {"one": {0, 1}, "two": {0, 1, 2}},
"b": {"one": {2, 3}, "three": {0, 1}},
}))
Expect(subject.Perform(StrategyRoundRobin)).To(Equal(map[string]map[string][]int32{
"a": {"one": {0, 2}, "two": {0, 1, 2}},
"b": {"one": {1, 3}, "three": {0, 1}},
}))
})
})
var _ = Describe("topicInfo", func() {
DescribeTable("Ranges",
func(memberIDs []string, partitions []int32, expected map[string][]int32) {
info := topicInfo{MemberIDs: memberIDs, Partitions: partitions}
Expect(info.Ranges()).To(Equal(expected))
},
Entry("three members, three partitions", []string{"M1", "M2", "M3"}, []int32{0, 1, 2}, map[string][]int32{
"M1": {0}, "M2": {1}, "M3": {2},
}),
Entry("member ID order", []string{"M3", "M1", "M2"}, []int32{0, 1, 2}, map[string][]int32{
"M1": {0}, "M2": {1}, "M3": {2},
}),
Entry("more members than partitions", []string{"M1", "M2", "M3"}, []int32{0, 1}, map[string][]int32{
"M1": {0}, "M3": {1},
}),
Entry("far more members than partitions", []string{"M1", "M2", "M3"}, []int32{0}, map[string][]int32{
"M2": {0},
}),
Entry("fewer members than partitions", []string{"M1", "M2", "M3"}, []int32{0, 1, 2, 3}, map[string][]int32{
"M1": {0}, "M2": {1, 2}, "M3": {3},
}),
Entry("uneven members/partitions ratio", []string{"M1", "M2", "M3"}, []int32{0, 2, 4, 6, 8}, map[string][]int32{
"M1": {0, 2}, "M2": {4}, "M3": {6, 8},
}),
)
DescribeTable("RoundRobin",
func(memberIDs []string, partitions []int32, expected map[string][]int32) {
info := topicInfo{MemberIDs: memberIDs, Partitions: partitions}
Expect(info.RoundRobin()).To(Equal(expected))
},
Entry("three members, three partitions", []string{"M1", "M2", "M3"}, []int32{0, 1, 2}, map[string][]int32{
"M1": {0}, "M2": {1}, "M3": {2},
}),
Entry("member ID order", []string{"M3", "M1", "M2"}, []int32{0, 1, 2}, map[string][]int32{
"M1": {0}, "M2": {1}, "M3": {2},
}),
Entry("more members than partitions", []string{"M1", "M2", "M3"}, []int32{0, 1}, map[string][]int32{
"M1": {0}, "M2": {1},
}),
Entry("far more members than partitions", []string{"M1", "M2", "M3"}, []int32{0}, map[string][]int32{
"M1": {0},
}),
Entry("fewer members than partitions", []string{"M1", "M2", "M3"}, []int32{0, 1, 2, 3}, map[string][]int32{
"M1": {0, 3}, "M2": {1}, "M3": {2},
}),
Entry("uneven members/partitions ratio", []string{"M1", "M2", "M3"}, []int32{0, 2, 4, 6, 8}, map[string][]int32{
"M1": {0, 6}, "M2": {2, 8}, "M3": {4},
}),
)
})

31
third/github.com/johng-cn/sarama-cluster/client_test.go
View File

@ -1,31 +0,0 @@
package cluster
import (
. "gitee.com/johng/gf/third/github.com/onsi/ginkgo"
. "gitee.com/johng/gf/third/github.com/onsi/gomega"
)
var _ = Describe("Client", func() {
var subject *Client
BeforeEach(func() {
var err error
subject, err = NewClient(testKafkaAddrs, nil)
Expect(err).NotTo(HaveOccurred())
})
It("should not allow to share clients across multiple consumers", func() {
c1, err := NewConsumerFromClient(subject, testGroup, testTopics)
Expect(err).NotTo(HaveOccurred())
defer c1.Close()
_, err = NewConsumerFromClient(subject, testGroup, testTopics)
Expect(err).To(MatchError("cluster: client is already used by another consumer"))
Expect(c1.Close()).To(Succeed())
c2, err := NewConsumerFromClient(subject, testGroup, testTopics)
Expect(err).NotTo(HaveOccurred())
Expect(c2.Close()).To(Succeed())
})
})

201
third/github.com/johng-cn/sarama-cluster/cluster_test.go
View File

@ -1,201 +0,0 @@
package cluster
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"testing"
"gitee.com/johng/gf/third/github.com/Shopify/sarama"
. "gitee.com/johng/gf/third/github.com/onsi/ginkgo"
. "gitee.com/johng/gf/third/github.com/onsi/gomega"
)
const (
testGroup = "sarama-cluster-group"
testKafkaData = "/tmp/sarama-cluster-test"
)
var (
testKafkaRoot = "kafka_2.12-1.1.0"
testKafkaAddrs = []string{"127.0.0.1:29092"}
testTopics = []string{"topic-a", "topic-b"}
testClient sarama.Client
testKafkaCmd, testZkCmd *exec.Cmd
)
func init() {
if dir := os.Getenv("KAFKA_DIR"); dir != "" {
testKafkaRoot = dir
}
}
var _ = Describe("offsetInfo", func() {
It("should calculate next offset", func() {
Expect(offsetInfo{-2, ""}.NextOffset(sarama.OffsetOldest)).To(Equal(sarama.OffsetOldest))
Expect(offsetInfo{-2, ""}.NextOffset(sarama.OffsetNewest)).To(Equal(sarama.OffsetNewest))
Expect(offsetInfo{-1, ""}.NextOffset(sarama.OffsetOldest)).To(Equal(sarama.OffsetOldest))
Expect(offsetInfo{-1, ""}.NextOffset(sarama.OffsetNewest)).To(Equal(sarama.OffsetNewest))
Expect(offsetInfo{0, ""}.NextOffset(sarama.OffsetOldest)).To(Equal(int64(0)))
Expect(offsetInfo{100, ""}.NextOffset(sarama.OffsetOldest)).To(Equal(int64(100)))
})
})
var _ = Describe("int32Slice", func() {
It("should diff", func() {
Expect(((int32Slice)(nil)).Diff(int32Slice{1, 3, 5})).To(BeNil())
Expect(int32Slice{1, 3, 5}.Diff((int32Slice)(nil))).To(Equal([]int32{1, 3, 5}))
Expect(int32Slice{1, 3, 5}.Diff(int32Slice{1, 3, 5})).To(BeNil())
Expect(int32Slice{1, 3, 5}.Diff(int32Slice{1, 2, 3, 4, 5})).To(BeNil())
Expect(int32Slice{1, 3, 5}.Diff(int32Slice{2, 3, 4})).To(Equal([]int32{1, 5}))
Expect(int32Slice{1, 3, 5}.Diff(int32Slice{1, 4})).To(Equal([]int32{3, 5}))
Expect(int32Slice{1, 3, 5}.Diff(int32Slice{2, 5})).To(Equal([]int32{1, 3}))
})
})
// --------------------------------------------------------------------
var _ = BeforeSuite(func() {
testZkCmd = testCmd(
testDataDir(testKafkaRoot, "bin", "kafka-run-class.sh"),
"org.apache.zookeeper.server.quorum.QuorumPeerMain",
testDataDir("zookeeper.properties"),
)
testKafkaCmd = testCmd(
testDataDir(testKafkaRoot, "bin", "kafka-run-class.sh"),
"-name", "kafkaServer", "kafka.Kafka",
testDataDir("server.properties"),
)
// Remove old test data before starting
Expect(os.RemoveAll(testKafkaData)).NotTo(HaveOccurred())
Expect(os.MkdirAll(testKafkaData, 0777)).To(Succeed())
Expect(testZkCmd.Start()).To(Succeed())
Expect(testKafkaCmd.Start()).To(Succeed())
// Wait for client
Eventually(func() error {
var err error
// sync-producer requires Return.Successes set to true
testConf := sarama.NewConfig()
testConf.Producer.Return.Successes = true
testClient, err = sarama.NewClient(testKafkaAddrs, testConf)
return err
}, "30s", "1s").Should(Succeed())
// Ensure we can retrieve partition info
Eventually(func() error {
_, err := testClient.Partitions(testTopics[0])
return err
}, "30s", "1s").Should(Succeed())
// Seed a few messages
Expect(testSeed(1000, testTopics)).To(Succeed())
})
var _ = AfterSuite(func() {
if testClient != nil {
_ = testClient.Close()
}
_ = testKafkaCmd.Process.Kill()
_ = testZkCmd.Process.Kill()
_ = testKafkaCmd.Wait()
_ = testZkCmd.Wait()
_ = os.RemoveAll(testKafkaData)
})
// --------------------------------------------------------------------
func TestSuite(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "sarama/cluster")
}
func testDataDir(tokens ...string) string {
tokens = append([]string{"testdata"}, tokens...)
return filepath.Join(tokens...)
}
func testSeed(n int, testTopics []string) error {
producer, err := sarama.NewSyncProducerFromClient(testClient)
if err != nil {
return err
}
defer producer.Close()
for i := 0; i < n; i++ {
kv := sarama.StringEncoder(fmt.Sprintf("PLAINDATA-%08d", i))
for _, t := range testTopics {
msg := &sarama.ProducerMessage{Topic: t, Key: kv, Value: kv}
if _, _, err := producer.SendMessage(msg); err != nil {
return err
}
}
}
return nil
}
func testCmd(name string, arg ...string) *exec.Cmd {
cmd := exec.Command(name, arg...)
if testing.Verbose() || os.Getenv("CI") != "" {
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
}
cmd.Env = []string{"KAFKA_HEAP_OPTS=-Xmx1G -Xms1G"}
return cmd
}
type testConsumerMessage struct {
sarama.ConsumerMessage
ConsumerID string
}
// --------------------------------------------------------------------
var _ sarama.Consumer = &mockConsumer{}
var _ sarama.PartitionConsumer = &mockPartitionConsumer{}
type mockClient struct {
sarama.Client
topics map[string][]int32
}
type mockConsumer struct{ sarama.Consumer }
type mockPartitionConsumer struct {
sarama.PartitionConsumer
Topic string
Partition int32
Offset int64
}
func (m *mockClient) Partitions(t string) ([]int32, error) {
pts, ok := m.topics[t]
if !ok {
return nil, sarama.ErrInvalidTopic
}
return pts, nil
}
func (*mockConsumer) ConsumePartition(topic string, partition int32, offset int64) (sarama.PartitionConsumer, error) {
if offset > -1 && offset < 1000 {
return nil, sarama.ErrOffsetOutOfRange
}
return &mockPartitionConsumer{
Topic: topic,
Partition: partition,
Offset: offset,
}, nil
}
func (*mockPartitionConsumer) Close() error { return nil }

26
third/github.com/johng-cn/sarama-cluster/config_test.go
View File

@ -1,26 +0,0 @@
package cluster
import (
"time"
. "gitee.com/johng/gf/third/github.com/onsi/ginkgo"
. "gitee.com/johng/gf/third/github.com/onsi/gomega"
)
var _ = Describe("Config", func() {
var subject *Config
BeforeEach(func() {
subject = NewConfig()
})
It("should init", func() {
Expect(subject.Group.Session.Timeout).To(Equal(30 * time.Second))
Expect(subject.Group.Heartbeat.Interval).To(Equal(3 * time.Second))
Expect(subject.Group.Return.Notifications).To(BeFalse())
Expect(subject.Metadata.Retry.Max).To(Equal(3))
Expect(subject.Group.Offsets.Synchronization.DwellTime).NotTo(BeZero())
// Expect(subject.Config.Version).To(Equal(sarama.V0_9_0_0))
})
})

12
third/github.com/johng-cn/sarama-cluster/consumer.go
View File

@ -374,7 +374,7 @@ func (c *Consumer) nextTick() {
// Issue rebalance start notification
if c.client.config.Group.Return.Notifications {
c.handleNotification(notification)
c.handleNotification(newNotification(c.subs.Info()))
}
// Rebalance, fetch new subscriptions
@ -705,7 +705,7 @@ func (c *Consumer) joinGroup() (*balancer, error) {
return nil, err
}
strategy, err = newBalancerFromMeta(c.client, members)
strategy, err = newBalancerFromMeta(c.client, Strategy(resp.GroupProtocol), members)
if err != nil {
return nil, err
}
@ -730,7 +730,7 @@ func (c *Consumer) syncGroup(strategy *balancer) (map[string][]int32, error) {
}
if strategy != nil {
for memberID, topics := range strategy.Perform(c.client.config.Group.PartitionStrategy) {
for memberID, topics := range strategy.Perform() {
if err := req.AddGroupAssignmentMember(memberID, &sarama.ConsumerGroupMemberAssignment{
Topics: topics,
}); err != nil {
@ -860,7 +860,11 @@ func (c *Consumer) createConsumer(tomb *loopTomb, topic string, partition int32,
})
if c.client.config.Group.Mode == ConsumerModePartitions {
c.partitions <- pc
select {
case c.partitions <- pc:
case <-c.dying:
pc.Close()
}
}
return nil
}

350
third/github.com/johng-cn/sarama-cluster/consumer_test.go
View File

@ -1,350 +0,0 @@
package cluster
import (
"fmt"
"regexp"
"sync/atomic"
"time"
"gitee.com/johng/gf/third/github.com/Shopify/sarama"
. "gitee.com/johng/gf/third/github.com/onsi/ginkgo"
. "gitee.com/johng/gf/third/github.com/onsi/gomega"
)
var _ = Describe("Consumer", func() {
var newConsumerOf = func(group string, topics ...string) (*Consumer, error) {
config := NewConfig()
config.Consumer.Return.Errors = true
config.Consumer.Offsets.Initial = sarama.OffsetOldest
return NewConsumer(testKafkaAddrs, group, topics, config)
}
var subscriptionsOf = func(c *Consumer) GomegaAsyncAssertion {
return Eventually(func() map[string][]int32 {
return c.Subscriptions()
}, "10s", "100ms")
}
It("should init and share", func() {
// start CS1
cs1, err := newConsumerOf(testGroup, testTopics...)
Expect(err).NotTo(HaveOccurred())
// CS1 should consume all 8 partitions
subscriptionsOf(cs1).Should(Equal(map[string][]int32{
"topic-a": {0, 1, 2, 3},
"topic-b": {0, 1, 2, 3},
}))
// start CS2
cs2, err := newConsumerOf(testGroup, testTopics...)
Expect(err).NotTo(HaveOccurred())
defer cs2.Close()
// CS1 and CS2 should consume 4 partitions each
subscriptionsOf(cs1).Should(HaveLen(2))
subscriptionsOf(cs1).Should(HaveKeyWithValue("topic-a", HaveLen(2)))
subscriptionsOf(cs1).Should(HaveKeyWithValue("topic-b", HaveLen(2)))
subscriptionsOf(cs2).Should(HaveLen(2))
subscriptionsOf(cs2).Should(HaveKeyWithValue("topic-a", HaveLen(2)))
subscriptionsOf(cs2).Should(HaveKeyWithValue("topic-b", HaveLen(2)))
// shutdown CS1, now CS2 should consume all 8 partitions
Expect(cs1.Close()).NotTo(HaveOccurred())
subscriptionsOf(cs2).Should(Equal(map[string][]int32{
"topic-a": {0, 1, 2, 3},
"topic-b": {0, 1, 2, 3},
}))
})
It("should allow more consumers than partitions", func() {
cs1, err := newConsumerOf(testGroup, "topic-a")
Expect(err).NotTo(HaveOccurred())
defer cs1.Close()
cs2, err := newConsumerOf(testGroup, "topic-a")
Expect(err).NotTo(HaveOccurred())
defer cs2.Close()
cs3, err := newConsumerOf(testGroup, "topic-a")
Expect(err).NotTo(HaveOccurred())
defer cs3.Close()
cs4, err := newConsumerOf(testGroup, "topic-a")
Expect(err).NotTo(HaveOccurred())
// start 4 consumers, one for each partition
subscriptionsOf(cs1).Should(HaveKeyWithValue("topic-a", HaveLen(1)))
subscriptionsOf(cs2).Should(HaveKeyWithValue("topic-a", HaveLen(1)))
subscriptionsOf(cs3).Should(HaveKeyWithValue("topic-a", HaveLen(1)))
subscriptionsOf(cs4).Should(HaveKeyWithValue("topic-a", HaveLen(1)))
// add a 5th consumer
cs5, err := newConsumerOf(testGroup, "topic-a")
Expect(err).NotTo(HaveOccurred())
defer cs5.Close()
// make sure no errors occurred
Expect(cs1.Errors()).ShouldNot(Receive())
Expect(cs2.Errors()).ShouldNot(Receive())
Expect(cs3.Errors()).ShouldNot(Receive())
Expect(cs4.Errors()).ShouldNot(Receive())
Expect(cs5.Errors()).ShouldNot(Receive())
// close 4th, make sure the 5th takes over
Expect(cs4.Close()).To(Succeed())
subscriptionsOf(cs1).Should(HaveKeyWithValue("topic-a", HaveLen(1)))
subscriptionsOf(cs2).Should(HaveKeyWithValue("topic-a", HaveLen(1)))
subscriptionsOf(cs3).Should(HaveKeyWithValue("topic-a", HaveLen(1)))
subscriptionsOf(cs4).Should(BeEmpty())
subscriptionsOf(cs5).Should(HaveKeyWithValue("topic-a", HaveLen(1)))
// there should still be no errors
Expect(cs1.Errors()).ShouldNot(Receive())
Expect(cs2.Errors()).ShouldNot(Receive())
Expect(cs3.Errors()).ShouldNot(Receive())
Expect(cs4.Errors()).ShouldNot(Receive())
Expect(cs5.Errors()).ShouldNot(Receive())
})
It("should be allowed to subscribe to partitions via white/black-lists", func() {
config := NewConfig()
config.Consumer.Return.Errors = true
config.Group.Topics.Whitelist = regexp.MustCompile(`topic-\w+`)
config.Group.Topics.Blacklist = regexp.MustCompile(`[bcd]$`)
cs, err := NewConsumer(testKafkaAddrs, testGroup, nil, config)
Expect(err).NotTo(HaveOccurred())
defer cs.Close()
subscriptionsOf(cs).Should(Equal(map[string][]int32{
"topic-a": {0, 1, 2, 3},
}))
})
It("should receive rebalance notifications", func() {
config := NewConfig()
config.Consumer.Return.Errors = true
config.Group.Return.Notifications = true
cs, err := NewConsumer(testKafkaAddrs, testGroup, testTopics, config)
Expect(err).NotTo(HaveOccurred())
defer cs.Close()
select {
case n := <-cs.Notifications():
Expect(n).To(Equal(&Notification{
Type: RebalanceStart,
Current: map[string][]int32{},
}))
case err := <-cs.Errors():
Expect(err).NotTo(HaveOccurred())
case <-cs.Messages():
Fail("expected notification to arrive before message")
}
select {
case n := <-cs.Notifications():
Expect(n).To(Equal(&Notification{
Type: RebalanceOK,
Claimed: map[string][]int32{
"topic-a": {0, 1, 2, 3},
"topic-b": {0, 1, 2, 3},
},
Released: map[string][]int32{},
Current: map[string][]int32{
"topic-a": {0, 1, 2, 3},
"topic-b": {0, 1, 2, 3},
},
}))
case err := <-cs.Errors():
Expect(err).NotTo(HaveOccurred())
case <-cs.Messages():
Fail("expected notification to arrive before message")
}
})
It("should support manual mark/commit", func() {
cs, err := newConsumerOf(testGroup, "topic-a")
Expect(err).NotTo(HaveOccurred())
defer cs.Close()
subscriptionsOf(cs).Should(Equal(map[string][]int32{
"topic-a": {0, 1, 2, 3}},
))
cs.MarkPartitionOffset("topic-a", 1, 3, "")
cs.MarkPartitionOffset("topic-a", 2, 4, "")
Expect(cs.CommitOffsets()).NotTo(HaveOccurred())
offsets, err := cs.fetchOffsets(cs.Subscriptions())
Expect(err).NotTo(HaveOccurred())
Expect(offsets).To(Equal(map[string]map[int32]offsetInfo{
"topic-a": {0: {Offset: -1}, 1: {Offset: 4}, 2: {Offset: 5}, 3: {Offset: -1}},
}))
})
It("should support manual mark/commit, reset/commit", func() {
cs, err := newConsumerOf(testGroup, "topic-a")
Expect(err).NotTo(HaveOccurred())
defer cs.Close()
subscriptionsOf(cs).Should(Equal(map[string][]int32{
"topic-a": {0, 1, 2, 3}},
))
cs.MarkPartitionOffset("topic-a", 1, 3, "")
cs.MarkPartitionOffset("topic-a", 2, 4, "")
cs.MarkPartitionOffset("topic-b", 1, 2, "") // should not throw NPE
Expect(cs.CommitOffsets()).NotTo(HaveOccurred())
cs.ResetPartitionOffset("topic-a", 1, 2, "")
cs.ResetPartitionOffset("topic-a", 2, 3, "")
cs.ResetPartitionOffset("topic-b", 1, 2, "") // should not throw NPE
Expect(cs.CommitOffsets()).NotTo(HaveOccurred())
offsets, err := cs.fetchOffsets(cs.Subscriptions())
Expect(err).NotTo(HaveOccurred())
Expect(offsets).To(Equal(map[string]map[int32]offsetInfo{
"topic-a": {0: {Offset: -1}, 1: {Offset: 3}, 2: {Offset: 4}, 3: {Offset: -1}},
}))
})
It("should not commit unprocessed offsets", func() {
const groupID = "panicking"
cs, err := newConsumerOf(groupID, "topic-a")
Expect(err).NotTo(HaveOccurred())
subscriptionsOf(cs).Should(Equal(map[string][]int32{
"topic-a": {0, 1, 2, 3},
}))
n := 0
Expect(func() {
for range cs.Messages() {
n++
panic("stop here!")
}
}).To(Panic())
Expect(cs.Close()).To(Succeed())
Expect(n).To(Equal(1))
bk, err := testClient.Coordinator(groupID)
Expect(err).NotTo(HaveOccurred())
req := &sarama.OffsetFetchRequest{
Version: 1,
ConsumerGroup: groupID,
}
req.AddPartition("topic-a", 0)
req.AddPartition("topic-a", 1)
req.AddPartition("topic-a", 2)
req.AddPartition("topic-a", 3)
Expect(bk.FetchOffset(req)).To(Equal(&sarama.OffsetFetchResponse{
Blocks: map[string]map[int32]*sarama.OffsetFetchResponseBlock{
"topic-a": {0: {Offset: -1}, 1: {Offset: -1}, 2: {Offset: -1}, 3: {Offset: -1}},
},
}))
})
It("should consume partitions", func() {
count := int32(0)
consume := func(consumerID string) {
defer GinkgoRecover()
config := NewConfig()
config.Group.Mode = ConsumerModePartitions
config.Consumer.Offsets.Initial = sarama.OffsetOldest
cs, err := NewConsumer(testKafkaAddrs, "partitions", testTopics, config)
Expect(err).NotTo(HaveOccurred())
defer cs.Close()
for pc := range cs.Partitions() {
go func(pc PartitionConsumer) {
defer pc.Close()
for msg := range pc.Messages() {
atomic.AddInt32(&count, 1)
cs.MarkOffset(msg, "")
}
}(pc)
}
}
go consume("A")
go consume("B")
go consume("C")
Eventually(func() int32 {
return atomic.LoadInt32(&count)
}, "30s", "100ms").Should(BeNumerically(">=", 2000))
})
It("should consume/commit/resume", func() {
acc := make(chan *testConsumerMessage, 20000)
consume := func(consumerID string, max int32) {
defer GinkgoRecover()
cs, err := NewConsumer(testKafkaAddrs, "fuzzing", testTopics, nil)
Expect(err).NotTo(HaveOccurred())
defer cs.Close()
cs.consumerID = consumerID
for msg := range cs.Messages() {
acc <- &testConsumerMessage{*msg, consumerID}
cs.MarkOffset(msg, "")
if atomic.AddInt32(&max, -1) <= 0 {
return
}
}
}
go consume("A", 1500)
go consume("B", 2000)
go consume("C", 1500)
go consume("D", 200)
go consume("E", 100)
time.Sleep(10 * time.Second) // wait for consumers to subscribe to topics
Expect(testSeed(5000, testTopics)).NotTo(HaveOccurred())
Eventually(func() int { return len(acc) }, "30s", "100ms").Should(BeNumerically(">=", 5000))
go consume("F", 300)
go consume("G", 400)
go consume("H", 1000)
go consume("I", 2000)
Expect(testSeed(5000, testTopics)).NotTo(HaveOccurred())
Eventually(func() int { return len(acc) }, "30s", "100ms").Should(BeNumerically(">=", 8000))
go consume("J", 1000)
Expect(testSeed(5000, testTopics)).NotTo(HaveOccurred())
Eventually(func() int { return len(acc) }, "30s", "100ms").Should(BeNumerically(">=", 9000))
go consume("K", 1000)
go consume("L", 3000)
Expect(testSeed(5000, testTopics)).NotTo(HaveOccurred())
Eventually(func() int { return len(acc) }, "30s", "100ms").Should(BeNumerically(">=", 12000))
go consume("M", 1000)
Expect(testSeed(5000, testTopics)).NotTo(HaveOccurred())
Eventually(func() int { return len(acc) }, "30s", "100ms").Should(BeNumerically(">=", 15000))
close(acc)
uniques := make(map[string][]string)
for msg := range acc {
key := fmt.Sprintf("%s/%d/%d", msg.Topic, msg.Partition, msg.Offset)
uniques[key] = append(uniques[key], msg.ConsumerID)
}
Expect(uniques).To(HaveLen(15000))
})
It("should allow close to be called multiple times", func() {
cs, err := newConsumerOf(testGroup, testTopics...)
Expect(err).NotTo(HaveOccurred())
Expect(cs.Close()).NotTo(HaveOccurred())
Expect(cs.Close()).NotTo(HaveOccurred())
})
})

123
third/github.com/johng-cn/sarama-cluster/examples_test.go
View File

@ -1,123 +0,0 @@
package cluster_test
import (
"fmt"
"log"
"os"
"os/signal"
"regexp"
cluster "gitee.com/johng/gf/third/github.com/bsm/sarama-cluster"
)
// This example shows how to use the consumer to read messages
// from a multiple topics through a multiplexed channel.
func ExampleConsumer() {
// init (custom) config, enable errors and notifications
config := cluster.NewConfig()
config.Consumer.Return.Errors = true
config.Group.Return.Notifications = true
// init consumer
brokers := []string{"127.0.0.1:9092"}
topics := []string{"my_topic", "other_topic"}
consumer, err := cluster.NewConsumer(brokers, "my-consumer-group", topics, config)
if err != nil {
panic(err)
}
defer consumer.Close()
// trap SIGINT to trigger a shutdown.
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt)
// consume errors
go func() {
for err := range consumer.Errors() {
log.Printf("Error: %s\n", err.Error())
}
}()
// consume notifications
go func() {
for ntf := range consumer.Notifications() {
log.Printf("Rebalanced: %+v\n", ntf)
}
}()
// consume messages, watch signals
for {
select {
case msg, ok := <-consumer.Messages():
if ok {
fmt.Fprintf(os.Stdout, "%s/%d/%d\t%s\t%s\n", msg.Topic, msg.Partition, msg.Offset, msg.Key, msg.Value)
consumer.MarkOffset(msg, "") // mark message as processed
}
case <-signals:
return
}
}
}
// This example shows how to use the consumer to read messages
// through individual partitions.
func ExampleConsumer_Partitions() {
// init (custom) config, set mode to ConsumerModePartitions
config := cluster.NewConfig()
config.Group.Mode = cluster.ConsumerModePartitions
// init consumer
brokers := []string{"127.0.0.1:9092"}
topics := []string{"my_topic", "other_topic"}
consumer, err := cluster.NewConsumer(brokers, "my-consumer-group", topics, config)
if err != nil {
panic(err)
}
defer consumer.Close()
// trap SIGINT to trigger a shutdown.
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt)
// consume partitions
for {
select {
case part, ok := <-consumer.Partitions():
if !ok {
return
}
// start a separate goroutine to consume messages
go func(pc cluster.PartitionConsumer) {
for msg := range pc.Messages() {
fmt.Fprintf(os.Stdout, "%s/%d/%d\t%s\t%s\n", msg.Topic, msg.Partition, msg.Offset, msg.Key, msg.Value)
consumer.MarkOffset(msg, "") // mark message as processed
}
}(part)
case <-signals:
return
}
}
}
// This example shows how to use the consumer with
// topic whitelists.
func ExampleConfig_whitelist() {
// init (custom) config, enable errors and notifications
config := cluster.NewConfig()
config.Group.Topics.Whitelist = regexp.MustCompile(`myservice.*`)
// init consumer
consumer, err := cluster.NewConsumer([]string{"127.0.0.1:9092"}, "my-consumer-group", nil, config)
if err != nil {
panic(err)
}
defer consumer.Close()
// consume messages
msg := <-consumer.Messages()
fmt.Fprintf(os.Stdout, "%s/%d/%d\t%s\t%s\n", msg.Topic, msg.Partition, msg.Offset, msg.Key, msg.Value)
}

87
third/github.com/johng-cn/sarama-cluster/offsets_test.go
View File

@ -1,87 +0,0 @@
package cluster
import (
. "gitee.com/johng/gf/third/github.com/onsi/ginkgo"
. "gitee.com/johng/gf/third/github.com/onsi/gomega"
)
var _ = Describe("OffsetStash", func() {
var subject *OffsetStash
BeforeEach(func() {
subject = NewOffsetStash()
})
It("should update", func() {
Expect(subject.offsets).To(HaveLen(0))
subject.MarkPartitionOffset("topic", 0, 0, "m3ta")
Expect(subject.offsets).To(HaveLen(1))
Expect(subject.offsets).To(HaveKeyWithValue(
topicPartition{Topic: "topic", Partition: 0},
offsetInfo{Offset: 0, Metadata: "m3ta"},
))
subject.MarkPartitionOffset("topic", 0, 200, "m3ta")
Expect(subject.offsets).To(HaveLen(1))
Expect(subject.offsets).To(HaveKeyWithValue(
topicPartition{Topic: "topic", Partition: 0},
offsetInfo{Offset: 200, Metadata: "m3ta"},
))
subject.MarkPartitionOffset("topic", 0, 199, "m3t@")
Expect(subject.offsets).To(HaveLen(1))
Expect(subject.offsets).To(HaveKeyWithValue(
topicPartition{Topic: "topic", Partition: 0},
offsetInfo{Offset: 200, Metadata: "m3ta"},
))
subject.MarkPartitionOffset("topic", 1, 300, "")
Expect(subject.offsets).To(HaveLen(2))
Expect(subject.offsets).To(HaveKeyWithValue(
topicPartition{Topic: "topic", Partition: 1},
offsetInfo{Offset: 300, Metadata: ""},
))
})
It("should reset", func() {
Expect(subject.offsets).To(HaveLen(0))
subject.MarkPartitionOffset("topic", 0, 0, "m3ta")
Expect(subject.offsets).To(HaveLen(1))
Expect(subject.offsets).To(HaveKeyWithValue(
topicPartition{Topic: "topic", Partition: 0},
offsetInfo{Offset: 0, Metadata: "m3ta"},
))
subject.MarkPartitionOffset("topic", 0, 200, "m3ta")
Expect(subject.offsets).To(HaveLen(1))
Expect(subject.offsets).To(HaveKeyWithValue(
topicPartition{Topic: "topic", Partition: 0},
offsetInfo{Offset: 200, Metadata: "m3ta"},
))
subject.ResetPartitionOffset("topic", 0, 199, "m3t@")
Expect(subject.offsets).To(HaveLen(1))
Expect(subject.offsets).To(HaveKeyWithValue(
topicPartition{Topic: "topic", Partition: 0},
offsetInfo{Offset: 199, Metadata: "m3t@"},
))
subject.MarkPartitionOffset("topic", 1, 300, "")
Expect(subject.offsets).To(HaveLen(2))
Expect(subject.offsets).To(HaveKeyWithValue(
topicPartition{Topic: "topic", Partition: 1},
offsetInfo{Offset: 300, Metadata: ""},
))
subject.ResetPartitionOffset("topic", 1, 200, "m3t@")
Expect(subject.offsets).To(HaveLen(2))
Expect(subject.offsets).To(HaveKeyWithValue(
topicPartition{Topic: "topic", Partition: 1},
offsetInfo{Offset: 200, Metadata: "m3t@"},
))
})
})

137
third/github.com/johng-cn/sarama-cluster/partitions_test.go
View File

@ -1,137 +0,0 @@
package cluster
import (
"gitee.com/johng/gf/third/github.com/Shopify/sarama"
. "gitee.com/johng/gf/third/github.com/onsi/ginkgo"
. "gitee.com/johng/gf/third/github.com/onsi/gomega"
)
var _ = Describe("partitionConsumer", func() {
var subject *partitionConsumer
BeforeEach(func() {
var err error
subject, err = newPartitionConsumer(&mockConsumer{}, "topic", 0, offsetInfo{2000, "m3ta"}, sarama.OffsetOldest)
Expect(err).NotTo(HaveOccurred())
})
AfterEach(func() {
close(subject.dead)
Expect(subject.Close()).NotTo(HaveOccurred())
})
It("should set state", func() {
Expect(subject.getState()).To(Equal(partitionState{
Info: offsetInfo{2000, "m3ta"},
}))
})
It("should recover from default offset if requested offset is out of bounds", func() {
pc, err := newPartitionConsumer(&mockConsumer{}, "topic", 0, offsetInfo{200, "m3ta"}, sarama.OffsetOldest)
Expect(err).NotTo(HaveOccurred())
defer pc.Close()
close(pc.dead)
state := pc.getState()
Expect(state.Info.Offset).To(Equal(int64(-1)))
Expect(state.Info.Metadata).To(Equal("m3ta"))
})
It("should update state", func() {
subject.MarkOffset(2001, "met@") // should set state
Expect(subject.getState()).To(Equal(partitionState{
Info: offsetInfo{2002, "met@"},
Dirty: true,
}))
subject.markCommitted(2002) // should reset dirty status
Expect(subject.getState()).To(Equal(partitionState{
Info: offsetInfo{2002, "met@"},
}))
subject.MarkOffset(2001, "me7a") // should not update state
Expect(subject.getState()).To(Equal(partitionState{
Info: offsetInfo{2002, "met@"},
}))
subject.MarkOffset(2002, "me7a") // should bump state
Expect(subject.getState()).To(Equal(partitionState{
Info: offsetInfo{2003, "me7a"},
Dirty: true,
}))
// After committing a later offset, try rewinding back to earlier offset with new metadata.
subject.ResetOffset(2001, "met@")
Expect(subject.getState()).To(Equal(partitionState{
Info: offsetInfo{2002, "met@"},
Dirty: true,
}))
subject.markCommitted(2002) // should not unset state
Expect(subject.getState()).To(Equal(partitionState{
Info: offsetInfo{2002, "met@"},
}))
subject.MarkOffset(2002, "me7a") // should bump state
Expect(subject.getState()).To(Equal(partitionState{
Info: offsetInfo{2003, "me7a"},
Dirty: true,
}))
subject.markCommitted(2003)
Expect(subject.getState()).To(Equal(partitionState{
Info: offsetInfo{2003, "me7a"},
}))
})
})
var _ = Describe("partitionMap", func() {
var subject *partitionMap
BeforeEach(func() {
subject = newPartitionMap()
})
It("should fetch/store", func() {
Expect(subject.Fetch("topic", 0)).To(BeNil())
pc, err := newPartitionConsumer(&mockConsumer{}, "topic", 0, offsetInfo{2000, "m3ta"}, sarama.OffsetNewest)
Expect(err).NotTo(HaveOccurred())
subject.Store("topic", 0, pc)
Expect(subject.Fetch("topic", 0)).To(Equal(pc))
Expect(subject.Fetch("topic", 1)).To(BeNil())
Expect(subject.Fetch("other", 0)).To(BeNil())
})
It("should return info", func() {
pc0, err := newPartitionConsumer(&mockConsumer{}, "topic", 0, offsetInfo{2000, "m3ta"}, sarama.OffsetNewest)
Expect(err).NotTo(HaveOccurred())
pc1, err := newPartitionConsumer(&mockConsumer{}, "topic", 1, offsetInfo{2000, "m3ta"}, sarama.OffsetNewest)
Expect(err).NotTo(HaveOccurred())
subject.Store("topic", 0, pc0)
subject.Store("topic", 1, pc1)
info := subject.Info()
Expect(info).To(HaveLen(1))
Expect(info).To(HaveKeyWithValue("topic", []int32{0, 1}))
})
It("should create snapshots", func() {
pc0, err := newPartitionConsumer(&mockConsumer{}, "topic", 0, offsetInfo{2000, "m3ta"}, sarama.OffsetNewest)
Expect(err).NotTo(HaveOccurred())
pc1, err := newPartitionConsumer(&mockConsumer{}, "topic", 1, offsetInfo{2000, "m3ta"}, sarama.OffsetNewest)
Expect(err).NotTo(HaveOccurred())
subject.Store("topic", 0, pc0)
subject.Store("topic", 1, pc1)
subject.Fetch("topic", 1).MarkOffset(2000, "met@")
Expect(subject.Snapshot()).To(Equal(map[topicPartition]partitionState{
{"topic", 0}: {Info: offsetInfo{2000, "m3ta"}, Dirty: false},
{"topic", 1}: {Info: offsetInfo{2001, "met@"}, Dirty: true},
}))
})
})