diff --git a/g/database/gdb/gdb.go b/g/database/gdb/gdb.go index 94193ea97..cdb49446b 100644 --- a/g/database/gdb/gdb.go +++ b/g/database/gdb/gdb.go @@ -4,8 +4,7 @@ // If a copy of the MIT was not distributed with this file, // You can obtain one at https://gitee.com/johng/gf. -// Package gdb provides ORM features for popular relationship databases. -// 数据库ORM. +// Package gdb provides ORM features for popular relationship databases/数据库ORM. // 默认内置支持MySQL, 其他数据库需要手动import对应的数据库引擎第三方包. package gdb diff --git a/g/database/gkafka/gkafka.go b/g/database/gkafka/gkafka.go index 180aeeae4..9f313549a 100644 --- a/g/database/gkafka/gkafka.go +++ b/g/database/gkafka/gkafka.go @@ -4,7 +4,7 @@ // If a copy of the MIT was not distributed with this file, // You can obtain one at https://gitee.com/johng/gf. -// Kafka Client. +// Package gkafka provides producer and consumer client for kafka server/Kafka客户端. package gkafka import ( diff --git a/third/github.com/Shopify/sarama/.gitignore b/third/github.com/Shopify/sarama/.gitignore index c6c482dca..6e362e4f2 100644 --- a/third/github.com/Shopify/sarama/.gitignore +++ b/third/github.com/Shopify/sarama/.gitignore @@ -24,3 +24,4 @@ _testmain.go *.exe coverage.txt +profile.out diff --git a/third/github.com/Shopify/sarama/.travis.yml b/third/github.com/Shopify/sarama/.travis.yml index ea295ec5f..eb54a0ddc 100644 --- a/third/github.com/Shopify/sarama/.travis.yml +++ b/third/github.com/Shopify/sarama/.travis.yml @@ -1,8 +1,7 @@ language: go go: -- 1.8.x -- 1.9.x - 1.10.x +- 1.11.x env: global: @@ -12,9 +11,9 @@ env: - KAFKA_HOSTNAME=localhost - DEBUG=true matrix: - - KAFKA_VERSION=0.11.0.2 - - KAFKA_VERSION=1.0.0 - - KAFKA_VERSION=1.1.0 + - KAFKA_VERSION=1.1.1 + - KAFKA_VERSION=2.0.1 + - KAFKA_VERSION=2.1.0 before_install: - export REPOSITORY_ROOT=${TRAVIS_BUILD_DIR} @@ -28,7 +27,7 @@ script: - make test - make vet - make errcheck -- make fmt +- if [[ "$TRAVIS_GO_VERSION" == 1.11* ]]; then make fmt; fi after_success: - bash <(curl -s https://codecov.io/bash) diff --git a/third/github.com/Shopify/sarama/CHANGELOG.md b/third/github.com/Shopify/sarama/CHANGELOG.md index 16d5829c9..9f955d5e1 100644 --- a/third/github.com/Shopify/sarama/CHANGELOG.md +++ b/third/github.com/Shopify/sarama/CHANGELOG.md @@ -1,5 +1,87 @@ # Changelog +#### Version 1.20.0 (2018-12-10) + +New Features: + - Add support for zstd compression + ([#1170](https://github.com/Shopify/sarama/pull/1170)). + - Add support for Idempotent Producer + ([#1152](https://github.com/Shopify/sarama/pull/1152)). + - Add support support for Kafka 2.1.0 + ([#1229](https://github.com/Shopify/sarama/pull/1229)). + - Add support support for OffsetCommit request/response pairs versions v1 to v5 + ([#1201](https://github.com/Shopify/sarama/pull/1201)). + - Add support support for OffsetFetch request/response pair up to version v5 + ([#1198](https://github.com/Shopify/sarama/pull/1198)). + +Improvements: + - Export broker's Rack setting + ([#1173](https://github.com/Shopify/sarama/pull/1173)). + - Always use latest patch version of Go on CI + ([#1202](https://github.com/Shopify/sarama/pull/1202)). + - Add error codes 61 to 72 + ([#1195](https://github.com/Shopify/sarama/pull/1195)). + +Bug Fixes: + - Fix build without cgo + ([#1182](https://github.com/Shopify/sarama/pull/1182)). + - Fix go vet suggestion in consumer group file + ([#1209](https://github.com/Shopify/sarama/pull/1209)). + - Fix typos in code and comments + ([#1228](https://github.com/Shopify/sarama/pull/1228)). + +#### Version 1.19.0 (2018-09-27) + +New Features: + - Implement a higher-level consumer group + ([#1099](https://github.com/Shopify/sarama/pull/1099)). + +Improvements: + - Add support for Go 1.11 + ([#1176](https://github.com/Shopify/sarama/pull/1176)). + +Bug Fixes: + - Fix encoding of `MetadataResponse` with version 2 and higher + ([#1174](https://github.com/Shopify/sarama/pull/1174)). + - Fix race condition in mock async producer + ([#1174](https://github.com/Shopify/sarama/pull/1174)). + +#### Version 1.18.0 (2018-09-07) + +New Features: + - Make `Partitioner.RequiresConsistency` vary per-message + ([#1112](https://github.com/Shopify/sarama/pull/1112)). + - Add customizable partitioner + ([#1118](https://github.com/Shopify/sarama/pull/1118)). + - Add `ClusterAdmin` support for `CreateTopic`, `DeleteTopic`, `CreatePartitions`, + `DeleteRecords`, `DescribeConfig`, `AlterConfig`, `CreateACL`, `ListAcls`, `DeleteACL` + ([#1055](https://github.com/Shopify/sarama/pull/1055)). + +Improvements: + - Add support for Kafka 2.0.0 + ([#1149](https://github.com/Shopify/sarama/pull/1149)). + - Allow setting `LocalAddr` when dialing an address to support multi-homed hosts + ([#1123](https://github.com/Shopify/sarama/pull/1123)). + - Simpler offset management + ([#1127](https://github.com/Shopify/sarama/pull/1127)). + +Bug Fixes: + - Fix mutation of `ProducerMessage.MetaData` when producing to Kafka + ([#1110](https://github.com/Shopify/sarama/pull/1110)). + - Fix consumer block when response did not contain all the + expected topic/partition blocks + ([#1086](https://github.com/Shopify/sarama/pull/1086)). + - Fix consumer block when response contains only constrol messages + ([#1115](https://github.com/Shopify/sarama/pull/1115)). + - Add timeout config for ClusterAdmin requests + ([#1142](https://github.com/Shopify/sarama/pull/1142)). + - Add version check when producing message with headers + ([#1117](https://github.com/Shopify/sarama/pull/1117)). + - Fix `MetadataRequest` for empty list of topics + ([#1132](https://github.com/Shopify/sarama/pull/1132)). + - Fix producer topic metadata on-demand fetch when topic error happens in metadata response + ([#1125](https://github.com/Shopify/sarama/pull/1125)). + #### Version 1.17.0 (2018-05-30) New Features: diff --git a/third/github.com/Shopify/sarama/Makefile b/third/github.com/Shopify/sarama/Makefile index b9a453dd2..8fcf219f4 100644 --- a/third/github.com/Shopify/sarama/Makefile +++ b/third/github.com/Shopify/sarama/Makefile @@ -4,7 +4,7 @@ default: fmt vet errcheck test test: echo "" > coverage.txt for d in `go list ./... | grep -v vendor`; do \ - go test -p 1 -v -timeout 90s -race -coverprofile=profile.out -covermode=atomic $$d || exit 1; \ + go test -p 1 -v -timeout 240s -race -coverprofile=profile.out -covermode=atomic $$d || exit 1; \ if [ -f profile.out ]; then \ cat profile.out >> coverage.txt; \ rm profile.out; \ diff --git a/third/github.com/Shopify/sarama/README.md b/third/github.com/Shopify/sarama/README.md index 4fc0cc600..f241b89c5 100644 --- a/third/github.com/Shopify/sarama/README.md +++ b/third/github.com/Shopify/sarama/README.md @@ -1,7 +1,7 @@ sarama ====== -[![GoDoc](https://godoc.org/github.com/Shopify/sarama?status.png)](https://godoc.org/github.com/Shopify/sarama) +[![GoDoc](https://godoc.org/github.com/Shopify/sarama?status.svg)](https://godoc.org/github.com/Shopify/sarama) [![Build Status](https://travis-ci.org/Shopify/sarama.svg?branch=master)](https://travis-ci.org/Shopify/sarama) [![Coverage](https://codecov.io/gh/Shopify/sarama/branch/master/graph/badge.svg)](https://codecov.io/gh/Shopify/sarama) @@ -21,7 +21,7 @@ You might also want to look at the [Frequently Asked Questions](https://github.c Sarama provides a "2 releases + 2 months" compatibility guarantee: we support the two latest stable releases of Kafka and Go, and we provide a two month grace period for older releases. This means we currently officially support -Go 1.8 through 1.10, and Kafka 0.11 through 1.1, although older releases are +Go 1.8 through 1.11, and Kafka 1.0 through 2.0, although older releases are still likely to work. Sarama follows semantic versioning and provides API stability via the gopkg.in service. diff --git a/third/github.com/Shopify/sarama/admin.go b/third/github.com/Shopify/sarama/admin.go index 68284641c..52725758d 100644 --- a/third/github.com/Shopify/sarama/admin.go +++ b/third/github.com/Shopify/sarama/admin.go @@ -118,6 +118,7 @@ func (ca *clusterAdmin) CreateTopic(topic string, detail *TopicDetail, validateO request := &CreateTopicsRequest{ TopicDetails: topicDetails, ValidateOnly: validateOnly, + Timeout: ca.conf.Admin.Timeout, } if ca.conf.Version.IsAtLeast(V0_11_0_0) { @@ -155,7 +156,10 @@ func (ca *clusterAdmin) DeleteTopic(topic string) error { return ErrInvalidTopic } - request := &DeleteTopicsRequest{Topics: []string{topic}} + request := &DeleteTopicsRequest{ + Topics: []string{topic}, + Timeout: ca.conf.Admin.Timeout, + } if ca.conf.Version.IsAtLeast(V0_11_0_0) { request.Version = 1 @@ -192,6 +196,7 @@ func (ca *clusterAdmin) CreatePartitions(topic string, count int32, assignment [ request := &CreatePartitionsRequest{ TopicPartitions: topicPartitions, + Timeout: ca.conf.Admin.Timeout, } b, err := ca.Controller() @@ -225,7 +230,9 @@ func (ca *clusterAdmin) DeleteRecords(topic string, partitionOffsets map[int32]i topics := make(map[string]*DeleteRecordsRequestTopic) topics[topic] = &DeleteRecordsRequestTopic{PartitionOffsets: partitionOffsets} request := &DeleteRecordsRequest{ - Topics: topics} + Topics: topics, + Timeout: ca.conf.Admin.Timeout, + } b, err := ca.Controller() if err != nil { diff --git a/third/github.com/Shopify/sarama/async_producer.go b/third/github.com/Shopify/sarama/async_producer.go index 952b9abc0..db8f45933 100644 --- a/third/github.com/Shopify/sarama/async_producer.go +++ b/third/github.com/Shopify/sarama/async_producer.go @@ -47,6 +47,50 @@ type AsyncProducer interface { Errors() <-chan *ProducerError } +// transactionManager keeps the state necessary to ensure idempotent production +type transactionManager struct { + producerID int64 + producerEpoch int16 + sequenceNumbers map[string]int32 + mutex sync.Mutex +} + +const ( + noProducerID = -1 + noProducerEpoch = -1 +) + +func (t *transactionManager) getAndIncrementSequenceNumber(topic string, partition int32) int32 { + key := fmt.Sprintf("%s-%d", topic, partition) + t.mutex.Lock() + defer t.mutex.Unlock() + sequence := t.sequenceNumbers[key] + t.sequenceNumbers[key] = sequence + 1 + return sequence +} + +func newTransactionManager(conf *Config, client Client) (*transactionManager, error) { + txnmgr := &transactionManager{ + producerID: noProducerID, + producerEpoch: noProducerEpoch, + } + + if conf.Producer.Idempotent { + initProducerIDResponse, err := client.InitProducerID() + if err != nil { + return nil, err + } + txnmgr.producerID = initProducerIDResponse.ProducerID + txnmgr.producerEpoch = initProducerIDResponse.ProducerEpoch + txnmgr.sequenceNumbers = make(map[string]int32) + txnmgr.mutex = sync.Mutex{} + + Logger.Printf("Obtained a ProducerId: %d and ProducerEpoch: %d\n", txnmgr.producerID, txnmgr.producerEpoch) + } + + return txnmgr, nil +} + type asyncProducer struct { client Client conf *Config @@ -56,9 +100,11 @@ type asyncProducer struct { input, successes, retries chan *ProducerMessage inFlight sync.WaitGroup - brokers map[*Broker]chan<- *ProducerMessage - brokerRefs map[chan<- *ProducerMessage]int + brokers map[*Broker]*brokerProducer + brokerRefs map[*brokerProducer]int brokerLock sync.Mutex + + txnmgr *transactionManager } // NewAsyncProducer creates a new AsyncProducer using the given broker addresses and configuration. @@ -84,6 +130,11 @@ func NewAsyncProducerFromClient(client Client) (AsyncProducer, error) { return nil, ErrClosedClient } + txnmgr, err := newTransactionManager(client.Config(), client) + if err != nil { + return nil, err + } + p := &asyncProducer{ client: client, conf: client.Config(), @@ -91,8 +142,9 @@ func NewAsyncProducerFromClient(client Client) (AsyncProducer, error) { input: make(chan *ProducerMessage), successes: make(chan *ProducerMessage), retries: make(chan *ProducerMessage), - brokers: make(map[*Broker]chan<- *ProducerMessage), - brokerRefs: make(map[chan<- *ProducerMessage]int), + brokers: make(map[*Broker]*brokerProducer), + brokerRefs: make(map[*brokerProducer]int), + txnmgr: txnmgr, } // launch our singleton dispatchers @@ -145,9 +197,10 @@ type ProducerMessage struct { // least version 0.10.0. Timestamp time.Time - retries int - flags flagSet - expectation chan *ProducerError + retries int + flags flagSet + expectation chan *ProducerError + sequenceNumber int32 } const producerMessageOverhead = 26 // the metadata overhead of CRC, flags, etc. @@ -328,6 +381,10 @@ func (tp *topicProducer) dispatch() { continue } } + // All messages being retried (sent or not) have already had their retry count updated + if tp.parent.conf.Producer.Idempotent && msg.retries == 0 { + msg.sequenceNumber = tp.parent.txnmgr.getAndIncrementSequenceNumber(msg.Topic, msg.Partition) + } handler := tp.handlers[msg.Partition] if handler == nil { @@ -394,9 +451,9 @@ type partitionProducer struct { partition int32 input <-chan *ProducerMessage - leader *Broker - breaker *breaker.Breaker - output chan<- *ProducerMessage + leader *Broker + breaker *breaker.Breaker + brokerProducer *brokerProducer // highWatermark tracks the "current" retry level, which is the only one where we actually let messages through, // all other messages get buffered in retryState[msg.retries].buf to preserve ordering @@ -431,9 +488,9 @@ func (pp *partitionProducer) dispatch() { // on the first message pp.leader, _ = pp.parent.client.Leader(pp.topic, pp.partition) if pp.leader != nil { - pp.output = pp.parent.getBrokerProducer(pp.leader) + pp.brokerProducer = pp.parent.getBrokerProducer(pp.leader) pp.parent.inFlight.Add(1) // we're generating a syn message; track it so we don't shut down while it's still inflight - pp.output <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: syn} + pp.brokerProducer.input <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: syn} } for msg := range pp.input { @@ -465,7 +522,7 @@ func (pp *partitionProducer) dispatch() { // if we made it this far then the current msg contains real data, and can be sent to the next goroutine // without breaking any of our ordering guarantees - if pp.output == nil { + if pp.brokerProducer == nil { if err := pp.updateLeader(); err != nil { pp.parent.returnError(msg, err) time.Sleep(pp.parent.conf.Producer.Retry.Backoff) @@ -474,11 +531,11 @@ func (pp *partitionProducer) dispatch() { Logger.Printf("producer/leader/%s/%d selected broker %d\n", pp.topic, pp.partition, pp.leader.ID()) } - pp.output <- msg + pp.brokerProducer.input <- msg } - if pp.output != nil { - pp.parent.unrefBrokerProducer(pp.leader, pp.output) + if pp.brokerProducer != nil { + pp.parent.unrefBrokerProducer(pp.leader, pp.brokerProducer) } } @@ -490,12 +547,12 @@ func (pp *partitionProducer) newHighWatermark(hwm int) { // back to us and we can safely flush the backlog (otherwise we risk re-ordering messages) pp.retryState[pp.highWatermark].expectChaser = true pp.parent.inFlight.Add(1) // we're generating a fin message; track it so we don't shut down while it's still inflight - pp.output <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: fin, retries: pp.highWatermark - 1} + pp.brokerProducer.input <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: fin, retries: pp.highWatermark - 1} // a new HWM means that our current broker selection is out of date Logger.Printf("producer/leader/%s/%d abandoning broker %d\n", pp.topic, pp.partition, pp.leader.ID()) - pp.parent.unrefBrokerProducer(pp.leader, pp.output) - pp.output = nil + pp.parent.unrefBrokerProducer(pp.leader, pp.brokerProducer) + pp.brokerProducer = nil } func (pp *partitionProducer) flushRetryBuffers() { @@ -503,7 +560,7 @@ func (pp *partitionProducer) flushRetryBuffers() { for { pp.highWatermark-- - if pp.output == nil { + if pp.brokerProducer == nil { if err := pp.updateLeader(); err != nil { pp.parent.returnErrors(pp.retryState[pp.highWatermark].buf, err) goto flushDone @@ -512,7 +569,7 @@ func (pp *partitionProducer) flushRetryBuffers() { } for _, msg := range pp.retryState[pp.highWatermark].buf { - pp.output <- msg + pp.brokerProducer.input <- msg } flushDone: @@ -537,16 +594,16 @@ func (pp *partitionProducer) updateLeader() error { return err } - pp.output = pp.parent.getBrokerProducer(pp.leader) + pp.brokerProducer = pp.parent.getBrokerProducer(pp.leader) pp.parent.inFlight.Add(1) // we're generating a syn message; track it so we don't shut down while it's still inflight - pp.output <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: syn} + pp.brokerProducer.input <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: syn} return nil }) } // one per broker; also constructs an associated flusher -func (p *asyncProducer) newBrokerProducer(broker *Broker) chan<- *ProducerMessage { +func (p *asyncProducer) newBrokerProducer(broker *Broker) *brokerProducer { var ( input = make(chan *ProducerMessage) bridge = make(chan *produceSet) @@ -580,7 +637,7 @@ func (p *asyncProducer) newBrokerProducer(broker *Broker) chan<- *ProducerMessag close(responses) }) - return input + return bp } type brokerProducerResponse struct { @@ -595,7 +652,7 @@ type brokerProducer struct { parent *asyncProducer broker *Broker - input <-chan *ProducerMessage + input chan *ProducerMessage output chan<- *produceSet responses <-chan *brokerProducerResponse @@ -740,16 +797,17 @@ func (bp *brokerProducer) handleResponse(response *brokerProducerResponse) { func (bp *brokerProducer) handleSuccess(sent *produceSet, response *ProduceResponse) { // we iterate through the blocks in the request set, not the response, so that we notice // if the response is missing a block completely - sent.eachPartition(func(topic string, partition int32, msgs []*ProducerMessage) { + var retryTopics []string + sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) { if response == nil { // this only happens when RequiredAcks is NoResponse, so we have to assume success - bp.parent.returnSuccesses(msgs) + bp.parent.returnSuccesses(pSet.msgs) return } block := response.GetBlock(topic, partition) if block == nil { - bp.parent.returnErrors(msgs, ErrIncompleteResponse) + bp.parent.returnErrors(pSet.msgs, ErrIncompleteResponse) return } @@ -757,45 +815,107 @@ func (bp *brokerProducer) handleSuccess(sent *produceSet, response *ProduceRespo // Success case ErrNoError: if bp.parent.conf.Version.IsAtLeast(V0_10_0_0) && !block.Timestamp.IsZero() { - for _, msg := range msgs { + for _, msg := range pSet.msgs { msg.Timestamp = block.Timestamp } } - for i, msg := range msgs { + for i, msg := range pSet.msgs { msg.Offset = block.Offset + int64(i) } - bp.parent.returnSuccesses(msgs) + bp.parent.returnSuccesses(pSet.msgs) + // Duplicate + case ErrDuplicateSequenceNumber: + bp.parent.returnSuccesses(pSet.msgs) // Retriable errors case ErrInvalidMessage, ErrUnknownTopicOrPartition, ErrLeaderNotAvailable, ErrNotLeaderForPartition, ErrRequestTimedOut, ErrNotEnoughReplicas, ErrNotEnoughReplicasAfterAppend: - Logger.Printf("producer/broker/%d state change to [retrying] on %s/%d because %v\n", - bp.broker.ID(), topic, partition, block.Err) - bp.currentRetries[topic][partition] = block.Err - bp.parent.retryMessages(msgs, block.Err) - bp.parent.retryMessages(bp.buffer.dropPartition(topic, partition), block.Err) + retryTopics = append(retryTopics, topic) // Other non-retriable errors default: - bp.parent.returnErrors(msgs, block.Err) + bp.parent.returnErrors(pSet.msgs, block.Err) } }) + + if len(retryTopics) > 0 { + if bp.parent.conf.Producer.Idempotent { + err := bp.parent.client.RefreshMetadata(retryTopics...) + if err != nil { + Logger.Printf("Failed refreshing metadata because of %v\n", err) + } + } + + sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) { + block := response.GetBlock(topic, partition) + if block == nil { + // handled in the previous "eachPartition" loop + return + } + + switch block.Err { + case ErrInvalidMessage, ErrUnknownTopicOrPartition, ErrLeaderNotAvailable, ErrNotLeaderForPartition, + ErrRequestTimedOut, ErrNotEnoughReplicas, ErrNotEnoughReplicasAfterAppend: + Logger.Printf("producer/broker/%d state change to [retrying] on %s/%d because %v\n", + bp.broker.ID(), topic, partition, block.Err) + if bp.currentRetries[topic] == nil { + bp.currentRetries[topic] = make(map[int32]error) + } + bp.currentRetries[topic][partition] = block.Err + if bp.parent.conf.Producer.Idempotent { + go bp.parent.retryBatch(topic, partition, pSet, block.Err) + } else { + bp.parent.retryMessages(pSet.msgs, block.Err) + } + // dropping the following messages has the side effect of incrementing their retry count + bp.parent.retryMessages(bp.buffer.dropPartition(topic, partition), block.Err) + } + }) + } +} + +func (p *asyncProducer) retryBatch(topic string, partition int32, pSet *partitionSet, kerr KError) { + Logger.Printf("Retrying batch for %v-%d because of %s\n", topic, partition, kerr) + produceSet := newProduceSet(p) + produceSet.msgs[topic] = make(map[int32]*partitionSet) + produceSet.msgs[topic][partition] = pSet + produceSet.bufferBytes += pSet.bufferBytes + produceSet.bufferCount += len(pSet.msgs) + for _, msg := range pSet.msgs { + if msg.retries >= p.conf.Producer.Retry.Max { + p.returnError(msg, kerr) + return + } + msg.retries++ + } + + // it's expected that a metadata refresh has been requested prior to calling retryBatch + leader, err := p.client.Leader(topic, partition) + if err != nil { + Logger.Printf("Failed retrying batch for %v-%d because of %v while looking up for new leader\n", topic, partition, err) + for _, msg := range pSet.msgs { + p.returnError(msg, kerr) + } + return + } + bp := p.getBrokerProducer(leader) + bp.output <- produceSet } func (bp *brokerProducer) handleError(sent *produceSet, err error) { switch err.(type) { case PacketEncodingError: - sent.eachPartition(func(topic string, partition int32, msgs []*ProducerMessage) { - bp.parent.returnErrors(msgs, err) + sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) { + bp.parent.returnErrors(pSet.msgs, err) }) default: Logger.Printf("producer/broker/%d state change to [closing] because %s\n", bp.broker.ID(), err) bp.parent.abandonBrokerConnection(bp.broker) _ = bp.broker.Close() bp.closing = err - sent.eachPartition(func(topic string, partition int32, msgs []*ProducerMessage) { - bp.parent.retryMessages(msgs, err) + sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) { + bp.parent.retryMessages(pSet.msgs, err) }) - bp.buffer.eachPartition(func(topic string, partition int32, msgs []*ProducerMessage) { - bp.parent.retryMessages(msgs, err) + bp.buffer.eachPartition(func(topic string, partition int32, pSet *partitionSet) { + bp.parent.retryMessages(pSet.msgs, err) }) bp.rollOver() } @@ -892,7 +1012,7 @@ func (p *asyncProducer) retryMessages(batch []*ProducerMessage, err error) { } } -func (p *asyncProducer) getBrokerProducer(broker *Broker) chan<- *ProducerMessage { +func (p *asyncProducer) getBrokerProducer(broker *Broker) *brokerProducer { p.brokerLock.Lock() defer p.brokerLock.Unlock() @@ -909,13 +1029,13 @@ func (p *asyncProducer) getBrokerProducer(broker *Broker) chan<- *ProducerMessag return bp } -func (p *asyncProducer) unrefBrokerProducer(broker *Broker, bp chan<- *ProducerMessage) { +func (p *asyncProducer) unrefBrokerProducer(broker *Broker, bp *brokerProducer) { p.brokerLock.Lock() defer p.brokerLock.Unlock() p.brokerRefs[bp]-- if p.brokerRefs[bp] == 0 { - close(bp) + close(bp.input) delete(p.brokerRefs, bp) if p.brokers[broker] == bp { diff --git a/third/github.com/Shopify/sarama/async_producer_test.go b/third/github.com/Shopify/sarama/async_producer_test.go index 478dca4cf..038aa0b0c 100644 --- a/third/github.com/Shopify/sarama/async_producer_test.go +++ b/third/github.com/Shopify/sarama/async_producer_test.go @@ -459,6 +459,7 @@ func TestAsyncProducerMultipleRetries(t *testing.T) { metadataLeader2 := new(MetadataResponse) metadataLeader2.AddBroker(leader2.Addr(), leader2.BrokerID()) metadataLeader2.AddTopicPartition("my_topic", 0, leader2.BrokerID(), nil, nil, ErrNoError) + seedBroker.Returns(metadataLeader2) leader2.Returns(prodNotLeader) seedBroker.Returns(metadataLeader1) @@ -753,6 +754,255 @@ func TestAsyncProducerNoReturns(t *testing.T) { leader.Close() } +func TestAsyncProducerIdempotentGoldenPath(t *testing.T) { + broker := NewMockBroker(t, 1) + + metadataResponse := &MetadataResponse{ + Version: 1, + ControllerID: 1, + } + metadataResponse.AddBroker(broker.Addr(), broker.BrokerID()) + metadataResponse.AddTopicPartition("my_topic", 0, broker.BrokerID(), nil, nil, ErrNoError) + broker.Returns(metadataResponse) + + initProducerID := &InitProducerIDResponse{ + ThrottleTime: 0, + ProducerID: 1000, + ProducerEpoch: 1, + } + broker.Returns(initProducerID) + + config := NewConfig() + config.Producer.Flush.Messages = 10 + config.Producer.Return.Successes = true + config.Producer.Retry.Max = 4 + config.Producer.RequiredAcks = WaitForAll + config.Producer.Retry.Backoff = 0 + config.Producer.Idempotent = true + config.Net.MaxOpenRequests = 1 + config.Version = V0_11_0_0 + producer, err := NewAsyncProducer([]string{broker.Addr()}, config) + if err != nil { + t.Fatal(err) + } + + for i := 0; i < 10; i++ { + producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} + } + + prodSuccess := &ProduceResponse{ + Version: 3, + ThrottleTime: 0, + } + prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError) + broker.Returns(prodSuccess) + expectResults(t, producer, 10, 0) + + broker.Close() + closeProducer(t, producer) +} + +func TestAsyncProducerIdempotentRetryCheckBatch(t *testing.T) { + //Logger = log.New(os.Stderr, "", log.LstdFlags) + tests := []struct { + name string + failAfterWrite bool + }{ + {"FailAfterWrite", true}, + {"FailBeforeWrite", false}, + } + + for _, test := range tests { + broker := NewMockBroker(t, 1) + + metadataResponse := &MetadataResponse{ + Version: 1, + ControllerID: 1, + } + metadataResponse.AddBroker(broker.Addr(), broker.BrokerID()) + metadataResponse.AddTopicPartition("my_topic", 0, broker.BrokerID(), nil, nil, ErrNoError) + + initProducerIDResponse := &InitProducerIDResponse{ + ThrottleTime: 0, + ProducerID: 1000, + ProducerEpoch: 1, + } + + prodNotLeaderResponse := &ProduceResponse{ + Version: 3, + ThrottleTime: 0, + } + prodNotLeaderResponse.AddTopicPartition("my_topic", 0, ErrNotEnoughReplicas) + + prodDuplicate := &ProduceResponse{ + Version: 3, + ThrottleTime: 0, + } + prodDuplicate.AddTopicPartition("my_topic", 0, ErrDuplicateSequenceNumber) + + prodOutOfSeq := &ProduceResponse{ + Version: 3, + ThrottleTime: 0, + } + prodOutOfSeq.AddTopicPartition("my_topic", 0, ErrOutOfOrderSequenceNumber) + + prodSuccessResponse := &ProduceResponse{ + Version: 3, + ThrottleTime: 0, + } + prodSuccessResponse.AddTopicPartition("my_topic", 0, ErrNoError) + + prodCounter := 0 + lastBatchFirstSeq := -1 + lastBatchSize := -1 + lastSequenceWrittenToDisk := -1 + handlerFailBeforeWrite := func(req *request) (res encoder) { + switch req.body.key() { + case 3: + return metadataResponse + case 22: + return initProducerIDResponse + case 0: + prodCounter++ + + preq := req.body.(*ProduceRequest) + batch := preq.records["my_topic"][0].RecordBatch + batchFirstSeq := int(batch.FirstSequence) + batchSize := len(batch.Records) + + if lastSequenceWrittenToDisk == batchFirstSeq-1 { //in sequence append + + if lastBatchFirstSeq == batchFirstSeq { //is a batch retry + if lastBatchSize == batchSize { //good retry + // mock write to disk + lastSequenceWrittenToDisk = batchFirstSeq + batchSize - 1 + return prodSuccessResponse + } + t.Errorf("[%s] Retried Batch firstSeq=%d with different size old=%d new=%d", test.name, batchFirstSeq, lastBatchSize, batchSize) + return prodOutOfSeq + } else { // not a retry + // save batch just received for future check + lastBatchFirstSeq = batchFirstSeq + lastBatchSize = batchSize + + if prodCounter%2 == 1 { + if test.failAfterWrite { + // mock write to disk + lastSequenceWrittenToDisk = batchFirstSeq + batchSize - 1 + } + return prodNotLeaderResponse + } + // mock write to disk + lastSequenceWrittenToDisk = batchFirstSeq + batchSize - 1 + return prodSuccessResponse + } + } else { + if lastBatchFirstSeq == batchFirstSeq && lastBatchSize == batchSize { // is a good batch retry + if lastSequenceWrittenToDisk == (batchFirstSeq + batchSize - 1) { // we already have the messages + return prodDuplicate + } + // mock write to disk + lastSequenceWrittenToDisk = batchFirstSeq + batchSize - 1 + return prodSuccessResponse + } else { //out of sequence / bad retried batch + if lastBatchFirstSeq == batchFirstSeq && lastBatchSize != batchSize { + t.Errorf("[%s] Retried Batch firstSeq=%d with different size old=%d new=%d", test.name, batchFirstSeq, lastBatchSize, batchSize) + } else if lastSequenceWrittenToDisk+1 != batchFirstSeq { + t.Errorf("[%s] Out of sequence message lastSequence=%d new batch starts at=%d", test.name, lastSequenceWrittenToDisk, batchFirstSeq) + } else { + t.Errorf("[%s] Unexpected error", test.name) + } + + return prodOutOfSeq + } + } + + } + return nil + } + + config := NewConfig() + config.Version = V0_11_0_0 + config.Producer.Idempotent = true + config.Net.MaxOpenRequests = 1 + config.Producer.RequiredAcks = WaitForAll + config.Producer.Return.Successes = true + config.Producer.Flush.Frequency = 50 * time.Millisecond + config.Producer.Retry.Backoff = 100 * time.Millisecond + + broker.setHandler(handlerFailBeforeWrite) + producer, err := NewAsyncProducer([]string{broker.Addr()}, config) + if err != nil { + t.Fatal(err) + } + + for i := 0; i < 3; i++ { + producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} + } + + go func() { + for i := 0; i < 7; i++ { + producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder("goroutine")} + time.Sleep(100 * time.Millisecond) + } + }() + + expectResults(t, producer, 10, 0) + + broker.Close() + closeProducer(t, producer) + } +} + +func TestAsyncProducerIdempotentErrorOnOutOfSeq(t *testing.T) { + broker := NewMockBroker(t, 1) + + metadataResponse := &MetadataResponse{ + Version: 1, + ControllerID: 1, + } + metadataResponse.AddBroker(broker.Addr(), broker.BrokerID()) + metadataResponse.AddTopicPartition("my_topic", 0, broker.BrokerID(), nil, nil, ErrNoError) + broker.Returns(metadataResponse) + + initProducerID := &InitProducerIDResponse{ + ThrottleTime: 0, + ProducerID: 1000, + ProducerEpoch: 1, + } + broker.Returns(initProducerID) + + config := NewConfig() + config.Producer.Flush.Messages = 10 + config.Producer.Return.Successes = true + config.Producer.Retry.Max = 400000 + config.Producer.RequiredAcks = WaitForAll + config.Producer.Retry.Backoff = 0 + config.Producer.Idempotent = true + config.Net.MaxOpenRequests = 1 + config.Version = V0_11_0_0 + + producer, err := NewAsyncProducer([]string{broker.Addr()}, config) + if err != nil { + t.Fatal(err) + } + + for i := 0; i < 10; i++ { + producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} + } + + prodOutOfSeq := &ProduceResponse{ + Version: 3, + ThrottleTime: 0, + } + prodOutOfSeq.AddTopicPartition("my_topic", 0, ErrOutOfOrderSequenceNumber) + broker.Returns(prodOutOfSeq) + expectResults(t, producer, 0, 10) + + broker.Close() + closeProducer(t, producer) +} + // This example shows how to use the producer while simultaneously // reading the Errors channel to know about any failures. func ExampleAsyncProducer_select() { diff --git a/third/github.com/Shopify/sarama/balance_strategy.go b/third/github.com/Shopify/sarama/balance_strategy.go new file mode 100644 index 000000000..e78988d71 --- /dev/null +++ b/third/github.com/Shopify/sarama/balance_strategy.go @@ -0,0 +1,129 @@ +package sarama + +import ( + "math" + "sort" +) + +// BalanceStrategyPlan is the results of any BalanceStrategy.Plan attempt. +// It contains an allocation of topic/partitions by memberID in the form of +// a `memberID -> topic -> partitions` map. +type BalanceStrategyPlan map[string]map[string][]int32 + +// Add assigns a topic with a number partitions to a member. +func (p BalanceStrategyPlan) Add(memberID, topic string, partitions ...int32) { + if len(partitions) == 0 { + return + } + if _, ok := p[memberID]; !ok { + p[memberID] = make(map[string][]int32, 1) + } + p[memberID][topic] = append(p[memberID][topic], partitions...) +} + +// -------------------------------------------------------------------- + +// BalanceStrategy is used to balance topics and partitions +// across memebers of a consumer group +type BalanceStrategy interface { + // Name uniquely identifies the strategy. + Name() string + + // Plan accepts a map of `memberID -> metadata` and a map of `topic -> partitions` + // and returns a distribution plan. + Plan(members map[string]ConsumerGroupMemberMetadata, topics map[string][]int32) (BalanceStrategyPlan, error) +} + +// -------------------------------------------------------------------- + +// BalanceStrategyRange is the default and assigns partitions as ranges to consumer group members. +// Example with one topic T with six partitions (0..5) and two members (M1, M2): +// M1: {T: [0, 1, 2]} +// M2: {T: [3, 4, 5]} +var BalanceStrategyRange = &balanceStrategy{ + name: "range", + coreFn: func(plan BalanceStrategyPlan, memberIDs []string, topic string, partitions []int32) { + step := float64(len(partitions)) / float64(len(memberIDs)) + + for i, memberID := range memberIDs { + pos := float64(i) + min := int(math.Floor(pos*step + 0.5)) + max := int(math.Floor((pos+1)*step + 0.5)) + plan.Add(memberID, topic, partitions[min:max]...) + } + }, +} + +// BalanceStrategyRoundRobin assigns partitions to members in alternating order. +// Example with topic T with six partitions (0..5) and two members (M1, M2): +// M1: {T: [0, 2, 4]} +// M2: {T: [1, 3, 5]} +var BalanceStrategyRoundRobin = &balanceStrategy{ + name: "roundrobin", + coreFn: func(plan BalanceStrategyPlan, memberIDs []string, topic string, partitions []int32) { + for i, part := range partitions { + memberID := memberIDs[i%len(memberIDs)] + plan.Add(memberID, topic, part) + } + }, +} + +// -------------------------------------------------------------------- + +type balanceStrategy struct { + name string + coreFn func(plan BalanceStrategyPlan, memberIDs []string, topic string, partitions []int32) +} + +// Name implements BalanceStrategy. +func (s *balanceStrategy) Name() string { return s.name } + +// Balance implements BalanceStrategy. +func (s *balanceStrategy) Plan(members map[string]ConsumerGroupMemberMetadata, topics map[string][]int32) (BalanceStrategyPlan, error) { + // Build members by topic map + mbt := make(map[string][]string) + for memberID, meta := range members { + for _, topic := range meta.Topics { + mbt[topic] = append(mbt[topic], memberID) + } + } + + // Sort members for each topic + for topic, memberIDs := range mbt { + sort.Sort(&balanceStrategySortable{ + topic: topic, + memberIDs: memberIDs, + }) + } + + // Assemble plan + plan := make(BalanceStrategyPlan, len(members)) + for topic, memberIDs := range mbt { + s.coreFn(plan, memberIDs, topic, topics[topic]) + } + return plan, nil +} + +type balanceStrategySortable struct { + topic string + memberIDs []string +} + +func (p balanceStrategySortable) Len() int { return len(p.memberIDs) } +func (p balanceStrategySortable) Swap(i, j int) { + p.memberIDs[i], p.memberIDs[j] = p.memberIDs[j], p.memberIDs[i] +} +func (p balanceStrategySortable) Less(i, j int) bool { + return balanceStrategyHashValue(p.topic, p.memberIDs[i]) < balanceStrategyHashValue(p.topic, p.memberIDs[j]) +} + +func balanceStrategyHashValue(vv ...string) uint32 { + h := uint32(2166136261) + for _, s := range vv { + for _, c := range s { + h ^= uint32(c) + h *= 16777619 + } + } + return h +} diff --git a/third/github.com/Shopify/sarama/balance_strategy_test.go b/third/github.com/Shopify/sarama/balance_strategy_test.go new file mode 100644 index 000000000..047157f37 --- /dev/null +++ b/third/github.com/Shopify/sarama/balance_strategy_test.go @@ -0,0 +1,102 @@ +package sarama + +import ( + "reflect" + "testing" +) + +func TestBalanceStrategyRange(t *testing.T) { + tests := []struct { + members map[string][]string + topics map[string][]int32 + expected BalanceStrategyPlan + }{ + { + members: map[string][]string{"M1": {"T1", "T2"}, "M2": {"T1", "T2"}}, + topics: map[string][]int32{"T1": {0, 1, 2, 3}, "T2": {0, 1, 2, 3}}, + expected: BalanceStrategyPlan{ + "M1": map[string][]int32{"T1": {0, 1}, "T2": {2, 3}}, + "M2": map[string][]int32{"T1": {2, 3}, "T2": {0, 1}}, + }, + }, + { + members: map[string][]string{"M1": {"T1", "T2"}, "M2": {"T1", "T2"}}, + topics: map[string][]int32{"T1": {0, 1, 2}, "T2": {0, 1, 2}}, + expected: BalanceStrategyPlan{ + "M1": map[string][]int32{"T1": {0, 1}, "T2": {2}}, + "M2": map[string][]int32{"T1": {2}, "T2": {0, 1}}, + }, + }, + { + members: map[string][]string{"M1": {"T1"}, "M2": {"T1", "T2"}}, + topics: map[string][]int32{"T1": {0, 1}, "T2": {0, 1}}, + expected: BalanceStrategyPlan{ + "M1": map[string][]int32{"T1": {0}}, + "M2": map[string][]int32{"T1": {1}, "T2": {0, 1}}, + }, + }, + } + + strategy := BalanceStrategyRange + if strategy.Name() != "range" { + t.Errorf("Unexpected stategy name\nexpected: range\nactual: %v", strategy.Name()) + } + + for _, test := range tests { + members := make(map[string]ConsumerGroupMemberMetadata) + for memberID, topics := range test.members { + members[memberID] = ConsumerGroupMemberMetadata{Topics: topics} + } + + actual, err := strategy.Plan(members, test.topics) + if err != nil { + t.Errorf("Unexpected error %v", err) + } else if !reflect.DeepEqual(actual, test.expected) { + t.Errorf("Plan does not match expectation\nexpected: %#v\nactual: %#v", test.expected, actual) + } + } +} + +func TestBalanceStrategyRoundRobin(t *testing.T) { + tests := []struct { + members map[string][]string + topics map[string][]int32 + expected BalanceStrategyPlan + }{ + { + members: map[string][]string{"M1": {"T1", "T2"}, "M2": {"T1", "T2"}}, + topics: map[string][]int32{"T1": {0, 1, 2, 3}, "T2": {0, 1, 2, 3}}, + expected: BalanceStrategyPlan{ + "M1": map[string][]int32{"T1": {0, 2}, "T2": {1, 3}}, + "M2": map[string][]int32{"T1": {1, 3}, "T2": {0, 2}}, + }, + }, + { + members: map[string][]string{"M1": {"T1", "T2"}, "M2": {"T1", "T2"}}, + topics: map[string][]int32{"T1": {0, 1, 2}, "T2": {0, 1, 2}}, + expected: BalanceStrategyPlan{ + "M1": map[string][]int32{"T1": {0, 2}, "T2": {1}}, + "M2": map[string][]int32{"T1": {1}, "T2": {0, 2}}, + }, + }, + } + + strategy := BalanceStrategyRoundRobin + if strategy.Name() != "roundrobin" { + t.Errorf("Unexpected stategy name\nexpected: range\nactual: %v", strategy.Name()) + } + + for _, test := range tests { + members := make(map[string]ConsumerGroupMemberMetadata) + for memberID, topics := range test.members { + members[memberID] = ConsumerGroupMemberMetadata{Topics: topics} + } + + actual, err := strategy.Plan(members, test.topics) + if err != nil { + t.Errorf("Unexpected error %v", err) + } else if !reflect.DeepEqual(actual, test.expected) { + t.Errorf("Plan does not match expectation\nexpected: %#v\nactual: %#v", test.expected, actual) + } + } +} diff --git a/third/github.com/Shopify/sarama/broker.go b/third/github.com/Shopify/sarama/broker.go index 11c81a5b5..28730e9de 100644 --- a/third/github.com/Shopify/sarama/broker.go +++ b/third/github.com/Shopify/sarama/broker.go @@ -86,6 +86,7 @@ func (b *Broker) Open(conf *Config) error { dialer := net.Dialer{ Timeout: conf.Net.DialTimeout, KeepAlive: conf.Net.KeepAlive, + LocalAddr: conf.Net.LocalAddr, } if conf.Net.TLS.Enable { @@ -207,6 +208,17 @@ func (b *Broker) Addr() string { return b.addr } +// Rack returns the broker's rack as retrieved from Kafka's metadata or the +// empty string if it is not known. The returned value corresponds to the +// broker's broker.rack configuration setting. Requires protocol version to be +// at least v0.10.0.0. +func (b *Broker) Rack() string { + if b.rack == nil { + return "" + } + return *b.rack +} + func (b *Broker) GetMetadata(request *MetadataRequest) (*MetadataResponse, error) { response := new(MetadataResponse) diff --git a/third/github.com/Shopify/sarama/broker_test.go b/third/github.com/Shopify/sarama/broker_test.go index 9263cef8b..daf126b8b 100644 --- a/third/github.com/Shopify/sarama/broker_test.go +++ b/third/github.com/Shopify/sarama/broker_test.go @@ -51,10 +51,20 @@ func TestBrokerAccessors(t *testing.T) { t.Error("New broker didn't have the correct address") } + if broker.Rack() != "" { + t.Error("New broker didn't have an unknown rack.") + } + broker.id = 34 if broker.ID() != 34 { t.Error("Manually setting broker ID did not take effect.") } + + rack := "dc1" + broker.rack = &rack + if broker.Rack() != rack { + t.Error("Manually setting broker rack did not take effect.") + } } func TestSimpleBrokerCommunication(t *testing.T) { diff --git a/third/github.com/Shopify/sarama/client.go b/third/github.com/Shopify/sarama/client.go index 019cb4373..79be5ce53 100644 --- a/third/github.com/Shopify/sarama/client.go +++ b/third/github.com/Shopify/sarama/client.go @@ -17,7 +17,7 @@ type Client interface { // altered after it has been created. Config() *Config - // Controller returns the cluster controller broker. + // Controller returns the cluster controller broker. Requires Kafka 0.10 or higher. Controller() (*Broker, error) // Brokers returns the current set of active brokers as retrieved from cluster metadata. @@ -67,6 +67,9 @@ type Client interface { // in local cache. This function only works on Kafka 0.8.2 and higher. RefreshCoordinator(consumerGroup string) error + // InitProducerID retrieves information required for Idempotent Producer + InitProducerID() (*InitProducerIDResponse, error) + // Close shuts down all broker connections managed by this client. It is required // to call this function before a client object passes out of scope, as it will // otherwise leak memory. You must close any Producers or Consumers using a client @@ -100,10 +103,11 @@ type client struct { seedBrokers []*Broker deadSeeds []*Broker - controllerID int32 // cluster controller broker id - brokers map[int32]*Broker // maps broker ids to brokers - metadata map[string]map[int32]*PartitionMetadata // maps topics to partition ids to metadata - coordinators map[string]int32 // Maps consumer group names to coordinating broker IDs + controllerID int32 // cluster controller broker id + brokers map[int32]*Broker // maps broker ids to brokers + metadata map[string]map[int32]*PartitionMetadata // maps topics to partition ids to metadata + metadataTopics map[string]none // topics that need to collect metadata + coordinators map[string]int32 // Maps consumer group names to coordinating broker IDs // If the number of partitions is large, we can get some churn calling cachedPartitions, // so the result is cached. It is important to update this value whenever metadata is changed @@ -136,6 +140,7 @@ func NewClient(addrs []string, conf *Config) (Client, error) { closed: make(chan none), brokers: make(map[int32]*Broker), metadata: make(map[string]map[int32]*PartitionMetadata), + metadataTopics: make(map[string]none), cachedPartitionsResults: make(map[string][maxPartitionIndex][]int32), coordinators: make(map[string]int32), } @@ -174,13 +179,33 @@ func (client *client) Config() *Config { func (client *client) Brokers() []*Broker { client.lock.RLock() defer client.lock.RUnlock() - brokers := make([]*Broker, 0) + brokers := make([]*Broker, 0, len(client.brokers)) for _, broker := range client.brokers { brokers = append(brokers, broker) } return brokers } +func (client *client) InitProducerID() (*InitProducerIDResponse, error) { + var err error + for broker := client.any(); broker != nil; broker = client.any() { + + req := &InitProducerIDRequest{} + + response, err := broker.InitProducerID(req) + switch err.(type) { + case nil: + return response, nil + default: + // some error, remove that broker and try again + Logger.Printf("Client got error from broker %d when issuing InitProducerID : %v\n", broker.ID(), err) + _ = broker.Close() + client.deregisterBroker(broker) + } + } + return nil, err +} + func (client *client) Close() error { if client.Closed() { // Chances are this is being called from a defer() and the error will go unobserved @@ -207,6 +232,7 @@ func (client *client) Close() error { client.brokers = nil client.metadata = nil + client.metadataTopics = nil return nil } @@ -231,6 +257,22 @@ func (client *client) Topics() ([]string, error) { return ret, nil } +func (client *client) MetadataTopics() ([]string, error) { + if client.Closed() { + return nil, ErrClosedClient + } + + client.lock.RLock() + defer client.lock.RUnlock() + + ret := make([]string, 0, len(client.metadataTopics)) + for topic := range client.metadataTopics { + ret = append(ret, topic) + } + + return ret, nil +} + func (client *client) Partitions(topic string) ([]int32, error) { if client.Closed() { return nil, ErrClosedClient @@ -388,6 +430,10 @@ func (client *client) Controller() (*Broker, error) { return nil, ErrClosedClient } + if !client.conf.Version.IsAtLeast(V0_10_0_0) { + return nil, ErrUnsupportedVersion + } + controller := client.cachedController() if controller == nil { if err := client.refreshMetadata(); err != nil { @@ -645,7 +691,7 @@ func (client *client) refreshMetadata() error { topics := []string{} if !client.conf.Metadata.Full { - if specificTopics, err := client.Topics(); err != nil { + if specificTopics, err := client.MetadataTopics(); err != nil { return err } else if len(specificTopics) == 0 { return ErrNoTopicsToUpdateMetadata @@ -700,7 +746,7 @@ func (client *client) tryRefreshMetadata(topics []string, attemptsRemaining int) return err default: // some other error, remove that broker and try again - Logger.Println("client/metadata got error from broker while fetching metadata:", err) + Logger.Printf("client/metadata got error from broker %d while fetching metadata: %v\n", broker.ID(), err) _ = broker.Close() client.deregisterBroker(broker) } @@ -728,9 +774,16 @@ func (client *client) updateMetadata(data *MetadataResponse, allKnownMetaData bo if allKnownMetaData { client.metadata = make(map[string]map[int32]*PartitionMetadata) + client.metadataTopics = make(map[string]none) client.cachedPartitionsResults = make(map[string][maxPartitionIndex][]int32) } for _, topic := range data.Topics { + // topics must be added firstly to `metadataTopics` to guarantee that all + // requested topics must be recorded to keep them trackable for periodically + // metadata refresh. + if _, exists := client.metadataTopics[topic.Name]; !exists { + client.metadataTopics[topic.Name] = none{} + } delete(client.metadata, topic.Name) delete(client.cachedPartitionsResults, topic.Name) diff --git a/third/github.com/Shopify/sarama/client_test.go b/third/github.com/Shopify/sarama/client_test.go index fc255a730..1d0924d05 100644 --- a/third/github.com/Shopify/sarama/client_test.go +++ b/third/github.com/Shopify/sarama/client_test.go @@ -481,10 +481,11 @@ func TestClientController(t *testing.T) { t.Fatal(err) } defer safeClose(t, client2) - if _, err = client2.Controller(); err != ErrControllerNotAvailable { - t.Errorf("Expected Contoller() to return %s, found %s", ErrControllerNotAvailable, err) + if _, err = client2.Controller(); err != ErrUnsupportedVersion { + t.Errorf("Expected Contoller() to return %s, found %s", ErrUnsupportedVersion, err) } } + func TestClientCoordinatorWithConsumerOffsetsTopic(t *testing.T) { seedBroker := NewMockBroker(t, 1) staleCoordinator := NewMockBroker(t, 2) diff --git a/third/github.com/Shopify/sarama/client_tls_test.go b/third/github.com/Shopify/sarama/client_tls_test.go index eef5f6e9c..eff84c771 100644 --- a/third/github.com/Shopify/sarama/client_tls_test.go +++ b/third/github.com/Shopify/sarama/client_tls_test.go @@ -33,12 +33,12 @@ func TestTLS(t *testing.T) { nva := time.Now().Add(1 * time.Hour) caTemplate := &x509.Certificate{ - Subject: pkix.Name{CommonName: "ca"}, - Issuer: pkix.Name{CommonName: "ca"}, - SerialNumber: big.NewInt(0), - NotAfter: nva, - NotBefore: nvb, - IsCA: true, + Subject: pkix.Name{CommonName: "ca"}, + Issuer: pkix.Name{CommonName: "ca"}, + SerialNumber: big.NewInt(0), + NotAfter: nva, + NotBefore: nvb, + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, } diff --git a/third/github.com/Shopify/sarama/compress.go b/third/github.com/Shopify/sarama/compress.go new file mode 100644 index 000000000..fb5ea83a0 --- /dev/null +++ b/third/github.com/Shopify/sarama/compress.go @@ -0,0 +1,75 @@ +package sarama + +import ( + "bytes" + "compress/gzip" + "fmt" + "sync" + + "gitee.com/johng/gf/third/github.com/eapache/go-xerial-snappy" + "gitee.com/johng/gf/third/github.com/pierrec/lz4" +) + +var ( + lz4WriterPool = sync.Pool{ + New: func() interface{} { + return lz4.NewWriter(nil) + }, + } + + gzipWriterPool = sync.Pool{ + New: func() interface{} { + return gzip.NewWriter(nil) + }, + } +) + +func compress(cc CompressionCodec, level int, data []byte) ([]byte, error) { + switch cc { + case CompressionNone: + return data, nil + case CompressionGZIP: + var ( + err error + buf bytes.Buffer + writer *gzip.Writer + ) + if level != CompressionLevelDefault { + writer, err = gzip.NewWriterLevel(&buf, level) + if err != nil { + return nil, err + } + } else { + writer = gzipWriterPool.Get().(*gzip.Writer) + defer gzipWriterPool.Put(writer) + writer.Reset(&buf) + } + if _, err := writer.Write(data); err != nil { + return nil, err + } + if err := writer.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil + case CompressionSnappy: + return snappy.Encode(data), nil + case CompressionLZ4: + writer := lz4WriterPool.Get().(*lz4.Writer) + defer lz4WriterPool.Put(writer) + + var buf bytes.Buffer + writer.Reset(&buf) + + if _, err := writer.Write(data); err != nil { + return nil, err + } + if err := writer.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil + case CompressionZSTD: + return zstdCompressLevel(nil, data, level) + default: + return nil, PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", cc)} + } +} diff --git a/third/github.com/Shopify/sarama/config.go b/third/github.com/Shopify/sarama/config.go index 749236a1e..7adab5c08 100644 --- a/third/github.com/Shopify/sarama/config.go +++ b/third/github.com/Shopify/sarama/config.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "fmt" "io/ioutil" + "net" "regexp" "time" @@ -17,6 +18,13 @@ var validID = regexp.MustCompile(`\A[A-Za-z0-9._-]+\z`) // Config is used to pass multiple configuration options to Sarama's constructors. type Config struct { + // Admin is the namespace for ClusterAdmin properties used by the administrative Kafka client. + Admin struct { + // The maximum duration the administrative Kafka client will wait for ClusterAdmin operations, + // including topics, brokers, configurations and ACLs (defaults to 3 seconds). + Timeout time.Duration + } + // Net is the namespace for network-level properties used by the Broker, and // shared by the Client/Producer/Consumer. Net struct { @@ -58,6 +66,12 @@ type Config struct { // KeepAlive specifies the keep-alive period for an active network connection. // If zero, keep-alives are disabled. (default is 0: disabled). KeepAlive time.Duration + + // LocalAddr is the local address to use when dialing an + // address. The address must be of a compatible type for the + // network being dialed. + // If nil, a local address is automatically chosen. + LocalAddr net.Addr } // Metadata is the namespace for metadata management properties used by the @@ -110,6 +124,9 @@ type Config struct { // (defaults to hashing the message key). Similar to the `partitioner.class` // setting for the JVM producer. Partitioner PartitionerConstructor + // If enabled, the producer will ensure that exactly one copy of each message is + // written. + Idempotent bool // Return specifies what channels will be populated. If they are set to true, // you must read from the respective channels to prevent deadlock. If, @@ -159,14 +176,55 @@ type Config struct { // Consumer is the namespace for configuration related to consuming messages, // used by the Consumer. - // - // Note that Sarama's Consumer type does not currently support automatic - // consumer-group rebalancing and offset tracking. For Zookeeper-based - // tracking (Kafka 0.8.2 and earlier), the https://github.com/wvanbergen/kafka - // library builds on Sarama to add this support. For Kafka-based tracking - // (Kafka 0.9 and later), the https://github.com/bsm/sarama-cluster library - // builds on Sarama to add this support. Consumer struct { + + // Group is the namespace for configuring consumer group. + Group struct { + Session struct { + // The timeout used to detect consumer failures when using Kafka's group management facility. + // The consumer sends periodic heartbeats to indicate its liveness to the broker. + // If no heartbeats are received by the broker before the expiration of this session timeout, + // then the broker will remove this consumer from the group and initiate a rebalance. + // Note that the value must be in the allowable range as configured in the broker configuration + // by `group.min.session.timeout.ms` and `group.max.session.timeout.ms` (default 10s) + Timeout time.Duration + } + Heartbeat struct { + // The expected time between heartbeats to the consumer coordinator when using Kafka's group + // management facilities. Heartbeats are used to ensure that the consumer's session stays active and + // to facilitate rebalancing when new consumers join or leave the group. + // The value must be set lower than Consumer.Group.Session.Timeout, but typically should be set no + // higher than 1/3 of that value. + // It can be adjusted even lower to control the expected time for normal rebalances (default 3s) + Interval time.Duration + } + Rebalance struct { + // Strategy for allocating topic partitions to members (default BalanceStrategyRange) + Strategy BalanceStrategy + // The maximum allowed time for each worker to join the group once a rebalance has begun. + // This is basically a limit on the amount of time needed for all tasks to flush any pending + // data and commit offsets. If the timeout is exceeded, then the worker will be removed from + // the group, which will cause offset commit failures (default 60s). + Timeout time.Duration + + Retry struct { + // When a new consumer joins a consumer group the set of consumers attempt to "rebalance" + // the load to assign partitions to each consumer. If the set of consumers changes while + // this assignment is taking place the rebalance will fail and retry. This setting controls + // the maximum number of attempts before giving up (default 4). + Max int + // Backoff time between retries during rebalance (default 2s) + Backoff time.Duration + } + } + Member struct { + // Custom metadata to include when joining the group. The user data for all joined members + // can be retrieved by sending a DescribeGroupRequest to the broker that is the + // coordinator for the group. + UserData []byte + } + } + Retry struct { // How long to wait after a failing to read from a partition before // trying again (default 2s). @@ -248,6 +306,12 @@ type Config struct { // broker version 0.9.0 or later. // (default is 0: disabled). Retention time.Duration + + Retry struct { + // The total number of times to retry failing commit + // requests during OffsetManager shutdown (default 3). + Max int + } } } @@ -279,6 +343,8 @@ type Config struct { func NewConfig() *Config { c := &Config{} + c.Admin.Timeout = 3 * time.Second + c.Net.MaxOpenRequests = 5 c.Net.DialTimeout = 30 * time.Second c.Net.ReadTimeout = 30 * time.Second @@ -307,6 +373,14 @@ func NewConfig() *Config { c.Consumer.Return.Errors = false c.Consumer.Offsets.CommitInterval = 1 * time.Second c.Consumer.Offsets.Initial = OffsetNewest + c.Consumer.Offsets.Retry.Max = 3 + + c.Consumer.Group.Session.Timeout = 10 * time.Second + c.Consumer.Group.Heartbeat.Interval = 3 * time.Second + c.Consumer.Group.Rebalance.Strategy = BalanceStrategyRange + c.Consumer.Group.Rebalance.Timeout = 60 * time.Second + c.Consumer.Group.Rebalance.Retry.Max = 4 + c.Consumer.Group.Rebalance.Retry.Backoff = 2 * time.Second c.ClientID = defaultClientID c.ChannelBufferSize = 256 @@ -355,6 +429,15 @@ func (c *Config) Validate() error { if c.Consumer.Offsets.Retention%time.Millisecond != 0 { Logger.Println("Consumer.Offsets.Retention only supports millisecond precision; nanoseconds will be truncated.") } + if c.Consumer.Group.Session.Timeout%time.Millisecond != 0 { + Logger.Println("Consumer.Group.Session.Timeout only supports millisecond precision; nanoseconds will be truncated.") + } + if c.Consumer.Group.Heartbeat.Interval%time.Millisecond != 0 { + Logger.Println("Consumer.Group.Heartbeat.Interval only supports millisecond precision; nanoseconds will be truncated.") + } + if c.Consumer.Group.Rebalance.Timeout%time.Millisecond != 0 { + Logger.Println("Consumer.Group.Rebalance.Timeout only supports millisecond precision; nanoseconds will be truncated.") + } if c.ClientID == defaultClientID { Logger.Println("ClientID is the default of 'sarama', you should consider setting it to something application-specific.") } @@ -377,6 +460,12 @@ func (c *Config) Validate() error { return ConfigurationError("Net.SASL.Password must not be empty when SASL is enabled") } + // validate the Admin values + switch { + case c.Admin.Timeout <= 0: + return ConfigurationError("Admin.Timeout must be > 0") + } + // validate the Metadata values switch { case c.Metadata.Retry.Max < 0: @@ -425,6 +514,21 @@ func (c *Config) Validate() error { } } + if c.Producer.Idempotent { + if !c.Version.IsAtLeast(V0_11_0_0) { + return ConfigurationError("Idempotent producer requires Version >= V0_11_0_0") + } + if c.Producer.Retry.Max == 0 { + return ConfigurationError("Idempotent producer requires Producer.Retry.Max >= 1") + } + if c.Producer.RequiredAcks != WaitForAll { + return ConfigurationError("Idempotent producer requires Producer.RequiredAcks to be WaitForAll") + } + if c.Net.MaxOpenRequests > 1 { + return ConfigurationError("Idempotent producer requires Net.MaxOpenRequests to be 1") + } + } + // validate the Consumer values switch { case c.Consumer.Fetch.Min <= 0: @@ -443,7 +547,26 @@ func (c *Config) Validate() error { return ConfigurationError("Consumer.Offsets.CommitInterval must be > 0") case c.Consumer.Offsets.Initial != OffsetOldest && c.Consumer.Offsets.Initial != OffsetNewest: return ConfigurationError("Consumer.Offsets.Initial must be OffsetOldest or OffsetNewest") + case c.Consumer.Offsets.Retry.Max < 0: + return ConfigurationError("Consumer.Offsets.Retry.Max must be >= 0") + } + // validate the Consumer Group values + switch { + case c.Consumer.Group.Session.Timeout <= 2*time.Millisecond: + return ConfigurationError("Consumer.Group.Session.Timeout must be >= 2ms") + case c.Consumer.Group.Heartbeat.Interval < 1*time.Millisecond: + return ConfigurationError("Consumer.Group.Heartbeat.Interval must be >= 1ms") + case c.Consumer.Group.Heartbeat.Interval >= c.Consumer.Group.Session.Timeout: + return ConfigurationError("Consumer.Group.Heartbeat.Interval must be < Consumer.Group.Session.Timeout") + case c.Consumer.Group.Rebalance.Strategy == nil: + return ConfigurationError("Consumer.Group.Rebalance.Strategy must not be empty") + case c.Consumer.Group.Rebalance.Timeout <= time.Millisecond: + return ConfigurationError("Consumer.Group.Rebalance.Timeout must be >= 1ms") + case c.Consumer.Group.Rebalance.Retry.Max < 0: + return ConfigurationError("Consumer.Group.Rebalance.Retry.Max must be >= 0") + case c.Consumer.Group.Rebalance.Retry.Backoff < 0: + return ConfigurationError("Consumer.Group.Rebalance.Retry.Backoff must be >= 0") } // validate misc shared values diff --git a/third/github.com/Shopify/sarama/config_test.go b/third/github.com/Shopify/sarama/config_test.go index 1234ffda1..ab13ca7e5 100644 --- a/third/github.com/Shopify/sarama/config_test.go +++ b/third/github.com/Shopify/sarama/config_test.go @@ -122,6 +122,28 @@ func TestMetadataConfigValidates(t *testing.T) { } } +func TestAdminConfigValidates(t *testing.T) { + tests := []struct { + name string + cfg func(*Config) // resorting to using a function as a param because of internal composite structs + err string + }{ + {"Timeout", + func(cfg *Config) { + cfg.Admin.Timeout = 0 + }, + "Admin.Timeout must be > 0"}, + } + + for i, test := range tests { + c := NewConfig() + test.cfg(c) + if err := c.Validate(); string(err.(ConfigurationError)) != test.err { + t.Errorf("[%d]:[%s] Expected %s, Got %s\n", i, test.name, test.err, err) + } + } +} + func TestProducerConfigValidates(t *testing.T) { tests := []struct { name string @@ -185,6 +207,32 @@ func TestProducerConfigValidates(t *testing.T) { cfg.Producer.Retry.Backoff = -1 }, "Producer.Retry.Backoff must be >= 0"}, + {"Idempotent Version", + func(cfg *Config) { + cfg.Producer.Idempotent = true + cfg.Version = V0_10_0_0 + }, + "Idempotent producer requires Version >= V0_11_0_0"}, + {"Idempotent with Producer.Retry.Max", + func(cfg *Config) { + cfg.Version = V0_11_0_0 + cfg.Producer.Idempotent = true + cfg.Producer.Retry.Max = 0 + }, + "Idempotent producer requires Producer.Retry.Max >= 1"}, + {"Idempotent with Producer.RequiredAcks", + func(cfg *Config) { + cfg.Version = V0_11_0_0 + cfg.Producer.Idempotent = true + }, + "Idempotent producer requires Producer.RequiredAcks to be WaitForAll"}, + {"Idempotent with Net.MaxOpenRequests", + func(cfg *Config) { + cfg.Version = V0_11_0_0 + cfg.Producer.Idempotent = true + cfg.Producer.RequiredAcks = WaitForAll + }, + "Idempotent producer requires Net.MaxOpenRequests to be 1"}, } for i, test := range tests { @@ -200,7 +248,7 @@ func TestLZ4ConfigValidation(t *testing.T) { config := NewConfig() config.Producer.Compression = CompressionLZ4 if err := config.Validate(); string(err.(ConfigurationError)) != "lz4 compression requires Version >= V0_10_0_0" { - t.Error("Expected invalid lz4/kakfa version error, got ", err) + t.Error("Expected invalid lz4/kafka version error, got ", err) } config.Version = V0_10_0_0 if err := config.Validate(); err != nil { diff --git a/third/github.com/Shopify/sarama/consumer_group.go b/third/github.com/Shopify/sarama/consumer_group.go new file mode 100644 index 000000000..bb6a2c2b9 --- /dev/null +++ b/third/github.com/Shopify/sarama/consumer_group.go @@ -0,0 +1,774 @@ +package sarama + +import ( + "context" + "errors" + "fmt" + "sort" + "sync" + "time" +) + +// ErrClosedConsumerGroup is the error returned when a method is called on a consumer group that has been closed. +var ErrClosedConsumerGroup = errors.New("kafka: tried to use a consumer group that was closed") + +// ConsumerGroup is responsible for dividing up processing of topics and partitions +// over a collection of processes (the members of the consumer group). +type ConsumerGroup interface { + // Consume joins a cluster of consumers for a given list of topics and + // starts a blocking ConsumerGroupSession through the ConsumerGroupHandler. + // + // The life-cycle of a session is represented by the following steps: + // + // 1. The consumers join the group (as explained in https://kafka.apache.org/documentation/#intro_consumers) + // and is assigned their "fair share" of partitions, aka 'claims'. + // 2. Before processing starts, the handler's Setup() hook is called to notify the user + // of the claims and allow any necessary preparation or alteration of state. + // 3. For each of the assigned claims the handler's ConsumeClaim() function is then called + // in a separate goroutine which requires it to be thread-safe. Any state must be carefully protected + // from concurrent reads/writes. + // 4. The session will persist until one of the ConsumeClaim() functions exits. This can be either when the + // parent context is cancelled or when a server-side rebalance cycle is initiated. + // 5. Once all the ConsumeClaim() loops have exited, the handler's Cleanup() hook is called + // to allow the user to perform any final tasks before a rebalance. + // 6. Finally, marked offsets are committed one last time before claims are released. + // + // Please note, that once a rebalance is triggered, sessions must be completed within + // Config.Consumer.Group.Rebalance.Timeout. This means that ConsumeClaim() functions must exit + // as quickly as possible to allow time for Cleanup() and the final offset commit. If the timeout + // is exceeded, the consumer will be removed from the group by Kafka, which will cause offset + // commit failures. + Consume(ctx context.Context, topics []string, handler ConsumerGroupHandler) error + + // Errors returns a read channel of errors that occurred during the consumer life-cycle. + // By default, errors are logged and not returned over this channel. + // If you want to implement any custom error handling, set your config's + // Consumer.Return.Errors setting to true, and read from this channel. + Errors() <-chan error + + // Close stops the ConsumerGroup and detaches any running sessions. It is required to call + // this function before the object passes out of scope, as it will otherwise leak memory. + Close() error +} + +type consumerGroup struct { + client Client + ownClient bool + + config *Config + consumer Consumer + groupID string + memberID string + errors chan error + + lock sync.Mutex + closed chan none + closeOnce sync.Once +} + +// NewConsumerGroup creates a new consumer group the given broker addresses and configuration. +func NewConsumerGroup(addrs []string, groupID string, config *Config) (ConsumerGroup, error) { + client, err := NewClient(addrs, config) + if err != nil { + return nil, err + } + + c, err := NewConsumerGroupFromClient(groupID, client) + if err != nil { + _ = client.Close() + return nil, err + } + + c.(*consumerGroup).ownClient = true + return c, nil +} + +// NewConsumerGroupFromClient creates a new consumer group using the given client. It is still +// necessary to call Close() on the underlying client when shutting down this consumer. +// PLEASE NOTE: consumer groups can only re-use but not share clients. +func NewConsumerGroupFromClient(groupID string, client Client) (ConsumerGroup, error) { + config := client.Config() + if !config.Version.IsAtLeast(V0_10_2_0) { + return nil, ConfigurationError("consumer groups require Version to be >= V0_10_2_0") + } + + consumer, err := NewConsumerFromClient(client) + if err != nil { + return nil, err + } + + return &consumerGroup{ + client: client, + consumer: consumer, + config: config, + groupID: groupID, + errors: make(chan error, config.ChannelBufferSize), + closed: make(chan none), + }, nil +} + +// Errors implements ConsumerGroup. +func (c *consumerGroup) Errors() <-chan error { return c.errors } + +// Close implements ConsumerGroup. +func (c *consumerGroup) Close() (err error) { + c.closeOnce.Do(func() { + close(c.closed) + + c.lock.Lock() + defer c.lock.Unlock() + + // leave group + if e := c.leave(); e != nil { + err = e + } + + // drain errors + go func() { + close(c.errors) + }() + for e := range c.errors { + err = e + } + + if c.ownClient { + if e := c.client.Close(); e != nil { + err = e + } + } + }) + return +} + +// Consume implements ConsumerGroup. +func (c *consumerGroup) Consume(ctx context.Context, topics []string, handler ConsumerGroupHandler) error { + // Ensure group is not closed + select { + case <-c.closed: + return ErrClosedConsumerGroup + default: + } + + c.lock.Lock() + defer c.lock.Unlock() + + // Quick exit when no topics are provided + if len(topics) == 0 { + return fmt.Errorf("no topics provided") + } + + // Refresh metadata for requested topics + if err := c.client.RefreshMetadata(topics...); err != nil { + return err + } + + // Get coordinator + coordinator, err := c.client.Coordinator(c.groupID) + if err != nil { + return err + } + + // Init session + sess, err := c.newSession(ctx, coordinator, topics, handler, c.config.Consumer.Group.Rebalance.Retry.Max) + if err == ErrClosedClient { + return ErrClosedConsumerGroup + } else if err != nil { + return err + } + + // Wait for session exit signal + <-sess.ctx.Done() + + // Gracefully release session claims + return sess.release(true) +} + +func (c *consumerGroup) newSession(ctx context.Context, coordinator *Broker, topics []string, handler ConsumerGroupHandler, retries int) (*consumerGroupSession, error) { + // Join consumer group + join, err := c.joinGroupRequest(coordinator, topics) + if err != nil { + _ = coordinator.Close() + return nil, err + } + switch join.Err { + case ErrNoError: + c.memberID = join.MemberId + case ErrUnknownMemberId, ErrIllegalGeneration: // reset member ID and retry immediately + c.memberID = "" + return c.newSession(ctx, coordinator, topics, handler, retries) + case ErrRebalanceInProgress: // retry after backoff + if retries <= 0 { + return nil, join.Err + } + + select { + case <-c.closed: + return nil, ErrClosedConsumerGroup + case <-time.After(c.config.Consumer.Group.Rebalance.Retry.Backoff): + } + + return c.newSession(ctx, coordinator, topics, handler, retries-1) + default: + return nil, join.Err + } + + // Prepare distribution plan if we joined as the leader + var plan BalanceStrategyPlan + if join.LeaderId == join.MemberId { + members, err := join.GetMembers() + if err != nil { + return nil, err + } + + plan, err = c.balance(members) + if err != nil { + return nil, err + } + } + + // Sync consumer group + sync, err := c.syncGroupRequest(coordinator, plan, join.GenerationId) + if err != nil { + _ = coordinator.Close() + return nil, err + } + switch sync.Err { + case ErrNoError: + case ErrUnknownMemberId, ErrIllegalGeneration: // reset member ID and retry immediately + c.memberID = "" + return c.newSession(ctx, coordinator, topics, handler, retries) + case ErrRebalanceInProgress: // retry after backoff + if retries <= 0 { + return nil, sync.Err + } + + select { + case <-c.closed: + return nil, ErrClosedConsumerGroup + case <-time.After(c.config.Consumer.Group.Rebalance.Retry.Backoff): + } + + return c.newSession(ctx, coordinator, topics, handler, retries-1) + default: + return nil, sync.Err + } + + // Retrieve and sort claims + var claims map[string][]int32 + if len(sync.MemberAssignment) > 0 { + members, err := sync.GetMemberAssignment() + if err != nil { + return nil, err + } + claims = members.Topics + + for _, partitions := range claims { + sort.Sort(int32Slice(partitions)) + } + } + + return newConsumerGroupSession(ctx, c, claims, join.MemberId, join.GenerationId, handler) +} + +func (c *consumerGroup) joinGroupRequest(coordinator *Broker, topics []string) (*JoinGroupResponse, error) { + req := &JoinGroupRequest{ + GroupId: c.groupID, + MemberId: c.memberID, + SessionTimeout: int32(c.config.Consumer.Group.Session.Timeout / time.Millisecond), + ProtocolType: "consumer", + } + if c.config.Version.IsAtLeast(V0_10_1_0) { + req.Version = 1 + req.RebalanceTimeout = int32(c.config.Consumer.Group.Rebalance.Timeout / time.Millisecond) + } + + meta := &ConsumerGroupMemberMetadata{ + Topics: topics, + UserData: c.config.Consumer.Group.Member.UserData, + } + strategy := c.config.Consumer.Group.Rebalance.Strategy + if err := req.AddGroupProtocolMetadata(strategy.Name(), meta); err != nil { + return nil, err + } + + return coordinator.JoinGroup(req) +} + +func (c *consumerGroup) syncGroupRequest(coordinator *Broker, plan BalanceStrategyPlan, generationID int32) (*SyncGroupResponse, error) { + req := &SyncGroupRequest{ + GroupId: c.groupID, + MemberId: c.memberID, + GenerationId: generationID, + } + for memberID, topics := range plan { + err := req.AddGroupAssignmentMember(memberID, &ConsumerGroupMemberAssignment{ + Topics: topics, + }) + if err != nil { + return nil, err + } + } + return coordinator.SyncGroup(req) +} + +func (c *consumerGroup) heartbeatRequest(coordinator *Broker, memberID string, generationID int32) (*HeartbeatResponse, error) { + req := &HeartbeatRequest{ + GroupId: c.groupID, + MemberId: memberID, + GenerationId: generationID, + } + + return coordinator.Heartbeat(req) +} + +func (c *consumerGroup) balance(members map[string]ConsumerGroupMemberMetadata) (BalanceStrategyPlan, error) { + topics := make(map[string][]int32) + for _, meta := range members { + for _, topic := range meta.Topics { + topics[topic] = nil + } + } + + for topic := range topics { + partitions, err := c.client.Partitions(topic) + if err != nil { + return nil, err + } + topics[topic] = partitions + } + + strategy := c.config.Consumer.Group.Rebalance.Strategy + return strategy.Plan(members, topics) +} + +// Leaves the cluster, called by Close, protected by lock. +func (c *consumerGroup) leave() error { + if c.memberID == "" { + return nil + } + + coordinator, err := c.client.Coordinator(c.groupID) + if err != nil { + return err + } + + resp, err := coordinator.LeaveGroup(&LeaveGroupRequest{ + GroupId: c.groupID, + MemberId: c.memberID, + }) + if err != nil { + _ = coordinator.Close() + return err + } + + // Unset memberID + c.memberID = "" + + // Check response + switch resp.Err { + case ErrRebalanceInProgress, ErrUnknownMemberId, ErrNoError: + return nil + default: + return resp.Err + } +} + +func (c *consumerGroup) handleError(err error, topic string, partition int32) { + select { + case <-c.closed: + return + default: + } + + if _, ok := err.(*ConsumerError); !ok && topic != "" && partition > -1 { + err = &ConsumerError{ + Topic: topic, + Partition: partition, + Err: err, + } + } + + if c.config.Consumer.Return.Errors { + select { + case c.errors <- err: + default: + } + } else { + Logger.Println(err) + } +} + +// -------------------------------------------------------------------- + +// ConsumerGroupSession represents a consumer group member session. +type ConsumerGroupSession interface { + // Claims returns information about the claimed partitions by topic. + Claims() map[string][]int32 + + // MemberID returns the cluster member ID. + MemberID() string + + // GenerationID returns the current generation ID. + GenerationID() int32 + + // MarkOffset marks the provided offset, alongside a metadata string + // that represents the state of the partition consumer at that point in time. The + // metadata string can be used by another consumer to restore that state, so it + // can resume consumption. + // + // To follow upstream conventions, you are expected to mark the offset of the + // next message to read, not the last message read. Thus, when calling `MarkOffset` + // you should typically add one to the offset of the last consumed message. + // + // Note: calling MarkOffset does not necessarily commit the offset to the backend + // store immediately for efficiency reasons, and it may never be committed if + // your application crashes. This means that you may end up processing the same + // message twice, and your processing should ideally be idempotent. + MarkOffset(topic string, partition int32, offset int64, metadata string) + + // ResetOffset resets to the provided offset, alongside a metadata string that + // represents the state of the partition consumer at that point in time. Reset + // acts as a counterpart to MarkOffset, the difference being that it allows to + // reset an offset to an earlier or smaller value, where MarkOffset only + // allows incrementing the offset. cf MarkOffset for more details. + ResetOffset(topic string, partition int32, offset int64, metadata string) + + // MarkMessage marks a message as consumed. + MarkMessage(msg *ConsumerMessage, metadata string) + + // Context returns the session context. + Context() context.Context +} + +type consumerGroupSession struct { + parent *consumerGroup + memberID string + generationID int32 + handler ConsumerGroupHandler + + claims map[string][]int32 + offsets *offsetManager + ctx context.Context + cancel func() + + waitGroup sync.WaitGroup + releaseOnce sync.Once + hbDying, hbDead chan none +} + +func newConsumerGroupSession(ctx context.Context, parent *consumerGroup, claims map[string][]int32, memberID string, generationID int32, handler ConsumerGroupHandler) (*consumerGroupSession, error) { + // init offset manager + offsets, err := newOffsetManagerFromClient(parent.groupID, memberID, generationID, parent.client) + if err != nil { + return nil, err + } + + // init context + ctx, cancel := context.WithCancel(ctx) + + // init session + sess := &consumerGroupSession{ + parent: parent, + memberID: memberID, + generationID: generationID, + handler: handler, + offsets: offsets, + claims: claims, + ctx: ctx, + cancel: cancel, + hbDying: make(chan none), + hbDead: make(chan none), + } + + // start heartbeat loop + go sess.heartbeatLoop() + + // create a POM for each claim + for topic, partitions := range claims { + for _, partition := range partitions { + pom, err := offsets.ManagePartition(topic, partition) + if err != nil { + _ = sess.release(false) + return nil, err + } + + // handle POM errors + go func(topic string, partition int32) { + for err := range pom.Errors() { + sess.parent.handleError(err, topic, partition) + } + }(topic, partition) + } + } + + // perform setup + if err := handler.Setup(sess); err != nil { + _ = sess.release(true) + return nil, err + } + + // start consuming + for topic, partitions := range claims { + for _, partition := range partitions { + sess.waitGroup.Add(1) + + go func(topic string, partition int32) { + defer sess.waitGroup.Done() + + // cancel the as session as soon as the first + // goroutine exits + defer sess.cancel() + + // consume a single topic/partition, blocking + sess.consume(topic, partition) + }(topic, partition) + } + } + return sess, nil +} + +func (s *consumerGroupSession) Claims() map[string][]int32 { return s.claims } +func (s *consumerGroupSession) MemberID() string { return s.memberID } +func (s *consumerGroupSession) GenerationID() int32 { return s.generationID } + +func (s *consumerGroupSession) MarkOffset(topic string, partition int32, offset int64, metadata string) { + if pom := s.offsets.findPOM(topic, partition); pom != nil { + pom.MarkOffset(offset, metadata) + } +} + +func (s *consumerGroupSession) ResetOffset(topic string, partition int32, offset int64, metadata string) { + if pom := s.offsets.findPOM(topic, partition); pom != nil { + pom.ResetOffset(offset, metadata) + } +} + +func (s *consumerGroupSession) MarkMessage(msg *ConsumerMessage, metadata string) { + s.MarkOffset(msg.Topic, msg.Partition, msg.Offset+1, metadata) +} + +func (s *consumerGroupSession) Context() context.Context { + return s.ctx +} + +func (s *consumerGroupSession) consume(topic string, partition int32) { + // quick exit if rebalance is due + select { + case <-s.ctx.Done(): + return + case <-s.parent.closed: + return + default: + } + + // get next offset + offset := s.parent.config.Consumer.Offsets.Initial + if pom := s.offsets.findPOM(topic, partition); pom != nil { + offset, _ = pom.NextOffset() + } + + // create new claim + claim, err := newConsumerGroupClaim(s, topic, partition, offset) + if err != nil { + s.parent.handleError(err, topic, partition) + return + } + + // handle errors + go func() { + for err := range claim.Errors() { + s.parent.handleError(err, topic, partition) + } + }() + + // trigger close when session is done + go func() { + select { + case <-s.ctx.Done(): + case <-s.parent.closed: + } + claim.AsyncClose() + }() + + // start processing + if err := s.handler.ConsumeClaim(s, claim); err != nil { + s.parent.handleError(err, topic, partition) + } + + // ensure consumer is closed & drained + claim.AsyncClose() + for _, err := range claim.waitClosed() { + s.parent.handleError(err, topic, partition) + } +} + +func (s *consumerGroupSession) release(withCleanup bool) (err error) { + // signal release, stop heartbeat + s.cancel() + + // wait for consumers to exit + s.waitGroup.Wait() + + // perform release + s.releaseOnce.Do(func() { + if withCleanup { + if e := s.handler.Cleanup(s); e != nil { + s.parent.handleError(err, "", -1) + err = e + } + } + + if e := s.offsets.Close(); e != nil { + err = e + } + + close(s.hbDying) + <-s.hbDead + }) + + return +} + +func (s *consumerGroupSession) heartbeatLoop() { + defer close(s.hbDead) + defer s.cancel() // trigger the end of the session on exit + + pause := time.NewTicker(s.parent.config.Consumer.Group.Heartbeat.Interval) + defer pause.Stop() + + retries := s.parent.config.Metadata.Retry.Max + for { + coordinator, err := s.parent.client.Coordinator(s.parent.groupID) + if err != nil { + if retries <= 0 { + s.parent.handleError(err, "", -1) + return + } + + select { + case <-s.hbDying: + return + case <-time.After(s.parent.config.Metadata.Retry.Backoff): + retries-- + } + continue + } + + resp, err := s.parent.heartbeatRequest(coordinator, s.memberID, s.generationID) + if err != nil { + _ = coordinator.Close() + retries-- + continue + } + + switch resp.Err { + case ErrNoError: + retries = s.parent.config.Metadata.Retry.Max + case ErrRebalanceInProgress, ErrUnknownMemberId, ErrIllegalGeneration: + return + default: + s.parent.handleError(err, "", -1) + return + } + + select { + case <-pause.C: + case <-s.hbDying: + return + } + } +} + +// -------------------------------------------------------------------- + +// ConsumerGroupHandler instances are used to handle individual topic/partition claims. +// It also provides hooks for your consumer group session life-cycle and allow you to +// trigger logic before or after the consume loop(s). +// +// PLEASE NOTE that handlers are likely be called from several goroutines concurrently, +// ensure that all state is safely protected against race conditions. +type ConsumerGroupHandler interface { + // Setup is run at the beginning of a new session, before ConsumeClaim. + Setup(ConsumerGroupSession) error + + // Cleanup is run at the end of a session, once all ConsumeClaim goroutines have exited + // but before the offsets are committed for the very last time. + Cleanup(ConsumerGroupSession) error + + // ConsumeClaim must start a consumer loop of ConsumerGroupClaim's Messages(). + // Once the Messages() channel is closed, the Handler must finish its processing + // loop and exit. + ConsumeClaim(ConsumerGroupSession, ConsumerGroupClaim) error +} + +// ConsumerGroupClaim processes Kafka messages from a given topic and partition within a consumer group. +type ConsumerGroupClaim interface { + // Topic returns the consumed topic name. + Topic() string + + // Partition returns the consumed partition. + Partition() int32 + + // InitialOffset returns the initial offset that was used as a starting point for this claim. + InitialOffset() int64 + + // HighWaterMarkOffset returns the high water mark offset of the partition, + // i.e. the offset that will be used for the next message that will be produced. + // You can use this to determine how far behind the processing is. + HighWaterMarkOffset() int64 + + // Messages returns the read channel for the messages that are returned by + // the broker. The messages channel will be closed when a new rebalance cycle + // is due. You must finish processing and mark offsets within + // Config.Consumer.Group.Session.Timeout before the topic/partition is eventually + // re-assigned to another group member. + Messages() <-chan *ConsumerMessage +} + +type consumerGroupClaim struct { + topic string + partition int32 + offset int64 + PartitionConsumer +} + +func newConsumerGroupClaim(sess *consumerGroupSession, topic string, partition int32, offset int64) (*consumerGroupClaim, error) { + pcm, err := sess.parent.consumer.ConsumePartition(topic, partition, offset) + if err == ErrOffsetOutOfRange { + offset = sess.parent.config.Consumer.Offsets.Initial + pcm, err = sess.parent.consumer.ConsumePartition(topic, partition, offset) + } + if err != nil { + return nil, err + } + + go func() { + for err := range pcm.Errors() { + sess.parent.handleError(err, topic, partition) + } + }() + + return &consumerGroupClaim{ + topic: topic, + partition: partition, + offset: offset, + PartitionConsumer: pcm, + }, nil +} + +func (c *consumerGroupClaim) Topic() string { return c.topic } +func (c *consumerGroupClaim) Partition() int32 { return c.partition } +func (c *consumerGroupClaim) InitialOffset() int64 { return c.offset } + +// Drains messages and errors, ensures the claim is fully closed. +func (c *consumerGroupClaim) waitClosed() (errs ConsumerErrors) { + go func() { + for range c.Messages() { + } + }() + + for err := range c.Errors() { + errs = append(errs, err) + } + return +} diff --git a/third/github.com/Shopify/sarama/consumer_group_members.go b/third/github.com/Shopify/sarama/consumer_group_members.go index 9d92d350a..2d02cc386 100644 --- a/third/github.com/Shopify/sarama/consumer_group_members.go +++ b/third/github.com/Shopify/sarama/consumer_group_members.go @@ -1,5 +1,6 @@ package sarama +//ConsumerGroupMemberMetadata holds the metadata for consumer group type ConsumerGroupMemberMetadata struct { Version int16 Topics []string @@ -36,6 +37,7 @@ func (m *ConsumerGroupMemberMetadata) decode(pd packetDecoder) (err error) { return nil } +//ConsumerGroupMemberAssignment holds the member assignment for a consume group type ConsumerGroupMemberAssignment struct { Version int16 Topics map[string][]int32 diff --git a/third/github.com/Shopify/sarama/consumer_group_test.go b/third/github.com/Shopify/sarama/consumer_group_test.go new file mode 100644 index 000000000..8bf44e661 --- /dev/null +++ b/third/github.com/Shopify/sarama/consumer_group_test.go @@ -0,0 +1,58 @@ +package sarama + +import ( + "context" + "fmt" +) + +type exampleConsumerGroupHandler struct{} + +func (exampleConsumerGroupHandler) Setup(_ ConsumerGroupSession) error { return nil } +func (exampleConsumerGroupHandler) Cleanup(_ ConsumerGroupSession) error { return nil } +func (h exampleConsumerGroupHandler) ConsumeClaim(sess ConsumerGroupSession, claim ConsumerGroupClaim) error { + for msg := range claim.Messages() { + fmt.Printf("Message topic:%q partition:%d offset:%d\n", msg.Topic, msg.Partition, msg.Offset) + sess.MarkMessage(msg, "") + } + return nil +} + +func ExampleConsumerGroup() { + // Init config, specify appropriate version + config := NewConfig() + config.Version = V1_0_0_0 + config.Consumer.Return.Errors = true + + // Start with a client + client, err := NewClient([]string{"localhost:9092"}, config) + if err != nil { + panic(err) + } + defer func() { _ = client.Close() }() + + // Start a new consumer group + group, err := NewConsumerGroupFromClient("my-group", client) + if err != nil { + panic(err) + } + defer func() { _ = group.Close() }() + + // Track errors + go func() { + for err := range group.Errors() { + fmt.Println("ERROR", err) + } + }() + + // Iterate over consumer sessions. + ctx := context.Background() + for { + topics := []string{"my-topic"} + handler := exampleConsumerGroupHandler{} + + err := group.Consume(ctx, topics, handler) + if err != nil { + panic(err) + } + } +} diff --git a/third/github.com/Shopify/sarama/consumer_metadata_request.go b/third/github.com/Shopify/sarama/consumer_metadata_request.go index 4de45e7bf..a8dcaefe8 100644 --- a/third/github.com/Shopify/sarama/consumer_metadata_request.go +++ b/third/github.com/Shopify/sarama/consumer_metadata_request.go @@ -1,5 +1,6 @@ package sarama +//ConsumerMetadataRequest is used for metadata requests type ConsumerMetadataRequest struct { ConsumerGroup string } diff --git a/third/github.com/Shopify/sarama/consumer_metadata_response.go b/third/github.com/Shopify/sarama/consumer_metadata_response.go index 442cbde7a..4d86e9303 100644 --- a/third/github.com/Shopify/sarama/consumer_metadata_response.go +++ b/third/github.com/Shopify/sarama/consumer_metadata_response.go @@ -5,6 +5,7 @@ import ( "strconv" ) +//ConsumerMetadataResponse holds the reponse for a consumer gorup meta data request type ConsumerMetadataResponse struct { Err KError Coordinator *Broker diff --git a/third/github.com/Shopify/sarama/decompress.go b/third/github.com/Shopify/sarama/decompress.go new file mode 100644 index 000000000..c6e126ea2 --- /dev/null +++ b/third/github.com/Shopify/sarama/decompress.go @@ -0,0 +1,63 @@ +package sarama + +import ( + "bytes" + "compress/gzip" + "fmt" + "io/ioutil" + "sync" + + "gitee.com/johng/gf/third/github.com/eapache/go-xerial-snappy" + "gitee.com/johng/gf/third/github.com/pierrec/lz4" +) + +var ( + lz4ReaderPool = sync.Pool{ + New: func() interface{} { + return lz4.NewReader(nil) + }, + } + + gzipReaderPool sync.Pool +) + +func decompress(cc CompressionCodec, data []byte) ([]byte, error) { + switch cc { + case CompressionNone: + return data, nil + case CompressionGZIP: + var ( + err error + reader *gzip.Reader + readerIntf = gzipReaderPool.Get() + ) + if readerIntf != nil { + reader = readerIntf.(*gzip.Reader) + } else { + reader, err = gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + } + + defer gzipReaderPool.Put(reader) + + if err := reader.Reset(bytes.NewReader(data)); err != nil { + return nil, err + } + + return ioutil.ReadAll(reader) + case CompressionSnappy: + return snappy.Decode(data) + case CompressionLZ4: + reader := lz4ReaderPool.Get().(*lz4.Reader) + defer lz4ReaderPool.Put(reader) + + reader.Reset(bytes.NewReader(data)) + return ioutil.ReadAll(reader) + case CompressionZSTD: + return zstdDecompress(nil, data) + default: + return nil, PacketDecodingError{fmt.Sprintf("invalid compression specified (%d)", cc)} + } +} diff --git a/third/github.com/Shopify/sarama/describe_configs_request.go b/third/github.com/Shopify/sarama/describe_configs_request.go index 7a7cffc3f..416a4fe65 100644 --- a/third/github.com/Shopify/sarama/describe_configs_request.go +++ b/third/github.com/Shopify/sarama/describe_configs_request.go @@ -1,15 +1,17 @@ package sarama +type DescribeConfigsRequest struct { + Version int16 + Resources []*ConfigResource + IncludeSynonyms bool +} + type ConfigResource struct { Type ConfigResourceType Name string ConfigNames []string } -type DescribeConfigsRequest struct { - Resources []*ConfigResource -} - func (r *DescribeConfigsRequest) encode(pe packetEncoder) error { if err := pe.putArrayLength(len(r.Resources)); err != nil { return err @@ -30,6 +32,10 @@ func (r *DescribeConfigsRequest) encode(pe packetEncoder) error { } } + if r.Version >= 1 { + pe.putBool(r.IncludeSynonyms) + } + return nil } @@ -74,6 +80,14 @@ func (r *DescribeConfigsRequest) decode(pd packetDecoder, version int16) (err er } r.Resources[i].ConfigNames = cfnames } + r.Version = version + if r.Version >= 1 { + b, err := pd.getBool() + if err != nil { + return err + } + r.IncludeSynonyms = b + } return nil } @@ -83,9 +97,16 @@ func (r *DescribeConfigsRequest) key() int16 { } func (r *DescribeConfigsRequest) version() int16 { - return 0 + return r.Version } func (r *DescribeConfigsRequest) requiredVersion() KafkaVersion { - return V0_11_0_0 + switch r.Version { + case 1: + return V1_0_0_0 + case 2: + return V2_0_0_0 + default: + return V0_11_0_0 + } } diff --git a/third/github.com/Shopify/sarama/describe_configs_request_test.go b/third/github.com/Shopify/sarama/describe_configs_request_test.go index ca0fd0495..fec2f1163 100644 --- a/third/github.com/Shopify/sarama/describe_configs_request_test.go +++ b/third/github.com/Shopify/sarama/describe_configs_request_test.go @@ -33,23 +33,33 @@ var ( } singleDescribeConfigsRequestAllConfigs = []byte{ + 0, 0, 0, 1, // 1 config + 2, // a topic + 0, 3, 'f', 'o', 'o', // topic name: foo + 255, 255, 255, 255, // all configs + } + + singleDescribeConfigsRequestAllConfigsv1 = []byte{ 0, 0, 0, 1, // 1 config 2, // a topic 0, 3, 'f', 'o', 'o', // topic name: foo 255, 255, 255, 255, // no configs + 1, //synoms } ) -func TestDescribeConfigsRequest(t *testing.T) { +func TestDescribeConfigsRequestv0(t *testing.T) { var request *DescribeConfigsRequest request = &DescribeConfigsRequest{ + Version: 0, Resources: []*ConfigResource{}, } testRequest(t, "no requests", request, emptyDescribeConfigsRequest) configs := []string{"segment.ms"} request = &DescribeConfigsRequest{ + Version: 0, Resources: []*ConfigResource{ &ConfigResource{ Type: TopicResource, @@ -62,6 +72,7 @@ func TestDescribeConfigsRequest(t *testing.T) { testRequest(t, "one config", request, singleDescribeConfigsRequest) request = &DescribeConfigsRequest{ + Version: 0, Resources: []*ConfigResource{ &ConfigResource{ Type: TopicResource, @@ -78,6 +89,7 @@ func TestDescribeConfigsRequest(t *testing.T) { testRequest(t, "two configs", request, doubleDescribeConfigsRequest) request = &DescribeConfigsRequest{ + Version: 0, Resources: []*ConfigResource{ &ConfigResource{ Type: TopicResource, @@ -88,3 +100,20 @@ func TestDescribeConfigsRequest(t *testing.T) { testRequest(t, "one topic, all configs", request, singleDescribeConfigsRequestAllConfigs) } + +func TestDescribeConfigsRequestv1(t *testing.T) { + var request *DescribeConfigsRequest + + request = &DescribeConfigsRequest{ + Version: 1, + Resources: []*ConfigResource{ + { + Type: TopicResource, + Name: "foo", + }, + }, + IncludeSynonyms: true, + } + + testRequest(t, "one topic, all configs", request, singleDescribeConfigsRequestAllConfigsv1) +} diff --git a/third/github.com/Shopify/sarama/describe_configs_response.go b/third/github.com/Shopify/sarama/describe_configs_response.go index 6e5d30e4f..63fb6ea81 100644 --- a/third/github.com/Shopify/sarama/describe_configs_response.go +++ b/third/github.com/Shopify/sarama/describe_configs_response.go @@ -1,8 +1,41 @@ package sarama -import "time" +import ( + "fmt" + "time" +) + +type ConfigSource int8 + +func (s ConfigSource) String() string { + switch s { + case SourceUnknown: + return "Unknown" + case SourceTopic: + return "Topic" + case SourceDynamicBroker: + return "DynamicBroker" + case SourceDynamicDefaultBroker: + return "DynamicDefaultBroker" + case SourceStaticBroker: + return "StaticBroker" + case SourceDefault: + return "Default" + } + return fmt.Sprintf("Source Invalid: %d", int(s)) +} + +const ( + SourceUnknown ConfigSource = 0 + SourceTopic ConfigSource = 1 + SourceDynamicBroker ConfigSource = 2 + SourceDynamicDefaultBroker ConfigSource = 3 + SourceStaticBroker ConfigSource = 4 + SourceDefault ConfigSource = 5 +) type DescribeConfigsResponse struct { + Version int16 ThrottleTime time.Duration Resources []*ResourceResponse } @@ -20,7 +53,15 @@ type ConfigEntry struct { Value string ReadOnly bool Default bool + Source ConfigSource Sensitive bool + Synonyms []*ConfigSynonym +} + +type ConfigSynonym struct { + ConfigName string + ConfigValue string + Source ConfigSource } func (r *DescribeConfigsResponse) encode(pe packetEncoder) (err error) { @@ -30,14 +71,16 @@ func (r *DescribeConfigsResponse) encode(pe packetEncoder) (err error) { } for _, c := range r.Resources { - if err = c.encode(pe); err != nil { + if err = c.encode(pe, r.Version); err != nil { return err } } + return nil } func (r *DescribeConfigsResponse) decode(pd packetDecoder, version int16) (err error) { + r.Version = version throttleTime, err := pd.getInt32() if err != nil { return err @@ -66,14 +109,21 @@ func (r *DescribeConfigsResponse) key() int16 { } func (r *DescribeConfigsResponse) version() int16 { - return 0 + return r.Version } func (r *DescribeConfigsResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 + switch r.Version { + case 1: + return V1_0_0_0 + case 2: + return V2_0_0_0 + default: + return V0_11_0_0 + } } -func (r *ResourceResponse) encode(pe packetEncoder) (err error) { +func (r *ResourceResponse) encode(pe packetEncoder, version int16) (err error) { pe.putInt16(r.ErrorCode) if err = pe.putString(r.ErrorMsg); err != nil { @@ -91,7 +141,7 @@ func (r *ResourceResponse) encode(pe packetEncoder) (err error) { } for _, c := range r.Configs { - if err = c.encode(pe); err != nil { + if err = c.encode(pe, version); err != nil { return err } } @@ -139,7 +189,7 @@ func (r *ResourceResponse) decode(pd packetDecoder, version int16) (err error) { return nil } -func (r *ConfigEntry) encode(pe packetEncoder) (err error) { +func (r *ConfigEntry) encode(pe packetEncoder, version int16) (err error) { if err = pe.putString(r.Name); err != nil { return err } @@ -149,12 +199,32 @@ func (r *ConfigEntry) encode(pe packetEncoder) (err error) { } pe.putBool(r.ReadOnly) - pe.putBool(r.Default) - pe.putBool(r.Sensitive) + + if version <= 0 { + pe.putBool(r.Default) + pe.putBool(r.Sensitive) + } else { + pe.putInt8(int8(r.Source)) + pe.putBool(r.Sensitive) + + if err := pe.putArrayLength(len(r.Synonyms)); err != nil { + return err + } + for _, c := range r.Synonyms { + if err = c.encode(pe, version); err != nil { + return err + } + } + } + return nil } +//https://cwiki.apache.org/confluence/display/KAFKA/KIP-226+-+Dynamic+Broker+Configuration func (r *ConfigEntry) decode(pd packetDecoder, version int16) (err error) { + if version == 0 { + r.Source = SourceUnknown + } name, err := pd.getString() if err != nil { return err @@ -173,16 +243,78 @@ func (r *ConfigEntry) decode(pd packetDecoder, version int16) (err error) { } r.ReadOnly = read - de, err := pd.getBool() - if err != nil { - return err + if version == 0 { + defaultB, err := pd.getBool() + if err != nil { + return err + } + r.Default = defaultB + } else { + source, err := pd.getInt8() + if err != nil { + return err + } + r.Source = ConfigSource(source) } - r.Default = de sensitive, err := pd.getBool() if err != nil { return err } r.Sensitive = sensitive + + if version > 0 { + n, err := pd.getArrayLength() + if err != nil { + return err + } + r.Synonyms = make([]*ConfigSynonym, n) + + for i := 0; i < n; i++ { + s := &ConfigSynonym{} + if err := s.decode(pd, version); err != nil { + return err + } + r.Synonyms[i] = s + } + + } + return nil +} + +func (c *ConfigSynonym) encode(pe packetEncoder, version int16) (err error) { + err = pe.putString(c.ConfigName) + if err != nil { + return err + } + + err = pe.putString(c.ConfigValue) + if err != nil { + return err + } + + pe.putInt8(int8(c.Source)) + + return nil +} + +func (c *ConfigSynonym) decode(pd packetDecoder, version int16) error { + name, err := pd.getString() + if err != nil { + return nil + } + c.ConfigName = name + + value, err := pd.getString() + if err != nil { + return nil + } + c.ConfigValue = value + + source, err := pd.getInt8() + if err != nil { + return nil + } + c.Source = ConfigSource(source) return nil } diff --git a/third/github.com/Shopify/sarama/describe_configs_response_test.go b/third/github.com/Shopify/sarama/describe_configs_response_test.go index e3dcbac39..9584d577c 100644 --- a/third/github.com/Shopify/sarama/describe_configs_response_test.go +++ b/third/github.com/Shopify/sarama/describe_configs_response_test.go @@ -10,7 +10,7 @@ var ( 0, 0, 0, 0, // no configs } - describeConfigsResponsePopulated = []byte{ + describeConfigsResponsePopulatedv0 = []byte{ 0, 0, 0, 0, //throttle 0, 0, 0, 1, // response 0, 0, //errorcode @@ -24,9 +24,44 @@ var ( 0, // Default 0, // Sensitive } + + describeConfigsResponsePopulatedv1 = []byte{ + 0, 0, 0, 0, //throttle + 0, 0, 0, 1, // response + 0, 0, //errorcode + 0, 0, //string + 2, // topic + 0, 3, 'f', 'o', 'o', + 0, 0, 0, 1, //configs + 0, 10, 's', 'e', 'g', 'm', 'e', 'n', 't', '.', 'm', 's', + 0, 4, '1', '0', '0', '0', + 0, // ReadOnly + 4, // Source + 0, // Sensitive + 0, 0, 0, 0, // No Synonym + } + + describeConfigsResponseWithSynonymv1 = []byte{ + 0, 0, 0, 0, //throttle + 0, 0, 0, 1, // response + 0, 0, //errorcode + 0, 0, //string + 2, // topic + 0, 3, 'f', 'o', 'o', + 0, 0, 0, 1, //configs + 0, 10, 's', 'e', 'g', 'm', 'e', 'n', 't', '.', 'm', 's', + 0, 4, '1', '0', '0', '0', + 0, // ReadOnly + 4, // Source + 0, // Sensitive + 0, 0, 0, 1, // 1 Synonym + 0, 14, 'l', 'o', 'g', '.', 's', 'e', 'g', 'm', 'e', 'n', 't', '.', 'm', 's', + 0, 4, '1', '0', '0', '0', + 4, // Source + } ) -func TestDescribeConfigsResponse(t *testing.T) { +func TestDescribeConfigsResponsev0(t *testing.T) { var response *DescribeConfigsResponse response = &DescribeConfigsResponse{ @@ -38,7 +73,7 @@ func TestDescribeConfigsResponse(t *testing.T) { } response = &DescribeConfigsResponse{ - Resources: []*ResourceResponse{ + Version: 0, Resources: []*ResourceResponse{ &ResourceResponse{ ErrorCode: 0, ErrorMsg: "", @@ -56,5 +91,81 @@ func TestDescribeConfigsResponse(t *testing.T) { }, }, } - testResponse(t, "response with error", response, describeConfigsResponsePopulated) + testResponse(t, "response with error", response, describeConfigsResponsePopulatedv0) +} + +func TestDescribeConfigsResponsev1(t *testing.T) { + var response *DescribeConfigsResponse + + response = &DescribeConfigsResponse{ + Resources: []*ResourceResponse{}, + } + testVersionDecodable(t, "empty", response, describeConfigsResponseEmpty, 0) + if len(response.Resources) != 0 { + t.Error("Expected no groups") + } + + response = &DescribeConfigsResponse{ + Version: 1, + Resources: []*ResourceResponse{ + &ResourceResponse{ + ErrorCode: 0, + ErrorMsg: "", + Type: TopicResource, + Name: "foo", + Configs: []*ConfigEntry{ + &ConfigEntry{ + Name: "segment.ms", + Value: "1000", + ReadOnly: false, + Source: SourceStaticBroker, + Sensitive: false, + Synonyms: []*ConfigSynonym{}, + }, + }, + }, + }, + } + testResponse(t, "response with error", response, describeConfigsResponsePopulatedv1) +} + +func TestDescribeConfigsResponseWithSynonym(t *testing.T) { + var response *DescribeConfigsResponse + + response = &DescribeConfigsResponse{ + Resources: []*ResourceResponse{}, + } + testVersionDecodable(t, "empty", response, describeConfigsResponseEmpty, 0) + if len(response.Resources) != 0 { + t.Error("Expected no groups") + } + + response = &DescribeConfigsResponse{ + Version: 1, + Resources: []*ResourceResponse{ + &ResourceResponse{ + ErrorCode: 0, + ErrorMsg: "", + Type: TopicResource, + Name: "foo", + Configs: []*ConfigEntry{ + &ConfigEntry{ + Name: "segment.ms", + Value: "1000", + ReadOnly: false, + Source: SourceStaticBroker, + Sensitive: false, + Synonyms: []*ConfigSynonym{ + { + ConfigName: "log.segment.ms", + ConfigValue: "1000", + Source: SourceStaticBroker, + }, + }, + }, + }, + }, + }, + } + testResponse(t, "response with error", response, describeConfigsResponseWithSynonymv1) } diff --git a/third/github.com/Shopify/sarama/dev.yml b/third/github.com/Shopify/sarama/dev.yml index 294fcdb41..97eed3ad9 100644 --- a/third/github.com/Shopify/sarama/dev.yml +++ b/third/github.com/Shopify/sarama/dev.yml @@ -2,7 +2,7 @@ name: sarama up: - go: - version: '1.9' + version: '1.11' commands: test: diff --git a/third/github.com/Shopify/sarama/errors.go b/third/github.com/Shopify/sarama/errors.go index c578ef5fb..c11421d9e 100644 --- a/third/github.com/Shopify/sarama/errors.go +++ b/third/github.com/Shopify/sarama/errors.go @@ -145,6 +145,18 @@ const ( ErrSASLAuthenticationFailed KError = 58 ErrUnknownProducerID KError = 59 ErrReassignmentInProgress KError = 60 + ErrDelegationTokenAuthDisabled KError = 61 + ErrDelegationTokenNotFound KError = 62 + ErrDelegationTokenOwnerMismatch KError = 63 + ErrDelegationTokenRequestNotAllowed KError = 64 + ErrDelegationTokenAuthorizationFailed KError = 65 + ErrDelegationTokenExpired KError = 66 + ErrInvalidPrincipalType KError = 67 + ErrNonEmptyGroup KError = 68 + ErrGroupIDNotFound KError = 69 + ErrFetchSessionIDNotFound KError = 70 + ErrInvalidFetchSessionEpoch KError = 71 + ErrListenerNotFound KError = 72 ) func (err KError) Error() string { @@ -275,6 +287,30 @@ func (err KError) Error() string { return "kafka server: The broker could not locate the producer metadata associated with the Producer ID." case ErrReassignmentInProgress: return "kafka server: A partition reassignment is in progress." + case ErrDelegationTokenAuthDisabled: + return "kafka server: Delegation Token feature is not enabled." + case ErrDelegationTokenNotFound: + return "kafka server: Delegation Token is not found on server." + case ErrDelegationTokenOwnerMismatch: + return "kafka server: Specified Principal is not valid Owner/Renewer." + case ErrDelegationTokenRequestNotAllowed: + return "kafka server: Delegation Token requests are not allowed on PLAINTEXT/1-way SSL channels and on delegation token authenticated channels." + case ErrDelegationTokenAuthorizationFailed: + return "kafka server: Delegation Token authorization failed." + case ErrDelegationTokenExpired: + return "kafka server: Delegation Token is expired." + case ErrInvalidPrincipalType: + return "kafka server: Supplied principalType is not supported." + case ErrNonEmptyGroup: + return "kafka server: The group is not empty." + case ErrGroupIDNotFound: + return "kafka server: The group id does not exist." + case ErrFetchSessionIDNotFound: + return "kafka server: The fetch session ID was not found." + case ErrInvalidFetchSessionEpoch: + return "kafka server: The fetch session epoch is invalid." + case ErrListenerNotFound: + return "kafka server: There is no listener on the leader broker that matches the listener on which metadata request was processed." } return fmt.Sprintf("Unknown error, how did this happen? Error code = %d", err) diff --git a/third/github.com/Shopify/sarama/fetch_response.go b/third/github.com/Shopify/sarama/fetch_response.go index ae91bb9eb..90acfc280 100644 --- a/third/github.com/Shopify/sarama/fetch_response.go +++ b/third/github.com/Shopify/sarama/fetch_response.go @@ -33,7 +33,7 @@ type FetchResponseBlock struct { HighWaterMarkOffset int64 LastStableOffset int64 AbortedTransactions []*AbortedTransaction - Records *Records // deprecated: use FetchResponseBlock.Records + Records *Records // deprecated: use FetchResponseBlock.RecordsSet RecordsSet []*Records Partial bool } @@ -104,15 +104,26 @@ func (b *FetchResponseBlock) decode(pd packetDecoder, version int16) (err error) return err } - // If we have at least one full records, we skip incomplete ones - if partial && len(b.RecordsSet) > 0 { - break + n, err := records.numRecords() + if err != nil { + return err } - b.RecordsSet = append(b.RecordsSet, records) + if n > 0 || (partial && len(b.RecordsSet) == 0) { + b.RecordsSet = append(b.RecordsSet, records) - if b.Records == nil { - b.Records = records + if b.Records == nil { + b.Records = records + } + } + + overflow, err := records.isOverflow() + if err != nil { + return err + } + + if partial || overflow { + break } } diff --git a/third/github.com/Shopify/sarama/fetch_response_test.go b/third/github.com/Shopify/sarama/fetch_response_test.go index c6b6b46e4..917027644 100644 --- a/third/github.com/Shopify/sarama/fetch_response_test.go +++ b/third/github.com/Shopify/sarama/fetch_response_test.go @@ -27,6 +27,29 @@ var ( 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x02, 0x00, 0xEE} + overflowMessageFetchResponse = []byte{ + 0x00, 0x00, 0x00, 0x01, + 0x00, 0x05, 't', 'o', 'p', 'i', 'c', + 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x05, + 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x10, 0x10, + 0x00, 0x00, 0x00, 0x30, + // messageSet + 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x10, + // message + 0x23, 0x96, 0x4a, 0xf7, // CRC + 0x00, + 0x00, + 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x02, 0x00, 0xEE, + // overflow messageSet + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0xFF, + // overflow bytes + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + oneRecordFetchResponse = []byte{ 0x00, 0x00, 0x00, 0x00, // ThrottleTime 0x00, 0x00, 0x00, 0x01, // Number of Topics @@ -148,6 +171,66 @@ func TestOneMessageFetchResponse(t *testing.T) { } } +func TestOverflowMessageFetchResponse(t *testing.T) { + response := FetchResponse{} + testVersionDecodable(t, "overflow message", &response, overflowMessageFetchResponse, 0) + + if len(response.Blocks) != 1 { + t.Fatal("Decoding produced incorrect number of topic blocks.") + } + + if len(response.Blocks["topic"]) != 1 { + t.Fatal("Decoding produced incorrect number of partition blocks for topic.") + } + + block := response.GetBlock("topic", 5) + if block == nil { + t.Fatal("GetBlock didn't return block.") + } + if block.Err != ErrOffsetOutOfRange { + t.Error("Decoding didn't produce correct error code.") + } + if block.HighWaterMarkOffset != 0x10101010 { + t.Error("Decoding didn't produce correct high water mark offset.") + } + partial, err := block.Records.isPartial() + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if partial { + t.Error("Decoding detected a partial trailing message where there wasn't one.") + } + overflow, err := block.Records.isOverflow() + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if !overflow { + t.Error("Decoding detected a partial trailing message where there wasn't one.") + } + + n, err := block.Records.numRecords() + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if n != 1 { + t.Fatal("Decoding produced incorrect number of messages.") + } + msgBlock := block.Records.MsgSet.Messages[0] + if msgBlock.Offset != 0x550000 { + t.Error("Decoding produced incorrect message offset.") + } + msg := msgBlock.Msg + if msg.Codec != CompressionNone { + t.Error("Decoding produced incorrect message compression.") + } + if msg.Key != nil { + t.Error("Decoding produced message key where there was none.") + } + if !bytes.Equal(msg.Value, []byte{0x00, 0xEE}) { + t.Error("Decoding produced incorrect message value.") + } +} + func TestOneRecordFetchResponse(t *testing.T) { response := FetchResponse{} testVersionDecodable(t, "one record", &response, oneRecordFetchResponse, 4) diff --git a/third/github.com/Shopify/sarama/functional_consumer_group_test.go b/third/github.com/Shopify/sarama/functional_consumer_group_test.go new file mode 100644 index 000000000..ae376086d --- /dev/null +++ b/third/github.com/Shopify/sarama/functional_consumer_group_test.go @@ -0,0 +1,418 @@ +// +build go1.9 + +package sarama + +import ( + "context" + "fmt" + "log" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestFuncConsumerGroupPartitioning(t *testing.T) { + checkKafkaVersion(t, "0.10.2") + setupFunctionalTest(t) + defer teardownFunctionalTest(t) + + groupID := testFuncConsumerGroupID(t) + + // start M1 + m1 := runTestFuncConsumerGroupMember(t, groupID, "M1", 0, nil) + defer m1.Stop() + m1.WaitForState(2) + m1.WaitForClaims(map[string]int{"test.4": 4}) + m1.WaitForHandlers(4) + + // start M2 + m2 := runTestFuncConsumerGroupMember(t, groupID, "M2", 0, nil, "test.1", "test.4") + defer m2.Stop() + m2.WaitForState(2) + + // assert that claims are shared among both members + m1.WaitForClaims(map[string]int{"test.4": 2}) + m1.WaitForHandlers(2) + m2.WaitForClaims(map[string]int{"test.1": 1, "test.4": 2}) + m2.WaitForHandlers(3) + + // shutdown M1, wait for M2 to take over + m1.AssertCleanShutdown() + m2.WaitForClaims(map[string]int{"test.1": 1, "test.4": 4}) + m2.WaitForHandlers(5) + + // shutdown M2 + m2.AssertCleanShutdown() +} + +func TestFuncConsumerGroupExcessConsumers(t *testing.T) { + checkKafkaVersion(t, "0.10.2") + setupFunctionalTest(t) + defer teardownFunctionalTest(t) + + groupID := testFuncConsumerGroupID(t) + + // start members + m1 := runTestFuncConsumerGroupMember(t, groupID, "M1", 0, nil) + defer m1.Stop() + m2 := runTestFuncConsumerGroupMember(t, groupID, "M2", 0, nil) + defer m2.Stop() + m3 := runTestFuncConsumerGroupMember(t, groupID, "M3", 0, nil) + defer m3.Stop() + m4 := runTestFuncConsumerGroupMember(t, groupID, "M4", 0, nil) + defer m4.Stop() + + m1.WaitForClaims(map[string]int{"test.4": 1}) + m2.WaitForClaims(map[string]int{"test.4": 1}) + m3.WaitForClaims(map[string]int{"test.4": 1}) + m4.WaitForClaims(map[string]int{"test.4": 1}) + + // start M5 + m5 := runTestFuncConsumerGroupMember(t, groupID, "M5", 0, nil) + defer m5.Stop() + m5.WaitForState(1) + m5.AssertNoErrs() + + // assert that claims are shared among both members + m4.AssertCleanShutdown() + m5.WaitForState(2) + m5.WaitForClaims(map[string]int{"test.4": 1}) + + // shutdown everything + m1.AssertCleanShutdown() + m2.AssertCleanShutdown() + m3.AssertCleanShutdown() + m5.AssertCleanShutdown() +} + +func TestFuncConsumerGroupFuzzy(t *testing.T) { + checkKafkaVersion(t, "0.10.2") + setupFunctionalTest(t) + defer teardownFunctionalTest(t) + + if err := testFuncConsumerGroupFuzzySeed("test.4"); err != nil { + t.Fatal(err) + } + + groupID := testFuncConsumerGroupID(t) + sink := &testFuncConsumerGroupSink{msgs: make(chan testFuncConsumerGroupMessage, 20000)} + waitForMessages := func(t *testing.T, n int) { + t.Helper() + + for i := 0; i < 600; i++ { + if sink.Len() >= n { + break + } + time.Sleep(100 * time.Millisecond) + } + if sz := sink.Len(); sz < n { + log.Fatalf("expected to consume %d messages, but consumed %d", n, sz) + } + } + + defer runTestFuncConsumerGroupMember(t, groupID, "M1", 1500, sink).Stop() + defer runTestFuncConsumerGroupMember(t, groupID, "M2", 3000, sink).Stop() + defer runTestFuncConsumerGroupMember(t, groupID, "M3", 1500, sink).Stop() + defer runTestFuncConsumerGroupMember(t, groupID, "M4", 200, sink).Stop() + defer runTestFuncConsumerGroupMember(t, groupID, "M5", 100, sink).Stop() + waitForMessages(t, 3000) + + defer runTestFuncConsumerGroupMember(t, groupID, "M6", 300, sink).Stop() + defer runTestFuncConsumerGroupMember(t, groupID, "M7", 400, sink).Stop() + defer runTestFuncConsumerGroupMember(t, groupID, "M8", 500, sink).Stop() + defer runTestFuncConsumerGroupMember(t, groupID, "M9", 2000, sink).Stop() + waitForMessages(t, 8000) + + defer runTestFuncConsumerGroupMember(t, groupID, "M10", 1000, sink).Stop() + waitForMessages(t, 10000) + + defer runTestFuncConsumerGroupMember(t, groupID, "M11", 1000, sink).Stop() + defer runTestFuncConsumerGroupMember(t, groupID, "M12", 2500, sink).Stop() + waitForMessages(t, 12000) + + defer runTestFuncConsumerGroupMember(t, groupID, "M13", 1000, sink).Stop() + waitForMessages(t, 15000) + + if umap := sink.Close(); len(umap) != 15000 { + dupes := make(map[string][]string) + for k, v := range umap { + if len(v) > 1 { + dupes[k] = v + } + } + t.Fatalf("expected %d unique messages to be consumed but got %d, including %d duplicates:\n%v", 15000, len(umap), len(dupes), dupes) + } +} + +// -------------------------------------------------------------------- + +func testFuncConsumerGroupID(t *testing.T) string { + return fmt.Sprintf("sarama.%s%d", t.Name(), time.Now().UnixNano()) +} + +func testFuncConsumerGroupFuzzySeed(topic string) error { + client, err := NewClient(kafkaBrokers, nil) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + + total := int64(0) + for pn := int32(0); pn < 4; pn++ { + newest, err := client.GetOffset(topic, pn, OffsetNewest) + if err != nil { + return err + } + oldest, err := client.GetOffset(topic, pn, OffsetOldest) + if err != nil { + return err + } + total = total + newest - oldest + } + if total >= 21000 { + return nil + } + + producer, err := NewAsyncProducerFromClient(client) + if err != nil { + return err + } + for i := total; i < 21000; i++ { + producer.Input() <- &ProducerMessage{Topic: topic, Value: ByteEncoder([]byte("testdata"))} + } + return producer.Close() +} + +type testFuncConsumerGroupMessage struct { + ClientID string + *ConsumerMessage +} + +type testFuncConsumerGroupSink struct { + msgs chan testFuncConsumerGroupMessage + count int32 +} + +func (s *testFuncConsumerGroupSink) Len() int { + if s == nil { + return -1 + } + return int(atomic.LoadInt32(&s.count)) +} + +func (s *testFuncConsumerGroupSink) Push(clientID string, m *ConsumerMessage) { + if s != nil { + s.msgs <- testFuncConsumerGroupMessage{ClientID: clientID, ConsumerMessage: m} + atomic.AddInt32(&s.count, 1) + } +} + +func (s *testFuncConsumerGroupSink) Close() map[string][]string { + close(s.msgs) + + res := make(map[string][]string) + for msg := range s.msgs { + key := fmt.Sprintf("%s-%d:%d", msg.Topic, msg.Partition, msg.Offset) + res[key] = append(res[key], msg.ClientID) + } + return res +} + +type testFuncConsumerGroupMember struct { + ConsumerGroup + clientID string + claims map[string]int + state int32 + handlers int32 + errs []error + maxMessages int32 + isCapped bool + sink *testFuncConsumerGroupSink + + t *testing.T + mu sync.RWMutex +} + +func runTestFuncConsumerGroupMember(t *testing.T, groupID, clientID string, maxMessages int32, sink *testFuncConsumerGroupSink, topics ...string) *testFuncConsumerGroupMember { + t.Helper() + + config := NewConfig() + config.ClientID = clientID + config.Version = V0_10_2_0 + config.Consumer.Return.Errors = true + config.Consumer.Offsets.Initial = OffsetOldest + config.Consumer.Group.Rebalance.Timeout = 10 * time.Second + + group, err := NewConsumerGroup(kafkaBrokers, groupID, config) + if err != nil { + t.Fatal(err) + return nil + } + + if len(topics) == 0 { + topics = []string{"test.4"} + } + + member := &testFuncConsumerGroupMember{ + ConsumerGroup: group, + clientID: clientID, + claims: make(map[string]int), + maxMessages: maxMessages, + isCapped: maxMessages != 0, + sink: sink, + t: t, + } + go member.loop(topics) + return member +} + +func (m *testFuncConsumerGroupMember) AssertCleanShutdown() { + m.t.Helper() + + if err := m.Close(); err != nil { + m.t.Fatalf("unexpected error on Close(): %v", err) + } + m.WaitForState(4) + m.WaitForHandlers(0) + m.AssertNoErrs() +} + +func (m *testFuncConsumerGroupMember) AssertNoErrs() { + m.t.Helper() + + var errs []error + m.mu.RLock() + errs = append(errs, m.errs...) + m.mu.RUnlock() + + if len(errs) != 0 { + m.t.Fatalf("unexpected consumer errors: %v", errs) + } +} + +func (m *testFuncConsumerGroupMember) WaitForState(expected int32) { + m.t.Helper() + + m.waitFor("state", expected, func() (interface{}, error) { + return atomic.LoadInt32(&m.state), nil + }) +} + +func (m *testFuncConsumerGroupMember) WaitForHandlers(expected int) { + m.t.Helper() + + m.waitFor("handlers", expected, func() (interface{}, error) { + return int(atomic.LoadInt32(&m.handlers)), nil + }) +} + +func (m *testFuncConsumerGroupMember) WaitForClaims(expected map[string]int) { + m.t.Helper() + + m.waitFor("claims", expected, func() (interface{}, error) { + m.mu.RLock() + claims := m.claims + m.mu.RUnlock() + return claims, nil + }) +} + +func (m *testFuncConsumerGroupMember) Stop() { _ = m.Close() } + +func (m *testFuncConsumerGroupMember) Setup(s ConsumerGroupSession) error { + // store claims + claims := make(map[string]int) + for topic, partitions := range s.Claims() { + claims[topic] = len(partitions) + } + m.mu.Lock() + m.claims = claims + m.mu.Unlock() + + // enter post-setup state + atomic.StoreInt32(&m.state, 2) + return nil +} +func (m *testFuncConsumerGroupMember) Cleanup(s ConsumerGroupSession) error { + // enter post-cleanup state + atomic.StoreInt32(&m.state, 3) + return nil +} +func (m *testFuncConsumerGroupMember) ConsumeClaim(s ConsumerGroupSession, c ConsumerGroupClaim) error { + atomic.AddInt32(&m.handlers, 1) + defer atomic.AddInt32(&m.handlers, -1) + + for msg := range c.Messages() { + if n := atomic.AddInt32(&m.maxMessages, -1); m.isCapped && n < 0 { + break + } + s.MarkMessage(msg, "") + m.sink.Push(m.clientID, msg) + } + return nil +} + +func (m *testFuncConsumerGroupMember) waitFor(kind string, expected interface{}, factory func() (interface{}, error)) { + m.t.Helper() + + deadline := time.NewTimer(60 * time.Second) + defer deadline.Stop() + + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + var actual interface{} + for { + var err error + if actual, err = factory(); err != nil { + m.t.Errorf("failed retrieve value, expected %s %#v but received error %v", kind, expected, err) + } + + if reflect.DeepEqual(expected, actual) { + return + } + + select { + case <-deadline.C: + m.t.Fatalf("ttl exceeded, expected %s %#v but got %#v", kind, expected, actual) + return + case <-ticker.C: + } + } +} + +func (m *testFuncConsumerGroupMember) loop(topics []string) { + defer atomic.StoreInt32(&m.state, 4) + + go func() { + for err := range m.Errors() { + _ = m.Close() + + m.mu.Lock() + m.errs = append(m.errs, err) + m.mu.Unlock() + } + }() + + ctx := context.Background() + for { + // set state to pre-consume + atomic.StoreInt32(&m.state, 1) + + if err := m.Consume(ctx, topics, m); err == ErrClosedConsumerGroup { + return + } else if err != nil { + m.mu.Lock() + m.errs = append(m.errs, err) + m.mu.Unlock() + return + } + + // return if capped + if n := atomic.LoadInt32(&m.maxMessages); m.isCapped && n < 0 { + return + } + } +} diff --git a/third/github.com/Shopify/sarama/functional_consumer_test.go b/third/github.com/Shopify/sarama/functional_consumer_test.go index 83bec0331..aa8eccf7c 100644 --- a/third/github.com/Shopify/sarama/functional_consumer_test.go +++ b/third/github.com/Shopify/sarama/functional_consumer_test.go @@ -81,7 +81,7 @@ func TestVersionMatrix(t *testing.T) { // protocol versions and compressions for the except of LZ4. testVersions := versionRange(V0_8_2_0) allCodecsButLZ4 := []CompressionCodec{CompressionNone, CompressionGZIP, CompressionSnappy} - producedMessages := produceMsgs(t, testVersions, allCodecsButLZ4, 17, 100) + producedMessages := produceMsgs(t, testVersions, allCodecsButLZ4, 17, 100, false) // When/Then consumeMsgs(t, testVersions, producedMessages) @@ -98,7 +98,20 @@ func TestVersionMatrixLZ4(t *testing.T) { // and all possible compressions. testVersions := versionRange(V0_10_0_0) allCodecs := []CompressionCodec{CompressionNone, CompressionGZIP, CompressionSnappy, CompressionLZ4} - producedMessages := produceMsgs(t, testVersions, allCodecs, 17, 100) + producedMessages := produceMsgs(t, testVersions, allCodecs, 17, 100, false) + + // When/Then + consumeMsgs(t, testVersions, producedMessages) +} + +func TestVersionMatrixIdempotent(t *testing.T) { + setupFunctionalTest(t) + defer teardownFunctionalTest(t) + + // Produce lot's of message with all possible combinations of supported + // protocol versions starting with v0.11 (first where idempotent was supported) + testVersions := versionRange(V0_11_0_0) + producedMessages := produceMsgs(t, testVersions, []CompressionCodec{CompressionNone}, 17, 100, true) // When/Then consumeMsgs(t, testVersions, producedMessages) @@ -133,7 +146,7 @@ func versionRange(lower KafkaVersion) []KafkaVersion { return versions } -func produceMsgs(t *testing.T, clientVersions []KafkaVersion, codecs []CompressionCodec, flush int, countPerVerCodec int) []*ProducerMessage { +func produceMsgs(t *testing.T, clientVersions []KafkaVersion, codecs []CompressionCodec, flush int, countPerVerCodec int, idempotent bool) []*ProducerMessage { var wg sync.WaitGroup var producedMessagesMu sync.Mutex var producedMessages []*ProducerMessage @@ -145,6 +158,11 @@ func produceMsgs(t *testing.T, clientVersions []KafkaVersion, codecs []Compressi prodCfg.Producer.Return.Errors = true prodCfg.Producer.Flush.MaxMessages = flush prodCfg.Producer.Compression = codec + prodCfg.Producer.Idempotent = idempotent + if idempotent { + prodCfg.Producer.RequiredAcks = WaitForAll + prodCfg.Net.MaxOpenRequests = 1 + } p, err := NewSyncProducer(kafkaBrokers, prodCfg) if err != nil { diff --git a/third/github.com/Shopify/sarama/length_field.go b/third/github.com/Shopify/sarama/length_field.go index 576b1a6f6..da199a70a 100644 --- a/third/github.com/Shopify/sarama/length_field.go +++ b/third/github.com/Shopify/sarama/length_field.go @@ -5,6 +5,19 @@ import "encoding/binary" // LengthField implements the PushEncoder and PushDecoder interfaces for calculating 4-byte lengths. type lengthField struct { startOffset int + length int32 +} + +func (l *lengthField) decode(pd packetDecoder) error { + var err error + l.length, err = pd.getInt32() + if err != nil { + return err + } + if l.length > int32(pd.remaining()) { + return ErrInsufficientData + } + return nil } func (l *lengthField) saveOffset(in int) { @@ -21,7 +34,7 @@ func (l *lengthField) run(curOffset int, buf []byte) error { } func (l *lengthField) check(curOffset int, buf []byte) error { - if uint32(curOffset-l.startOffset-4) != binary.BigEndian.Uint32(buf[l.startOffset:]) { + if int32(curOffset-l.startOffset-4) != l.length { return PacketDecodingError{"length field invalid"} } diff --git a/third/github.com/Shopify/sarama/message.go b/third/github.com/Shopify/sarama/message.go index 0dad672ad..51d3309c0 100644 --- a/third/github.com/Shopify/sarama/message.go +++ b/third/github.com/Shopify/sarama/message.go @@ -1,27 +1,22 @@ package sarama import ( - "bytes" - "compress/gzip" "fmt" - "io/ioutil" "time" - - "gitee.com/johng/gf/third/github.com/eapache/go-xerial-snappy" - "gitee.com/johng/gf/third/github.com/pierrec/lz4" ) // CompressionCodec represents the various compression codecs recognized by Kafka in messages. type CompressionCodec int8 -// only the last two bits are really used -const compressionCodecMask int8 = 0x03 +// The lowest 3 bits contain the compression codec used for the message +const compressionCodecMask int8 = 0x07 const ( CompressionNone CompressionCodec = 0 CompressionGZIP CompressionCodec = 1 CompressionSnappy CompressionCodec = 2 CompressionLZ4 CompressionCodec = 3 + CompressionZSTD CompressionCodec = 4 ) func (cc CompressionCodec) String() string { @@ -76,47 +71,12 @@ func (m *Message) encode(pe packetEncoder) error { payload = m.compressedCache m.compressedCache = nil } else if m.Value != nil { - switch m.Codec { - case CompressionNone: - payload = m.Value - case CompressionGZIP: - var buf bytes.Buffer - var writer *gzip.Writer - if m.CompressionLevel != CompressionLevelDefault { - writer, err = gzip.NewWriterLevel(&buf, m.CompressionLevel) - if err != nil { - return err - } - } else { - writer = gzip.NewWriter(&buf) - } - if _, err = writer.Write(m.Value); err != nil { - return err - } - if err = writer.Close(); err != nil { - return err - } - m.compressedCache = buf.Bytes() - payload = m.compressedCache - case CompressionSnappy: - tmp := snappy.Encode(m.Value) - m.compressedCache = tmp - payload = m.compressedCache - case CompressionLZ4: - var buf bytes.Buffer - writer := lz4.NewWriter(&buf) - if _, err = writer.Write(m.Value); err != nil { - return err - } - if err = writer.Close(); err != nil { - return err - } - m.compressedCache = buf.Bytes() - payload = m.compressedCache - default: - return PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", m.Codec)} + payload, err = compress(m.Codec, m.CompressionLevel, m.Value) + if err != nil { + return err } + m.compressedCache = payload // Keep in mind the compressed payload size for metric gathering m.compressedSize = len(payload) } @@ -172,44 +132,18 @@ func (m *Message) decode(pd packetDecoder) (err error) { switch m.Codec { case CompressionNone: // nothing to do - case CompressionGZIP: + default: if m.Value == nil { break } - reader, err := gzip.NewReader(bytes.NewReader(m.Value)) + + m.Value, err = decompress(m.Codec, m.Value) if err != nil { return err } - if m.Value, err = ioutil.ReadAll(reader); err != nil { - return err - } if err := m.decodeSet(); err != nil { return err } - case CompressionSnappy: - if m.Value == nil { - break - } - if m.Value, err = snappy.Decode(m.Value); err != nil { - return err - } - if err := m.decodeSet(); err != nil { - return err - } - case CompressionLZ4: - if m.Value == nil { - break - } - reader := lz4.NewReader(bytes.NewReader(m.Value)) - if m.Value, err = ioutil.ReadAll(reader); err != nil { - return err - } - if err := m.decodeSet(); err != nil { - return err - } - - default: - return PacketDecodingError{fmt.Sprintf("invalid compression specified (%d)", m.Codec)} } return pd.pop() diff --git a/third/github.com/Shopify/sarama/message_set.go b/third/github.com/Shopify/sarama/message_set.go index 27db52fdf..600c7c4df 100644 --- a/third/github.com/Shopify/sarama/message_set.go +++ b/third/github.com/Shopify/sarama/message_set.go @@ -47,6 +47,7 @@ func (msb *MessageBlock) decode(pd packetDecoder) (err error) { type MessageSet struct { PartialTrailingMessage bool // whether the set on the wire contained an incomplete trailing MessageBlock + OverflowMessage bool // whether the set on the wire contained an overflow message Messages []*MessageBlock } @@ -85,7 +86,12 @@ func (ms *MessageSet) decode(pd packetDecoder) (err error) { case ErrInsufficientData: // As an optimization the server is allowed to return a partial message at the // end of the message set. Clients should handle this case. So we just ignore such things. - ms.PartialTrailingMessage = true + if msb.Offset == -1 { + // This is an overflow message caused by chunked down conversion + ms.OverflowMessage = true + } else { + ms.PartialTrailingMessage = true + } return nil default: return err diff --git a/third/github.com/Shopify/sarama/message_test.go b/third/github.com/Shopify/sarama/message_test.go index 0eb02f263..aa103d43f 100644 --- a/third/github.com/Shopify/sarama/message_test.go +++ b/third/github.com/Shopify/sarama/message_test.go @@ -52,6 +52,17 @@ var ( 5, 93, 204, 2, // LZ4 checksum } + emptyZSTDMessage = []byte{ + 180, 172, 84, 179, // CRC + 0x01, // version byte + 0x04, // attribute flags: zstd + 0, 0, 1, 88, 141, 205, 89, 56, // timestamp + 0xFF, 0xFF, 0xFF, 0xFF, // key + 0x00, 0x00, 0x00, 0x09, // len + // ZSTD data + 0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x00, 0x01, 0x00, 0x00, + } + emptyBulkSnappyMessage = []byte{ 180, 47, 53, 209, //CRC 0x00, // magic version byte @@ -86,6 +97,17 @@ var ( 112, 185, 52, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 121, 87, 72, 224, 0, 0, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 14, 121, 87, 72, 224, 0, 0, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 71, 129, 23, 111, // LZ4 checksum } + + emptyBulkZSTDMessage = []byte{ + 203, 151, 133, 28, // CRC + 0x01, // Version + 0x04, // attribute flags (ZSTD) + 255, 255, 249, 209, 212, 181, 73, 201, // timestamp + 0xFF, 0xFF, 0xFF, 0xFF, // key + 0x00, 0x00, 0x00, 0x26, // len + // ZSTD data + 0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x34, 0xcd, 0x0, 0x0, 0x78, 0x0, 0x0, 0xe, 0x79, 0x57, 0x48, 0xe0, 0x0, 0x0, 0xff, 0xff, 0xff, 0xff, 0x0, 0x1, 0x3, 0x0, 0x3d, 0xbd, 0x0, 0x3b, 0x15, 0x0, 0xb, 0xd2, 0x34, 0xc1, 0x78, + } ) func TestMessageEncoding(t *testing.T) { @@ -101,6 +123,12 @@ func TestMessageEncoding(t *testing.T) { message.Timestamp = time.Unix(1479847795, 0) message.Version = 1 testEncodable(t, "empty lz4", &message, emptyLZ4Message) + + message.Value = []byte{} + message.Codec = CompressionZSTD + message.Timestamp = time.Unix(1479847795, 0) + message.Version = 1 + testEncodable(t, "empty zstd", &message, emptyZSTDMessage) } func TestMessageDecoding(t *testing.T) { @@ -179,6 +207,22 @@ func TestMessageDecodingBulkLZ4(t *testing.T) { } } +func TestMessageDecodingBulkZSTD(t *testing.T) { + message := Message{} + testDecodable(t, "bulk zstd", &message, emptyBulkZSTDMessage) + if message.Codec != CompressionZSTD { + t.Errorf("Decoding produced codec %d, but expected %d.", message.Codec, CompressionZSTD) + } + if message.Key != nil { + t.Errorf("Decoding produced key %+v, but none was expected.", message.Key) + } + if message.Set == nil { + t.Error("Decoding produced no set, but one was expected.") + } else if len(message.Set.Messages) != 2 { + t.Errorf("Decoding produced a set with %d messages, but 2 were expected.", len(message.Set.Messages)) + } +} + func TestMessageDecodingVersion1(t *testing.T) { message := Message{Version: 1} testDecodable(t, "decoding empty v1 message", &message, emptyV1Message) diff --git a/third/github.com/Shopify/sarama/metadata_request.go b/third/github.com/Shopify/sarama/metadata_request.go index 48adfa28c..17dc4289a 100644 --- a/third/github.com/Shopify/sarama/metadata_request.go +++ b/third/github.com/Shopify/sarama/metadata_request.go @@ -10,7 +10,7 @@ func (r *MetadataRequest) encode(pe packetEncoder) error { if r.Version < 0 || r.Version > 5 { return PacketEncodingError{"invalid or unsupported MetadataRequest version field"} } - if r.Version == 0 || r.Topics != nil || len(r.Topics) > 0 { + if r.Version == 0 || len(r.Topics) > 0 { err := pe.putArrayLength(len(r.Topics)) if err != nil { return err diff --git a/third/github.com/Shopify/sarama/metadata_response.go b/third/github.com/Shopify/sarama/metadata_response.go index bf8a67bbc..c402d05fa 100644 --- a/third/github.com/Shopify/sarama/metadata_response.go +++ b/third/github.com/Shopify/sarama/metadata_response.go @@ -207,6 +207,10 @@ func (r *MetadataResponse) decode(pd packetDecoder, version int16) (err error) { } func (r *MetadataResponse) encode(pe packetEncoder) error { + if r.Version >= 3 { + pe.putInt32(r.ThrottleTimeMs) + } + err := pe.putArrayLength(len(r.Brokers)) if err != nil { return err @@ -218,6 +222,13 @@ func (r *MetadataResponse) encode(pe packetEncoder) error { } } + if r.Version >= 2 { + err := pe.putNullableString(r.ClusterID) + if err != nil { + return err + } + } + if r.Version >= 1 { pe.putInt32(r.ControllerID) } diff --git a/third/github.com/Shopify/sarama/mockresponses.go b/third/github.com/Shopify/sarama/mockresponses.go index 172044199..fe55200c6 100644 --- a/third/github.com/Shopify/sarama/mockresponses.go +++ b/third/github.com/Shopify/sarama/mockresponses.go @@ -523,7 +523,7 @@ func (mr *MockOffsetFetchResponse) SetOffset(group, topic string, partition int3 partitions = make(map[int32]*OffsetFetchResponseBlock) topics[topic] = partitions } - partitions[partition] = &OffsetFetchResponseBlock{offset, metadata, kerror} + partitions[partition] = &OffsetFetchResponseBlock{offset, 0, metadata, kerror} return mr } diff --git a/third/github.com/Shopify/sarama/mocks/async_producer.go b/third/github.com/Shopify/sarama/mocks/async_producer.go index 6ed9c9843..f428b4f17 100644 --- a/third/github.com/Shopify/sarama/mocks/async_producer.go +++ b/third/github.com/Shopify/sarama/mocks/async_producer.go @@ -44,6 +44,7 @@ func NewAsyncProducer(t ErrorReporter, config *sarama.Config) *AsyncProducer { defer func() { close(mp.successes) close(mp.errors) + close(mp.closed) }() for msg := range mp.input { @@ -86,8 +87,6 @@ func NewAsyncProducer(t ErrorReporter, config *sarama.Config) *AsyncProducer { mp.t.Errorf("Expected to exhaust all expectations, but %d are left.", len(mp.expectations)) } mp.l.Unlock() - - close(mp.closed) }() return mp diff --git a/third/github.com/Shopify/sarama/offset_commit_request.go b/third/github.com/Shopify/sarama/offset_commit_request.go index 37e99fbf5..1ec583e6d 100644 --- a/third/github.com/Shopify/sarama/offset_commit_request.go +++ b/third/github.com/Shopify/sarama/offset_commit_request.go @@ -52,12 +52,14 @@ type OffsetCommitRequest struct { // - 0 (kafka 0.8.1 and later) // - 1 (kafka 0.8.2 and later) // - 2 (kafka 0.9.0 and later) + // - 3 (kafka 0.11.0 and later) + // - 4 (kafka 2.0.0 and later) Version int16 blocks map[string]map[int32]*offsetCommitRequestBlock } func (r *OffsetCommitRequest) encode(pe packetEncoder) error { - if r.Version < 0 || r.Version > 2 { + if r.Version < 0 || r.Version > 4 { return PacketEncodingError{"invalid or unsupported OffsetCommitRequest version field"} } @@ -174,6 +176,10 @@ func (r *OffsetCommitRequest) requiredVersion() KafkaVersion { return V0_8_2_0 case 2: return V0_9_0_0 + case 3: + return V0_11_0_0 + case 4: + return V2_0_0_0 default: return MinVersion } diff --git a/third/github.com/Shopify/sarama/offset_commit_request_test.go b/third/github.com/Shopify/sarama/offset_commit_request_test.go index afc25b7b3..efb3d33f1 100644 --- a/third/github.com/Shopify/sarama/offset_commit_request_test.go +++ b/third/github.com/Shopify/sarama/offset_commit_request_test.go @@ -1,6 +1,9 @@ package sarama -import "testing" +import ( + "fmt" + "testing" +) var ( offsetCommitRequestNoBlocksV0 = []byte{ @@ -76,15 +79,17 @@ func TestOffsetCommitRequestV1(t *testing.T) { testRequest(t, "one block v1", request, offsetCommitRequestOneBlockV1) } -func TestOffsetCommitRequestV2(t *testing.T) { - request := new(OffsetCommitRequest) - request.ConsumerGroup = "foobar" - request.ConsumerID = "cons" - request.ConsumerGroupGeneration = 0x1122 - request.RetentionTime = 0x4433 - request.Version = 2 - testRequest(t, "no blocks v2", request, offsetCommitRequestNoBlocksV2) +func TestOffsetCommitRequestV2ToV4(t *testing.T) { + for version := 2; version <= 4; version++ { + request := new(OffsetCommitRequest) + request.ConsumerGroup = "foobar" + request.ConsumerID = "cons" + request.ConsumerGroupGeneration = 0x1122 + request.RetentionTime = 0x4433 + request.Version = int16(version) + testRequest(t, fmt.Sprintf("no blocks v%d", version), request, offsetCommitRequestNoBlocksV2) - request.AddBlock("topic", 0x5221, 0xDEADBEEF, 0, "metadata") - testRequest(t, "one block v2", request, offsetCommitRequestOneBlockV2) + request.AddBlock("topic", 0x5221, 0xDEADBEEF, 0, "metadata") + testRequest(t, fmt.Sprintf("one block v%d", version), request, offsetCommitRequestOneBlockV2) + } } diff --git a/third/github.com/Shopify/sarama/offset_commit_response.go b/third/github.com/Shopify/sarama/offset_commit_response.go index a4b18acdf..e842298db 100644 --- a/third/github.com/Shopify/sarama/offset_commit_response.go +++ b/third/github.com/Shopify/sarama/offset_commit_response.go @@ -1,7 +1,9 @@ package sarama type OffsetCommitResponse struct { - Errors map[string]map[int32]KError + Version int16 + ThrottleTimeMs int32 + Errors map[string]map[int32]KError } func (r *OffsetCommitResponse) AddError(topic string, partition int32, kerror KError) { @@ -17,6 +19,9 @@ func (r *OffsetCommitResponse) AddError(topic string, partition int32, kerror KE } func (r *OffsetCommitResponse) encode(pe packetEncoder) error { + if r.Version >= 3 { + pe.putInt32(r.ThrottleTimeMs) + } if err := pe.putArrayLength(len(r.Errors)); err != nil { return err } @@ -36,6 +41,15 @@ func (r *OffsetCommitResponse) encode(pe packetEncoder) error { } func (r *OffsetCommitResponse) decode(pd packetDecoder, version int16) (err error) { + r.Version = version + + if version >= 3 { + r.ThrottleTimeMs, err = pd.getInt32() + if err != nil { + return err + } + } + numTopics, err := pd.getArrayLength() if err != nil || numTopics == 0 { return err @@ -77,9 +91,20 @@ func (r *OffsetCommitResponse) key() int16 { } func (r *OffsetCommitResponse) version() int16 { - return 0 + return r.Version } func (r *OffsetCommitResponse) requiredVersion() KafkaVersion { - return MinVersion + switch r.Version { + case 1: + return V0_8_2_0 + case 2: + return V0_9_0_0 + case 3: + return V0_11_0_0 + case 4: + return V2_0_0_0 + default: + return MinVersion + } } diff --git a/third/github.com/Shopify/sarama/offset_commit_response_test.go b/third/github.com/Shopify/sarama/offset_commit_response_test.go index 074ec9232..3c85713c7 100644 --- a/third/github.com/Shopify/sarama/offset_commit_response_test.go +++ b/third/github.com/Shopify/sarama/offset_commit_response_test.go @@ -1,6 +1,7 @@ package sarama import ( + "fmt" "testing" ) @@ -22,3 +23,17 @@ func TestNormalOffsetCommitResponse(t *testing.T) { // unpredictable map traversal order. testResponse(t, "normal", &response, nil) } + +func TestOffsetCommitResponseWithThrottleTime(t *testing.T) { + for version := 3; version <= 4; version++ { + response := OffsetCommitResponse{ + Version: int16(version), + ThrottleTimeMs: 123, + } + response.AddError("t", 0, ErrNotLeaderForPartition) + response.Errors["m"] = make(map[int32]KError) + // The response encoded form cannot be checked for it varies due to + // unpredictable map traversal order. + testResponse(t, fmt.Sprintf("v%d with throttle time", version), &response, nil) + } +} diff --git a/third/github.com/Shopify/sarama/offset_fetch_request.go b/third/github.com/Shopify/sarama/offset_fetch_request.go index 5a05014b4..68608241f 100644 --- a/third/github.com/Shopify/sarama/offset_fetch_request.go +++ b/third/github.com/Shopify/sarama/offset_fetch_request.go @@ -1,28 +1,33 @@ package sarama type OffsetFetchRequest struct { - ConsumerGroup string Version int16 + ConsumerGroup string partitions map[string][]int32 } func (r *OffsetFetchRequest) encode(pe packetEncoder) (err error) { - if r.Version < 0 || r.Version > 1 { + if r.Version < 0 || r.Version > 5 { return PacketEncodingError{"invalid or unsupported OffsetFetchRequest version field"} } if err = pe.putString(r.ConsumerGroup); err != nil { return err } - if err = pe.putArrayLength(len(r.partitions)); err != nil { - return err - } - for topic, partitions := range r.partitions { - if err = pe.putString(topic); err != nil { + + if r.Version >= 2 && r.partitions == nil { + pe.putInt32(-1) + } else { + if err = pe.putArrayLength(len(r.partitions)); err != nil { return err } - if err = pe.putInt32Array(partitions); err != nil { - return err + for topic, partitions := range r.partitions { + if err = pe.putString(topic); err != nil { + return err + } + if err = pe.putInt32Array(partitions); err != nil { + return err + } } } return nil @@ -37,7 +42,7 @@ func (r *OffsetFetchRequest) decode(pd packetDecoder, version int16) (err error) if err != nil { return err } - if partitionCount == 0 { + if (partitionCount == 0 && version < 2) || partitionCount < 0 { return nil } r.partitions = make(map[string][]int32) @@ -67,11 +72,25 @@ func (r *OffsetFetchRequest) requiredVersion() KafkaVersion { switch r.Version { case 1: return V0_8_2_0 + case 2: + return V0_10_2_0 + case 3: + return V0_11_0_0 + case 4: + return V2_0_0_0 + case 5: + return V2_1_0_0 default: return MinVersion } } +func (r *OffsetFetchRequest) ZeroPartitions() { + if r.partitions == nil && r.Version >= 2 { + r.partitions = make(map[string][]int32) + } +} + func (r *OffsetFetchRequest) AddPartition(topic string, partitionID int32) { if r.partitions == nil { r.partitions = make(map[string][]int32) diff --git a/third/github.com/Shopify/sarama/offset_fetch_request_test.go b/third/github.com/Shopify/sarama/offset_fetch_request_test.go index 025d725c9..55b46eea7 100644 --- a/third/github.com/Shopify/sarama/offset_fetch_request_test.go +++ b/third/github.com/Shopify/sarama/offset_fetch_request_test.go @@ -1,6 +1,9 @@ package sarama -import "testing" +import ( + "fmt" + "testing" +) var ( offsetFetchRequestNoGroupNoPartitions = []byte{ @@ -17,15 +20,36 @@ var ( 0x00, 0x0D, 't', 'o', 'p', 'i', 'c', 'T', 'h', 'e', 'F', 'i', 'r', 's', 't', 0x00, 0x00, 0x00, 0x01, 0x4F, 0x4F, 0x4F, 0x4F} + + offsetFetchRequestAllPartitions = []byte{ + 0x00, 0x04, 'b', 'l', 'a', 'h', + 0xff, 0xff, 0xff, 0xff} ) -func TestOffsetFetchRequest(t *testing.T) { - request := new(OffsetFetchRequest) - testRequest(t, "no group, no partitions", request, offsetFetchRequestNoGroupNoPartitions) +func TestOffsetFetchRequestNoPartitions(t *testing.T) { + for version := 0; version <= 5; version++ { + request := new(OffsetFetchRequest) + request.Version = int16(version) + request.ZeroPartitions() + testRequest(t, fmt.Sprintf("no group, no partitions %d", version), request, offsetFetchRequestNoGroupNoPartitions) - request.ConsumerGroup = "blah" - testRequest(t, "no partitions", request, offsetFetchRequestNoPartitions) - - request.AddPartition("topicTheFirst", 0x4F4F4F4F) - testRequest(t, "one partition", request, offsetFetchRequestOnePartition) + request.ConsumerGroup = "blah" + testRequest(t, fmt.Sprintf("no partitions %d", version), request, offsetFetchRequestNoPartitions) + } +} +func TestOffsetFetchRequest(t *testing.T) { + for version := 0; version <= 5; version++ { + request := new(OffsetFetchRequest) + request.Version = int16(version) + request.ConsumerGroup = "blah" + request.AddPartition("topicTheFirst", 0x4F4F4F4F) + testRequest(t, fmt.Sprintf("one partition %d", version), request, offsetFetchRequestOnePartition) + } +} + +func TestOffsetFetchRequestAllPartitions(t *testing.T) { + for version := 2; version <= 5; version++ { + request := &OffsetFetchRequest{Version: int16(version), ConsumerGroup: "blah"} + testRequest(t, fmt.Sprintf("all partitions %d", version), request, offsetFetchRequestAllPartitions) + } } diff --git a/third/github.com/Shopify/sarama/offset_fetch_response.go b/third/github.com/Shopify/sarama/offset_fetch_response.go index 11e4b1f3f..9e2570280 100644 --- a/third/github.com/Shopify/sarama/offset_fetch_response.go +++ b/third/github.com/Shopify/sarama/offset_fetch_response.go @@ -1,17 +1,25 @@ package sarama type OffsetFetchResponseBlock struct { - Offset int64 - Metadata string - Err KError + Offset int64 + LeaderEpoch int32 + Metadata string + Err KError } -func (b *OffsetFetchResponseBlock) decode(pd packetDecoder) (err error) { +func (b *OffsetFetchResponseBlock) decode(pd packetDecoder, version int16) (err error) { b.Offset, err = pd.getInt64() if err != nil { return err } + if version >= 5 { + b.LeaderEpoch, err = pd.getInt32() + if err != nil { + return err + } + } + b.Metadata, err = pd.getString() if err != nil { return err @@ -26,9 +34,13 @@ func (b *OffsetFetchResponseBlock) decode(pd packetDecoder) (err error) { return nil } -func (b *OffsetFetchResponseBlock) encode(pe packetEncoder) (err error) { +func (b *OffsetFetchResponseBlock) encode(pe packetEncoder, version int16) (err error) { pe.putInt64(b.Offset) + if version >= 5 { + pe.putInt32(b.LeaderEpoch) + } + err = pe.putString(b.Metadata) if err != nil { return err @@ -40,10 +52,17 @@ func (b *OffsetFetchResponseBlock) encode(pe packetEncoder) (err error) { } type OffsetFetchResponse struct { - Blocks map[string]map[int32]*OffsetFetchResponseBlock + Version int16 + ThrottleTimeMs int32 + Blocks map[string]map[int32]*OffsetFetchResponseBlock + Err KError } func (r *OffsetFetchResponse) encode(pe packetEncoder) error { + if r.Version >= 3 { + pe.putInt32(r.ThrottleTimeMs) + } + if err := pe.putArrayLength(len(r.Blocks)); err != nil { return err } @@ -56,53 +75,75 @@ func (r *OffsetFetchResponse) encode(pe packetEncoder) error { } for partition, block := range partitions { pe.putInt32(partition) - if err := block.encode(pe); err != nil { + if err := block.encode(pe, r.Version); err != nil { return err } } } + if r.Version >= 2 { + pe.putInt16(int16(r.Err)) + } return nil } func (r *OffsetFetchResponse) decode(pd packetDecoder, version int16) (err error) { + r.Version = version + + if version >= 3 { + r.ThrottleTimeMs, err = pd.getInt32() + if err != nil { + return err + } + } + numTopics, err := pd.getArrayLength() - if err != nil || numTopics == 0 { + if err != nil { return err } - r.Blocks = make(map[string]map[int32]*OffsetFetchResponseBlock, numTopics) - for i := 0; i < numTopics; i++ { - name, err := pd.getString() - if err != nil { - return err - } - - numBlocks, err := pd.getArrayLength() - if err != nil { - return err - } - - if numBlocks == 0 { - r.Blocks[name] = nil - continue - } - r.Blocks[name] = make(map[int32]*OffsetFetchResponseBlock, numBlocks) - - for j := 0; j < numBlocks; j++ { - id, err := pd.getInt32() + if numTopics > 0 { + r.Blocks = make(map[string]map[int32]*OffsetFetchResponseBlock, numTopics) + for i := 0; i < numTopics; i++ { + name, err := pd.getString() if err != nil { return err } - block := new(OffsetFetchResponseBlock) - err = block.decode(pd) + numBlocks, err := pd.getArrayLength() if err != nil { return err } - r.Blocks[name][id] = block + + if numBlocks == 0 { + r.Blocks[name] = nil + continue + } + r.Blocks[name] = make(map[int32]*OffsetFetchResponseBlock, numBlocks) + + for j := 0; j < numBlocks; j++ { + id, err := pd.getInt32() + if err != nil { + return err + } + + block := new(OffsetFetchResponseBlock) + err = block.decode(pd, version) + if err != nil { + return err + } + r.Blocks[name][id] = block + } } } + if version >= 2 { + kerr, err := pd.getInt16() + if err != nil { + return err + } + r.Err = KError(kerr) + } + return nil } @@ -111,11 +152,24 @@ func (r *OffsetFetchResponse) key() int16 { } func (r *OffsetFetchResponse) version() int16 { - return 0 + return r.Version } func (r *OffsetFetchResponse) requiredVersion() KafkaVersion { - return MinVersion + switch r.Version { + case 1: + return V0_8_2_0 + case 2: + return V0_10_2_0 + case 3: + return V0_11_0_0 + case 4: + return V2_0_0_0 + case 5: + return V2_1_0_0 + default: + return MinVersion + } } func (r *OffsetFetchResponse) GetBlock(topic string, partition int32) *OffsetFetchResponseBlock { diff --git a/third/github.com/Shopify/sarama/offset_fetch_response_test.go b/third/github.com/Shopify/sarama/offset_fetch_response_test.go index 7614ae424..b564f70f9 100644 --- a/third/github.com/Shopify/sarama/offset_fetch_response_test.go +++ b/third/github.com/Shopify/sarama/offset_fetch_response_test.go @@ -1,22 +1,65 @@ package sarama -import "testing" +import ( + "fmt" + "testing" +) var ( emptyOffsetFetchResponse = []byte{ 0x00, 0x00, 0x00, 0x00} + + emptyOffsetFetchResponseV2 = []byte{ + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x2A} + + emptyOffsetFetchResponseV3 = []byte{ + 0x00, 0x00, 0x00, 0x09, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x2A} ) func TestEmptyOffsetFetchResponse(t *testing.T) { - response := OffsetFetchResponse{} - testResponse(t, "empty", &response, emptyOffsetFetchResponse) + for version := 0; version <= 1; version++ { + response := OffsetFetchResponse{Version: int16(version)} + testResponse(t, fmt.Sprintf("empty v%d", version), &response, emptyOffsetFetchResponse) + } + + responseV2 := OffsetFetchResponse{Version: 2, Err: ErrInvalidRequest} + testResponse(t, "empty V2", &responseV2, emptyOffsetFetchResponseV2) + + for version := 3; version <= 5; version++ { + responseV3 := OffsetFetchResponse{Version: int16(version), Err: ErrInvalidRequest, ThrottleTimeMs: 9} + testResponse(t, fmt.Sprintf("empty v%d", version), &responseV3, emptyOffsetFetchResponseV3) + } } func TestNormalOffsetFetchResponse(t *testing.T) { - response := OffsetFetchResponse{} - response.AddBlock("t", 0, &OffsetFetchResponseBlock{0, "md", ErrRequestTimedOut}) - response.Blocks["m"] = nil // The response encoded form cannot be checked for it varies due to // unpredictable map traversal order. - testResponse(t, "normal", &response, nil) + // Hence the 'nil' as byte[] parameter in the 'testResponse(..)' calls + + for version := 0; version <= 1; version++ { + response := OffsetFetchResponse{Version: int16(version)} + response.AddBlock("t", 0, &OffsetFetchResponseBlock{0, 0, "md", ErrRequestTimedOut}) + response.Blocks["m"] = nil + testResponse(t, fmt.Sprintf("Normal v%d", version), &response, nil) + } + + responseV2 := OffsetFetchResponse{Version: 2, Err: ErrInvalidRequest} + responseV2.AddBlock("t", 0, &OffsetFetchResponseBlock{0, 0, "md", ErrRequestTimedOut}) + responseV2.Blocks["m"] = nil + testResponse(t, "normal V2", &responseV2, nil) + + for version := 3; version <= 4; version++ { + responseV3 := OffsetFetchResponse{Version: int16(version), Err: ErrInvalidRequest, ThrottleTimeMs: 9} + responseV3.AddBlock("t", 0, &OffsetFetchResponseBlock{0, 0, "md", ErrRequestTimedOut}) + responseV3.Blocks["m"] = nil + testResponse(t, fmt.Sprintf("Normal v%d", version), &responseV3, nil) + } + + responseV5 := OffsetFetchResponse{Version: 5, Err: ErrInvalidRequest, ThrottleTimeMs: 9} + responseV5.AddBlock("t", 0, &OffsetFetchResponseBlock{Offset: 10, LeaderEpoch: 100, Metadata: "md", Err: ErrRequestTimedOut}) + responseV5.Blocks["m"] = nil + testResponse(t, "normal V5", &responseV5, nil) } diff --git a/third/github.com/Shopify/sarama/offset_manager.go b/third/github.com/Shopify/sarama/offset_manager.go index 6c01f959e..8ea857f83 100644 --- a/third/github.com/Shopify/sarama/offset_manager.go +++ b/third/github.com/Shopify/sarama/offset_manager.go @@ -25,27 +25,49 @@ type offsetManager struct { client Client conf *Config group string + ticker *time.Ticker - lock sync.Mutex - poms map[string]map[int32]*partitionOffsetManager - boms map[*Broker]*brokerOffsetManager + memberID string + generation int32 + + broker *Broker + brokerLock sync.RWMutex + + poms map[string]map[int32]*partitionOffsetManager + pomsLock sync.RWMutex + + closeOnce sync.Once + closing chan none + closed chan none } // NewOffsetManagerFromClient creates a new OffsetManager from the given client. // It is still necessary to call Close() on the underlying client when finished with the partition manager. func NewOffsetManagerFromClient(group string, client Client) (OffsetManager, error) { + return newOffsetManagerFromClient(group, "", GroupGenerationUndefined, client) +} + +func newOffsetManagerFromClient(group, memberID string, generation int32, client Client) (*offsetManager, error) { // Check that we are not dealing with a closed Client before processing any other arguments if client.Closed() { return nil, ErrClosedClient } + conf := client.Config() om := &offsetManager{ client: client, - conf: client.Config(), + conf: conf, group: group, + ticker: time.NewTicker(conf.Consumer.Offsets.CommitInterval), poms: make(map[string]map[int32]*partitionOffsetManager), - boms: make(map[*Broker]*brokerOffsetManager), + + memberID: memberID, + generation: generation, + + closing: make(chan none), + closed: make(chan none), } + go withRecover(om.mainLoop) return om, nil } @@ -56,8 +78,8 @@ func (om *offsetManager) ManagePartition(topic string, partition int32) (Partiti return nil, err } - om.lock.Lock() - defer om.lock.Unlock() + om.pomsLock.Lock() + defer om.pomsLock.Unlock() topicManagers := om.poms[topic] if topicManagers == nil { @@ -74,53 +96,307 @@ func (om *offsetManager) ManagePartition(topic string, partition int32) (Partiti } func (om *offsetManager) Close() error { + om.closeOnce.Do(func() { + // exit the mainLoop + close(om.closing) + <-om.closed + + // mark all POMs as closed + om.asyncClosePOMs() + + // flush one last time + for attempt := 0; attempt <= om.conf.Consumer.Offsets.Retry.Max; attempt++ { + om.flushToBroker() + if om.releasePOMs(false) == 0 { + break + } + } + + om.releasePOMs(true) + om.brokerLock.Lock() + om.broker = nil + om.brokerLock.Unlock() + }) return nil } -func (om *offsetManager) refBrokerOffsetManager(broker *Broker) *brokerOffsetManager { - om.lock.Lock() - defer om.lock.Unlock() - - bom := om.boms[broker] - if bom == nil { - bom = om.newBrokerOffsetManager(broker) - om.boms[broker] = bom +func (om *offsetManager) fetchInitialOffset(topic string, partition int32, retries int) (int64, string, error) { + broker, err := om.coordinator() + if err != nil { + if retries <= 0 { + return 0, "", err + } + return om.fetchInitialOffset(topic, partition, retries-1) } - bom.refs++ + req := new(OffsetFetchRequest) + req.Version = 1 + req.ConsumerGroup = om.group + req.AddPartition(topic, partition) - return bom + resp, err := broker.FetchOffset(req) + if err != nil { + if retries <= 0 { + return 0, "", err + } + om.releaseCoordinator(broker) + return om.fetchInitialOffset(topic, partition, retries-1) + } + + block := resp.GetBlock(topic, partition) + if block == nil { + return 0, "", ErrIncompleteResponse + } + + switch block.Err { + case ErrNoError: + return block.Offset, block.Metadata, nil + case ErrNotCoordinatorForConsumer: + if retries <= 0 { + return 0, "", block.Err + } + om.releaseCoordinator(broker) + return om.fetchInitialOffset(topic, partition, retries-1) + case ErrOffsetsLoadInProgress: + if retries <= 0 { + return 0, "", block.Err + } + select { + case <-om.closing: + return 0, "", block.Err + case <-time.After(om.conf.Metadata.Retry.Backoff): + } + return om.fetchInitialOffset(topic, partition, retries-1) + default: + return 0, "", block.Err + } } -func (om *offsetManager) unrefBrokerOffsetManager(bom *brokerOffsetManager) { - om.lock.Lock() - defer om.lock.Unlock() +func (om *offsetManager) coordinator() (*Broker, error) { + om.brokerLock.RLock() + broker := om.broker + om.brokerLock.RUnlock() - bom.refs-- + if broker != nil { + return broker, nil + } - if bom.refs == 0 { - close(bom.updateSubscriptions) - if om.boms[bom.broker] == bom { - delete(om.boms, bom.broker) + om.brokerLock.Lock() + defer om.brokerLock.Unlock() + + if broker := om.broker; broker != nil { + return broker, nil + } + + if err := om.client.RefreshCoordinator(om.group); err != nil { + return nil, err + } + + broker, err := om.client.Coordinator(om.group) + if err != nil { + return nil, err + } + + om.broker = broker + return broker, nil +} + +func (om *offsetManager) releaseCoordinator(b *Broker) { + om.brokerLock.Lock() + if om.broker == b { + om.broker = nil + } + om.brokerLock.Unlock() +} + +func (om *offsetManager) mainLoop() { + defer om.ticker.Stop() + defer close(om.closed) + + for { + select { + case <-om.ticker.C: + om.flushToBroker() + om.releasePOMs(false) + case <-om.closing: + return } } } -func (om *offsetManager) abandonBroker(bom *brokerOffsetManager) { - om.lock.Lock() - defer om.lock.Unlock() +func (om *offsetManager) flushToBroker() { + req := om.constructRequest() + if req == nil { + return + } - delete(om.boms, bom.broker) + broker, err := om.coordinator() + if err != nil { + om.handleError(err) + return + } + + resp, err := broker.CommitOffset(req) + if err != nil { + om.handleError(err) + om.releaseCoordinator(broker) + _ = broker.Close() + return + } + + om.handleResponse(broker, req, resp) } -func (om *offsetManager) abandonPartitionOffsetManager(pom *partitionOffsetManager) { - om.lock.Lock() - defer om.lock.Unlock() +func (om *offsetManager) constructRequest() *OffsetCommitRequest { + var r *OffsetCommitRequest + var perPartitionTimestamp int64 + if om.conf.Consumer.Offsets.Retention == 0 { + perPartitionTimestamp = ReceiveTime + r = &OffsetCommitRequest{ + Version: 1, + ConsumerGroup: om.group, + ConsumerID: om.memberID, + ConsumerGroupGeneration: om.generation, + } + } else { + r = &OffsetCommitRequest{ + Version: 2, + RetentionTime: int64(om.conf.Consumer.Offsets.Retention / time.Millisecond), + ConsumerGroup: om.group, + ConsumerID: om.memberID, + ConsumerGroupGeneration: om.generation, + } - delete(om.poms[pom.topic], pom.partition) - if len(om.poms[pom.topic]) == 0 { - delete(om.poms, pom.topic) } + + om.pomsLock.RLock() + defer om.pomsLock.RUnlock() + + for _, topicManagers := range om.poms { + for _, pom := range topicManagers { + pom.lock.Lock() + if pom.dirty { + r.AddBlock(pom.topic, pom.partition, pom.offset, perPartitionTimestamp, pom.metadata) + } + pom.lock.Unlock() + } + } + + if len(r.blocks) > 0 { + return r + } + + return nil +} + +func (om *offsetManager) handleResponse(broker *Broker, req *OffsetCommitRequest, resp *OffsetCommitResponse) { + om.pomsLock.RLock() + defer om.pomsLock.RUnlock() + + for _, topicManagers := range om.poms { + for _, pom := range topicManagers { + if req.blocks[pom.topic] == nil || req.blocks[pom.topic][pom.partition] == nil { + continue + } + + var err KError + var ok bool + + if resp.Errors[pom.topic] == nil { + pom.handleError(ErrIncompleteResponse) + continue + } + if err, ok = resp.Errors[pom.topic][pom.partition]; !ok { + pom.handleError(ErrIncompleteResponse) + continue + } + + switch err { + case ErrNoError: + block := req.blocks[pom.topic][pom.partition] + pom.updateCommitted(block.offset, block.metadata) + case ErrNotLeaderForPartition, ErrLeaderNotAvailable, + ErrConsumerCoordinatorNotAvailable, ErrNotCoordinatorForConsumer: + // not a critical error, we just need to redispatch + om.releaseCoordinator(broker) + case ErrOffsetMetadataTooLarge, ErrInvalidCommitOffsetSize: + // nothing we can do about this, just tell the user and carry on + pom.handleError(err) + case ErrOffsetsLoadInProgress: + // nothing wrong but we didn't commit, we'll get it next time round + break + case ErrUnknownTopicOrPartition: + // let the user know *and* try redispatching - if topic-auto-create is + // enabled, redispatching should trigger a metadata req and create the + // topic; if not then re-dispatching won't help, but we've let the user + // know and it shouldn't hurt either (see https://github.com/Shopify/sarama/issues/706) + fallthrough + default: + // dunno, tell the user and try redispatching + pom.handleError(err) + om.releaseCoordinator(broker) + } + } + } +} + +func (om *offsetManager) handleError(err error) { + om.pomsLock.RLock() + defer om.pomsLock.RUnlock() + + for _, topicManagers := range om.poms { + for _, pom := range topicManagers { + pom.handleError(err) + } + } +} + +func (om *offsetManager) asyncClosePOMs() { + om.pomsLock.RLock() + defer om.pomsLock.RUnlock() + + for _, topicManagers := range om.poms { + for _, pom := range topicManagers { + pom.AsyncClose() + } + } +} + +// Releases/removes closed POMs once they are clean (or when forced) +func (om *offsetManager) releasePOMs(force bool) (remaining int) { + om.pomsLock.Lock() + defer om.pomsLock.Unlock() + + for topic, topicManagers := range om.poms { + for partition, pom := range topicManagers { + pom.lock.Lock() + releaseDue := pom.done && (force || !pom.dirty) + pom.lock.Unlock() + + if releaseDue { + pom.release() + + delete(om.poms[topic], partition) + if len(om.poms[topic]) == 0 { + delete(om.poms, topic) + } + } + } + remaining += len(om.poms[topic]) + } + return +} + +func (om *offsetManager) findPOM(topic string, partition int32) *partitionOffsetManager { + om.pomsLock.RLock() + defer om.pomsLock.RUnlock() + + if partitions, ok := om.poms[topic]; ok { + if pom, ok := partitions[partition]; ok { + return pom + } + } + return nil } // Partition Offset Manager @@ -187,138 +463,26 @@ type partitionOffsetManager struct { offset int64 metadata string dirty bool - clean sync.Cond - broker *brokerOffsetManager + done bool - errors chan *ConsumerError - rebalance chan none - dying chan none + releaseOnce sync.Once + errors chan *ConsumerError } func (om *offsetManager) newPartitionOffsetManager(topic string, partition int32) (*partitionOffsetManager, error) { - pom := &partitionOffsetManager{ + offset, metadata, err := om.fetchInitialOffset(topic, partition, om.conf.Metadata.Retry.Max) + if err != nil { + return nil, err + } + + return &partitionOffsetManager{ parent: om, topic: topic, partition: partition, errors: make(chan *ConsumerError, om.conf.ChannelBufferSize), - rebalance: make(chan none, 1), - dying: make(chan none), - } - pom.clean.L = &pom.lock - - if err := pom.selectBroker(); err != nil { - return nil, err - } - - if err := pom.fetchInitialOffset(om.conf.Metadata.Retry.Max); err != nil { - return nil, err - } - - pom.broker.updateSubscriptions <- pom - - go withRecover(pom.mainLoop) - - return pom, nil -} - -func (pom *partitionOffsetManager) mainLoop() { - for { - select { - case <-pom.rebalance: - if err := pom.selectBroker(); err != nil { - pom.handleError(err) - pom.rebalance <- none{} - } else { - pom.broker.updateSubscriptions <- pom - } - case <-pom.dying: - if pom.broker != nil { - select { - case <-pom.rebalance: - case pom.broker.updateSubscriptions <- pom: - } - pom.parent.unrefBrokerOffsetManager(pom.broker) - } - pom.parent.abandonPartitionOffsetManager(pom) - close(pom.errors) - return - } - } -} - -func (pom *partitionOffsetManager) selectBroker() error { - if pom.broker != nil { - pom.parent.unrefBrokerOffsetManager(pom.broker) - pom.broker = nil - } - - var broker *Broker - var err error - - if err = pom.parent.client.RefreshCoordinator(pom.parent.group); err != nil { - return err - } - - if broker, err = pom.parent.client.Coordinator(pom.parent.group); err != nil { - return err - } - - pom.broker = pom.parent.refBrokerOffsetManager(broker) - return nil -} - -func (pom *partitionOffsetManager) fetchInitialOffset(retries int) error { - request := new(OffsetFetchRequest) - request.Version = 1 - request.ConsumerGroup = pom.parent.group - request.AddPartition(pom.topic, pom.partition) - - response, err := pom.broker.broker.FetchOffset(request) - if err != nil { - return err - } - - block := response.GetBlock(pom.topic, pom.partition) - if block == nil { - return ErrIncompleteResponse - } - - switch block.Err { - case ErrNoError: - pom.offset = block.Offset - pom.metadata = block.Metadata - return nil - case ErrNotCoordinatorForConsumer: - if retries <= 0 { - return block.Err - } - if err := pom.selectBroker(); err != nil { - return err - } - return pom.fetchInitialOffset(retries - 1) - case ErrOffsetsLoadInProgress: - if retries <= 0 { - return block.Err - } - time.Sleep(pom.parent.conf.Metadata.Retry.Backoff) - return pom.fetchInitialOffset(retries - 1) - default: - return block.Err - } -} - -func (pom *partitionOffsetManager) handleError(err error) { - cErr := &ConsumerError{ - Topic: pom.topic, - Partition: pom.partition, - Err: err, - } - - if pom.parent.conf.Consumer.Return.Errors { - pom.errors <- cErr - } else { - Logger.Println(cErr) - } + offset: offset, + metadata: metadata, + }, nil } func (pom *partitionOffsetManager) Errors() <-chan *ConsumerError { @@ -353,7 +517,6 @@ func (pom *partitionOffsetManager) updateCommitted(offset int64, metadata string if pom.offset == offset && pom.metadata == metadata { pom.dirty = false - pom.clean.Signal() } } @@ -369,16 +532,9 @@ func (pom *partitionOffsetManager) NextOffset() (int64, string) { } func (pom *partitionOffsetManager) AsyncClose() { - go func() { - pom.lock.Lock() - defer pom.lock.Unlock() - - for pom.dirty { - pom.clean.Wait() - } - - close(pom.dying) - }() + pom.lock.Lock() + pom.done = true + pom.lock.Unlock() } func (pom *partitionOffsetManager) Close() error { @@ -395,166 +551,22 @@ func (pom *partitionOffsetManager) Close() error { return nil } -// Broker Offset Manager - -type brokerOffsetManager struct { - parent *offsetManager - broker *Broker - timer *time.Ticker - updateSubscriptions chan *partitionOffsetManager - subscriptions map[*partitionOffsetManager]none - refs int -} - -func (om *offsetManager) newBrokerOffsetManager(broker *Broker) *brokerOffsetManager { - bom := &brokerOffsetManager{ - parent: om, - broker: broker, - timer: time.NewTicker(om.conf.Consumer.Offsets.CommitInterval), - updateSubscriptions: make(chan *partitionOffsetManager), - subscriptions: make(map[*partitionOffsetManager]none), +func (pom *partitionOffsetManager) handleError(err error) { + cErr := &ConsumerError{ + Topic: pom.topic, + Partition: pom.partition, + Err: err, } - go withRecover(bom.mainLoop) - - return bom -} - -func (bom *brokerOffsetManager) mainLoop() { - for { - select { - case <-bom.timer.C: - if len(bom.subscriptions) > 0 { - bom.flushToBroker() - } - case s, ok := <-bom.updateSubscriptions: - if !ok { - bom.timer.Stop() - return - } - if _, ok := bom.subscriptions[s]; ok { - delete(bom.subscriptions, s) - } else { - bom.subscriptions[s] = none{} - } - } - } -} - -func (bom *brokerOffsetManager) flushToBroker() { - request := bom.constructRequest() - if request == nil { - return - } - - response, err := bom.broker.CommitOffset(request) - - if err != nil { - bom.abort(err) - return - } - - for s := range bom.subscriptions { - if request.blocks[s.topic] == nil || request.blocks[s.topic][s.partition] == nil { - continue - } - - var err KError - var ok bool - - if response.Errors[s.topic] == nil { - s.handleError(ErrIncompleteResponse) - delete(bom.subscriptions, s) - s.rebalance <- none{} - continue - } - if err, ok = response.Errors[s.topic][s.partition]; !ok { - s.handleError(ErrIncompleteResponse) - delete(bom.subscriptions, s) - s.rebalance <- none{} - continue - } - - switch err { - case ErrNoError: - block := request.blocks[s.topic][s.partition] - s.updateCommitted(block.offset, block.metadata) - case ErrNotLeaderForPartition, ErrLeaderNotAvailable, - ErrConsumerCoordinatorNotAvailable, ErrNotCoordinatorForConsumer: - // not a critical error, we just need to redispatch - delete(bom.subscriptions, s) - s.rebalance <- none{} - case ErrOffsetMetadataTooLarge, ErrInvalidCommitOffsetSize: - // nothing we can do about this, just tell the user and carry on - s.handleError(err) - case ErrOffsetsLoadInProgress: - // nothing wrong but we didn't commit, we'll get it next time round - break - case ErrUnknownTopicOrPartition: - // let the user know *and* try redispatching - if topic-auto-create is - // enabled, redispatching should trigger a metadata request and create the - // topic; if not then re-dispatching won't help, but we've let the user - // know and it shouldn't hurt either (see https://github.com/Shopify/sarama/issues/706) - fallthrough - default: - // dunno, tell the user and try redispatching - s.handleError(err) - delete(bom.subscriptions, s) - s.rebalance <- none{} - } - } -} - -func (bom *brokerOffsetManager) constructRequest() *OffsetCommitRequest { - var r *OffsetCommitRequest - var perPartitionTimestamp int64 - if bom.parent.conf.Consumer.Offsets.Retention == 0 { - perPartitionTimestamp = ReceiveTime - r = &OffsetCommitRequest{ - Version: 1, - ConsumerGroup: bom.parent.group, - ConsumerGroupGeneration: GroupGenerationUndefined, - } + if pom.parent.conf.Consumer.Return.Errors { + pom.errors <- cErr } else { - r = &OffsetCommitRequest{ - Version: 2, - RetentionTime: int64(bom.parent.conf.Consumer.Offsets.Retention / time.Millisecond), - ConsumerGroup: bom.parent.group, - ConsumerGroupGeneration: GroupGenerationUndefined, - } - + Logger.Println(cErr) } - - for s := range bom.subscriptions { - s.lock.Lock() - if s.dirty { - r.AddBlock(s.topic, s.partition, s.offset, perPartitionTimestamp, s.metadata) - } - s.lock.Unlock() - } - - if len(r.blocks) > 0 { - return r - } - - return nil } -func (bom *brokerOffsetManager) abort(err error) { - _ = bom.broker.Close() // we don't care about the error this might return, we already have one - bom.parent.abandonBroker(bom) - - for pom := range bom.subscriptions { - pom.handleError(err) - pom.rebalance <- none{} - } - - for s := range bom.updateSubscriptions { - if _, ok := bom.subscriptions[s]; !ok { - s.handleError(err) - s.rebalance <- none{} - } - } - - bom.subscriptions = make(map[*partitionOffsetManager]none) +func (pom *partitionOffsetManager) release() { + pom.releaseOnce.Do(func() { + go close(pom.errors) + }) } diff --git a/third/github.com/Shopify/sarama/offset_manager_test.go b/third/github.com/Shopify/sarama/offset_manager_test.go index 21e4947c6..86d6f4eb4 100644 --- a/third/github.com/Shopify/sarama/offset_manager_test.go +++ b/third/github.com/Shopify/sarama/offset_manager_test.go @@ -5,13 +5,16 @@ import ( "time" ) -func initOffsetManager(t *testing.T) (om OffsetManager, +func initOffsetManager(t *testing.T, retention time.Duration) (om OffsetManager, testClient Client, broker, coordinator *MockBroker) { config := NewConfig() config.Metadata.Retry.Max = 1 config.Consumer.Offsets.CommitInterval = 1 * time.Millisecond config.Version = V0_9_0_0 + if retention > 0 { + config.Consumer.Offsets.Retention = retention + } broker = NewMockBroker(t, 1) coordinator = NewMockBroker(t, 2) @@ -64,31 +67,30 @@ func initPartitionOffsetManager(t *testing.T, om OffsetManager, func TestNewOffsetManager(t *testing.T) { seedBroker := NewMockBroker(t, 1) seedBroker.Returns(new(MetadataResponse)) + defer seedBroker.Close() testClient, err := NewClient([]string{seedBroker.Addr()}, nil) if err != nil { t.Fatal(err) } - _, err = NewOffsetManagerFromClient("group", testClient) + om, err := NewOffsetManagerFromClient("group", testClient) if err != nil { t.Error(err) } - + safeClose(t, om) safeClose(t, testClient) _, err = NewOffsetManagerFromClient("group", testClient) if err != ErrClosedClient { t.Errorf("Error expected for closed client; actual value: %v", err) } - - seedBroker.Close() } // Test recovery from ErrNotCoordinatorForConsumer // on first fetchInitialOffset call func TestOffsetManagerFetchInitialFail(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t) + om, testClient, broker, coordinator := initOffsetManager(t, 0) // Error on first fetchInitialOffset call responseBlock := OffsetFetchResponseBlock{ @@ -131,7 +133,7 @@ func TestOffsetManagerFetchInitialFail(t *testing.T) { // Test fetchInitialOffset retry on ErrOffsetsLoadInProgress func TestOffsetManagerFetchInitialLoadInProgress(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t) + om, testClient, broker, coordinator := initOffsetManager(t, 0) // Error on first fetchInitialOffset call responseBlock := OffsetFetchResponseBlock{ @@ -164,7 +166,7 @@ func TestOffsetManagerFetchInitialLoadInProgress(t *testing.T) { } func TestPartitionOffsetManagerInitialOffset(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t) + om, testClient, broker, coordinator := initOffsetManager(t, 0) testClient.Config().Consumer.Offsets.Initial = OffsetOldest // Kafka returns -1 if no offset has been stored for this partition yet. @@ -186,7 +188,7 @@ func TestPartitionOffsetManagerInitialOffset(t *testing.T) { } func TestPartitionOffsetManagerNextOffset(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t) + om, testClient, broker, coordinator := initOffsetManager(t, 0) pom := initPartitionOffsetManager(t, om, coordinator, 5, "test_meta") offset, meta := pom.NextOffset() @@ -205,7 +207,7 @@ func TestPartitionOffsetManagerNextOffset(t *testing.T) { } func TestPartitionOffsetManagerResetOffset(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t) + om, testClient, broker, coordinator := initOffsetManager(t, 0) pom := initPartitionOffsetManager(t, om, coordinator, 5, "original_meta") ocResponse := new(OffsetCommitResponse) @@ -231,9 +233,7 @@ func TestPartitionOffsetManagerResetOffset(t *testing.T) { } func TestPartitionOffsetManagerResetOffsetWithRetention(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t) - testClient.Config().Consumer.Offsets.Retention = time.Hour - + om, testClient, broker, coordinator := initOffsetManager(t, time.Hour) pom := initPartitionOffsetManager(t, om, coordinator, 5, "original_meta") ocResponse := new(OffsetCommitResponse) @@ -269,7 +269,7 @@ func TestPartitionOffsetManagerResetOffsetWithRetention(t *testing.T) { } func TestPartitionOffsetManagerMarkOffset(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t) + om, testClient, broker, coordinator := initOffsetManager(t, 0) pom := initPartitionOffsetManager(t, om, coordinator, 5, "original_meta") ocResponse := new(OffsetCommitResponse) @@ -294,9 +294,7 @@ func TestPartitionOffsetManagerMarkOffset(t *testing.T) { } func TestPartitionOffsetManagerMarkOffsetWithRetention(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t) - testClient.Config().Consumer.Offsets.Retention = time.Hour - + om, testClient, broker, coordinator := initOffsetManager(t, time.Hour) pom := initPartitionOffsetManager(t, om, coordinator, 5, "original_meta") ocResponse := new(OffsetCommitResponse) @@ -331,7 +329,7 @@ func TestPartitionOffsetManagerMarkOffsetWithRetention(t *testing.T) { } func TestPartitionOffsetManagerCommitErr(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t) + om, testClient, broker, coordinator := initOffsetManager(t, 0) pom := initPartitionOffsetManager(t, om, coordinator, 5, "meta") // Error on one partition @@ -353,24 +351,14 @@ func TestPartitionOffsetManagerCommitErr(t *testing.T) { ocResponse2 := new(OffsetCommitResponse) newCoordinator.Returns(ocResponse2) - // For RefreshCoordinator() - broker.Returns(&ConsumerMetadataResponse{ - CoordinatorID: newCoordinator.BrokerID(), - CoordinatorHost: "127.0.0.1", - CoordinatorPort: newCoordinator.Port(), - }) + // No error, no need to refresh coordinator // Error on the wrong partition for this pom ocResponse3 := new(OffsetCommitResponse) ocResponse3.AddError("my_topic", 1, ErrNoError) newCoordinator.Returns(ocResponse3) - // For RefreshCoordinator() - broker.Returns(&ConsumerMetadataResponse{ - CoordinatorID: newCoordinator.BrokerID(), - CoordinatorHost: "127.0.0.1", - CoordinatorPort: newCoordinator.Port(), - }) + // No error, no need to refresh coordinator // ErrUnknownTopicOrPartition/ErrNotLeaderForPartition/ErrLeaderNotAvailable block ocResponse4 := new(OffsetCommitResponse) @@ -405,7 +393,7 @@ func TestPartitionOffsetManagerCommitErr(t *testing.T) { // Test of recovery from abort func TestAbortPartitionOffsetManager(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t) + om, testClient, broker, coordinator := initOffsetManager(t, 0) pom := initPartitionOffsetManager(t, om, coordinator, 5, "meta") // this triggers an error in the CommitOffset request, diff --git a/third/github.com/Shopify/sarama/offset_request.go b/third/github.com/Shopify/sarama/offset_request.go index 4c5df75df..326c3720c 100644 --- a/third/github.com/Shopify/sarama/offset_request.go +++ b/third/github.com/Shopify/sarama/offset_request.go @@ -27,12 +27,20 @@ func (b *offsetRequestBlock) decode(pd packetDecoder, version int16) (err error) } type OffsetRequest struct { - Version int16 - blocks map[string]map[int32]*offsetRequestBlock + Version int16 + replicaID int32 + isReplicaIDSet bool + blocks map[string]map[int32]*offsetRequestBlock } func (r *OffsetRequest) encode(pe packetEncoder) error { - pe.putInt32(-1) // replica ID is always -1 for clients + if r.isReplicaIDSet { + pe.putInt32(r.replicaID) + } else { + // default replica ID is always -1 for clients + pe.putInt32(-1) + } + err := pe.putArrayLength(len(r.blocks)) if err != nil { return err @@ -59,10 +67,14 @@ func (r *OffsetRequest) encode(pe packetEncoder) error { func (r *OffsetRequest) decode(pd packetDecoder, version int16) error { r.Version = version - // Ignore replica ID - if _, err := pd.getInt32(); err != nil { + replicaID, err := pd.getInt32() + if err != nil { return err } + if replicaID >= 0 { + r.SetReplicaID(replicaID) + } + blockCount, err := pd.getArrayLength() if err != nil { return err @@ -113,6 +125,18 @@ func (r *OffsetRequest) requiredVersion() KafkaVersion { } } +func (r *OffsetRequest) SetReplicaID(id int32) { + r.replicaID = id + r.isReplicaIDSet = true +} + +func (r *OffsetRequest) ReplicaID() int32 { + if r.isReplicaIDSet { + return r.replicaID + } + return -1 +} + func (r *OffsetRequest) AddBlock(topic string, partitionID int32, time int64, maxOffsets int32) { if r.blocks == nil { r.blocks = make(map[string]map[int32]*offsetRequestBlock) diff --git a/third/github.com/Shopify/sarama/offset_request_test.go b/third/github.com/Shopify/sarama/offset_request_test.go index 9ce562c99..8ca818e49 100644 --- a/third/github.com/Shopify/sarama/offset_request_test.go +++ b/third/github.com/Shopify/sarama/offset_request_test.go @@ -23,6 +23,10 @@ var ( 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01} + + offsetRequestReplicaID = []byte{ + 0x00, 0x00, 0x00, 0x2a, + 0x00, 0x00, 0x00, 0x00} ) func TestOffsetRequest(t *testing.T) { @@ -41,3 +45,15 @@ func TestOffsetRequestV1(t *testing.T) { request.AddBlock("bar", 4, 1, 2) // Last argument is ignored for V1 testRequest(t, "one block", request, offsetRequestOneBlockV1) } + +func TestOffsetRequestReplicaID(t *testing.T) { + request := new(OffsetRequest) + replicaID := int32(42) + request.SetReplicaID(replicaID) + + if found := request.ReplicaID(); found != replicaID { + t.Errorf("replicaID: expected %v, found %v", replicaID, found) + } + + testRequest(t, "with replica ID", request, offsetRequestReplicaID) +} diff --git a/third/github.com/Shopify/sarama/produce_response.go b/third/github.com/Shopify/sarama/produce_response.go index 667e34c66..4c5cd3569 100644 --- a/third/github.com/Shopify/sarama/produce_response.go +++ b/third/github.com/Shopify/sarama/produce_response.go @@ -179,5 +179,11 @@ func (r *ProduceResponse) AddTopicPartition(topic string, partition int32, err K byTopic = make(map[int32]*ProduceResponseBlock) r.Blocks[topic] = byTopic } - byTopic[partition] = &ProduceResponseBlock{Err: err} + block := &ProduceResponseBlock{ + Err: err, + } + if r.Version >= 2 { + block.Timestamp = time.Now() + } + byTopic[partition] = block } diff --git a/third/github.com/Shopify/sarama/produce_set.go b/third/github.com/Shopify/sarama/produce_set.go index 13be2b3c9..219ec5f26 100644 --- a/third/github.com/Shopify/sarama/produce_set.go +++ b/third/github.com/Shopify/sarama/produce_set.go @@ -2,6 +2,7 @@ package sarama import ( "encoding/binary" + "errors" "time" ) @@ -61,9 +62,13 @@ func (ps *produceSet) add(msg *ProducerMessage) error { batch := &RecordBatch{ FirstTimestamp: timestamp, Version: 2, - ProducerID: -1, /* No producer id */ Codec: ps.parent.conf.Producer.Compression, CompressionLevel: ps.parent.conf.Producer.CompressionLevel, + ProducerID: ps.parent.txnmgr.producerID, + ProducerEpoch: ps.parent.txnmgr.producerEpoch, + } + if ps.parent.conf.Producer.Idempotent { + batch.FirstSequence = msg.sequenceNumber } set = &partitionSet{recordsToSend: newDefaultRecords(batch)} size = recordBatchOverhead @@ -72,9 +77,12 @@ func (ps *produceSet) add(msg *ProducerMessage) error { } partitions[msg.Partition] = set } - set.msgs = append(set.msgs, msg) + if ps.parent.conf.Version.IsAtLeast(V0_11_0_0) { + if ps.parent.conf.Producer.Idempotent && msg.sequenceNumber < set.recordsToSend.RecordBatch.FirstSequence { + return errors.New("Assertion failed: Message out of sequence added to a batch") + } // We are being conservative here to avoid having to prep encode the record size += maximumRecordOverhead rec := &Record{ @@ -120,8 +128,8 @@ func (ps *produceSet) buildRequest() *ProduceRequest { req.Version = 3 } - for topic, partitionSet := range ps.msgs { - for partition, set := range partitionSet { + for topic, partitionSets := range ps.msgs { + for partition, set := range partitionSets { if req.Version >= 3 { // If the API version we're hitting is 3 or greater, we need to calculate // offsets for each record in the batch relative to FirstOffset. @@ -137,7 +145,6 @@ func (ps *produceSet) buildRequest() *ProduceRequest { record.OffsetDelta = int64(i) } } - req.AddBatch(topic, partition, rb) continue } @@ -183,10 +190,10 @@ func (ps *produceSet) buildRequest() *ProduceRequest { return req } -func (ps *produceSet) eachPartition(cb func(topic string, partition int32, msgs []*ProducerMessage)) { +func (ps *produceSet) eachPartition(cb func(topic string, partition int32, pSet *partitionSet)) { for topic, partitionSet := range ps.msgs { for partition, set := range partitionSet { - cb(topic, partition, set.msgs) + cb(topic, partition, set) } } } diff --git a/third/github.com/Shopify/sarama/produce_set_test.go b/third/github.com/Shopify/sarama/produce_set_test.go index 6663f36f7..51d4cef30 100644 --- a/third/github.com/Shopify/sarama/produce_set_test.go +++ b/third/github.com/Shopify/sarama/produce_set_test.go @@ -7,8 +7,11 @@ import ( ) func makeProduceSet() (*asyncProducer, *produceSet) { + conf := NewConfig() + txnmgr, _ := newTransactionManager(conf, nil) parent := &asyncProducer{ - conf: NewConfig(), + conf: conf, + txnmgr: txnmgr, } return parent, newProduceSet(parent) } @@ -72,8 +75,8 @@ func TestProduceSetPartitionTracking(t *testing.T) { seenT1P1 := false seenT2P0 := false - ps.eachPartition(func(topic string, partition int32, msgs []*ProducerMessage) { - if len(msgs) != 1 { + ps.eachPartition(func(topic string, partition int32, pSet *partitionSet) { + if len(pSet.msgs) != 1 { t.Error("Wrong message count") } @@ -253,3 +256,91 @@ func TestProduceSetV3RequestBuilding(t *testing.T) { } } } + +func TestProduceSetIdempotentRequestBuilding(t *testing.T) { + const pID = 1000 + const pEpoch = 1234 + + config := NewConfig() + config.Producer.RequiredAcks = WaitForAll + config.Producer.Idempotent = true + config.Version = V0_11_0_0 + + parent := &asyncProducer{ + conf: config, + txnmgr: &transactionManager{ + producerID: pID, + producerEpoch: pEpoch, + }, + } + ps := newProduceSet(parent) + + now := time.Now() + msg := &ProducerMessage{ + Topic: "t1", + Partition: 0, + Key: StringEncoder(TestMessage), + Value: StringEncoder(TestMessage), + Headers: []RecordHeader{ + RecordHeader{ + Key: []byte("header-1"), + Value: []byte("value-1"), + }, + RecordHeader{ + Key: []byte("header-2"), + Value: []byte("value-2"), + }, + RecordHeader{ + Key: []byte("header-3"), + Value: []byte("value-3"), + }, + }, + Timestamp: now, + sequenceNumber: 123, + } + for i := 0; i < 10; i++ { + safeAddMessage(t, ps, msg) + msg.Timestamp = msg.Timestamp.Add(time.Second) + } + + req := ps.buildRequest() + + if req.Version != 3 { + t.Error("Wrong request version") + } + + batch := req.records["t1"][0].RecordBatch + if batch.FirstTimestamp != now { + t.Errorf("Wrong first timestamp: %v", batch.FirstTimestamp) + } + if batch.ProducerID != pID { + t.Errorf("Wrong producerID: %v", batch.ProducerID) + } + if batch.ProducerEpoch != pEpoch { + t.Errorf("Wrong producerEpoch: %v", batch.ProducerEpoch) + } + if batch.FirstSequence != 123 { + t.Errorf("Wrong first sequence: %v", batch.FirstSequence) + } + for i := 0; i < 10; i++ { + rec := batch.Records[i] + if rec.TimestampDelta != time.Duration(i)*time.Second { + t.Errorf("Wrong timestamp delta: %v", rec.TimestampDelta) + } + + if rec.OffsetDelta != int64(i) { + t.Errorf("Wrong relative inner offset, expected %d, got %d", i, rec.OffsetDelta) + } + + for j, h := range batch.Records[i].Headers { + exp := fmt.Sprintf("header-%d", j+1) + if string(h.Key) != exp { + t.Errorf("Wrong header key, expected %v, got %v", exp, h.Key) + } + exp = fmt.Sprintf("value-%d", j+1) + if string(h.Value) != exp { + t.Errorf("Wrong header value, expected %v, got %v", exp, h.Value) + } + } + } +} diff --git a/third/github.com/Shopify/sarama/record_batch.go b/third/github.com/Shopify/sarama/record_batch.go index 1b7425ddd..e0f183f7a 100644 --- a/third/github.com/Shopify/sarama/record_batch.go +++ b/third/github.com/Shopify/sarama/record_batch.go @@ -1,14 +1,8 @@ package sarama import ( - "bytes" - "compress/gzip" "fmt" - "io/ioutil" "time" - - "gitee.com/johng/gf/third/github.com/eapache/go-xerial-snappy" - "gitee.com/johng/gf/third/github.com/pierrec/lz4" ) const recordBatchOverhead = 49 @@ -174,27 +168,9 @@ func (b *RecordBatch) decode(pd packetDecoder) (err error) { return err } - switch b.Codec { - case CompressionNone: - case CompressionGZIP: - reader, err := gzip.NewReader(bytes.NewReader(recBuffer)) - if err != nil { - return err - } - if recBuffer, err = ioutil.ReadAll(reader); err != nil { - return err - } - case CompressionSnappy: - if recBuffer, err = snappy.Decode(recBuffer); err != nil { - return err - } - case CompressionLZ4: - reader := lz4.NewReader(bytes.NewReader(recBuffer)) - if recBuffer, err = ioutil.ReadAll(reader); err != nil { - return err - } - default: - return PacketDecodingError{fmt.Sprintf("invalid compression specified (%d)", b.Codec)} + recBuffer, err = decompress(b.Codec, recBuffer) + if err != nil { + return err } b.recordsLen = len(recBuffer) @@ -215,44 +191,8 @@ func (b *RecordBatch) encodeRecords(pe packetEncoder) error { } b.recordsLen = len(raw) - switch b.Codec { - case CompressionNone: - b.compressedRecords = raw - case CompressionGZIP: - var buf bytes.Buffer - var writer *gzip.Writer - if b.CompressionLevel != CompressionLevelDefault { - writer, err = gzip.NewWriterLevel(&buf, b.CompressionLevel) - if err != nil { - return err - } - } else { - writer = gzip.NewWriter(&buf) - } - if _, err := writer.Write(raw); err != nil { - return err - } - if err := writer.Close(); err != nil { - return err - } - b.compressedRecords = buf.Bytes() - case CompressionSnappy: - b.compressedRecords = snappy.Encode(raw) - case CompressionLZ4: - var buf bytes.Buffer - writer := lz4.NewWriter(&buf) - if _, err := writer.Write(raw); err != nil { - return err - } - if err := writer.Close(); err != nil { - return err - } - b.compressedRecords = buf.Bytes() - default: - return PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", b.Codec)} - } - - return nil + b.compressedRecords, err = compress(b.Codec, b.CompressionLevel, raw) + return err } func (b *RecordBatch) computeAttributes() int16 { diff --git a/third/github.com/Shopify/sarama/records.go b/third/github.com/Shopify/sarama/records.go index 301055bb0..192f5927b 100644 --- a/third/github.com/Shopify/sarama/records.go +++ b/third/github.com/Shopify/sarama/records.go @@ -163,6 +163,27 @@ func (r *Records) isControl() (bool, error) { return false, fmt.Errorf("unknown records type: %v", r.recordsType) } +func (r *Records) isOverflow() (bool, error) { + if r.recordsType == unknownRecords { + if empty, err := r.setTypeFromFields(); err != nil || empty { + return false, err + } + } + + switch r.recordsType { + case unknownRecords: + return false, nil + case legacyRecords: + if r.MsgSet == nil { + return false, nil + } + return r.MsgSet.OverflowMessage, nil + case defaultRecords: + return false, nil + } + return false, fmt.Errorf("unknown records type: %v", r.recordsType) +} + func magicValue(pd packetDecoder) (int8, error) { dec, err := pd.peek(magicOffset, magicLength) if err != nil { diff --git a/third/github.com/Shopify/sarama/tools/README.md b/third/github.com/Shopify/sarama/tools/README.md new file mode 100644 index 000000000..3464c4ad8 --- /dev/null +++ b/third/github.com/Shopify/sarama/tools/README.md @@ -0,0 +1,10 @@ +# Sarama tools + +This folder contains applications that are useful for exploration of your Kafka cluster, or instrumentation. +Some of these tools mirror tools that ship with Kafka, but these tools won't require installing the JVM to function. + +- [kafka-console-producer](./kafka-console-producer): a command line tool to produce a single message to your Kafka custer. +- [kafka-console-partitionconsumer](./kafka-console-partitionconsumer): (deprecated) a command line tool to consume a single partition of a topic on your Kafka cluster. +- [kafka-console-consumer](./kafka-console-consumer): a command line tool to consume arbitrary partitions of a topic on your Kafka cluster. + +To install all tools, run `go get github.com/Shopify/sarama/tools/...` diff --git a/third/github.com/Shopify/sarama/tools/kafka-console-consumer/.gitignore b/third/github.com/Shopify/sarama/tools/kafka-console-consumer/.gitignore new file mode 100644 index 000000000..67da9dfa9 --- /dev/null +++ b/third/github.com/Shopify/sarama/tools/kafka-console-consumer/.gitignore @@ -0,0 +1,2 @@ +kafka-console-consumer +kafka-console-consumer.test diff --git a/third/github.com/Shopify/sarama/tools/kafka-console-consumer/README.md b/third/github.com/Shopify/sarama/tools/kafka-console-consumer/README.md new file mode 100644 index 000000000..4e77f0b70 --- /dev/null +++ b/third/github.com/Shopify/sarama/tools/kafka-console-consumer/README.md @@ -0,0 +1,29 @@ +# kafka-console-consumer + +A simple command line tool to consume partitions of a topic and print the +messages on the standard output. + +### Installation + + go get github.com/Shopify/sarama/tools/kafka-console-consumer + +### Usage + + # Minimum invocation + kafka-console-consumer -topic=test -brokers=kafka1:9092 + + # It will pick up a KAFKA_PEERS environment variable + export KAFKA_PEERS=kafka1:9092,kafka2:9092,kafka3:9092 + kafka-console-consumer -topic=test + + # You can specify the offset you want to start at. It can be either + # `oldest`, `newest`. The default is `newest`. + kafka-console-consumer -topic=test -offset=oldest + kafka-console-consumer -topic=test -offset=newest + + # You can specify the partition(s) you want to consume as a comma-separated + # list. The default is `all`. + kafka-console-consumer -topic=test -partitions=1,2,3 + + # Display all command line options + kafka-console-consumer -help diff --git a/third/github.com/Shopify/sarama/tools/kafka-console-consumer/kafka-console-consumer.go b/third/github.com/Shopify/sarama/tools/kafka-console-consumer/kafka-console-consumer.go new file mode 100644 index 000000000..45f0d0f2d --- /dev/null +++ b/third/github.com/Shopify/sarama/tools/kafka-console-consumer/kafka-console-consumer.go @@ -0,0 +1,145 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + "os/signal" + "strconv" + "strings" + "sync" + + "gitee.com/johng/gf/third/github.com/Shopify/sarama" +) + +var ( + brokerList = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The comma separated list of brokers in the Kafka cluster") + topic = flag.String("topic", "", "REQUIRED: the topic to consume") + partitions = flag.String("partitions", "all", "The partitions to consume, can be 'all' or comma-separated numbers") + offset = flag.String("offset", "newest", "The offset to start with. Can be `oldest`, `newest`") + verbose = flag.Bool("verbose", false, "Whether to turn on sarama logging") + bufferSize = flag.Int("buffer-size", 256, "The buffer size of the message channel.") + + logger = log.New(os.Stderr, "", log.LstdFlags) +) + +func main() { + flag.Parse() + + if *brokerList == "" { + printUsageErrorAndExit("You have to provide -brokers as a comma-separated list, or set the KAFKA_PEERS environment variable.") + } + + if *topic == "" { + printUsageErrorAndExit("-topic is required") + } + + if *verbose { + sarama.Logger = logger + } + + var initialOffset int64 + switch *offset { + case "oldest": + initialOffset = sarama.OffsetOldest + case "newest": + initialOffset = sarama.OffsetNewest + default: + printUsageErrorAndExit("-offset should be `oldest` or `newest`") + } + + c, err := sarama.NewConsumer(strings.Split(*brokerList, ","), nil) + if err != nil { + printErrorAndExit(69, "Failed to start consumer: %s", err) + } + + partitionList, err := getPartitions(c) + if err != nil { + printErrorAndExit(69, "Failed to get the list of partitions: %s", err) + } + + var ( + messages = make(chan *sarama.ConsumerMessage, *bufferSize) + closing = make(chan struct{}) + wg sync.WaitGroup + ) + + go func() { + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Kill, os.Interrupt) + <-signals + logger.Println("Initiating shutdown of consumer...") + close(closing) + }() + + for _, partition := range partitionList { + pc, err := c.ConsumePartition(*topic, partition, initialOffset) + if err != nil { + printErrorAndExit(69, "Failed to start consumer for partition %d: %s", partition, err) + } + + go func(pc sarama.PartitionConsumer) { + <-closing + pc.AsyncClose() + }(pc) + + wg.Add(1) + go func(pc sarama.PartitionConsumer) { + defer wg.Done() + for message := range pc.Messages() { + messages <- message + } + }(pc) + } + + go func() { + for msg := range messages { + fmt.Printf("Partition:\t%d\n", msg.Partition) + fmt.Printf("Offset:\t%d\n", msg.Offset) + fmt.Printf("Key:\t%s\n", string(msg.Key)) + fmt.Printf("Value:\t%s\n", string(msg.Value)) + fmt.Println() + } + }() + + wg.Wait() + logger.Println("Done consuming topic", *topic) + close(messages) + + if err := c.Close(); err != nil { + logger.Println("Failed to close consumer: ", err) + } +} + +func getPartitions(c sarama.Consumer) ([]int32, error) { + if *partitions == "all" { + return c.Partitions(*topic) + } + + tmp := strings.Split(*partitions, ",") + var pList []int32 + for i := range tmp { + val, err := strconv.ParseInt(tmp[i], 10, 32) + if err != nil { + return nil, err + } + pList = append(pList, int32(val)) + } + + return pList, nil +} + +func printErrorAndExit(code int, format string, values ...interface{}) { + fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...)) + fmt.Fprintln(os.Stderr) + os.Exit(code) +} + +func printUsageErrorAndExit(format string, values ...interface{}) { + fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...)) + fmt.Fprintln(os.Stderr) + fmt.Fprintln(os.Stderr, "Available command line options:") + flag.PrintDefaults() + os.Exit(64) +} diff --git a/third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/.gitignore b/third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/.gitignore new file mode 100644 index 000000000..5837fe8ca --- /dev/null +++ b/third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/.gitignore @@ -0,0 +1,2 @@ +kafka-console-partitionconsumer +kafka-console-partitionconsumer.test diff --git a/third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/README.md b/third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/README.md new file mode 100644 index 000000000..646dd5f5c --- /dev/null +++ b/third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/README.md @@ -0,0 +1,28 @@ +# kafka-console-partitionconsumer + +NOTE: this tool is deprecated in favour of the more general and more powerful +`kafka-console-consumer`. + +A simple command line tool to consume a partition of a topic and print the messages +on the standard output. + +### Installation + + go get github.com/Shopify/sarama/tools/kafka-console-partitionconsumer + +### Usage + + # Minimum invocation + kafka-console-partitionconsumer -topic=test -partition=4 -brokers=kafka1:9092 + + # It will pick up a KAFKA_PEERS environment variable + export KAFKA_PEERS=kafka1:9092,kafka2:9092,kafka3:9092 + kafka-console-partitionconsumer -topic=test -partition=4 + + # You can specify the offset you want to start at. It can be either + # `oldest`, `newest`, or a specific offset number + kafka-console-partitionconsumer -topic=test -partition=3 -offset=oldest + kafka-console-partitionconsumer -topic=test -partition=2 -offset=1337 + + # Display all command line options + kafka-console-partitionconsumer -help diff --git a/third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/kafka-console-partitionconsumer.go b/third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/kafka-console-partitionconsumer.go new file mode 100644 index 000000000..55c3ce20a --- /dev/null +++ b/third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/kafka-console-partitionconsumer.go @@ -0,0 +1,102 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + "os/signal" + "strconv" + "strings" + + "gitee.com/johng/gf/third/github.com/Shopify/sarama" +) + +var ( + brokerList = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The comma separated list of brokers in the Kafka cluster") + topic = flag.String("topic", "", "REQUIRED: the topic to consume") + partition = flag.Int("partition", -1, "REQUIRED: the partition to consume") + offset = flag.String("offset", "newest", "The offset to start with. Can be `oldest`, `newest`, or an actual offset") + verbose = flag.Bool("verbose", false, "Whether to turn on sarama logging") + + logger = log.New(os.Stderr, "", log.LstdFlags) +) + +func main() { + flag.Parse() + + if *brokerList == "" { + printUsageErrorAndExit("You have to provide -brokers as a comma-separated list, or set the KAFKA_PEERS environment variable.") + } + + if *topic == "" { + printUsageErrorAndExit("-topic is required") + } + + if *partition == -1 { + printUsageErrorAndExit("-partition is required") + } + + if *verbose { + sarama.Logger = logger + } + + var ( + initialOffset int64 + offsetError error + ) + switch *offset { + case "oldest": + initialOffset = sarama.OffsetOldest + case "newest": + initialOffset = sarama.OffsetNewest + default: + initialOffset, offsetError = strconv.ParseInt(*offset, 10, 64) + } + + if offsetError != nil { + printUsageErrorAndExit("Invalid initial offset: %s", *offset) + } + + c, err := sarama.NewConsumer(strings.Split(*brokerList, ","), nil) + if err != nil { + printErrorAndExit(69, "Failed to start consumer: %s", err) + } + + pc, err := c.ConsumePartition(*topic, int32(*partition), initialOffset) + if err != nil { + printErrorAndExit(69, "Failed to start partition consumer: %s", err) + } + + go func() { + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Kill, os.Interrupt) + <-signals + pc.AsyncClose() + }() + + for msg := range pc.Messages() { + fmt.Printf("Offset:\t%d\n", msg.Offset) + fmt.Printf("Key:\t%s\n", string(msg.Key)) + fmt.Printf("Value:\t%s\n", string(msg.Value)) + fmt.Println() + } + + if err := c.Close(); err != nil { + logger.Println("Failed to close consumer: ", err) + } +} + +func printErrorAndExit(code int, format string, values ...interface{}) { + fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...)) + fmt.Fprintln(os.Stderr) + os.Exit(code) +} + +func printUsageErrorAndExit(format string, values ...interface{}) { + fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...)) + fmt.Fprintln(os.Stderr) + fmt.Fprintln(os.Stderr, "Available command line options:") + flag.PrintDefaults() + os.Exit(64) +} diff --git a/third/github.com/Shopify/sarama/tools/kafka-console-producer/.gitignore b/third/github.com/Shopify/sarama/tools/kafka-console-producer/.gitignore new file mode 100644 index 000000000..2b9e563a1 --- /dev/null +++ b/third/github.com/Shopify/sarama/tools/kafka-console-producer/.gitignore @@ -0,0 +1,2 @@ +kafka-console-producer +kafka-console-producer.test diff --git a/third/github.com/Shopify/sarama/tools/kafka-console-producer/README.md b/third/github.com/Shopify/sarama/tools/kafka-console-producer/README.md new file mode 100644 index 000000000..6b3a65f21 --- /dev/null +++ b/third/github.com/Shopify/sarama/tools/kafka-console-producer/README.md @@ -0,0 +1,34 @@ +# kafka-console-producer + +A simple command line tool to produce a single message to Kafka. + +### Installation + + go get github.com/Shopify/sarama/tools/kafka-console-producer + + +### Usage + + # Minimum invocation + kafka-console-producer -topic=test -value=value -brokers=kafka1:9092 + + # It will pick up a KAFKA_PEERS environment variable + export KAFKA_PEERS=kafka1:9092,kafka2:9092,kafka3:9092 + kafka-console-producer -topic=test -value=value + + # It will read the value from stdin by using pipes + echo "hello world" | kafka-console-producer -topic=test + + # Specify a key: + echo "hello world" | kafka-console-producer -topic=test -key=key + + # Partitioning: by default, kafka-console-producer will partition as follows: + # - manual partitioning if a -partition is provided + # - hash partitioning by key if a -key is provided + # - random partioning otherwise. + # + # You can override this using the -partitioner argument: + echo "hello world" | kafka-console-producer -topic=test -key=key -partitioner=random + + # Display all command line options + kafka-console-producer -help diff --git a/third/github.com/Shopify/sarama/tools/kafka-console-producer/kafka-console-producer.go b/third/github.com/Shopify/sarama/tools/kafka-console-producer/kafka-console-producer.go new file mode 100644 index 000000000..7fce88233 --- /dev/null +++ b/third/github.com/Shopify/sarama/tools/kafka-console-producer/kafka-console-producer.go @@ -0,0 +1,124 @@ +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "strings" + + "gitee.com/johng/gf/third/github.com/Shopify/sarama" + "gitee.com/johng/gf/third/github.com/rcrowley/go-metrics" +) + +var ( + brokerList = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The comma separated list of brokers in the Kafka cluster. You can also set the KAFKA_PEERS environment variable") + topic = flag.String("topic", "", "REQUIRED: the topic to produce to") + key = flag.String("key", "", "The key of the message to produce. Can be empty.") + value = flag.String("value", "", "REQUIRED: the value of the message to produce. You can also provide the value on stdin.") + partitioner = flag.String("partitioner", "", "The partitioning scheme to use. Can be `hash`, `manual`, or `random`") + partition = flag.Int("partition", -1, "The partition to produce to.") + verbose = flag.Bool("verbose", false, "Turn on sarama logging to stderr") + showMetrics = flag.Bool("metrics", false, "Output metrics on successful publish to stderr") + silent = flag.Bool("silent", false, "Turn off printing the message's topic, partition, and offset to stdout") + + logger = log.New(os.Stderr, "", log.LstdFlags) +) + +func main() { + flag.Parse() + + if *brokerList == "" { + printUsageErrorAndExit("no -brokers specified. Alternatively, set the KAFKA_PEERS environment variable") + } + + if *topic == "" { + printUsageErrorAndExit("no -topic specified") + } + + if *verbose { + sarama.Logger = logger + } + + config := sarama.NewConfig() + config.Producer.RequiredAcks = sarama.WaitForAll + config.Producer.Return.Successes = true + + switch *partitioner { + case "": + if *partition >= 0 { + config.Producer.Partitioner = sarama.NewManualPartitioner + } else { + config.Producer.Partitioner = sarama.NewHashPartitioner + } + case "hash": + config.Producer.Partitioner = sarama.NewHashPartitioner + case "random": + config.Producer.Partitioner = sarama.NewRandomPartitioner + case "manual": + config.Producer.Partitioner = sarama.NewManualPartitioner + if *partition == -1 { + printUsageErrorAndExit("-partition is required when partitioning manually") + } + default: + printUsageErrorAndExit(fmt.Sprintf("Partitioner %s not supported.", *partitioner)) + } + + message := &sarama.ProducerMessage{Topic: *topic, Partition: int32(*partition)} + + if *key != "" { + message.Key = sarama.StringEncoder(*key) + } + + if *value != "" { + message.Value = sarama.StringEncoder(*value) + } else if stdinAvailable() { + bytes, err := ioutil.ReadAll(os.Stdin) + if err != nil { + printErrorAndExit(66, "Failed to read data from the standard input: %s", err) + } + message.Value = sarama.ByteEncoder(bytes) + } else { + printUsageErrorAndExit("-value is required, or you have to provide the value on stdin") + } + + producer, err := sarama.NewSyncProducer(strings.Split(*brokerList, ","), config) + if err != nil { + printErrorAndExit(69, "Failed to open Kafka producer: %s", err) + } + defer func() { + if err := producer.Close(); err != nil { + logger.Println("Failed to close Kafka producer cleanly:", err) + } + }() + + partition, offset, err := producer.SendMessage(message) + if err != nil { + printErrorAndExit(69, "Failed to produce message: %s", err) + } else if !*silent { + fmt.Printf("topic=%s\tpartition=%d\toffset=%d\n", *topic, partition, offset) + } + if *showMetrics { + metrics.WriteOnce(config.MetricRegistry, os.Stderr) + } +} + +func printErrorAndExit(code int, format string, values ...interface{}) { + fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...)) + fmt.Fprintln(os.Stderr) + os.Exit(code) +} + +func printUsageErrorAndExit(message string) { + fmt.Fprintln(os.Stderr, "ERROR:", message) + fmt.Fprintln(os.Stderr) + fmt.Fprintln(os.Stderr, "Available command line options:") + flag.PrintDefaults() + os.Exit(64) +} + +func stdinAvailable() bool { + stat, _ := os.Stdin.Stat() + return (stat.Mode() & os.ModeCharDevice) == 0 +} diff --git a/third/github.com/Shopify/sarama/utils.go b/third/github.com/Shopify/sarama/utils.go index 702e22627..7dcbf034f 100644 --- a/third/github.com/Shopify/sarama/utils.go +++ b/third/github.com/Shopify/sarama/utils.go @@ -155,6 +155,10 @@ var ( V0_11_0_2 = newKafkaVersion(0, 11, 0, 2) V1_0_0_0 = newKafkaVersion(1, 0, 0, 0) V1_1_0_0 = newKafkaVersion(1, 1, 0, 0) + V1_1_1_0 = newKafkaVersion(1, 1, 1, 0) + V2_0_0_0 = newKafkaVersion(2, 0, 0, 0) + V2_0_1_0 = newKafkaVersion(2, 0, 1, 0) + V2_1_0_0 = newKafkaVersion(2, 1, 0, 0) SupportedVersions = []KafkaVersion{ V0_8_2_0, @@ -173,9 +177,13 @@ var ( V0_11_0_2, V1_0_0_0, V1_1_0_0, + V1_1_1_0, + V2_0_0_0, + V2_0_1_0, + V2_1_0_0, } MinVersion = V0_8_2_0 - MaxVersion = V1_1_0_0 + MaxVersion = V2_1_0_0 ) func ParseKafkaVersion(s string) (KafkaVersion, error) { @@ -206,7 +214,7 @@ func scanKafkaVersion(s string, pattern string, format string, v [3]*uint) error func (v KafkaVersion) String() string { if v.version[0] == 0 { return fmt.Sprintf("0.%d.%d.%d", v.version[1], v.version[2], v.version[3]) - } else { - return fmt.Sprintf("%d.%d.%d", v.version[0], v.version[1], v.version[2]) } + + return fmt.Sprintf("%d.%d.%d", v.version[0], v.version[1], v.version[2]) } diff --git a/third/github.com/Shopify/sarama/vagrant/boot_cluster.sh b/third/github.com/Shopify/sarama/vagrant/boot_cluster.sh new file mode 100755 index 000000000..95e47dde4 --- /dev/null +++ b/third/github.com/Shopify/sarama/vagrant/boot_cluster.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +set -ex + +# Launch and wait for toxiproxy +${REPOSITORY_ROOT}/vagrant/run_toxiproxy.sh & +while ! nc -q 1 localhost 2181 ${KAFKA_INSTALL_ROOT}/zookeeper-${ZK_PORT}/myid +done diff --git a/third/github.com/Shopify/sarama/vagrant/kafka.conf b/third/github.com/Shopify/sarama/vagrant/kafka.conf new file mode 100644 index 000000000..25101df5a --- /dev/null +++ b/third/github.com/Shopify/sarama/vagrant/kafka.conf @@ -0,0 +1,9 @@ +start on started zookeeper-ZK_PORT +stop on stopping zookeeper-ZK_PORT + +# Use a script instead of exec (using env stanza leaks KAFKA_HEAP_OPTS from zookeeper) +script + sleep 2 + export KAFKA_HEAP_OPTS="-Xmx320m" + exec /opt/kafka-KAFKAID/bin/kafka-server-start.sh /opt/kafka-KAFKAID/config/server.properties +end script diff --git a/third/github.com/Shopify/sarama/vagrant/provision.sh b/third/github.com/Shopify/sarama/vagrant/provision.sh new file mode 100755 index 000000000..13a8d5623 --- /dev/null +++ b/third/github.com/Shopify/sarama/vagrant/provision.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +set -ex + +apt-get update +yes | apt-get install default-jre + +export KAFKA_INSTALL_ROOT=/opt +export KAFKA_HOSTNAME=192.168.100.67 +export KAFKA_VERSION=1.0.0 +export REPOSITORY_ROOT=/vagrant + +sh /vagrant/vagrant/install_cluster.sh +sh /vagrant/vagrant/setup_services.sh +sh /vagrant/vagrant/create_topics.sh diff --git a/third/github.com/Shopify/sarama/vagrant/run_toxiproxy.sh b/third/github.com/Shopify/sarama/vagrant/run_toxiproxy.sh new file mode 100755 index 000000000..e52c00e7b --- /dev/null +++ b/third/github.com/Shopify/sarama/vagrant/run_toxiproxy.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +set -ex + +${KAFKA_INSTALL_ROOT}/toxiproxy -port 8474 -host 0.0.0.0 & +PID=$! + +while ! nc -q 1 localhost 8474 + +# The number of threads handling network requests +num.network.threads=2 + +# The number of threads doing disk I/O +num.io.threads=8 + +# The send buffer (SO_SNDBUF) used by the socket server +socket.send.buffer.bytes=1048576 + +# The receive buffer (SO_RCVBUF) used by the socket server +socket.receive.buffer.bytes=1048576 + +# The maximum size of a request that the socket server will accept (protection against OOM) +socket.request.max.bytes=104857600 + + +############################# Log Basics ############################# + +# A comma seperated list of directories under which to store log files +log.dirs=KAFKA_DATADIR + +# The default number of log partitions per topic. More partitions allow greater +# parallelism for consumption, but this will also result in more files across +# the brokers. +num.partitions=2 + +# Create new topics with a replication factor of 2 so failover can be tested +# more easily. +default.replication.factor=2 + +auto.create.topics.enable=false +delete.topic.enable=true + +############################# Log Flush Policy ############################# + +# Messages are immediately written to the filesystem but by default we only fsync() to sync +# the OS cache lazily. The following configurations control the flush of data to disk. +# There are a few important trade-offs here: +# 1. Durability: Unflushed data may be lost if you are not using replication. +# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# The settings below allow one to configure the flush policy to flush data after a period of time or +# every N messages (or both). This can be done globally and overridden on a per-topic basis. + +# The number of messages to accept before forcing a flush of data to disk +#log.flush.interval.messages=10000 + +# The maximum amount of time a message can sit in a log before we force a flush +#log.flush.interval.ms=1000 + +############################# Log Retention Policy ############################# + +# The following configurations control the disposal of log segments. The policy can +# be set to delete segments after a period of time, or after a given size has accumulated. +# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens +# from the end of the log. + +# The minimum age of a log file to be eligible for deletion +log.retention.hours=168 + +# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining +# segments don't drop below log.retention.bytes. +log.retention.bytes=268435456 + +# The maximum size of a log segment file. When this size is reached a new log segment will be created. +log.segment.bytes=268435456 + +# The interval at which log segments are checked to see if they can be deleted according +# to the retention policies +log.retention.check.interval.ms=60000 + +# By default the log cleaner is disabled and the log retention policy will default to just delete segments after their retention expires. +# If log.cleaner.enable=true is set the cleaner will be enabled and individual logs can then be marked for log compaction. +log.cleaner.enable=false + +############################# Zookeeper ############################# + +# Zookeeper connection string (see zookeeper docs for details). +# This is a comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". +# You can also append an optional chroot string to the urls to specify the +# root directory for all kafka znodes. +zookeeper.connect=localhost:ZK_PORT + +# Timeout in ms for connecting to zookeeper +zookeeper.session.timeout.ms=3000 +zookeeper.connection.timeout.ms=3000 diff --git a/third/github.com/Shopify/sarama/vagrant/setup_services.sh b/third/github.com/Shopify/sarama/vagrant/setup_services.sh new file mode 100755 index 000000000..81d8ea05d --- /dev/null +++ b/third/github.com/Shopify/sarama/vagrant/setup_services.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +set -ex + +stop toxiproxy || true +cp ${REPOSITORY_ROOT}/vagrant/toxiproxy.conf /etc/init/toxiproxy.conf +cp ${REPOSITORY_ROOT}/vagrant/run_toxiproxy.sh ${KAFKA_INSTALL_ROOT}/ +start toxiproxy + +for i in 1 2 3 4 5; do + ZK_PORT=`expr $i + 2180` + KAFKA_PORT=`expr $i + 9090` + + stop zookeeper-${ZK_PORT} || true + + # set up zk service + cp ${REPOSITORY_ROOT}/vagrant/zookeeper.conf /etc/init/zookeeper-${ZK_PORT}.conf + sed -i s/KAFKAID/${KAFKA_PORT}/g /etc/init/zookeeper-${ZK_PORT}.conf + + # set up kafka service + cp ${REPOSITORY_ROOT}/vagrant/kafka.conf /etc/init/kafka-${KAFKA_PORT}.conf + sed -i s/KAFKAID/${KAFKA_PORT}/g /etc/init/kafka-${KAFKA_PORT}.conf + sed -i s/ZK_PORT/${ZK_PORT}/g /etc/init/kafka-${KAFKA_PORT}.conf + + start zookeeper-${ZK_PORT} +done + +# Wait for the last kafka node to finish booting +while ! nc -q 1 localhost 29095 -1 && offset < 1000 { - return nil, sarama.ErrOffsetOutOfRange - } - return &mockPartitionConsumer{ - Topic: topic, - Partition: partition, - Offset: offset, - }, nil -} - -func (*mockPartitionConsumer) Close() error { return nil } diff --git a/third/github.com/johng-cn/sarama-cluster/config_test.go b/third/github.com/johng-cn/sarama-cluster/config_test.go deleted file mode 100644 index c24e7c2aa..000000000 --- a/third/github.com/johng-cn/sarama-cluster/config_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package cluster - -import ( - "time" - - . "gitee.com/johng/gf/third/github.com/onsi/ginkgo" - . "gitee.com/johng/gf/third/github.com/onsi/gomega" -) - -var _ = Describe("Config", func() { - var subject *Config - - BeforeEach(func() { - subject = NewConfig() - }) - - It("should init", func() { - Expect(subject.Group.Session.Timeout).To(Equal(30 * time.Second)) - Expect(subject.Group.Heartbeat.Interval).To(Equal(3 * time.Second)) - Expect(subject.Group.Return.Notifications).To(BeFalse()) - Expect(subject.Metadata.Retry.Max).To(Equal(3)) - Expect(subject.Group.Offsets.Synchronization.DwellTime).NotTo(BeZero()) - // Expect(subject.Config.Version).To(Equal(sarama.V0_9_0_0)) - }) - -}) diff --git a/third/github.com/johng-cn/sarama-cluster/consumer.go b/third/github.com/johng-cn/sarama-cluster/consumer.go index 41d766b70..0cd83f817 100644 --- a/third/github.com/johng-cn/sarama-cluster/consumer.go +++ b/third/github.com/johng-cn/sarama-cluster/consumer.go @@ -374,7 +374,7 @@ func (c *Consumer) nextTick() { // Issue rebalance start notification if c.client.config.Group.Return.Notifications { - c.handleNotification(notification) + c.handleNotification(newNotification(c.subs.Info())) } // Rebalance, fetch new subscriptions @@ -705,7 +705,7 @@ func (c *Consumer) joinGroup() (*balancer, error) { return nil, err } - strategy, err = newBalancerFromMeta(c.client, members) + strategy, err = newBalancerFromMeta(c.client, Strategy(resp.GroupProtocol), members) if err != nil { return nil, err } @@ -730,7 +730,7 @@ func (c *Consumer) syncGroup(strategy *balancer) (map[string][]int32, error) { } if strategy != nil { - for memberID, topics := range strategy.Perform(c.client.config.Group.PartitionStrategy) { + for memberID, topics := range strategy.Perform() { if err := req.AddGroupAssignmentMember(memberID, &sarama.ConsumerGroupMemberAssignment{ Topics: topics, }); err != nil { @@ -860,7 +860,11 @@ func (c *Consumer) createConsumer(tomb *loopTomb, topic string, partition int32, }) if c.client.config.Group.Mode == ConsumerModePartitions { - c.partitions <- pc + select { + case c.partitions <- pc: + case <-c.dying: + pc.Close() + } } return nil } diff --git a/third/github.com/johng-cn/sarama-cluster/consumer_test.go b/third/github.com/johng-cn/sarama-cluster/consumer_test.go deleted file mode 100644 index b1c989d4b..000000000 --- a/third/github.com/johng-cn/sarama-cluster/consumer_test.go +++ /dev/null @@ -1,350 +0,0 @@ -package cluster - -import ( - "fmt" - "regexp" - "sync/atomic" - "time" - - "gitee.com/johng/gf/third/github.com/Shopify/sarama" - . "gitee.com/johng/gf/third/github.com/onsi/ginkgo" - . "gitee.com/johng/gf/third/github.com/onsi/gomega" -) - -var _ = Describe("Consumer", func() { - - var newConsumerOf = func(group string, topics ...string) (*Consumer, error) { - config := NewConfig() - config.Consumer.Return.Errors = true - config.Consumer.Offsets.Initial = sarama.OffsetOldest - return NewConsumer(testKafkaAddrs, group, topics, config) - } - - var subscriptionsOf = func(c *Consumer) GomegaAsyncAssertion { - return Eventually(func() map[string][]int32 { - return c.Subscriptions() - }, "10s", "100ms") - } - - It("should init and share", func() { - // start CS1 - cs1, err := newConsumerOf(testGroup, testTopics...) - Expect(err).NotTo(HaveOccurred()) - - // CS1 should consume all 8 partitions - subscriptionsOf(cs1).Should(Equal(map[string][]int32{ - "topic-a": {0, 1, 2, 3}, - "topic-b": {0, 1, 2, 3}, - })) - - // start CS2 - cs2, err := newConsumerOf(testGroup, testTopics...) - Expect(err).NotTo(HaveOccurred()) - defer cs2.Close() - - // CS1 and CS2 should consume 4 partitions each - subscriptionsOf(cs1).Should(HaveLen(2)) - subscriptionsOf(cs1).Should(HaveKeyWithValue("topic-a", HaveLen(2))) - subscriptionsOf(cs1).Should(HaveKeyWithValue("topic-b", HaveLen(2))) - - subscriptionsOf(cs2).Should(HaveLen(2)) - subscriptionsOf(cs2).Should(HaveKeyWithValue("topic-a", HaveLen(2))) - subscriptionsOf(cs2).Should(HaveKeyWithValue("topic-b", HaveLen(2))) - - // shutdown CS1, now CS2 should consume all 8 partitions - Expect(cs1.Close()).NotTo(HaveOccurred()) - subscriptionsOf(cs2).Should(Equal(map[string][]int32{ - "topic-a": {0, 1, 2, 3}, - "topic-b": {0, 1, 2, 3}, - })) - }) - - It("should allow more consumers than partitions", func() { - cs1, err := newConsumerOf(testGroup, "topic-a") - Expect(err).NotTo(HaveOccurred()) - defer cs1.Close() - cs2, err := newConsumerOf(testGroup, "topic-a") - Expect(err).NotTo(HaveOccurred()) - defer cs2.Close() - cs3, err := newConsumerOf(testGroup, "topic-a") - Expect(err).NotTo(HaveOccurred()) - defer cs3.Close() - cs4, err := newConsumerOf(testGroup, "topic-a") - Expect(err).NotTo(HaveOccurred()) - - // start 4 consumers, one for each partition - subscriptionsOf(cs1).Should(HaveKeyWithValue("topic-a", HaveLen(1))) - subscriptionsOf(cs2).Should(HaveKeyWithValue("topic-a", HaveLen(1))) - subscriptionsOf(cs3).Should(HaveKeyWithValue("topic-a", HaveLen(1))) - subscriptionsOf(cs4).Should(HaveKeyWithValue("topic-a", HaveLen(1))) - - // add a 5th consumer - cs5, err := newConsumerOf(testGroup, "topic-a") - Expect(err).NotTo(HaveOccurred()) - defer cs5.Close() - - // make sure no errors occurred - Expect(cs1.Errors()).ShouldNot(Receive()) - Expect(cs2.Errors()).ShouldNot(Receive()) - Expect(cs3.Errors()).ShouldNot(Receive()) - Expect(cs4.Errors()).ShouldNot(Receive()) - Expect(cs5.Errors()).ShouldNot(Receive()) - - // close 4th, make sure the 5th takes over - Expect(cs4.Close()).To(Succeed()) - subscriptionsOf(cs1).Should(HaveKeyWithValue("topic-a", HaveLen(1))) - subscriptionsOf(cs2).Should(HaveKeyWithValue("topic-a", HaveLen(1))) - subscriptionsOf(cs3).Should(HaveKeyWithValue("topic-a", HaveLen(1))) - subscriptionsOf(cs4).Should(BeEmpty()) - subscriptionsOf(cs5).Should(HaveKeyWithValue("topic-a", HaveLen(1))) - - // there should still be no errors - Expect(cs1.Errors()).ShouldNot(Receive()) - Expect(cs2.Errors()).ShouldNot(Receive()) - Expect(cs3.Errors()).ShouldNot(Receive()) - Expect(cs4.Errors()).ShouldNot(Receive()) - Expect(cs5.Errors()).ShouldNot(Receive()) - }) - - It("should be allowed to subscribe to partitions via white/black-lists", func() { - config := NewConfig() - config.Consumer.Return.Errors = true - config.Group.Topics.Whitelist = regexp.MustCompile(`topic-\w+`) - config.Group.Topics.Blacklist = regexp.MustCompile(`[bcd]$`) - - cs, err := NewConsumer(testKafkaAddrs, testGroup, nil, config) - Expect(err).NotTo(HaveOccurred()) - defer cs.Close() - - subscriptionsOf(cs).Should(Equal(map[string][]int32{ - "topic-a": {0, 1, 2, 3}, - })) - }) - - It("should receive rebalance notifications", func() { - config := NewConfig() - config.Consumer.Return.Errors = true - config.Group.Return.Notifications = true - - cs, err := NewConsumer(testKafkaAddrs, testGroup, testTopics, config) - Expect(err).NotTo(HaveOccurred()) - defer cs.Close() - - select { - case n := <-cs.Notifications(): - Expect(n).To(Equal(&Notification{ - Type: RebalanceStart, - Current: map[string][]int32{}, - })) - case err := <-cs.Errors(): - Expect(err).NotTo(HaveOccurred()) - case <-cs.Messages(): - Fail("expected notification to arrive before message") - } - - select { - case n := <-cs.Notifications(): - Expect(n).To(Equal(&Notification{ - Type: RebalanceOK, - Claimed: map[string][]int32{ - "topic-a": {0, 1, 2, 3}, - "topic-b": {0, 1, 2, 3}, - }, - Released: map[string][]int32{}, - Current: map[string][]int32{ - "topic-a": {0, 1, 2, 3}, - "topic-b": {0, 1, 2, 3}, - }, - })) - case err := <-cs.Errors(): - Expect(err).NotTo(HaveOccurred()) - case <-cs.Messages(): - Fail("expected notification to arrive before message") - } - }) - - It("should support manual mark/commit", func() { - cs, err := newConsumerOf(testGroup, "topic-a") - Expect(err).NotTo(HaveOccurred()) - defer cs.Close() - - subscriptionsOf(cs).Should(Equal(map[string][]int32{ - "topic-a": {0, 1, 2, 3}}, - )) - - cs.MarkPartitionOffset("topic-a", 1, 3, "") - cs.MarkPartitionOffset("topic-a", 2, 4, "") - Expect(cs.CommitOffsets()).NotTo(HaveOccurred()) - - offsets, err := cs.fetchOffsets(cs.Subscriptions()) - Expect(err).NotTo(HaveOccurred()) - Expect(offsets).To(Equal(map[string]map[int32]offsetInfo{ - "topic-a": {0: {Offset: -1}, 1: {Offset: 4}, 2: {Offset: 5}, 3: {Offset: -1}}, - })) - }) - - It("should support manual mark/commit, reset/commit", func() { - cs, err := newConsumerOf(testGroup, "topic-a") - Expect(err).NotTo(HaveOccurred()) - defer cs.Close() - - subscriptionsOf(cs).Should(Equal(map[string][]int32{ - "topic-a": {0, 1, 2, 3}}, - )) - - cs.MarkPartitionOffset("topic-a", 1, 3, "") - cs.MarkPartitionOffset("topic-a", 2, 4, "") - cs.MarkPartitionOffset("topic-b", 1, 2, "") // should not throw NPE - Expect(cs.CommitOffsets()).NotTo(HaveOccurred()) - - cs.ResetPartitionOffset("topic-a", 1, 2, "") - cs.ResetPartitionOffset("topic-a", 2, 3, "") - cs.ResetPartitionOffset("topic-b", 1, 2, "") // should not throw NPE - Expect(cs.CommitOffsets()).NotTo(HaveOccurred()) - - offsets, err := cs.fetchOffsets(cs.Subscriptions()) - Expect(err).NotTo(HaveOccurred()) - Expect(offsets).To(Equal(map[string]map[int32]offsetInfo{ - "topic-a": {0: {Offset: -1}, 1: {Offset: 3}, 2: {Offset: 4}, 3: {Offset: -1}}, - })) - }) - - It("should not commit unprocessed offsets", func() { - const groupID = "panicking" - - cs, err := newConsumerOf(groupID, "topic-a") - Expect(err).NotTo(HaveOccurred()) - - subscriptionsOf(cs).Should(Equal(map[string][]int32{ - "topic-a": {0, 1, 2, 3}, - })) - - n := 0 - Expect(func() { - for range cs.Messages() { - n++ - panic("stop here!") - } - }).To(Panic()) - Expect(cs.Close()).To(Succeed()) - Expect(n).To(Equal(1)) - - bk, err := testClient.Coordinator(groupID) - Expect(err).NotTo(HaveOccurred()) - - req := &sarama.OffsetFetchRequest{ - Version: 1, - ConsumerGroup: groupID, - } - req.AddPartition("topic-a", 0) - req.AddPartition("topic-a", 1) - req.AddPartition("topic-a", 2) - req.AddPartition("topic-a", 3) - Expect(bk.FetchOffset(req)).To(Equal(&sarama.OffsetFetchResponse{ - Blocks: map[string]map[int32]*sarama.OffsetFetchResponseBlock{ - "topic-a": {0: {Offset: -1}, 1: {Offset: -1}, 2: {Offset: -1}, 3: {Offset: -1}}, - }, - })) - }) - - It("should consume partitions", func() { - count := int32(0) - consume := func(consumerID string) { - defer GinkgoRecover() - - config := NewConfig() - config.Group.Mode = ConsumerModePartitions - config.Consumer.Offsets.Initial = sarama.OffsetOldest - - cs, err := NewConsumer(testKafkaAddrs, "partitions", testTopics, config) - Expect(err).NotTo(HaveOccurred()) - defer cs.Close() - - for pc := range cs.Partitions() { - go func(pc PartitionConsumer) { - defer pc.Close() - - for msg := range pc.Messages() { - atomic.AddInt32(&count, 1) - cs.MarkOffset(msg, "") - } - }(pc) - } - } - - go consume("A") - go consume("B") - go consume("C") - - Eventually(func() int32 { - return atomic.LoadInt32(&count) - }, "30s", "100ms").Should(BeNumerically(">=", 2000)) - }) - - It("should consume/commit/resume", func() { - acc := make(chan *testConsumerMessage, 20000) - consume := func(consumerID string, max int32) { - defer GinkgoRecover() - - cs, err := NewConsumer(testKafkaAddrs, "fuzzing", testTopics, nil) - Expect(err).NotTo(HaveOccurred()) - defer cs.Close() - cs.consumerID = consumerID - - for msg := range cs.Messages() { - acc <- &testConsumerMessage{*msg, consumerID} - cs.MarkOffset(msg, "") - - if atomic.AddInt32(&max, -1) <= 0 { - return - } - } - } - - go consume("A", 1500) - go consume("B", 2000) - go consume("C", 1500) - go consume("D", 200) - go consume("E", 100) - time.Sleep(10 * time.Second) // wait for consumers to subscribe to topics - Expect(testSeed(5000, testTopics)).NotTo(HaveOccurred()) - Eventually(func() int { return len(acc) }, "30s", "100ms").Should(BeNumerically(">=", 5000)) - - go consume("F", 300) - go consume("G", 400) - go consume("H", 1000) - go consume("I", 2000) - Expect(testSeed(5000, testTopics)).NotTo(HaveOccurred()) - Eventually(func() int { return len(acc) }, "30s", "100ms").Should(BeNumerically(">=", 8000)) - - go consume("J", 1000) - Expect(testSeed(5000, testTopics)).NotTo(HaveOccurred()) - Eventually(func() int { return len(acc) }, "30s", "100ms").Should(BeNumerically(">=", 9000)) - - go consume("K", 1000) - go consume("L", 3000) - Expect(testSeed(5000, testTopics)).NotTo(HaveOccurred()) - Eventually(func() int { return len(acc) }, "30s", "100ms").Should(BeNumerically(">=", 12000)) - - go consume("M", 1000) - Expect(testSeed(5000, testTopics)).NotTo(HaveOccurred()) - Eventually(func() int { return len(acc) }, "30s", "100ms").Should(BeNumerically(">=", 15000)) - - close(acc) - - uniques := make(map[string][]string) - for msg := range acc { - key := fmt.Sprintf("%s/%d/%d", msg.Topic, msg.Partition, msg.Offset) - uniques[key] = append(uniques[key], msg.ConsumerID) - } - Expect(uniques).To(HaveLen(15000)) - }) - - It("should allow close to be called multiple times", func() { - cs, err := newConsumerOf(testGroup, testTopics...) - Expect(err).NotTo(HaveOccurred()) - Expect(cs.Close()).NotTo(HaveOccurred()) - Expect(cs.Close()).NotTo(HaveOccurred()) - }) - -}) diff --git a/third/github.com/johng-cn/sarama-cluster/examples_test.go b/third/github.com/johng-cn/sarama-cluster/examples_test.go deleted file mode 100644 index 2a55ee9be..000000000 --- a/third/github.com/johng-cn/sarama-cluster/examples_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package cluster_test - -import ( - "fmt" - "log" - "os" - "os/signal" - "regexp" - - cluster "gitee.com/johng/gf/third/github.com/bsm/sarama-cluster" -) - -// This example shows how to use the consumer to read messages -// from a multiple topics through a multiplexed channel. -func ExampleConsumer() { - - // init (custom) config, enable errors and notifications - config := cluster.NewConfig() - config.Consumer.Return.Errors = true - config.Group.Return.Notifications = true - - // init consumer - brokers := []string{"127.0.0.1:9092"} - topics := []string{"my_topic", "other_topic"} - consumer, err := cluster.NewConsumer(brokers, "my-consumer-group", topics, config) - if err != nil { - panic(err) - } - defer consumer.Close() - - // trap SIGINT to trigger a shutdown. - signals := make(chan os.Signal, 1) - signal.Notify(signals, os.Interrupt) - - // consume errors - go func() { - for err := range consumer.Errors() { - log.Printf("Error: %s\n", err.Error()) - } - }() - - // consume notifications - go func() { - for ntf := range consumer.Notifications() { - log.Printf("Rebalanced: %+v\n", ntf) - } - }() - - // consume messages, watch signals - for { - select { - case msg, ok := <-consumer.Messages(): - if ok { - fmt.Fprintf(os.Stdout, "%s/%d/%d\t%s\t%s\n", msg.Topic, msg.Partition, msg.Offset, msg.Key, msg.Value) - consumer.MarkOffset(msg, "") // mark message as processed - } - case <-signals: - return - } - } -} - -// This example shows how to use the consumer to read messages -// through individual partitions. -func ExampleConsumer_Partitions() { - - // init (custom) config, set mode to ConsumerModePartitions - config := cluster.NewConfig() - config.Group.Mode = cluster.ConsumerModePartitions - - // init consumer - brokers := []string{"127.0.0.1:9092"} - topics := []string{"my_topic", "other_topic"} - consumer, err := cluster.NewConsumer(brokers, "my-consumer-group", topics, config) - if err != nil { - panic(err) - } - defer consumer.Close() - - // trap SIGINT to trigger a shutdown. - signals := make(chan os.Signal, 1) - signal.Notify(signals, os.Interrupt) - - // consume partitions - for { - select { - case part, ok := <-consumer.Partitions(): - if !ok { - return - } - - // start a separate goroutine to consume messages - go func(pc cluster.PartitionConsumer) { - for msg := range pc.Messages() { - fmt.Fprintf(os.Stdout, "%s/%d/%d\t%s\t%s\n", msg.Topic, msg.Partition, msg.Offset, msg.Key, msg.Value) - consumer.MarkOffset(msg, "") // mark message as processed - } - }(part) - case <-signals: - return - } - } -} - -// This example shows how to use the consumer with -// topic whitelists. -func ExampleConfig_whitelist() { - - // init (custom) config, enable errors and notifications - config := cluster.NewConfig() - config.Group.Topics.Whitelist = regexp.MustCompile(`myservice.*`) - - // init consumer - consumer, err := cluster.NewConsumer([]string{"127.0.0.1:9092"}, "my-consumer-group", nil, config) - if err != nil { - panic(err) - } - defer consumer.Close() - - // consume messages - msg := <-consumer.Messages() - fmt.Fprintf(os.Stdout, "%s/%d/%d\t%s\t%s\n", msg.Topic, msg.Partition, msg.Offset, msg.Key, msg.Value) -} diff --git a/third/github.com/johng-cn/sarama-cluster/offsets_test.go b/third/github.com/johng-cn/sarama-cluster/offsets_test.go deleted file mode 100644 index 9328204b1..000000000 --- a/third/github.com/johng-cn/sarama-cluster/offsets_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package cluster - -import ( - . "gitee.com/johng/gf/third/github.com/onsi/ginkgo" - . "gitee.com/johng/gf/third/github.com/onsi/gomega" -) - -var _ = Describe("OffsetStash", func() { - var subject *OffsetStash - - BeforeEach(func() { - subject = NewOffsetStash() - }) - - It("should update", func() { - Expect(subject.offsets).To(HaveLen(0)) - - subject.MarkPartitionOffset("topic", 0, 0, "m3ta") - Expect(subject.offsets).To(HaveLen(1)) - Expect(subject.offsets).To(HaveKeyWithValue( - topicPartition{Topic: "topic", Partition: 0}, - offsetInfo{Offset: 0, Metadata: "m3ta"}, - )) - - subject.MarkPartitionOffset("topic", 0, 200, "m3ta") - Expect(subject.offsets).To(HaveLen(1)) - Expect(subject.offsets).To(HaveKeyWithValue( - topicPartition{Topic: "topic", Partition: 0}, - offsetInfo{Offset: 200, Metadata: "m3ta"}, - )) - - subject.MarkPartitionOffset("topic", 0, 199, "m3t@") - Expect(subject.offsets).To(HaveLen(1)) - Expect(subject.offsets).To(HaveKeyWithValue( - topicPartition{Topic: "topic", Partition: 0}, - offsetInfo{Offset: 200, Metadata: "m3ta"}, - )) - - subject.MarkPartitionOffset("topic", 1, 300, "") - Expect(subject.offsets).To(HaveLen(2)) - Expect(subject.offsets).To(HaveKeyWithValue( - topicPartition{Topic: "topic", Partition: 1}, - offsetInfo{Offset: 300, Metadata: ""}, - )) - }) - - It("should reset", func() { - Expect(subject.offsets).To(HaveLen(0)) - - subject.MarkPartitionOffset("topic", 0, 0, "m3ta") - Expect(subject.offsets).To(HaveLen(1)) - Expect(subject.offsets).To(HaveKeyWithValue( - topicPartition{Topic: "topic", Partition: 0}, - offsetInfo{Offset: 0, Metadata: "m3ta"}, - )) - - subject.MarkPartitionOffset("topic", 0, 200, "m3ta") - Expect(subject.offsets).To(HaveLen(1)) - Expect(subject.offsets).To(HaveKeyWithValue( - topicPartition{Topic: "topic", Partition: 0}, - offsetInfo{Offset: 200, Metadata: "m3ta"}, - )) - - subject.ResetPartitionOffset("topic", 0, 199, "m3t@") - Expect(subject.offsets).To(HaveLen(1)) - Expect(subject.offsets).To(HaveKeyWithValue( - topicPartition{Topic: "topic", Partition: 0}, - offsetInfo{Offset: 199, Metadata: "m3t@"}, - )) - - subject.MarkPartitionOffset("topic", 1, 300, "") - Expect(subject.offsets).To(HaveLen(2)) - Expect(subject.offsets).To(HaveKeyWithValue( - topicPartition{Topic: "topic", Partition: 1}, - offsetInfo{Offset: 300, Metadata: ""}, - )) - - subject.ResetPartitionOffset("topic", 1, 200, "m3t@") - Expect(subject.offsets).To(HaveLen(2)) - Expect(subject.offsets).To(HaveKeyWithValue( - topicPartition{Topic: "topic", Partition: 1}, - offsetInfo{Offset: 200, Metadata: "m3t@"}, - )) - - }) - -}) diff --git a/third/github.com/johng-cn/sarama-cluster/partitions_test.go b/third/github.com/johng-cn/sarama-cluster/partitions_test.go deleted file mode 100644 index 8857e17b7..000000000 --- a/third/github.com/johng-cn/sarama-cluster/partitions_test.go +++ /dev/null @@ -1,137 +0,0 @@ -package cluster - -import ( - "gitee.com/johng/gf/third/github.com/Shopify/sarama" - . "gitee.com/johng/gf/third/github.com/onsi/ginkgo" - . "gitee.com/johng/gf/third/github.com/onsi/gomega" -) - -var _ = Describe("partitionConsumer", func() { - var subject *partitionConsumer - - BeforeEach(func() { - var err error - subject, err = newPartitionConsumer(&mockConsumer{}, "topic", 0, offsetInfo{2000, "m3ta"}, sarama.OffsetOldest) - Expect(err).NotTo(HaveOccurred()) - }) - - AfterEach(func() { - close(subject.dead) - Expect(subject.Close()).NotTo(HaveOccurred()) - }) - - It("should set state", func() { - Expect(subject.getState()).To(Equal(partitionState{ - Info: offsetInfo{2000, "m3ta"}, - })) - }) - - It("should recover from default offset if requested offset is out of bounds", func() { - pc, err := newPartitionConsumer(&mockConsumer{}, "topic", 0, offsetInfo{200, "m3ta"}, sarama.OffsetOldest) - Expect(err).NotTo(HaveOccurred()) - defer pc.Close() - close(pc.dead) - - state := pc.getState() - Expect(state.Info.Offset).To(Equal(int64(-1))) - Expect(state.Info.Metadata).To(Equal("m3ta")) - }) - - It("should update state", func() { - subject.MarkOffset(2001, "met@") // should set state - Expect(subject.getState()).To(Equal(partitionState{ - Info: offsetInfo{2002, "met@"}, - Dirty: true, - })) - - subject.markCommitted(2002) // should reset dirty status - Expect(subject.getState()).To(Equal(partitionState{ - Info: offsetInfo{2002, "met@"}, - })) - - subject.MarkOffset(2001, "me7a") // should not update state - Expect(subject.getState()).To(Equal(partitionState{ - Info: offsetInfo{2002, "met@"}, - })) - - subject.MarkOffset(2002, "me7a") // should bump state - Expect(subject.getState()).To(Equal(partitionState{ - Info: offsetInfo{2003, "me7a"}, - Dirty: true, - })) - - // After committing a later offset, try rewinding back to earlier offset with new metadata. - subject.ResetOffset(2001, "met@") - Expect(subject.getState()).To(Equal(partitionState{ - Info: offsetInfo{2002, "met@"}, - Dirty: true, - })) - - subject.markCommitted(2002) // should not unset state - Expect(subject.getState()).To(Equal(partitionState{ - Info: offsetInfo{2002, "met@"}, - })) - - subject.MarkOffset(2002, "me7a") // should bump state - Expect(subject.getState()).To(Equal(partitionState{ - Info: offsetInfo{2003, "me7a"}, - Dirty: true, - })) - - subject.markCommitted(2003) - Expect(subject.getState()).To(Equal(partitionState{ - Info: offsetInfo{2003, "me7a"}, - })) - }) - -}) - -var _ = Describe("partitionMap", func() { - var subject *partitionMap - - BeforeEach(func() { - subject = newPartitionMap() - }) - - It("should fetch/store", func() { - Expect(subject.Fetch("topic", 0)).To(BeNil()) - - pc, err := newPartitionConsumer(&mockConsumer{}, "topic", 0, offsetInfo{2000, "m3ta"}, sarama.OffsetNewest) - Expect(err).NotTo(HaveOccurred()) - - subject.Store("topic", 0, pc) - Expect(subject.Fetch("topic", 0)).To(Equal(pc)) - Expect(subject.Fetch("topic", 1)).To(BeNil()) - Expect(subject.Fetch("other", 0)).To(BeNil()) - }) - - It("should return info", func() { - pc0, err := newPartitionConsumer(&mockConsumer{}, "topic", 0, offsetInfo{2000, "m3ta"}, sarama.OffsetNewest) - Expect(err).NotTo(HaveOccurred()) - pc1, err := newPartitionConsumer(&mockConsumer{}, "topic", 1, offsetInfo{2000, "m3ta"}, sarama.OffsetNewest) - Expect(err).NotTo(HaveOccurred()) - subject.Store("topic", 0, pc0) - subject.Store("topic", 1, pc1) - - info := subject.Info() - Expect(info).To(HaveLen(1)) - Expect(info).To(HaveKeyWithValue("topic", []int32{0, 1})) - }) - - It("should create snapshots", func() { - pc0, err := newPartitionConsumer(&mockConsumer{}, "topic", 0, offsetInfo{2000, "m3ta"}, sarama.OffsetNewest) - Expect(err).NotTo(HaveOccurred()) - pc1, err := newPartitionConsumer(&mockConsumer{}, "topic", 1, offsetInfo{2000, "m3ta"}, sarama.OffsetNewest) - Expect(err).NotTo(HaveOccurred()) - - subject.Store("topic", 0, pc0) - subject.Store("topic", 1, pc1) - subject.Fetch("topic", 1).MarkOffset(2000, "met@") - - Expect(subject.Snapshot()).To(Equal(map[topicPartition]partitionState{ - {"topic", 0}: {Info: offsetInfo{2000, "m3ta"}, Dirty: false}, - {"topic", 1}: {Info: offsetInfo{2001, "met@"}, Dirty: true}, - })) - }) - -})