From 5d3fd91f0bc00a40ee632fa546b1a70345947fff Mon Sep 17 00:00:00 2001 From: John Date: Thu, 14 Feb 2019 19:21:38 +0800 Subject: [PATCH] remove sarama --- third/github.com/Shopify/sarama/.gitignore | 27 - third/github.com/Shopify/sarama/.travis.yml | 35 - third/github.com/Shopify/sarama/CHANGELOG.md | 623 ---------- third/github.com/Shopify/sarama/LICENSE | 20 - third/github.com/Shopify/sarama/Makefile | 30 - third/github.com/Shopify/sarama/README.md | 39 - third/github.com/Shopify/sarama/Vagrantfile | 20 - .../github.com/Shopify/sarama/acl_bindings.go | 119 -- .../Shopify/sarama/acl_create_request.go | 76 -- .../Shopify/sarama/acl_create_request_test.go | 34 - .../Shopify/sarama/acl_create_response.go | 88 -- .../sarama/acl_create_response_test.go | 41 - .../Shopify/sarama/acl_delete_request.go | 48 - .../Shopify/sarama/acl_delete_request_test.go | 69 -- .../Shopify/sarama/acl_delete_response.go | 155 --- .../sarama/acl_delete_response_test.go | 38 - .../Shopify/sarama/acl_describe_request.go | 25 - .../sarama/acl_describe_request_test.go | 35 - .../Shopify/sarama/acl_describe_response.go | 80 -- .../sarama/acl_describe_response_test.go | 45 - third/github.com/Shopify/sarama/acl_filter.go | 61 - third/github.com/Shopify/sarama/acl_types.go | 42 - .../sarama/add_offsets_to_txn_request.go | 52 - .../sarama/add_offsets_to_txn_request_test.go | 23 - .../sarama/add_offsets_to_txn_response.go | 44 - .../add_offsets_to_txn_response_test.go | 22 - .../sarama/add_partitions_to_txn_request.go | 76 -- .../add_partitions_to_txn_request_test.go | 27 - .../sarama/add_partitions_to_txn_response.go | 108 -- .../add_partitions_to_txn_response_test.go | 31 - third/github.com/Shopify/sarama/admin.go | 382 ------ third/github.com/Shopify/sarama/admin_test.go | 501 -------- .../Shopify/sarama/alter_configs_request.go | 120 -- .../sarama/alter_configs_request_test.go | 86 -- .../Shopify/sarama/alter_configs_response.go | 95 -- .../sarama/alter_configs_response_test.go | 45 - .../Shopify/sarama/api_versions_request.go | 24 - .../sarama/api_versions_request_test.go | 14 - .../Shopify/sarama/api_versions_response.go | 87 -- .../sarama/api_versions_response_test.go | 32 - .../Shopify/sarama/async_producer.go | 1052 ---------------- .../Shopify/sarama/async_producer_test.go | 1095 ----------------- .../Shopify/sarama/balance_strategy.go | 129 -- .../Shopify/sarama/balance_strategy_test.go | 102 -- third/github.com/Shopify/sarama/broker.go | 895 -------------- .../github.com/Shopify/sarama/broker_test.go | 368 ------ third/github.com/Shopify/sarama/client.go | 899 -------------- .../github.com/Shopify/sarama/client_test.go | 661 ---------- .../Shopify/sarama/client_tls_test.go | 206 ---- third/github.com/Shopify/sarama/compress.go | 75 -- third/github.com/Shopify/sarama/config.go | 581 --------- .../Shopify/sarama/config_resource_type.go | 15 - .../github.com/Shopify/sarama/config_test.go | 281 ----- third/github.com/Shopify/sarama/consumer.go | 807 ------------ .../Shopify/sarama/consumer_group.go | 774 ------------ .../Shopify/sarama/consumer_group_members.go | 96 -- .../sarama/consumer_group_members_test.go | 73 -- .../Shopify/sarama/consumer_group_test.go | 58 - .../sarama/consumer_metadata_request.go | 34 - .../sarama/consumer_metadata_request_test.go | 23 - .../sarama/consumer_metadata_response.go | 78 -- .../sarama/consumer_metadata_response_test.go | 44 - .../Shopify/sarama/consumer_test.go | 1036 ---------------- .../github.com/Shopify/sarama/crc32_field.go | 69 -- .../sarama/create_partitions_request.go | 121 -- .../sarama/create_partitions_request_test.go | 50 - .../sarama/create_partitions_response.go | 94 -- .../sarama/create_partitions_response_test.go | 52 - .../Shopify/sarama/create_topics_request.go | 174 --- .../sarama/create_topics_request_test.go | 50 - .../Shopify/sarama/create_topics_response.go | 112 -- .../sarama/create_topics_response_test.go | 52 - third/github.com/Shopify/sarama/decompress.go | 63 - .../Shopify/sarama/delete_groups_request.go | 30 - .../sarama/delete_groups_request_test.go | 34 - .../Shopify/sarama/delete_groups_response.go | 70 -- .../sarama/delete_groups_response_test.go | 57 - .../Shopify/sarama/delete_records_request.go | 126 -- .../sarama/delete_records_request_test.go | 36 - .../Shopify/sarama/delete_records_response.go | 158 --- .../sarama/delete_records_response_test.go | 39 - .../Shopify/sarama/delete_topics_request.go | 48 - .../sarama/delete_topics_request_test.go | 33 - .../Shopify/sarama/delete_topics_response.go | 78 -- .../sarama/delete_topics_response_test.go | 36 - .../sarama/describe_configs_request.go | 112 -- .../sarama/describe_configs_request_test.go | 119 -- .../sarama/describe_configs_response.go | 320 ----- .../sarama/describe_configs_response_test.go | 171 --- .../Shopify/sarama/describe_groups_request.go | 30 - .../sarama/describe_groups_request_test.go | 34 - .../sarama/describe_groups_response.go | 187 --- .../sarama/describe_groups_response_test.go | 91 -- third/github.com/Shopify/sarama/dev.yml | 10 - .../Shopify/sarama/encoder_decoder.go | 89 -- .../Shopify/sarama/end_txn_request.go | 50 - .../Shopify/sarama/end_txn_request_test.go | 23 - .../Shopify/sarama/end_txn_response.go | 44 - .../Shopify/sarama/end_txn_response_test.go | 22 - third/github.com/Shopify/sarama/errors.go | 317 ----- .../Shopify/sarama/fetch_request.go | 170 --- .../Shopify/sarama/fetch_request_test.go | 48 - .../Shopify/sarama/fetch_response.go | 396 ------ .../Shopify/sarama/fetch_response_test.go | 331 ----- .../sarama/find_coordinator_request.go | 61 - .../sarama/find_coordinator_request_test.go | 33 - .../sarama/find_coordinator_response.go | 92 -- .../sarama/find_coordinator_response_test.go | 83 -- .../Shopify/sarama/functional_client_test.go | 90 -- .../sarama/functional_consumer_group_test.go | 418 ------- .../sarama/functional_consumer_test.go | 244 ---- .../sarama/functional_offset_manager_test.go | 47 - .../sarama/functional_producer_test.go | 323 ----- .../Shopify/sarama/functional_test.go | 148 --- .../Shopify/sarama/heartbeat_request.go | 47 - .../Shopify/sarama/heartbeat_request_test.go | 21 - .../Shopify/sarama/heartbeat_response.go | 32 - .../Shopify/sarama/heartbeat_response_test.go | 18 - .../sarama/init_producer_id_request.go | 43 - .../sarama/init_producer_id_request_test.go | 31 - .../sarama/init_producer_id_response.go | 55 - .../sarama/init_producer_id_response_test.go | 37 - .../Shopify/sarama/join_group_request.go | 163 --- .../Shopify/sarama/join_group_request_test.go | 83 -- .../Shopify/sarama/join_group_response.go | 135 -- .../sarama/join_group_response_test.go | 172 --- .../Shopify/sarama/leave_group_request.go | 40 - .../sarama/leave_group_request_test.go | 19 - .../Shopify/sarama/leave_group_response.go | 32 - .../sarama/leave_group_response_test.go | 24 - .../github.com/Shopify/sarama/length_field.go | 82 -- .../Shopify/sarama/list_groups_request.go | 24 - .../sarama/list_groups_request_test.go | 7 - .../Shopify/sarama/list_groups_response.go | 69 -- .../sarama/list_groups_response_test.go | 58 - third/github.com/Shopify/sarama/message.go | 157 --- .../github.com/Shopify/sarama/message_set.go | 108 -- .../github.com/Shopify/sarama/message_test.go | 240 ---- .../Shopify/sarama/metadata_request.go | 88 -- .../Shopify/sarama/metadata_request_test.go | 76 -- .../Shopify/sarama/metadata_response.go | 321 ----- .../Shopify/sarama/metadata_response_test.go | 290 ----- third/github.com/Shopify/sarama/metrics.go | 51 - .../github.com/Shopify/sarama/metrics_test.go | 172 --- third/github.com/Shopify/sarama/mockbroker.go | 330 ----- .../Shopify/sarama/mockresponses.go | 727 ----------- .../github.com/Shopify/sarama/mocks/README.md | 13 - .../Shopify/sarama/mocks/async_producer.go | 173 --- .../sarama/mocks/async_producer_test.go | 132 -- .../Shopify/sarama/mocks/consumer.go | 315 ----- .../Shopify/sarama/mocks/consumer_test.go | 249 ---- .../github.com/Shopify/sarama/mocks/mocks.go | 48 - .../Shopify/sarama/mocks/sync_producer.go | 157 --- .../sarama/mocks/sync_producer_test.go | 250 ---- .../Shopify/sarama/offset_commit_request.go | 210 ---- .../sarama/offset_commit_request_test.go | 95 -- .../Shopify/sarama/offset_commit_response.go | 110 -- .../sarama/offset_commit_response_test.go | 39 - .../Shopify/sarama/offset_fetch_request.go | 100 -- .../sarama/offset_fetch_request_test.go | 55 - .../Shopify/sarama/offset_fetch_response.go | 197 --- .../sarama/offset_fetch_response_test.go | 65 - .../Shopify/sarama/offset_manager.go | 572 --------- .../Shopify/sarama/offset_manager_test.go | 421 ------- .../Shopify/sarama/offset_request.go | 156 --- .../Shopify/sarama/offset_request_test.go | 59 - .../Shopify/sarama/offset_response.go | 174 --- .../Shopify/sarama/offset_response_test.go | 111 -- .../Shopify/sarama/packet_decoder.go | 60 - .../Shopify/sarama/packet_encoder.go | 65 - .../github.com/Shopify/sarama/partitioner.go | 217 ---- .../Shopify/sarama/partitioner_test.go | 283 ----- .../github.com/Shopify/sarama/prep_encoder.go | 153 --- .../Shopify/sarama/produce_request.go | 252 ---- .../Shopify/sarama/produce_request_test.go | 106 -- .../Shopify/sarama/produce_response.go | 189 --- .../Shopify/sarama/produce_response_test.go | 128 -- .../github.com/Shopify/sarama/produce_set.go | 259 ---- .../Shopify/sarama/produce_set_test.go | 346 ------ .../github.com/Shopify/sarama/real_decoder.go | 324 ----- .../github.com/Shopify/sarama/real_encoder.go | 156 --- third/github.com/Shopify/sarama/record.go | 113 -- .../github.com/Shopify/sarama/record_batch.go | 208 ---- .../github.com/Shopify/sarama/record_test.go | 292 ----- third/github.com/Shopify/sarama/records.go | 194 --- .../github.com/Shopify/sarama/records_test.go | 143 --- third/github.com/Shopify/sarama/request.go | 149 --- .../github.com/Shopify/sarama/request_test.go | 105 -- .../Shopify/sarama/response_header.go | 21 - .../Shopify/sarama/response_header_test.go | 21 - third/github.com/Shopify/sarama/sarama.go | 99 -- .../Shopify/sarama/sasl_handshake_request.go | 33 - .../sarama/sasl_handshake_request_test.go | 17 - .../Shopify/sarama/sasl_handshake_response.go | 38 - .../sarama/sasl_handshake_response_test.go | 24 - .../Shopify/sarama/sync_group_request.go | 100 -- .../Shopify/sarama/sync_group_request_test.go | 38 - .../Shopify/sarama/sync_group_response.go | 41 - .../sarama/sync_group_response_test.go | 40 - .../Shopify/sarama/sync_producer.go | 149 --- .../Shopify/sarama/sync_producer_test.go | 199 --- third/github.com/Shopify/sarama/timestamp.go | 40 - .../github.com/Shopify/sarama/tools/README.md | 10 - .../tools/kafka-console-consumer/.gitignore | 2 - .../tools/kafka-console-consumer/README.md | 29 - .../kafka-console-consumer.go | 145 --- .../.gitignore | 2 - .../kafka-console-partitionconsumer/README.md | 28 - .../kafka-console-partitionconsumer.go | 102 -- .../tools/kafka-console-producer/.gitignore | 2 - .../tools/kafka-console-producer/README.md | 34 - .../kafka-console-producer.go | 124 -- .../sarama/txn_offset_commit_request.go | 126 -- .../sarama/txn_offset_commit_request_test.go | 35 - .../sarama/txn_offset_commit_response.go | 83 -- .../sarama/txn_offset_commit_response_test.go | 31 - third/github.com/Shopify/sarama/utils.go | 220 ---- third/github.com/Shopify/sarama/utils_test.go | 41 - .../Shopify/sarama/vagrant/boot_cluster.sh | 22 - .../Shopify/sarama/vagrant/create_topics.sh | 8 - .../Shopify/sarama/vagrant/halt_cluster.sh | 15 - .../Shopify/sarama/vagrant/install_cluster.sh | 49 - .../Shopify/sarama/vagrant/kafka.conf | 9 - .../Shopify/sarama/vagrant/provision.sh | 15 - .../Shopify/sarama/vagrant/run_toxiproxy.sh | 22 - .../Shopify/sarama/vagrant/server.properties | 127 -- .../Shopify/sarama/vagrant/setup_services.sh | 29 - .../Shopify/sarama/vagrant/toxiproxy.conf | 6 - .../Shopify/sarama/vagrant/zookeeper.conf | 7 - .../sarama/vagrant/zookeeper.properties | 36 - third/github.com/Shopify/sarama/zstd_cgo.go | 13 - .../Shopify/sarama/zstd_fallback.go | 17 - 232 files changed, 32030 deletions(-) delete mode 100644 third/github.com/Shopify/sarama/.gitignore delete mode 100644 third/github.com/Shopify/sarama/.travis.yml delete mode 100644 third/github.com/Shopify/sarama/CHANGELOG.md delete mode 100644 third/github.com/Shopify/sarama/LICENSE delete mode 100644 third/github.com/Shopify/sarama/Makefile delete mode 100644 third/github.com/Shopify/sarama/README.md delete mode 100644 third/github.com/Shopify/sarama/Vagrantfile delete mode 100644 third/github.com/Shopify/sarama/acl_bindings.go delete mode 100644 third/github.com/Shopify/sarama/acl_create_request.go delete mode 100644 third/github.com/Shopify/sarama/acl_create_request_test.go delete mode 100644 third/github.com/Shopify/sarama/acl_create_response.go delete mode 100644 third/github.com/Shopify/sarama/acl_create_response_test.go delete mode 100644 third/github.com/Shopify/sarama/acl_delete_request.go delete mode 100644 third/github.com/Shopify/sarama/acl_delete_request_test.go delete mode 100644 third/github.com/Shopify/sarama/acl_delete_response.go delete mode 100644 third/github.com/Shopify/sarama/acl_delete_response_test.go delete mode 100644 third/github.com/Shopify/sarama/acl_describe_request.go delete mode 100644 third/github.com/Shopify/sarama/acl_describe_request_test.go delete mode 100644 third/github.com/Shopify/sarama/acl_describe_response.go delete mode 100644 third/github.com/Shopify/sarama/acl_describe_response_test.go delete mode 100644 third/github.com/Shopify/sarama/acl_filter.go delete mode 100644 third/github.com/Shopify/sarama/acl_types.go delete mode 100644 third/github.com/Shopify/sarama/add_offsets_to_txn_request.go delete mode 100644 third/github.com/Shopify/sarama/add_offsets_to_txn_request_test.go delete mode 100644 third/github.com/Shopify/sarama/add_offsets_to_txn_response.go delete mode 100644 third/github.com/Shopify/sarama/add_offsets_to_txn_response_test.go delete mode 100644 third/github.com/Shopify/sarama/add_partitions_to_txn_request.go delete mode 100644 third/github.com/Shopify/sarama/add_partitions_to_txn_request_test.go delete mode 100644 third/github.com/Shopify/sarama/add_partitions_to_txn_response.go delete mode 100644 third/github.com/Shopify/sarama/add_partitions_to_txn_response_test.go delete mode 100644 third/github.com/Shopify/sarama/admin.go delete mode 100644 third/github.com/Shopify/sarama/admin_test.go delete mode 100644 third/github.com/Shopify/sarama/alter_configs_request.go delete mode 100644 third/github.com/Shopify/sarama/alter_configs_request_test.go delete mode 100644 third/github.com/Shopify/sarama/alter_configs_response.go delete mode 100644 third/github.com/Shopify/sarama/alter_configs_response_test.go delete mode 100644 third/github.com/Shopify/sarama/api_versions_request.go delete mode 100644 third/github.com/Shopify/sarama/api_versions_request_test.go delete mode 100644 third/github.com/Shopify/sarama/api_versions_response.go delete mode 100644 third/github.com/Shopify/sarama/api_versions_response_test.go delete mode 100644 third/github.com/Shopify/sarama/async_producer.go delete mode 100644 third/github.com/Shopify/sarama/async_producer_test.go delete mode 100644 third/github.com/Shopify/sarama/balance_strategy.go delete mode 100644 third/github.com/Shopify/sarama/balance_strategy_test.go delete mode 100644 third/github.com/Shopify/sarama/broker.go delete mode 100644 third/github.com/Shopify/sarama/broker_test.go delete mode 100644 third/github.com/Shopify/sarama/client.go delete mode 100644 third/github.com/Shopify/sarama/client_test.go delete mode 100644 third/github.com/Shopify/sarama/client_tls_test.go delete mode 100644 third/github.com/Shopify/sarama/compress.go delete mode 100644 third/github.com/Shopify/sarama/config.go delete mode 100644 third/github.com/Shopify/sarama/config_resource_type.go delete mode 100644 third/github.com/Shopify/sarama/config_test.go delete mode 100644 third/github.com/Shopify/sarama/consumer.go delete mode 100644 third/github.com/Shopify/sarama/consumer_group.go delete mode 100644 third/github.com/Shopify/sarama/consumer_group_members.go delete mode 100644 third/github.com/Shopify/sarama/consumer_group_members_test.go delete mode 100644 third/github.com/Shopify/sarama/consumer_group_test.go delete mode 100644 third/github.com/Shopify/sarama/consumer_metadata_request.go delete mode 100644 third/github.com/Shopify/sarama/consumer_metadata_request_test.go delete mode 100644 third/github.com/Shopify/sarama/consumer_metadata_response.go delete mode 100644 third/github.com/Shopify/sarama/consumer_metadata_response_test.go delete mode 100644 third/github.com/Shopify/sarama/consumer_test.go delete mode 100644 third/github.com/Shopify/sarama/crc32_field.go delete mode 100644 third/github.com/Shopify/sarama/create_partitions_request.go delete mode 100644 third/github.com/Shopify/sarama/create_partitions_request_test.go delete mode 100644 third/github.com/Shopify/sarama/create_partitions_response.go delete mode 100644 third/github.com/Shopify/sarama/create_partitions_response_test.go delete mode 100644 third/github.com/Shopify/sarama/create_topics_request.go delete mode 100644 third/github.com/Shopify/sarama/create_topics_request_test.go delete mode 100644 third/github.com/Shopify/sarama/create_topics_response.go delete mode 100644 third/github.com/Shopify/sarama/create_topics_response_test.go delete mode 100644 third/github.com/Shopify/sarama/decompress.go delete mode 100644 third/github.com/Shopify/sarama/delete_groups_request.go delete mode 100644 third/github.com/Shopify/sarama/delete_groups_request_test.go delete mode 100644 third/github.com/Shopify/sarama/delete_groups_response.go delete mode 100644 third/github.com/Shopify/sarama/delete_groups_response_test.go delete mode 100644 third/github.com/Shopify/sarama/delete_records_request.go delete mode 100644 third/github.com/Shopify/sarama/delete_records_request_test.go delete mode 100644 third/github.com/Shopify/sarama/delete_records_response.go delete mode 100644 third/github.com/Shopify/sarama/delete_records_response_test.go delete mode 100644 third/github.com/Shopify/sarama/delete_topics_request.go delete mode 100644 third/github.com/Shopify/sarama/delete_topics_request_test.go delete mode 100644 third/github.com/Shopify/sarama/delete_topics_response.go delete mode 100644 third/github.com/Shopify/sarama/delete_topics_response_test.go delete mode 100644 third/github.com/Shopify/sarama/describe_configs_request.go delete mode 100644 third/github.com/Shopify/sarama/describe_configs_request_test.go delete mode 100644 third/github.com/Shopify/sarama/describe_configs_response.go delete mode 100644 third/github.com/Shopify/sarama/describe_configs_response_test.go delete mode 100644 third/github.com/Shopify/sarama/describe_groups_request.go delete mode 100644 third/github.com/Shopify/sarama/describe_groups_request_test.go delete mode 100644 third/github.com/Shopify/sarama/describe_groups_response.go delete mode 100644 third/github.com/Shopify/sarama/describe_groups_response_test.go delete mode 100644 third/github.com/Shopify/sarama/dev.yml delete mode 100644 third/github.com/Shopify/sarama/encoder_decoder.go delete mode 100644 third/github.com/Shopify/sarama/end_txn_request.go delete mode 100644 third/github.com/Shopify/sarama/end_txn_request_test.go delete mode 100644 third/github.com/Shopify/sarama/end_txn_response.go delete mode 100644 third/github.com/Shopify/sarama/end_txn_response_test.go delete mode 100644 third/github.com/Shopify/sarama/errors.go delete mode 100644 third/github.com/Shopify/sarama/fetch_request.go delete mode 100644 third/github.com/Shopify/sarama/fetch_request_test.go delete mode 100644 third/github.com/Shopify/sarama/fetch_response.go delete mode 100644 third/github.com/Shopify/sarama/fetch_response_test.go delete mode 100644 third/github.com/Shopify/sarama/find_coordinator_request.go delete mode 100644 third/github.com/Shopify/sarama/find_coordinator_request_test.go delete mode 100644 third/github.com/Shopify/sarama/find_coordinator_response.go delete mode 100644 third/github.com/Shopify/sarama/find_coordinator_response_test.go delete mode 100644 third/github.com/Shopify/sarama/functional_client_test.go delete mode 100644 third/github.com/Shopify/sarama/functional_consumer_group_test.go delete mode 100644 third/github.com/Shopify/sarama/functional_consumer_test.go delete mode 100644 third/github.com/Shopify/sarama/functional_offset_manager_test.go delete mode 100644 third/github.com/Shopify/sarama/functional_producer_test.go delete mode 100644 third/github.com/Shopify/sarama/functional_test.go delete mode 100644 third/github.com/Shopify/sarama/heartbeat_request.go delete mode 100644 third/github.com/Shopify/sarama/heartbeat_request_test.go delete mode 100644 third/github.com/Shopify/sarama/heartbeat_response.go delete mode 100644 third/github.com/Shopify/sarama/heartbeat_response_test.go delete mode 100644 third/github.com/Shopify/sarama/init_producer_id_request.go delete mode 100644 third/github.com/Shopify/sarama/init_producer_id_request_test.go delete mode 100644 third/github.com/Shopify/sarama/init_producer_id_response.go delete mode 100644 third/github.com/Shopify/sarama/init_producer_id_response_test.go delete mode 100644 third/github.com/Shopify/sarama/join_group_request.go delete mode 100644 third/github.com/Shopify/sarama/join_group_request_test.go delete mode 100644 third/github.com/Shopify/sarama/join_group_response.go delete mode 100644 third/github.com/Shopify/sarama/join_group_response_test.go delete mode 100644 third/github.com/Shopify/sarama/leave_group_request.go delete mode 100644 third/github.com/Shopify/sarama/leave_group_request_test.go delete mode 100644 third/github.com/Shopify/sarama/leave_group_response.go delete mode 100644 third/github.com/Shopify/sarama/leave_group_response_test.go delete mode 100644 third/github.com/Shopify/sarama/length_field.go delete mode 100644 third/github.com/Shopify/sarama/list_groups_request.go delete mode 100644 third/github.com/Shopify/sarama/list_groups_request_test.go delete mode 100644 third/github.com/Shopify/sarama/list_groups_response.go delete mode 100644 third/github.com/Shopify/sarama/list_groups_response_test.go delete mode 100644 third/github.com/Shopify/sarama/message.go delete mode 100644 third/github.com/Shopify/sarama/message_set.go delete mode 100644 third/github.com/Shopify/sarama/message_test.go delete mode 100644 third/github.com/Shopify/sarama/metadata_request.go delete mode 100644 third/github.com/Shopify/sarama/metadata_request_test.go delete mode 100644 third/github.com/Shopify/sarama/metadata_response.go delete mode 100644 third/github.com/Shopify/sarama/metadata_response_test.go delete mode 100644 third/github.com/Shopify/sarama/metrics.go delete mode 100644 third/github.com/Shopify/sarama/metrics_test.go delete mode 100644 third/github.com/Shopify/sarama/mockbroker.go delete mode 100644 third/github.com/Shopify/sarama/mockresponses.go delete mode 100644 third/github.com/Shopify/sarama/mocks/README.md delete mode 100644 third/github.com/Shopify/sarama/mocks/async_producer.go delete mode 100644 third/github.com/Shopify/sarama/mocks/async_producer_test.go delete mode 100644 third/github.com/Shopify/sarama/mocks/consumer.go delete mode 100644 third/github.com/Shopify/sarama/mocks/consumer_test.go delete mode 100644 third/github.com/Shopify/sarama/mocks/mocks.go delete mode 100644 third/github.com/Shopify/sarama/mocks/sync_producer.go delete mode 100644 third/github.com/Shopify/sarama/mocks/sync_producer_test.go delete mode 100644 third/github.com/Shopify/sarama/offset_commit_request.go delete mode 100644 third/github.com/Shopify/sarama/offset_commit_request_test.go delete mode 100644 third/github.com/Shopify/sarama/offset_commit_response.go delete mode 100644 third/github.com/Shopify/sarama/offset_commit_response_test.go delete mode 100644 third/github.com/Shopify/sarama/offset_fetch_request.go delete mode 100644 third/github.com/Shopify/sarama/offset_fetch_request_test.go delete mode 100644 third/github.com/Shopify/sarama/offset_fetch_response.go delete mode 100644 third/github.com/Shopify/sarama/offset_fetch_response_test.go delete mode 100644 third/github.com/Shopify/sarama/offset_manager.go delete mode 100644 third/github.com/Shopify/sarama/offset_manager_test.go delete mode 100644 third/github.com/Shopify/sarama/offset_request.go delete mode 100644 third/github.com/Shopify/sarama/offset_request_test.go delete mode 100644 third/github.com/Shopify/sarama/offset_response.go delete mode 100644 third/github.com/Shopify/sarama/offset_response_test.go delete mode 100644 third/github.com/Shopify/sarama/packet_decoder.go delete mode 100644 third/github.com/Shopify/sarama/packet_encoder.go delete mode 100644 third/github.com/Shopify/sarama/partitioner.go delete mode 100644 third/github.com/Shopify/sarama/partitioner_test.go delete mode 100644 third/github.com/Shopify/sarama/prep_encoder.go delete mode 100644 third/github.com/Shopify/sarama/produce_request.go delete mode 100644 third/github.com/Shopify/sarama/produce_request_test.go delete mode 100644 third/github.com/Shopify/sarama/produce_response.go delete mode 100644 third/github.com/Shopify/sarama/produce_response_test.go delete mode 100644 third/github.com/Shopify/sarama/produce_set.go delete mode 100644 third/github.com/Shopify/sarama/produce_set_test.go delete mode 100644 third/github.com/Shopify/sarama/real_decoder.go delete mode 100644 third/github.com/Shopify/sarama/real_encoder.go delete mode 100644 third/github.com/Shopify/sarama/record.go delete mode 100644 third/github.com/Shopify/sarama/record_batch.go delete mode 100644 third/github.com/Shopify/sarama/record_test.go delete mode 100644 third/github.com/Shopify/sarama/records.go delete mode 100644 third/github.com/Shopify/sarama/records_test.go delete mode 100644 third/github.com/Shopify/sarama/request.go delete mode 100644 third/github.com/Shopify/sarama/request_test.go delete mode 100644 third/github.com/Shopify/sarama/response_header.go delete mode 100644 third/github.com/Shopify/sarama/response_header_test.go delete mode 100644 third/github.com/Shopify/sarama/sarama.go delete mode 100644 third/github.com/Shopify/sarama/sasl_handshake_request.go delete mode 100644 third/github.com/Shopify/sarama/sasl_handshake_request_test.go delete mode 100644 third/github.com/Shopify/sarama/sasl_handshake_response.go delete mode 100644 third/github.com/Shopify/sarama/sasl_handshake_response_test.go delete mode 100644 third/github.com/Shopify/sarama/sync_group_request.go delete mode 100644 third/github.com/Shopify/sarama/sync_group_request_test.go delete mode 100644 third/github.com/Shopify/sarama/sync_group_response.go delete mode 100644 third/github.com/Shopify/sarama/sync_group_response_test.go delete mode 100644 third/github.com/Shopify/sarama/sync_producer.go delete mode 100644 third/github.com/Shopify/sarama/sync_producer_test.go delete mode 100644 third/github.com/Shopify/sarama/timestamp.go delete mode 100644 third/github.com/Shopify/sarama/tools/README.md delete mode 100644 third/github.com/Shopify/sarama/tools/kafka-console-consumer/.gitignore delete mode 100644 third/github.com/Shopify/sarama/tools/kafka-console-consumer/README.md delete mode 100644 third/github.com/Shopify/sarama/tools/kafka-console-consumer/kafka-console-consumer.go delete mode 100644 third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/.gitignore delete mode 100644 third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/README.md delete mode 100644 third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/kafka-console-partitionconsumer.go delete mode 100644 third/github.com/Shopify/sarama/tools/kafka-console-producer/.gitignore delete mode 100644 third/github.com/Shopify/sarama/tools/kafka-console-producer/README.md delete mode 100644 third/github.com/Shopify/sarama/tools/kafka-console-producer/kafka-console-producer.go delete mode 100644 third/github.com/Shopify/sarama/txn_offset_commit_request.go delete mode 100644 third/github.com/Shopify/sarama/txn_offset_commit_request_test.go delete mode 100644 third/github.com/Shopify/sarama/txn_offset_commit_response.go delete mode 100644 third/github.com/Shopify/sarama/txn_offset_commit_response_test.go delete mode 100644 third/github.com/Shopify/sarama/utils.go delete mode 100644 third/github.com/Shopify/sarama/utils_test.go delete mode 100755 third/github.com/Shopify/sarama/vagrant/boot_cluster.sh delete mode 100755 third/github.com/Shopify/sarama/vagrant/create_topics.sh delete mode 100755 third/github.com/Shopify/sarama/vagrant/halt_cluster.sh delete mode 100755 third/github.com/Shopify/sarama/vagrant/install_cluster.sh delete mode 100644 third/github.com/Shopify/sarama/vagrant/kafka.conf delete mode 100755 third/github.com/Shopify/sarama/vagrant/provision.sh delete mode 100755 third/github.com/Shopify/sarama/vagrant/run_toxiproxy.sh delete mode 100644 third/github.com/Shopify/sarama/vagrant/server.properties delete mode 100755 third/github.com/Shopify/sarama/vagrant/setup_services.sh delete mode 100644 third/github.com/Shopify/sarama/vagrant/toxiproxy.conf delete mode 100644 third/github.com/Shopify/sarama/vagrant/zookeeper.conf delete mode 100644 third/github.com/Shopify/sarama/vagrant/zookeeper.properties delete mode 100644 third/github.com/Shopify/sarama/zstd_cgo.go delete mode 100644 third/github.com/Shopify/sarama/zstd_fallback.go diff --git a/third/github.com/Shopify/sarama/.gitignore b/third/github.com/Shopify/sarama/.gitignore deleted file mode 100644 index 6e362e4f2..000000000 --- a/third/github.com/Shopify/sarama/.gitignore +++ /dev/null @@ -1,27 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so -*.test - -# Folders -_obj -_test -.vagrant - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe - -coverage.txt -profile.out diff --git a/third/github.com/Shopify/sarama/.travis.yml b/third/github.com/Shopify/sarama/.travis.yml deleted file mode 100644 index eb54a0ddc..000000000 --- a/third/github.com/Shopify/sarama/.travis.yml +++ /dev/null @@ -1,35 +0,0 @@ -language: go -go: -- 1.10.x -- 1.11.x - -env: - global: - - KAFKA_PEERS=localhost:9091,localhost:9092,localhost:9093,localhost:9094,localhost:9095 - - TOXIPROXY_ADDR=http://localhost:8474 - - KAFKA_INSTALL_ROOT=/home/travis/kafka - - KAFKA_HOSTNAME=localhost - - DEBUG=true - matrix: - - KAFKA_VERSION=1.1.1 - - KAFKA_VERSION=2.0.1 - - KAFKA_VERSION=2.1.0 - -before_install: -- export REPOSITORY_ROOT=${TRAVIS_BUILD_DIR} -- vagrant/install_cluster.sh -- vagrant/boot_cluster.sh -- vagrant/create_topics.sh - -install: make install_dependencies - -script: -- make test -- make vet -- make errcheck -- if [[ "$TRAVIS_GO_VERSION" == 1.11* ]]; then make fmt; fi - -after_success: -- bash <(curl -s https://codecov.io/bash) - -after_script: vagrant/halt_cluster.sh diff --git a/third/github.com/Shopify/sarama/CHANGELOG.md b/third/github.com/Shopify/sarama/CHANGELOG.md deleted file mode 100644 index 9f955d5e1..000000000 --- a/third/github.com/Shopify/sarama/CHANGELOG.md +++ /dev/null @@ -1,623 +0,0 @@ -# Changelog - -#### Version 1.20.0 (2018-12-10) - -New Features: - - Add support for zstd compression - ([#1170](https://github.com/Shopify/sarama/pull/1170)). - - Add support for Idempotent Producer - ([#1152](https://github.com/Shopify/sarama/pull/1152)). - - Add support support for Kafka 2.1.0 - ([#1229](https://github.com/Shopify/sarama/pull/1229)). - - Add support support for OffsetCommit request/response pairs versions v1 to v5 - ([#1201](https://github.com/Shopify/sarama/pull/1201)). - - Add support support for OffsetFetch request/response pair up to version v5 - ([#1198](https://github.com/Shopify/sarama/pull/1198)). - -Improvements: - - Export broker's Rack setting - ([#1173](https://github.com/Shopify/sarama/pull/1173)). - - Always use latest patch version of Go on CI - ([#1202](https://github.com/Shopify/sarama/pull/1202)). - - Add error codes 61 to 72 - ([#1195](https://github.com/Shopify/sarama/pull/1195)). - -Bug Fixes: - - Fix build without cgo - ([#1182](https://github.com/Shopify/sarama/pull/1182)). - - Fix go vet suggestion in consumer group file - ([#1209](https://github.com/Shopify/sarama/pull/1209)). - - Fix typos in code and comments - ([#1228](https://github.com/Shopify/sarama/pull/1228)). - -#### Version 1.19.0 (2018-09-27) - -New Features: - - Implement a higher-level consumer group - ([#1099](https://github.com/Shopify/sarama/pull/1099)). - -Improvements: - - Add support for Go 1.11 - ([#1176](https://github.com/Shopify/sarama/pull/1176)). - -Bug Fixes: - - Fix encoding of `MetadataResponse` with version 2 and higher - ([#1174](https://github.com/Shopify/sarama/pull/1174)). - - Fix race condition in mock async producer - ([#1174](https://github.com/Shopify/sarama/pull/1174)). - -#### Version 1.18.0 (2018-09-07) - -New Features: - - Make `Partitioner.RequiresConsistency` vary per-message - ([#1112](https://github.com/Shopify/sarama/pull/1112)). - - Add customizable partitioner - ([#1118](https://github.com/Shopify/sarama/pull/1118)). - - Add `ClusterAdmin` support for `CreateTopic`, `DeleteTopic`, `CreatePartitions`, - `DeleteRecords`, `DescribeConfig`, `AlterConfig`, `CreateACL`, `ListAcls`, `DeleteACL` - ([#1055](https://github.com/Shopify/sarama/pull/1055)). - -Improvements: - - Add support for Kafka 2.0.0 - ([#1149](https://github.com/Shopify/sarama/pull/1149)). - - Allow setting `LocalAddr` when dialing an address to support multi-homed hosts - ([#1123](https://github.com/Shopify/sarama/pull/1123)). - - Simpler offset management - ([#1127](https://github.com/Shopify/sarama/pull/1127)). - -Bug Fixes: - - Fix mutation of `ProducerMessage.MetaData` when producing to Kafka - ([#1110](https://github.com/Shopify/sarama/pull/1110)). - - Fix consumer block when response did not contain all the - expected topic/partition blocks - ([#1086](https://github.com/Shopify/sarama/pull/1086)). - - Fix consumer block when response contains only constrol messages - ([#1115](https://github.com/Shopify/sarama/pull/1115)). - - Add timeout config for ClusterAdmin requests - ([#1142](https://github.com/Shopify/sarama/pull/1142)). - - Add version check when producing message with headers - ([#1117](https://github.com/Shopify/sarama/pull/1117)). - - Fix `MetadataRequest` for empty list of topics - ([#1132](https://github.com/Shopify/sarama/pull/1132)). - - Fix producer topic metadata on-demand fetch when topic error happens in metadata response - ([#1125](https://github.com/Shopify/sarama/pull/1125)). - -#### Version 1.17.0 (2018-05-30) - -New Features: - - Add support for gzip compression levels - ([#1044](https://github.com/Shopify/sarama/pull/1044)). - - Add support for Metadata request/response pairs versions v1 to v5 - ([#1047](https://github.com/Shopify/sarama/pull/1047), - [#1069](https://github.com/Shopify/sarama/pull/1069)). - - Add versioning to JoinGroup request/response pairs - ([#1098](https://github.com/Shopify/sarama/pull/1098)) - - Add support for CreatePartitions, DeleteGroups, DeleteRecords request/response pairs - ([#1065](https://github.com/Shopify/sarama/pull/1065), - [#1096](https://github.com/Shopify/sarama/pull/1096), - [#1027](https://github.com/Shopify/sarama/pull/1027)). - - Add `Controller()` method to Client interface - ([#1063](https://github.com/Shopify/sarama/pull/1063)). - -Improvements: - - ConsumerMetadataReq/Resp has been migrated to FindCoordinatorReq/Resp - ([#1010](https://github.com/Shopify/sarama/pull/1010)). - - Expose missing protocol parts: `msgSet` and `recordBatch` - ([#1049](https://github.com/Shopify/sarama/pull/1049)). - - Add support for v1 DeleteTopics Request - ([#1052](https://github.com/Shopify/sarama/pull/1052)). - - Add support for Go 1.10 - ([#1064](https://github.com/Shopify/sarama/pull/1064)). - - Claim support for Kafka 1.1.0 - ([#1073](https://github.com/Shopify/sarama/pull/1073)). - -Bug Fixes: - - Fix FindCoordinatorResponse.encode to allow nil Coordinator - ([#1050](https://github.com/Shopify/sarama/pull/1050), - [#1051](https://github.com/Shopify/sarama/pull/1051)). - - Clear all metadata when we have the latest topic info - ([#1033](https://github.com/Shopify/sarama/pull/1033)). - - Make `PartitionConsumer.Close` idempotent - ([#1092](https://github.com/Shopify/sarama/pull/1092)). - -#### Version 1.16.0 (2018-02-12) - -New Features: - - Add support for the Create/Delete Topics request/response pairs - ([#1007](https://github.com/Shopify/sarama/pull/1007), - [#1008](https://github.com/Shopify/sarama/pull/1008)). - - Add support for the Describe/Create/Delete ACL request/response pairs - ([#1009](https://github.com/Shopify/sarama/pull/1009)). - - Add support for the five transaction-related request/response pairs - ([#1016](https://github.com/Shopify/sarama/pull/1016)). - -Improvements: - - Permit setting version on mock producer responses - ([#999](https://github.com/Shopify/sarama/pull/999)). - - Add `NewMockBrokerListener` helper for testing TLS connections - ([#1019](https://github.com/Shopify/sarama/pull/1019)). - - Changed the default value for `Consumer.Fetch.Default` from 32KiB to 1MiB - which results in much higher throughput in most cases - ([#1024](https://github.com/Shopify/sarama/pull/1024)). - - Reuse the `time.Ticker` across fetch requests in the PartitionConsumer to - reduce CPU and memory usage when processing many partitions - ([#1028](https://github.com/Shopify/sarama/pull/1028)). - - Assign relative offsets to messages in the producer to save the brokers a - recompression pass - ([#1002](https://github.com/Shopify/sarama/pull/1002), - [#1015](https://github.com/Shopify/sarama/pull/1015)). - -Bug Fixes: - - Fix producing uncompressed batches with the new protocol format - ([#1032](https://github.com/Shopify/sarama/issues/1032)). - - Fix consuming compacted topics with the new protocol format - ([#1005](https://github.com/Shopify/sarama/issues/1005)). - - Fix consuming topics with a mix of protocol formats - ([#1021](https://github.com/Shopify/sarama/issues/1021)). - - Fix consuming when the broker includes multiple batches in a single response - ([#1022](https://github.com/Shopify/sarama/issues/1022)). - - Fix detection of `PartialTrailingMessage` when the partial message was - truncated before the magic value indicating its version - ([#1030](https://github.com/Shopify/sarama/pull/1030)). - - Fix expectation-checking in the mock of `SyncProducer.SendMessages` - ([#1035](https://github.com/Shopify/sarama/pull/1035)). - -#### Version 1.15.0 (2017-12-08) - -New Features: - - Claim official support for Kafka 1.0, though it did already work - ([#984](https://github.com/Shopify/sarama/pull/984)). - - Helper methods for Kafka version numbers to/from strings - ([#989](https://github.com/Shopify/sarama/pull/989)). - - Implement CreatePartitions request/response - ([#985](https://github.com/Shopify/sarama/pull/985)). - -Improvements: - - Add error codes 45-60 - ([#986](https://github.com/Shopify/sarama/issues/986)). - -Bug Fixes: - - Fix slow consuming for certain Kafka 0.11/1.0 configurations - ([#982](https://github.com/Shopify/sarama/pull/982)). - - Correctly determine when a FetchResponse contains the new message format - ([#990](https://github.com/Shopify/sarama/pull/990)). - - Fix producing with multiple headers - ([#996](https://github.com/Shopify/sarama/pull/996)). - - Fix handling of truncated record batches - ([#998](https://github.com/Shopify/sarama/pull/998)). - - Fix leaking metrics when closing brokers - ([#991](https://github.com/Shopify/sarama/pull/991)). - -#### Version 1.14.0 (2017-11-13) - -New Features: - - Add support for the new Kafka 0.11 record-batch format, including the wire - protocol and the necessary behavioural changes in the producer and consumer. - Transactions and idempotency are not yet supported, but producing and - consuming should work with all the existing bells and whistles (batching, - compression, etc) as well as the new custom headers. Thanks to Vlad Hanciuta - of Arista Networks for this work. Part of - ([#901](https://github.com/Shopify/sarama/issues/901)). - -Bug Fixes: - - Fix encoding of ProduceResponse versions in test - ([#970](https://github.com/Shopify/sarama/pull/970)). - - Return partial replicas list when we have it - ([#975](https://github.com/Shopify/sarama/pull/975)). - -#### Version 1.13.0 (2017-10-04) - -New Features: - - Support for FetchRequest version 3 - ([#905](https://github.com/Shopify/sarama/pull/905)). - - Permit setting version on mock FetchResponses - ([#939](https://github.com/Shopify/sarama/pull/939)). - - Add a configuration option to support storing only minimal metadata for - extremely large clusters - ([#937](https://github.com/Shopify/sarama/pull/937)). - - Add `PartitionOffsetManager.ResetOffset` for backtracking tracked offsets - ([#932](https://github.com/Shopify/sarama/pull/932)). - -Improvements: - - Provide the block-level timestamp when consuming compressed messages - ([#885](https://github.com/Shopify/sarama/issues/885)). - - `Client.Replicas` and `Client.InSyncReplicas` now respect the order returned - by the broker, which can be meaningful - ([#930](https://github.com/Shopify/sarama/pull/930)). - - Use a `Ticker` to reduce consumer timer overhead at the cost of higher - variance in the actual timeout - ([#933](https://github.com/Shopify/sarama/pull/933)). - -Bug Fixes: - - Gracefully handle messages with negative timestamps - ([#907](https://github.com/Shopify/sarama/pull/907)). - - Raise a proper error when encountering an unknown message version - ([#940](https://github.com/Shopify/sarama/pull/940)). - -#### Version 1.12.0 (2017-05-08) - -New Features: - - Added support for the `ApiVersions` request and response pair, and Kafka - version 0.10.2 ([#867](https://github.com/Shopify/sarama/pull/867)). Note - that you still need to specify the Kafka version in the Sarama configuration - for the time being. - - Added a `Brokers` method to the Client which returns the complete set of - active brokers ([#813](https://github.com/Shopify/sarama/pull/813)). - - Added an `InSyncReplicas` method to the Client which returns the set of all - in-sync broker IDs for the given partition, now that the Kafka versions for - which this was misleading are no longer in our supported set - ([#872](https://github.com/Shopify/sarama/pull/872)). - - Added a `NewCustomHashPartitioner` method which allows constructing a hash - partitioner with a custom hash method in case the default (FNV-1a) is not - suitable - ([#837](https://github.com/Shopify/sarama/pull/837), - [#841](https://github.com/Shopify/sarama/pull/841)). - -Improvements: - - Recognize more Kafka error codes - ([#859](https://github.com/Shopify/sarama/pull/859)). - -Bug Fixes: - - Fix an issue where decoding a malformed FetchRequest would not return the - correct error ([#818](https://github.com/Shopify/sarama/pull/818)). - - Respect ordering of group protocols in JoinGroupRequests. This fix is - transparent if you're using the `AddGroupProtocol` or - `AddGroupProtocolMetadata` helpers; otherwise you will need to switch from - the `GroupProtocols` field (now deprecated) to use `OrderedGroupProtocols` - ([#812](https://github.com/Shopify/sarama/issues/812)). - - Fix an alignment-related issue with atomics on 32-bit architectures - ([#859](https://github.com/Shopify/sarama/pull/859)). - -#### Version 1.11.0 (2016-12-20) - -_Important:_ As of Sarama 1.11 it is necessary to set the config value of -`Producer.Return.Successes` to true in order to use the SyncProducer. Previous -versions would silently override this value when instantiating a SyncProducer -which led to unexpected values and data races. - -New Features: - - Metrics! Thanks to Sébastien Launay for all his work on this feature - ([#701](https://github.com/Shopify/sarama/pull/701), - [#746](https://github.com/Shopify/sarama/pull/746), - [#766](https://github.com/Shopify/sarama/pull/766)). - - Add support for LZ4 compression - ([#786](https://github.com/Shopify/sarama/pull/786)). - - Add support for ListOffsetRequest v1 and Kafka 0.10.1 - ([#775](https://github.com/Shopify/sarama/pull/775)). - - Added a `HighWaterMarks` method to the Consumer which aggregates the - `HighWaterMarkOffset` values of its child topic/partitions - ([#769](https://github.com/Shopify/sarama/pull/769)). - -Bug Fixes: - - Fixed producing when using timestamps, compression and Kafka 0.10 - ([#759](https://github.com/Shopify/sarama/pull/759)). - - Added missing decoder methods to DescribeGroups response - ([#756](https://github.com/Shopify/sarama/pull/756)). - - Fix producer shutdown when `Return.Errors` is disabled - ([#787](https://github.com/Shopify/sarama/pull/787)). - - Don't mutate configuration in SyncProducer - ([#790](https://github.com/Shopify/sarama/pull/790)). - - Fix crash on SASL initialization failure - ([#795](https://github.com/Shopify/sarama/pull/795)). - -#### Version 1.10.1 (2016-08-30) - -Bug Fixes: - - Fix the documentation for `HashPartitioner` which was incorrect - ([#717](https://github.com/Shopify/sarama/pull/717)). - - Permit client creation even when it is limited by ACLs - ([#722](https://github.com/Shopify/sarama/pull/722)). - - Several fixes to the consumer timer optimization code, regressions introduced - in v1.10.0. Go's timers are finicky - ([#730](https://github.com/Shopify/sarama/pull/730), - [#733](https://github.com/Shopify/sarama/pull/733), - [#734](https://github.com/Shopify/sarama/pull/734)). - - Handle consuming compressed relative offsets with Kafka 0.10 - ([#735](https://github.com/Shopify/sarama/pull/735)). - -#### Version 1.10.0 (2016-08-02) - -_Important:_ As of Sarama 1.10 it is necessary to tell Sarama the version of -Kafka you are running against (via the `config.Version` value) in order to use -features that may not be compatible with old Kafka versions. If you don't -specify this value it will default to 0.8.2 (the minimum supported), and trying -to use more recent features (like the offset manager) will fail with an error. - -_Also:_ The offset-manager's behaviour has been changed to match the upstream -java consumer (see [#705](https://github.com/Shopify/sarama/pull/705) and -[#713](https://github.com/Shopify/sarama/pull/713)). If you use the -offset-manager, please ensure that you are committing one *greater* than the -last consumed message offset or else you may end up consuming duplicate -messages. - -New Features: - - Support for Kafka 0.10 - ([#672](https://github.com/Shopify/sarama/pull/672), - [#678](https://github.com/Shopify/sarama/pull/678), - [#681](https://github.com/Shopify/sarama/pull/681), and others). - - Support for configuring the target Kafka version - ([#676](https://github.com/Shopify/sarama/pull/676)). - - Batch producing support in the SyncProducer - ([#677](https://github.com/Shopify/sarama/pull/677)). - - Extend producer mock to allow setting expectations on message contents - ([#667](https://github.com/Shopify/sarama/pull/667)). - -Improvements: - - Support `nil` compressed messages for deleting in compacted topics - ([#634](https://github.com/Shopify/sarama/pull/634)). - - Pre-allocate decoding errors, greatly reducing heap usage and GC time against - misbehaving brokers ([#690](https://github.com/Shopify/sarama/pull/690)). - - Re-use consumer expiry timers, removing one allocation per consumed message - ([#707](https://github.com/Shopify/sarama/pull/707)). - -Bug Fixes: - - Actually default the client ID to "sarama" like we say we do - ([#664](https://github.com/Shopify/sarama/pull/664)). - - Fix a rare issue where `Client.Leader` could return the wrong error - ([#685](https://github.com/Shopify/sarama/pull/685)). - - Fix a possible tight loop in the consumer - ([#693](https://github.com/Shopify/sarama/pull/693)). - - Match upstream's offset-tracking behaviour - ([#705](https://github.com/Shopify/sarama/pull/705)). - - Report UnknownTopicOrPartition errors from the offset manager - ([#706](https://github.com/Shopify/sarama/pull/706)). - - Fix possible negative partition value from the HashPartitioner - ([#709](https://github.com/Shopify/sarama/pull/709)). - -#### Version 1.9.0 (2016-05-16) - -New Features: - - Add support for custom offset manager retention durations - ([#602](https://github.com/Shopify/sarama/pull/602)). - - Publish low-level mocks to enable testing of third-party producer/consumer - implementations ([#570](https://github.com/Shopify/sarama/pull/570)). - - Declare support for Golang 1.6 - ([#611](https://github.com/Shopify/sarama/pull/611)). - - Support for SASL plain-text auth - ([#648](https://github.com/Shopify/sarama/pull/648)). - -Improvements: - - Simplified broker locking scheme slightly - ([#604](https://github.com/Shopify/sarama/pull/604)). - - Documentation cleanup - ([#605](https://github.com/Shopify/sarama/pull/605), - [#621](https://github.com/Shopify/sarama/pull/621), - [#654](https://github.com/Shopify/sarama/pull/654)). - -Bug Fixes: - - Fix race condition shutting down the OffsetManager - ([#658](https://github.com/Shopify/sarama/pull/658)). - -#### Version 1.8.0 (2016-02-01) - -New Features: - - Full support for Kafka 0.9: - - All protocol messages and fields - ([#586](https://github.com/Shopify/sarama/pull/586), - [#588](https://github.com/Shopify/sarama/pull/588), - [#590](https://github.com/Shopify/sarama/pull/590)). - - Verified that TLS support works - ([#581](https://github.com/Shopify/sarama/pull/581)). - - Fixed the OffsetManager compatibility - ([#585](https://github.com/Shopify/sarama/pull/585)). - -Improvements: - - Optimize for fewer system calls when reading from the network - ([#584](https://github.com/Shopify/sarama/pull/584)). - - Automatically retry `InvalidMessage` errors to match upstream behaviour - ([#589](https://github.com/Shopify/sarama/pull/589)). - -#### Version 1.7.0 (2015-12-11) - -New Features: - - Preliminary support for Kafka 0.9 - ([#572](https://github.com/Shopify/sarama/pull/572)). This comes with several - caveats: - - Protocol-layer support is mostly in place - ([#577](https://github.com/Shopify/sarama/pull/577)), however Kafka 0.9 - renamed some messages and fields, which we did not in order to preserve API - compatibility. - - The producer and consumer work against 0.9, but the offset manager does - not ([#573](https://github.com/Shopify/sarama/pull/573)). - - TLS support may or may not work - ([#581](https://github.com/Shopify/sarama/pull/581)). - -Improvements: - - Don't wait for request timeouts on dead brokers, greatly speeding recovery - when the TCP connection is left hanging - ([#548](https://github.com/Shopify/sarama/pull/548)). - - Refactored part of the producer. The new version provides a much more elegant - solution to [#449](https://github.com/Shopify/sarama/pull/449). It is also - slightly more efficient, and much more precise in calculating batch sizes - when compression is used - ([#549](https://github.com/Shopify/sarama/pull/549), - [#550](https://github.com/Shopify/sarama/pull/550), - [#551](https://github.com/Shopify/sarama/pull/551)). - -Bug Fixes: - - Fix race condition in consumer test mock - ([#553](https://github.com/Shopify/sarama/pull/553)). - -#### Version 1.6.1 (2015-09-25) - -Bug Fixes: - - Fix panic that could occur if a user-supplied message value failed to encode - ([#449](https://github.com/Shopify/sarama/pull/449)). - -#### Version 1.6.0 (2015-09-04) - -New Features: - - Implementation of a consumer offset manager using the APIs introduced in - Kafka 0.8.2. The API is designed mainly for integration into a future - high-level consumer, not for direct use, although it is *possible* to use it - directly. - ([#461](https://github.com/Shopify/sarama/pull/461)). - -Improvements: - - CRC32 calculation is much faster on machines with SSE4.2 instructions, - removing a major hotspot from most profiles - ([#255](https://github.com/Shopify/sarama/pull/255)). - -Bug Fixes: - - Make protocol decoding more robust against some malformed packets generated - by go-fuzz ([#523](https://github.com/Shopify/sarama/pull/523), - [#525](https://github.com/Shopify/sarama/pull/525)) or found in other ways - ([#528](https://github.com/Shopify/sarama/pull/528)). - - Fix a potential race condition panic in the consumer on shutdown - ([#529](https://github.com/Shopify/sarama/pull/529)). - -#### Version 1.5.0 (2015-08-17) - -New Features: - - TLS-encrypted network connections are now supported. This feature is subject - to change when Kafka releases built-in TLS support, but for now this is - enough to work with TLS-terminating proxies - ([#154](https://github.com/Shopify/sarama/pull/154)). - -Improvements: - - The consumer will not block if a single partition is not drained by the user; - all other partitions will continue to consume normally - ([#485](https://github.com/Shopify/sarama/pull/485)). - - Formatting of error strings has been much improved - ([#495](https://github.com/Shopify/sarama/pull/495)). - - Internal refactoring of the producer for code cleanliness and to enable - future work ([#300](https://github.com/Shopify/sarama/pull/300)). - -Bug Fixes: - - Fix a potential deadlock in the consumer on shutdown - ([#475](https://github.com/Shopify/sarama/pull/475)). - -#### Version 1.4.3 (2015-07-21) - -Bug Fixes: - - Don't include the partitioner in the producer's "fetch partitions" - circuit-breaker ([#466](https://github.com/Shopify/sarama/pull/466)). - - Don't retry messages until the broker is closed when abandoning a broker in - the producer ([#468](https://github.com/Shopify/sarama/pull/468)). - - Update the import path for snappy-go, it has moved again and the API has - changed slightly ([#486](https://github.com/Shopify/sarama/pull/486)). - -#### Version 1.4.2 (2015-05-27) - -Bug Fixes: - - Update the import path for snappy-go, it has moved from google code to github - ([#456](https://github.com/Shopify/sarama/pull/456)). - -#### Version 1.4.1 (2015-05-25) - -Improvements: - - Optimizations when decoding snappy messages, thanks to John Potocny - ([#446](https://github.com/Shopify/sarama/pull/446)). - -Bug Fixes: - - Fix hypothetical race conditions on producer shutdown - ([#450](https://github.com/Shopify/sarama/pull/450), - [#451](https://github.com/Shopify/sarama/pull/451)). - -#### Version 1.4.0 (2015-05-01) - -New Features: - - The consumer now implements `Topics()` and `Partitions()` methods to enable - users to dynamically choose what topics/partitions to consume without - instantiating a full client - ([#431](https://github.com/Shopify/sarama/pull/431)). - - The partition-consumer now exposes the high water mark offset value returned - by the broker via the `HighWaterMarkOffset()` method ([#339](https://github.com/Shopify/sarama/pull/339)). - - Added a `kafka-console-consumer` tool capable of handling multiple - partitions, and deprecated the now-obsolete `kafka-console-partitionConsumer` - ([#439](https://github.com/Shopify/sarama/pull/439), - [#442](https://github.com/Shopify/sarama/pull/442)). - -Improvements: - - The producer's logging during retry scenarios is more consistent, more - useful, and slightly less verbose - ([#429](https://github.com/Shopify/sarama/pull/429)). - - The client now shuffles its initial list of seed brokers in order to prevent - thundering herd on the first broker in the list - ([#441](https://github.com/Shopify/sarama/pull/441)). - -Bug Fixes: - - The producer now correctly manages its state if retries occur when it is - shutting down, fixing several instances of confusing behaviour and at least - one potential deadlock ([#419](https://github.com/Shopify/sarama/pull/419)). - - The consumer now handles messages for different partitions asynchronously, - making it much more resilient to specific user code ordering - ([#325](https://github.com/Shopify/sarama/pull/325)). - -#### Version 1.3.0 (2015-04-16) - -New Features: - - The client now tracks consumer group coordinators using - ConsumerMetadataRequests similar to how it tracks partition leadership using - regular MetadataRequests ([#411](https://github.com/Shopify/sarama/pull/411)). - This adds two methods to the client API: - - `Coordinator(consumerGroup string) (*Broker, error)` - - `RefreshCoordinator(consumerGroup string) error` - -Improvements: - - ConsumerMetadataResponses now automatically create a Broker object out of the - ID/address/port combination for the Coordinator; accessing the fields - individually has been deprecated - ([#413](https://github.com/Shopify/sarama/pull/413)). - - Much improved handling of `OffsetOutOfRange` errors in the consumer. - Consumers will fail to start if the provided offset is out of range - ([#418](https://github.com/Shopify/sarama/pull/418)) - and they will automatically shut down if the offset falls out of range - ([#424](https://github.com/Shopify/sarama/pull/424)). - - Small performance improvement in encoding and decoding protocol messages - ([#427](https://github.com/Shopify/sarama/pull/427)). - -Bug Fixes: - - Fix a rare race condition in the client's background metadata refresher if - it happens to be activated while the client is being closed - ([#422](https://github.com/Shopify/sarama/pull/422)). - -#### Version 1.2.0 (2015-04-07) - -Improvements: - - The producer's behaviour when `Flush.Frequency` is set is now more intuitive - ([#389](https://github.com/Shopify/sarama/pull/389)). - - The producer is now somewhat more memory-efficient during and after retrying - messages due to an improved queue implementation - ([#396](https://github.com/Shopify/sarama/pull/396)). - - The consumer produces much more useful logging output when leadership - changes ([#385](https://github.com/Shopify/sarama/pull/385)). - - The client's `GetOffset` method will now automatically refresh metadata and - retry once in the event of stale information or similar - ([#394](https://github.com/Shopify/sarama/pull/394)). - - Broker connections now have support for using TCP keepalives - ([#407](https://github.com/Shopify/sarama/issues/407)). - -Bug Fixes: - - The OffsetCommitRequest message now correctly implements all three possible - API versions ([#390](https://github.com/Shopify/sarama/pull/390), - [#400](https://github.com/Shopify/sarama/pull/400)). - -#### Version 1.1.0 (2015-03-20) - -Improvements: - - Wrap the producer's partitioner call in a circuit-breaker so that repeatedly - broken topics don't choke throughput - ([#373](https://github.com/Shopify/sarama/pull/373)). - -Bug Fixes: - - Fix the producer's internal reference counting in certain unusual scenarios - ([#367](https://github.com/Shopify/sarama/pull/367)). - - Fix the consumer's internal reference counting in certain unusual scenarios - ([#369](https://github.com/Shopify/sarama/pull/369)). - - Fix a condition where the producer's internal control messages could have - gotten stuck ([#368](https://github.com/Shopify/sarama/pull/368)). - - Fix an issue where invalid partition lists would be cached when asking for - metadata for a non-existant topic ([#372](https://github.com/Shopify/sarama/pull/372)). - - -#### Version 1.0.0 (2015-03-17) - -Version 1.0.0 is the first tagged version, and is almost a complete rewrite. The primary differences with previous untagged versions are: - -- The producer has been rewritten; there is now a `SyncProducer` with a blocking API, and an `AsyncProducer` that is non-blocking. -- The consumer has been rewritten to only open one connection per broker instead of one connection per partition. -- The main types of Sarama are now interfaces to make depedency injection easy; mock implementations for `Consumer`, `SyncProducer` and `AsyncProducer` are provided in the `github.com/Shopify/sarama/mocks` package. -- For most uses cases, it is no longer necessary to open a `Client`; this will be done for you. -- All the configuration values have been unified in the `Config` struct. -- Much improved test suite. diff --git a/third/github.com/Shopify/sarama/LICENSE b/third/github.com/Shopify/sarama/LICENSE deleted file mode 100644 index d2bf4352f..000000000 --- a/third/github.com/Shopify/sarama/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -Copyright (c) 2013 Shopify - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/third/github.com/Shopify/sarama/Makefile b/third/github.com/Shopify/sarama/Makefile deleted file mode 100644 index 8fcf219f4..000000000 --- a/third/github.com/Shopify/sarama/Makefile +++ /dev/null @@ -1,30 +0,0 @@ -default: fmt vet errcheck test - -# Taken from https://github.com/codecov/example-go#caveat-multiple-files -test: - echo "" > coverage.txt - for d in `go list ./... | grep -v vendor`; do \ - go test -p 1 -v -timeout 240s -race -coverprofile=profile.out -covermode=atomic $$d || exit 1; \ - if [ -f profile.out ]; then \ - cat profile.out >> coverage.txt; \ - rm profile.out; \ - fi \ - done - -vet: - go vet ./... - -# See https://github.com/kisielk/errcheck/pull/141 for details on ignorepkg -errcheck: - errcheck -ignorepkg fmt github.com/Shopify/sarama/... - -fmt: - @if [ -n "$$(go fmt ./...)" ]; then echo 'Please run go fmt on your code.' && exit 1; fi - -install_dependencies: install_errcheck get - -install_errcheck: - go get github.com/kisielk/errcheck - -get: - go get -t diff --git a/third/github.com/Shopify/sarama/README.md b/third/github.com/Shopify/sarama/README.md deleted file mode 100644 index f241b89c5..000000000 --- a/third/github.com/Shopify/sarama/README.md +++ /dev/null @@ -1,39 +0,0 @@ -sarama -====== - -[![GoDoc](https://godoc.org/github.com/Shopify/sarama?status.svg)](https://godoc.org/github.com/Shopify/sarama) -[![Build Status](https://travis-ci.org/Shopify/sarama.svg?branch=master)](https://travis-ci.org/Shopify/sarama) -[![Coverage](https://codecov.io/gh/Shopify/sarama/branch/master/graph/badge.svg)](https://codecov.io/gh/Shopify/sarama) - -Sarama is an MIT-licensed Go client library for [Apache Kafka](https://kafka.apache.org/) version 0.8 (and later). - -### Getting started - -- API documentation and examples are available via [godoc](https://godoc.org/github.com/Shopify/sarama). -- Mocks for testing are available in the [mocks](./mocks) subpackage. -- The [examples](./examples) directory contains more elaborate example applications. -- The [tools](./tools) directory contains command line tools that can be useful for testing, diagnostics, and instrumentation. - -You might also want to look at the [Frequently Asked Questions](https://github.com/Shopify/sarama/wiki/Frequently-Asked-Questions). - -### Compatibility and API stability - -Sarama provides a "2 releases + 2 months" compatibility guarantee: we support -the two latest stable releases of Kafka and Go, and we provide a two month -grace period for older releases. This means we currently officially support -Go 1.8 through 1.11, and Kafka 1.0 through 2.0, although older releases are -still likely to work. - -Sarama follows semantic versioning and provides API stability via the gopkg.in service. -You can import a version with a guaranteed stable API via http://gopkg.in/Shopify/sarama.v1. -A changelog is available [here](CHANGELOG.md). - -### Contributing - -* Get started by checking our [contribution guidelines](https://github.com/Shopify/sarama/blob/master/.github/CONTRIBUTING.md). -* Read the [Sarama wiki](https://github.com/Shopify/sarama/wiki) for more - technical and design details. -* The [Kafka Protocol Specification](https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol) - contains a wealth of useful information. -* For more general issues, there is [a google group](https://groups.google.com/forum/#!forum/kafka-clients) for Kafka client developers. -* If you have any questions, just ask! diff --git a/third/github.com/Shopify/sarama/Vagrantfile b/third/github.com/Shopify/sarama/Vagrantfile deleted file mode 100644 index f4b848a30..000000000 --- a/third/github.com/Shopify/sarama/Vagrantfile +++ /dev/null @@ -1,20 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! -VAGRANTFILE_API_VERSION = "2" - -# We have 5 * 192MB ZK processes and 5 * 320MB Kafka processes => 2560MB -MEMORY = 3072 - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "ubuntu/trusty64" - - config.vm.provision :shell, path: "vagrant/provision.sh" - - config.vm.network "private_network", ip: "192.168.100.67" - - config.vm.provider "virtualbox" do |v| - v.memory = MEMORY - end -end diff --git a/third/github.com/Shopify/sarama/acl_bindings.go b/third/github.com/Shopify/sarama/acl_bindings.go deleted file mode 100644 index 51517359a..000000000 --- a/third/github.com/Shopify/sarama/acl_bindings.go +++ /dev/null @@ -1,119 +0,0 @@ -package sarama - -type Resource struct { - ResourceType AclResourceType - ResourceName string -} - -func (r *Resource) encode(pe packetEncoder) error { - pe.putInt8(int8(r.ResourceType)) - - if err := pe.putString(r.ResourceName); err != nil { - return err - } - - return nil -} - -func (r *Resource) decode(pd packetDecoder, version int16) (err error) { - resourceType, err := pd.getInt8() - if err != nil { - return err - } - r.ResourceType = AclResourceType(resourceType) - - if r.ResourceName, err = pd.getString(); err != nil { - return err - } - - return nil -} - -type Acl struct { - Principal string - Host string - Operation AclOperation - PermissionType AclPermissionType -} - -func (a *Acl) encode(pe packetEncoder) error { - if err := pe.putString(a.Principal); err != nil { - return err - } - - if err := pe.putString(a.Host); err != nil { - return err - } - - pe.putInt8(int8(a.Operation)) - pe.putInt8(int8(a.PermissionType)) - - return nil -} - -func (a *Acl) decode(pd packetDecoder, version int16) (err error) { - if a.Principal, err = pd.getString(); err != nil { - return err - } - - if a.Host, err = pd.getString(); err != nil { - return err - } - - operation, err := pd.getInt8() - if err != nil { - return err - } - a.Operation = AclOperation(operation) - - permissionType, err := pd.getInt8() - if err != nil { - return err - } - a.PermissionType = AclPermissionType(permissionType) - - return nil -} - -type ResourceAcls struct { - Resource - Acls []*Acl -} - -func (r *ResourceAcls) encode(pe packetEncoder) error { - if err := r.Resource.encode(pe); err != nil { - return err - } - - if err := pe.putArrayLength(len(r.Acls)); err != nil { - return err - } - for _, acl := range r.Acls { - if err := acl.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (r *ResourceAcls) decode(pd packetDecoder, version int16) error { - if err := r.Resource.decode(pd, version); err != nil { - return err - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Acls = make([]*Acl, n) - for i := 0; i < n; i++ { - r.Acls[i] = new(Acl) - if err := r.Acls[i].decode(pd, version); err != nil { - return err - } - } - - return nil -} diff --git a/third/github.com/Shopify/sarama/acl_create_request.go b/third/github.com/Shopify/sarama/acl_create_request.go deleted file mode 100644 index 0b6ecbec3..000000000 --- a/third/github.com/Shopify/sarama/acl_create_request.go +++ /dev/null @@ -1,76 +0,0 @@ -package sarama - -type CreateAclsRequest struct { - AclCreations []*AclCreation -} - -func (c *CreateAclsRequest) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(c.AclCreations)); err != nil { - return err - } - - for _, aclCreation := range c.AclCreations { - if err := aclCreation.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (c *CreateAclsRequest) decode(pd packetDecoder, version int16) (err error) { - n, err := pd.getArrayLength() - if err != nil { - return err - } - - c.AclCreations = make([]*AclCreation, n) - - for i := 0; i < n; i++ { - c.AclCreations[i] = new(AclCreation) - if err := c.AclCreations[i].decode(pd, version); err != nil { - return err - } - } - - return nil -} - -func (d *CreateAclsRequest) key() int16 { - return 30 -} - -func (d *CreateAclsRequest) version() int16 { - return 0 -} - -func (d *CreateAclsRequest) requiredVersion() KafkaVersion { - return V0_11_0_0 -} - -type AclCreation struct { - Resource - Acl -} - -func (a *AclCreation) encode(pe packetEncoder) error { - if err := a.Resource.encode(pe); err != nil { - return err - } - if err := a.Acl.encode(pe); err != nil { - return err - } - - return nil -} - -func (a *AclCreation) decode(pd packetDecoder, version int16) (err error) { - if err := a.Resource.decode(pd, version); err != nil { - return err - } - if err := a.Acl.decode(pd, version); err != nil { - return err - } - - return nil -} diff --git a/third/github.com/Shopify/sarama/acl_create_request_test.go b/third/github.com/Shopify/sarama/acl_create_request_test.go deleted file mode 100644 index fb4b35c16..000000000 --- a/third/github.com/Shopify/sarama/acl_create_request_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package sarama - -import "testing" - -var ( - aclCreateRequest = []byte{ - 0, 0, 0, 1, - 3, // resource type = group - 0, 5, 'g', 'r', 'o', 'u', 'p', - 0, 9, 'p', 'r', 'i', 'n', 'c', 'i', 'p', 'a', 'l', - 0, 4, 'h', 'o', 's', 't', - 2, // all - 2, // deny - } -) - -func TestCreateAclsRequest(t *testing.T) { - req := &CreateAclsRequest{ - AclCreations: []*AclCreation{{ - Resource: Resource{ - ResourceType: AclResourceGroup, - ResourceName: "group", - }, - Acl: Acl{ - Principal: "principal", - Host: "host", - Operation: AclOperationAll, - PermissionType: AclPermissionDeny, - }}, - }, - } - - testRequest(t, "create request", req, aclCreateRequest) -} diff --git a/third/github.com/Shopify/sarama/acl_create_response.go b/third/github.com/Shopify/sarama/acl_create_response.go deleted file mode 100644 index 8a56f3573..000000000 --- a/third/github.com/Shopify/sarama/acl_create_response.go +++ /dev/null @@ -1,88 +0,0 @@ -package sarama - -import "time" - -type CreateAclsResponse struct { - ThrottleTime time.Duration - AclCreationResponses []*AclCreationResponse -} - -func (c *CreateAclsResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(c.ThrottleTime / time.Millisecond)) - - if err := pe.putArrayLength(len(c.AclCreationResponses)); err != nil { - return err - } - - for _, aclCreationResponse := range c.AclCreationResponses { - if err := aclCreationResponse.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (c *CreateAclsResponse) decode(pd packetDecoder, version int16) (err error) { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - c.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - c.AclCreationResponses = make([]*AclCreationResponse, n) - for i := 0; i < n; i++ { - c.AclCreationResponses[i] = new(AclCreationResponse) - if err := c.AclCreationResponses[i].decode(pd, version); err != nil { - return err - } - } - - return nil -} - -func (d *CreateAclsResponse) key() int16 { - return 30 -} - -func (d *CreateAclsResponse) version() int16 { - return 0 -} - -func (d *CreateAclsResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} - -type AclCreationResponse struct { - Err KError - ErrMsg *string -} - -func (a *AclCreationResponse) encode(pe packetEncoder) error { - pe.putInt16(int16(a.Err)) - - if err := pe.putNullableString(a.ErrMsg); err != nil { - return err - } - - return nil -} - -func (a *AclCreationResponse) decode(pd packetDecoder, version int16) (err error) { - kerr, err := pd.getInt16() - if err != nil { - return err - } - a.Err = KError(kerr) - - if a.ErrMsg, err = pd.getNullableString(); err != nil { - return err - } - - return nil -} diff --git a/third/github.com/Shopify/sarama/acl_create_response_test.go b/third/github.com/Shopify/sarama/acl_create_response_test.go deleted file mode 100644 index 65b934d9a..000000000 --- a/third/github.com/Shopify/sarama/acl_create_response_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -var ( - createResponseWithError = []byte{ - 0, 0, 0, 100, - 0, 0, 0, 1, - 0, 42, - 0, 5, 'e', 'r', 'r', 'o', 'r', - } - - createResponseArray = []byte{ - 0, 0, 0, 100, - 0, 0, 0, 2, - 0, 42, - 0, 5, 'e', 'r', 'r', 'o', 'r', - 0, 0, - 255, 255, - } -) - -func TestCreateAclsResponse(t *testing.T) { - errmsg := "error" - resp := &CreateAclsResponse{ - ThrottleTime: 100 * time.Millisecond, - AclCreationResponses: []*AclCreationResponse{{ - Err: ErrInvalidRequest, - ErrMsg: &errmsg, - }}, - } - - testResponse(t, "response with error", resp, createResponseWithError) - - resp.AclCreationResponses = append(resp.AclCreationResponses, new(AclCreationResponse)) - - testResponse(t, "response array", resp, createResponseArray) -} diff --git a/third/github.com/Shopify/sarama/acl_delete_request.go b/third/github.com/Shopify/sarama/acl_delete_request.go deleted file mode 100644 index 4133dceab..000000000 --- a/third/github.com/Shopify/sarama/acl_delete_request.go +++ /dev/null @@ -1,48 +0,0 @@ -package sarama - -type DeleteAclsRequest struct { - Filters []*AclFilter -} - -func (d *DeleteAclsRequest) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(d.Filters)); err != nil { - return err - } - - for _, filter := range d.Filters { - if err := filter.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (d *DeleteAclsRequest) decode(pd packetDecoder, version int16) (err error) { - n, err := pd.getArrayLength() - if err != nil { - return err - } - - d.Filters = make([]*AclFilter, n) - for i := 0; i < n; i++ { - d.Filters[i] = new(AclFilter) - if err := d.Filters[i].decode(pd, version); err != nil { - return err - } - } - - return nil -} - -func (d *DeleteAclsRequest) key() int16 { - return 31 -} - -func (d *DeleteAclsRequest) version() int16 { - return 0 -} - -func (d *DeleteAclsRequest) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/third/github.com/Shopify/sarama/acl_delete_request_test.go b/third/github.com/Shopify/sarama/acl_delete_request_test.go deleted file mode 100644 index 2efdcb48e..000000000 --- a/third/github.com/Shopify/sarama/acl_delete_request_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package sarama - -import "testing" - -var ( - aclDeleteRequestNulls = []byte{ - 0, 0, 0, 1, - 1, - 255, 255, - 255, 255, - 255, 255, - 11, - 3, - } - - aclDeleteRequest = []byte{ - 0, 0, 0, 1, - 1, // any - 0, 6, 'f', 'i', 'l', 't', 'e', 'r', - 0, 9, 'p', 'r', 'i', 'n', 'c', 'i', 'p', 'a', 'l', - 0, 4, 'h', 'o', 's', 't', - 4, // write - 3, // allow - } - - aclDeleteRequestArray = []byte{ - 0, 0, 0, 2, - 1, - 0, 6, 'f', 'i', 'l', 't', 'e', 'r', - 0, 9, 'p', 'r', 'i', 'n', 'c', 'i', 'p', 'a', 'l', - 0, 4, 'h', 'o', 's', 't', - 4, // write - 3, // allow - 2, - 0, 5, 't', 'o', 'p', 'i', 'c', - 255, 255, - 255, 255, - 6, - 2, - } -) - -func TestDeleteAclsRequest(t *testing.T) { - req := &DeleteAclsRequest{ - Filters: []*AclFilter{{ - ResourceType: AclResourceAny, - Operation: AclOperationAlterConfigs, - PermissionType: AclPermissionAllow, - }}, - } - - testRequest(t, "delete request nulls", req, aclDeleteRequestNulls) - - req.Filters[0].ResourceName = nullString("filter") - req.Filters[0].Principal = nullString("principal") - req.Filters[0].Host = nullString("host") - req.Filters[0].Operation = AclOperationWrite - - testRequest(t, "delete request", req, aclDeleteRequest) - - req.Filters = append(req.Filters, &AclFilter{ - ResourceType: AclResourceTopic, - ResourceName: nullString("topic"), - Operation: AclOperationDelete, - PermissionType: AclPermissionDeny, - }) - - testRequest(t, "delete request array", req, aclDeleteRequestArray) -} diff --git a/third/github.com/Shopify/sarama/acl_delete_response.go b/third/github.com/Shopify/sarama/acl_delete_response.go deleted file mode 100644 index b5e1c45eb..000000000 --- a/third/github.com/Shopify/sarama/acl_delete_response.go +++ /dev/null @@ -1,155 +0,0 @@ -package sarama - -import "time" - -type DeleteAclsResponse struct { - ThrottleTime time.Duration - FilterResponses []*FilterResponse -} - -func (a *DeleteAclsResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(a.ThrottleTime / time.Millisecond)) - - if err := pe.putArrayLength(len(a.FilterResponses)); err != nil { - return err - } - - for _, filterResponse := range a.FilterResponses { - if err := filterResponse.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (a *DeleteAclsResponse) decode(pd packetDecoder, version int16) (err error) { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - n, err := pd.getArrayLength() - if err != nil { - return err - } - a.FilterResponses = make([]*FilterResponse, n) - - for i := 0; i < n; i++ { - a.FilterResponses[i] = new(FilterResponse) - if err := a.FilterResponses[i].decode(pd, version); err != nil { - return err - } - } - - return nil -} - -func (d *DeleteAclsResponse) key() int16 { - return 31 -} - -func (d *DeleteAclsResponse) version() int16 { - return 0 -} - -func (d *DeleteAclsResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} - -type FilterResponse struct { - Err KError - ErrMsg *string - MatchingAcls []*MatchingAcl -} - -func (f *FilterResponse) encode(pe packetEncoder) error { - pe.putInt16(int16(f.Err)) - if err := pe.putNullableString(f.ErrMsg); err != nil { - return err - } - - if err := pe.putArrayLength(len(f.MatchingAcls)); err != nil { - return err - } - for _, matchingAcl := range f.MatchingAcls { - if err := matchingAcl.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (f *FilterResponse) decode(pd packetDecoder, version int16) (err error) { - kerr, err := pd.getInt16() - if err != nil { - return err - } - f.Err = KError(kerr) - - if f.ErrMsg, err = pd.getNullableString(); err != nil { - return err - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - f.MatchingAcls = make([]*MatchingAcl, n) - for i := 0; i < n; i++ { - f.MatchingAcls[i] = new(MatchingAcl) - if err := f.MatchingAcls[i].decode(pd, version); err != nil { - return err - } - } - - return nil -} - -type MatchingAcl struct { - Err KError - ErrMsg *string - Resource - Acl -} - -func (m *MatchingAcl) encode(pe packetEncoder) error { - pe.putInt16(int16(m.Err)) - if err := pe.putNullableString(m.ErrMsg); err != nil { - return err - } - - if err := m.Resource.encode(pe); err != nil { - return err - } - - if err := m.Acl.encode(pe); err != nil { - return err - } - - return nil -} - -func (m *MatchingAcl) decode(pd packetDecoder, version int16) (err error) { - kerr, err := pd.getInt16() - if err != nil { - return err - } - m.Err = KError(kerr) - - if m.ErrMsg, err = pd.getNullableString(); err != nil { - return err - } - - if err := m.Resource.decode(pd, version); err != nil { - return err - } - - if err := m.Acl.decode(pd, version); err != nil { - return err - } - - return nil -} diff --git a/third/github.com/Shopify/sarama/acl_delete_response_test.go b/third/github.com/Shopify/sarama/acl_delete_response_test.go deleted file mode 100644 index 0d9dea684..000000000 --- a/third/github.com/Shopify/sarama/acl_delete_response_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -var ( - deleteAclsResponse = []byte{ - 0, 0, 0, 100, - 0, 0, 0, 1, - 0, 0, // no error - 255, 255, // no error message - 0, 0, 0, 1, // 1 matching acl - 0, 0, // no error - 255, 255, // no error message - 2, // resource type - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 9, 'p', 'r', 'i', 'n', 'c', 'i', 'p', 'a', 'l', - 0, 4, 'h', 'o', 's', 't', - 4, - 3, - } -) - -func TestDeleteAclsResponse(t *testing.T) { - resp := &DeleteAclsResponse{ - ThrottleTime: 100 * time.Millisecond, - FilterResponses: []*FilterResponse{{ - MatchingAcls: []*MatchingAcl{{ - Resource: Resource{ResourceType: AclResourceTopic, ResourceName: "topic"}, - Acl: Acl{Principal: "principal", Host: "host", Operation: AclOperationWrite, PermissionType: AclPermissionAllow}, - }}, - }}, - } - - testResponse(t, "", resp, deleteAclsResponse) -} diff --git a/third/github.com/Shopify/sarama/acl_describe_request.go b/third/github.com/Shopify/sarama/acl_describe_request.go deleted file mode 100644 index 02a5a1f0e..000000000 --- a/third/github.com/Shopify/sarama/acl_describe_request.go +++ /dev/null @@ -1,25 +0,0 @@ -package sarama - -type DescribeAclsRequest struct { - AclFilter -} - -func (d *DescribeAclsRequest) encode(pe packetEncoder) error { - return d.AclFilter.encode(pe) -} - -func (d *DescribeAclsRequest) decode(pd packetDecoder, version int16) (err error) { - return d.AclFilter.decode(pd, version) -} - -func (d *DescribeAclsRequest) key() int16 { - return 29 -} - -func (d *DescribeAclsRequest) version() int16 { - return 0 -} - -func (d *DescribeAclsRequest) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/third/github.com/Shopify/sarama/acl_describe_request_test.go b/third/github.com/Shopify/sarama/acl_describe_request_test.go deleted file mode 100644 index 3af14c616..000000000 --- a/third/github.com/Shopify/sarama/acl_describe_request_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package sarama - -import ( - "testing" -) - -var ( - aclDescribeRequest = []byte{ - 2, // resource type - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 9, 'p', 'r', 'i', 'n', 'c', 'i', 'p', 'a', 'l', - 0, 4, 'h', 'o', 's', 't', - 5, // acl operation - 3, // acl permission type - } -) - -func TestAclDescribeRequest(t *testing.T) { - resourcename := "topic" - principal := "principal" - host := "host" - - req := &DescribeAclsRequest{ - AclFilter{ - ResourceType: AclResourceTopic, - ResourceName: &resourcename, - Principal: &principal, - Host: &host, - Operation: AclOperationCreate, - PermissionType: AclPermissionAllow, - }, - } - - testRequest(t, "", req, aclDescribeRequest) -} diff --git a/third/github.com/Shopify/sarama/acl_describe_response.go b/third/github.com/Shopify/sarama/acl_describe_response.go deleted file mode 100644 index 5bc9497f4..000000000 --- a/third/github.com/Shopify/sarama/acl_describe_response.go +++ /dev/null @@ -1,80 +0,0 @@ -package sarama - -import "time" - -type DescribeAclsResponse struct { - ThrottleTime time.Duration - Err KError - ErrMsg *string - ResourceAcls []*ResourceAcls -} - -func (d *DescribeAclsResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(d.ThrottleTime / time.Millisecond)) - pe.putInt16(int16(d.Err)) - - if err := pe.putNullableString(d.ErrMsg); err != nil { - return err - } - - if err := pe.putArrayLength(len(d.ResourceAcls)); err != nil { - return err - } - - for _, resourceAcl := range d.ResourceAcls { - if err := resourceAcl.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (d *DescribeAclsResponse) decode(pd packetDecoder, version int16) (err error) { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - d.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - kerr, err := pd.getInt16() - if err != nil { - return err - } - d.Err = KError(kerr) - - errmsg, err := pd.getString() - if err != nil { - return err - } - if errmsg != "" { - d.ErrMsg = &errmsg - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - d.ResourceAcls = make([]*ResourceAcls, n) - - for i := 0; i < n; i++ { - d.ResourceAcls[i] = new(ResourceAcls) - if err := d.ResourceAcls[i].decode(pd, version); err != nil { - return err - } - } - - return nil -} - -func (d *DescribeAclsResponse) key() int16 { - return 29 -} - -func (d *DescribeAclsResponse) version() int16 { - return 0 -} - -func (d *DescribeAclsResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/third/github.com/Shopify/sarama/acl_describe_response_test.go b/third/github.com/Shopify/sarama/acl_describe_response_test.go deleted file mode 100644 index f0652cfee..000000000 --- a/third/github.com/Shopify/sarama/acl_describe_response_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -var aclDescribeResponseError = []byte{ - 0, 0, 0, 100, - 0, 8, // error - 0, 5, 'e', 'r', 'r', 'o', 'r', - 0, 0, 0, 1, // 1 resource - 2, // cluster type - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 0, 0, 1, // 1 acl - 0, 9, 'p', 'r', 'i', 'n', 'c', 'i', 'p', 'a', 'l', - 0, 4, 'h', 'o', 's', 't', - 4, // write - 3, // allow -} - -func TestAclDescribeResponse(t *testing.T) { - errmsg := "error" - resp := &DescribeAclsResponse{ - ThrottleTime: 100 * time.Millisecond, - Err: ErrBrokerNotAvailable, - ErrMsg: &errmsg, - ResourceAcls: []*ResourceAcls{{ - Resource: Resource{ - ResourceName: "topic", - ResourceType: AclResourceTopic, - }, - Acls: []*Acl{ - { - Principal: "principal", - Host: "host", - Operation: AclOperationWrite, - PermissionType: AclPermissionAllow, - }, - }, - }}, - } - - testResponse(t, "describe", resp, aclDescribeResponseError) -} diff --git a/third/github.com/Shopify/sarama/acl_filter.go b/third/github.com/Shopify/sarama/acl_filter.go deleted file mode 100644 index 970635421..000000000 --- a/third/github.com/Shopify/sarama/acl_filter.go +++ /dev/null @@ -1,61 +0,0 @@ -package sarama - -type AclFilter struct { - ResourceType AclResourceType - ResourceName *string - Principal *string - Host *string - Operation AclOperation - PermissionType AclPermissionType -} - -func (a *AclFilter) encode(pe packetEncoder) error { - pe.putInt8(int8(a.ResourceType)) - if err := pe.putNullableString(a.ResourceName); err != nil { - return err - } - if err := pe.putNullableString(a.Principal); err != nil { - return err - } - if err := pe.putNullableString(a.Host); err != nil { - return err - } - pe.putInt8(int8(a.Operation)) - pe.putInt8(int8(a.PermissionType)) - - return nil -} - -func (a *AclFilter) decode(pd packetDecoder, version int16) (err error) { - resourceType, err := pd.getInt8() - if err != nil { - return err - } - a.ResourceType = AclResourceType(resourceType) - - if a.ResourceName, err = pd.getNullableString(); err != nil { - return err - } - - if a.Principal, err = pd.getNullableString(); err != nil { - return err - } - - if a.Host, err = pd.getNullableString(); err != nil { - return err - } - - operation, err := pd.getInt8() - if err != nil { - return err - } - a.Operation = AclOperation(operation) - - permissionType, err := pd.getInt8() - if err != nil { - return err - } - a.PermissionType = AclPermissionType(permissionType) - - return nil -} diff --git a/third/github.com/Shopify/sarama/acl_types.go b/third/github.com/Shopify/sarama/acl_types.go deleted file mode 100644 index 19da6f2f4..000000000 --- a/third/github.com/Shopify/sarama/acl_types.go +++ /dev/null @@ -1,42 +0,0 @@ -package sarama - -type AclOperation int - -// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclOperation.java -const ( - AclOperationUnknown AclOperation = 0 - AclOperationAny AclOperation = 1 - AclOperationAll AclOperation = 2 - AclOperationRead AclOperation = 3 - AclOperationWrite AclOperation = 4 - AclOperationCreate AclOperation = 5 - AclOperationDelete AclOperation = 6 - AclOperationAlter AclOperation = 7 - AclOperationDescribe AclOperation = 8 - AclOperationClusterAction AclOperation = 9 - AclOperationDescribeConfigs AclOperation = 10 - AclOperationAlterConfigs AclOperation = 11 - AclOperationIdempotentWrite AclOperation = 12 -) - -type AclPermissionType int - -// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclPermissionType.java -const ( - AclPermissionUnknown AclPermissionType = 0 - AclPermissionAny AclPermissionType = 1 - AclPermissionDeny AclPermissionType = 2 - AclPermissionAllow AclPermissionType = 3 -) - -type AclResourceType int - -// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/ResourceType.java -const ( - AclResourceUnknown AclResourceType = 0 - AclResourceAny AclResourceType = 1 - AclResourceTopic AclResourceType = 2 - AclResourceGroup AclResourceType = 3 - AclResourceCluster AclResourceType = 4 - AclResourceTransactionalID AclResourceType = 5 -) diff --git a/third/github.com/Shopify/sarama/add_offsets_to_txn_request.go b/third/github.com/Shopify/sarama/add_offsets_to_txn_request.go deleted file mode 100644 index 6da166c63..000000000 --- a/third/github.com/Shopify/sarama/add_offsets_to_txn_request.go +++ /dev/null @@ -1,52 +0,0 @@ -package sarama - -type AddOffsetsToTxnRequest struct { - TransactionalID string - ProducerID int64 - ProducerEpoch int16 - GroupID string -} - -func (a *AddOffsetsToTxnRequest) encode(pe packetEncoder) error { - if err := pe.putString(a.TransactionalID); err != nil { - return err - } - - pe.putInt64(a.ProducerID) - - pe.putInt16(a.ProducerEpoch) - - if err := pe.putString(a.GroupID); err != nil { - return err - } - - return nil -} - -func (a *AddOffsetsToTxnRequest) decode(pd packetDecoder, version int16) (err error) { - if a.TransactionalID, err = pd.getString(); err != nil { - return err - } - if a.ProducerID, err = pd.getInt64(); err != nil { - return err - } - if a.ProducerEpoch, err = pd.getInt16(); err != nil { - return err - } - if a.GroupID, err = pd.getString(); err != nil { - return err - } - return nil -} - -func (a *AddOffsetsToTxnRequest) key() int16 { - return 25 -} - -func (a *AddOffsetsToTxnRequest) version() int16 { - return 0 -} - -func (a *AddOffsetsToTxnRequest) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/third/github.com/Shopify/sarama/add_offsets_to_txn_request_test.go b/third/github.com/Shopify/sarama/add_offsets_to_txn_request_test.go deleted file mode 100644 index e96b3d33f..000000000 --- a/third/github.com/Shopify/sarama/add_offsets_to_txn_request_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package sarama - -import "testing" - -var ( - addOffsetsToTxnRequest = []byte{ - 0, 3, 't', 'x', 'n', - 0, 0, 0, 0, 0, 0, 31, 64, - 0, 0, - 0, 7, 'g', 'r', 'o', 'u', 'p', 'i', 'd', - } -) - -func TestAddOffsetsToTxnRequest(t *testing.T) { - req := &AddOffsetsToTxnRequest{ - TransactionalID: "txn", - ProducerID: 8000, - ProducerEpoch: 0, - GroupID: "groupid", - } - - testRequest(t, "", req, addOffsetsToTxnRequest) -} diff --git a/third/github.com/Shopify/sarama/add_offsets_to_txn_response.go b/third/github.com/Shopify/sarama/add_offsets_to_txn_response.go deleted file mode 100644 index 3a46151a0..000000000 --- a/third/github.com/Shopify/sarama/add_offsets_to_txn_response.go +++ /dev/null @@ -1,44 +0,0 @@ -package sarama - -import ( - "time" -) - -type AddOffsetsToTxnResponse struct { - ThrottleTime time.Duration - Err KError -} - -func (a *AddOffsetsToTxnResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(a.ThrottleTime / time.Millisecond)) - pe.putInt16(int16(a.Err)) - return nil -} - -func (a *AddOffsetsToTxnResponse) decode(pd packetDecoder, version int16) (err error) { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - kerr, err := pd.getInt16() - if err != nil { - return err - } - a.Err = KError(kerr) - - return nil -} - -func (a *AddOffsetsToTxnResponse) key() int16 { - return 25 -} - -func (a *AddOffsetsToTxnResponse) version() int16 { - return 0 -} - -func (a *AddOffsetsToTxnResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/third/github.com/Shopify/sarama/add_offsets_to_txn_response_test.go b/third/github.com/Shopify/sarama/add_offsets_to_txn_response_test.go deleted file mode 100644 index 4504966fe..000000000 --- a/third/github.com/Shopify/sarama/add_offsets_to_txn_response_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -var ( - addOffsetsToTxnResponse = []byte{ - 0, 0, 0, 100, - 0, 47, - } -) - -func TestAddOffsetsToTxnResponse(t *testing.T) { - resp := &AddOffsetsToTxnResponse{ - ThrottleTime: 100 * time.Millisecond, - Err: ErrInvalidProducerEpoch, - } - - testResponse(t, "", resp, addOffsetsToTxnResponse) -} diff --git a/third/github.com/Shopify/sarama/add_partitions_to_txn_request.go b/third/github.com/Shopify/sarama/add_partitions_to_txn_request.go deleted file mode 100644 index a8a59225e..000000000 --- a/third/github.com/Shopify/sarama/add_partitions_to_txn_request.go +++ /dev/null @@ -1,76 +0,0 @@ -package sarama - -type AddPartitionsToTxnRequest struct { - TransactionalID string - ProducerID int64 - ProducerEpoch int16 - TopicPartitions map[string][]int32 -} - -func (a *AddPartitionsToTxnRequest) encode(pe packetEncoder) error { - if err := pe.putString(a.TransactionalID); err != nil { - return err - } - pe.putInt64(a.ProducerID) - pe.putInt16(a.ProducerEpoch) - - if err := pe.putArrayLength(len(a.TopicPartitions)); err != nil { - return err - } - for topic, partitions := range a.TopicPartitions { - if err := pe.putString(topic); err != nil { - return err - } - if err := pe.putInt32Array(partitions); err != nil { - return err - } - } - - return nil -} - -func (a *AddPartitionsToTxnRequest) decode(pd packetDecoder, version int16) (err error) { - if a.TransactionalID, err = pd.getString(); err != nil { - return err - } - if a.ProducerID, err = pd.getInt64(); err != nil { - return err - } - if a.ProducerEpoch, err = pd.getInt16(); err != nil { - return err - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - a.TopicPartitions = make(map[string][]int32) - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - - partitions, err := pd.getInt32Array() - if err != nil { - return err - } - - a.TopicPartitions[topic] = partitions - } - - return nil -} - -func (a *AddPartitionsToTxnRequest) key() int16 { - return 24 -} - -func (a *AddPartitionsToTxnRequest) version() int16 { - return 0 -} - -func (a *AddPartitionsToTxnRequest) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/third/github.com/Shopify/sarama/add_partitions_to_txn_request_test.go b/third/github.com/Shopify/sarama/add_partitions_to_txn_request_test.go deleted file mode 100644 index 76a2eee43..000000000 --- a/third/github.com/Shopify/sarama/add_partitions_to_txn_request_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package sarama - -import "testing" - -var ( - addPartitionsToTxnRequest = []byte{ - 0, 3, 't', 'x', 'n', - 0, 0, 0, 0, 0, 0, 31, 64, // ProducerID - 0, 0, 0, 0, // ProducerEpoch - 0, 1, // 1 topic - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 0, 0, 1, 0, 0, 0, 1, - } -) - -func TestAddPartitionsToTxnRequest(t *testing.T) { - req := &AddPartitionsToTxnRequest{ - TransactionalID: "txn", - ProducerID: 8000, - ProducerEpoch: 0, - TopicPartitions: map[string][]int32{ - "topic": []int32{1}, - }, - } - - testRequest(t, "", req, addPartitionsToTxnRequest) -} diff --git a/third/github.com/Shopify/sarama/add_partitions_to_txn_response.go b/third/github.com/Shopify/sarama/add_partitions_to_txn_response.go deleted file mode 100644 index 581c556c5..000000000 --- a/third/github.com/Shopify/sarama/add_partitions_to_txn_response.go +++ /dev/null @@ -1,108 +0,0 @@ -package sarama - -import ( - "time" -) - -type AddPartitionsToTxnResponse struct { - ThrottleTime time.Duration - Errors map[string][]*PartitionError -} - -func (a *AddPartitionsToTxnResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(a.ThrottleTime / time.Millisecond)) - if err := pe.putArrayLength(len(a.Errors)); err != nil { - return err - } - - for topic, e := range a.Errors { - if err := pe.putString(topic); err != nil { - return err - } - if err := pe.putArrayLength(len(e)); err != nil { - return err - } - for _, partitionError := range e { - if err := partitionError.encode(pe); err != nil { - return err - } - } - } - - return nil -} - -func (a *AddPartitionsToTxnResponse) decode(pd packetDecoder, version int16) (err error) { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - a.Errors = make(map[string][]*PartitionError) - - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - - m, err := pd.getArrayLength() - if err != nil { - return err - } - - a.Errors[topic] = make([]*PartitionError, m) - - for j := 0; j < m; j++ { - a.Errors[topic][j] = new(PartitionError) - if err := a.Errors[topic][j].decode(pd, version); err != nil { - return err - } - } - } - - return nil -} - -func (a *AddPartitionsToTxnResponse) key() int16 { - return 24 -} - -func (a *AddPartitionsToTxnResponse) version() int16 { - return 0 -} - -func (a *AddPartitionsToTxnResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} - -type PartitionError struct { - Partition int32 - Err KError -} - -func (p *PartitionError) encode(pe packetEncoder) error { - pe.putInt32(p.Partition) - pe.putInt16(int16(p.Err)) - return nil -} - -func (p *PartitionError) decode(pd packetDecoder, version int16) (err error) { - if p.Partition, err = pd.getInt32(); err != nil { - return err - } - - kerr, err := pd.getInt16() - if err != nil { - return err - } - p.Err = KError(kerr) - - return nil -} diff --git a/third/github.com/Shopify/sarama/add_partitions_to_txn_response_test.go b/third/github.com/Shopify/sarama/add_partitions_to_txn_response_test.go deleted file mode 100644 index 7b5b82f80..000000000 --- a/third/github.com/Shopify/sarama/add_partitions_to_txn_response_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -var ( - addPartitionsToTxnResponse = []byte{ - 0, 0, 0, 100, - 0, 0, 0, 1, - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 0, 0, 1, // 1 partition error - 0, 0, 0, 2, // partition 2 - 0, 48, // error - } -) - -func TestAddPartitionsToTxnResponse(t *testing.T) { - resp := &AddPartitionsToTxnResponse{ - ThrottleTime: 100 * time.Millisecond, - Errors: map[string][]*PartitionError{ - "topic": []*PartitionError{&PartitionError{ - Err: ErrInvalidTxnState, - Partition: 2, - }}, - }, - } - - testResponse(t, "", resp, addPartitionsToTxnResponse) -} diff --git a/third/github.com/Shopify/sarama/admin.go b/third/github.com/Shopify/sarama/admin.go deleted file mode 100644 index 52725758d..000000000 --- a/third/github.com/Shopify/sarama/admin.go +++ /dev/null @@ -1,382 +0,0 @@ -package sarama - -import "errors" - -// ClusterAdmin is the administrative client for Kafka, which supports managing and inspecting topics, -// brokers, configurations and ACLs. The minimum broker version required is 0.10.0.0. -// Methods with stricter requirements will specify the minimum broker version required. -// You MUST call Close() on a client to avoid leaks -type ClusterAdmin interface { - // Creates a new topic. This operation is supported by brokers with version 0.10.1.0 or higher. - // It may take several seconds after CreateTopic returns success for all the brokers - // to become aware that the topic has been created. During this time, listTopics - // may not return information about the new topic.The validateOnly option is supported from version 0.10.2.0. - CreateTopic(topic string, detail *TopicDetail, validateOnly bool) error - - // Delete a topic. It may take several seconds after the DeleteTopic to returns success - // and for all the brokers to become aware that the topics are gone. - // During this time, listTopics may continue to return information about the deleted topic. - // If delete.topic.enable is false on the brokers, deleteTopic will mark - // the topic for deletion, but not actually delete them. - // This operation is supported by brokers with version 0.10.1.0 or higher. - DeleteTopic(topic string) error - - // Increase the number of partitions of the topics according to the corresponding values. - // If partitions are increased for a topic that has a key, the partition logic or ordering of - // the messages will be affected. It may take several seconds after this method returns - // success for all the brokers to become aware that the partitions have been created. - // During this time, ClusterAdmin#describeTopics may not return information about the - // new partitions. This operation is supported by brokers with version 1.0.0 or higher. - CreatePartitions(topic string, count int32, assignment [][]int32, validateOnly bool) error - - // Delete records whose offset is smaller than the given offset of the corresponding partition. - // This operation is supported by brokers with version 0.11.0.0 or higher. - DeleteRecords(topic string, partitionOffsets map[int32]int64) error - - // Get the configuration for the specified resources. - // The returned configuration includes default values and the Default is true - // can be used to distinguish them from user supplied values. - // Config entries where ReadOnly is true cannot be updated. - // The value of config entries where Sensitive is true is always nil so - // sensitive information is not disclosed. - // This operation is supported by brokers with version 0.11.0.0 or higher. - DescribeConfig(resource ConfigResource) ([]ConfigEntry, error) - - // Update the configuration for the specified resources with the default options. - // This operation is supported by brokers with version 0.11.0.0 or higher. - // The resources with their configs (topic is the only resource type with configs - // that can be updated currently Updates are not transactional so they may succeed - // for some resources while fail for others. The configs for a particular resource are updated automatically. - AlterConfig(resourceType ConfigResourceType, name string, entries map[string]*string, validateOnly bool) error - - // Creates access control lists (ACLs) which are bound to specific resources. - // This operation is not transactional so it may succeed for some ACLs while fail for others. - // If you attempt to add an ACL that duplicates an existing ACL, no error will be raised, but - // no changes will be made. This operation is supported by brokers with version 0.11.0.0 or higher. - CreateACL(resource Resource, acl Acl) error - - // Lists access control lists (ACLs) according to the supplied filter. - // it may take some time for changes made by createAcls or deleteAcls to be reflected in the output of ListAcls - // This operation is supported by brokers with version 0.11.0.0 or higher. - ListAcls(filter AclFilter) ([]ResourceAcls, error) - - // Deletes access control lists (ACLs) according to the supplied filters. - // This operation is not transactional so it may succeed for some ACLs while fail for others. - // This operation is supported by brokers with version 0.11.0.0 or higher. - DeleteACL(filter AclFilter, validateOnly bool) ([]MatchingAcl, error) - - // Close shuts down the admin and closes underlying client. - Close() error -} - -type clusterAdmin struct { - client Client - conf *Config -} - -// NewClusterAdmin creates a new ClusterAdmin using the given broker addresses and configuration. -func NewClusterAdmin(addrs []string, conf *Config) (ClusterAdmin, error) { - client, err := NewClient(addrs, conf) - if err != nil { - return nil, err - } - - //make sure we can retrieve the controller - _, err = client.Controller() - if err != nil { - return nil, err - } - - ca := &clusterAdmin{ - client: client, - conf: client.Config(), - } - return ca, nil -} - -func (ca *clusterAdmin) Close() error { - return ca.client.Close() -} - -func (ca *clusterAdmin) Controller() (*Broker, error) { - return ca.client.Controller() -} - -func (ca *clusterAdmin) CreateTopic(topic string, detail *TopicDetail, validateOnly bool) error { - - if topic == "" { - return ErrInvalidTopic - } - - if detail == nil { - return errors.New("You must specify topic details") - } - - topicDetails := make(map[string]*TopicDetail) - topicDetails[topic] = detail - - request := &CreateTopicsRequest{ - TopicDetails: topicDetails, - ValidateOnly: validateOnly, - Timeout: ca.conf.Admin.Timeout, - } - - if ca.conf.Version.IsAtLeast(V0_11_0_0) { - request.Version = 1 - } - if ca.conf.Version.IsAtLeast(V1_0_0_0) { - request.Version = 2 - } - - b, err := ca.Controller() - if err != nil { - return err - } - - rsp, err := b.CreateTopics(request) - if err != nil { - return err - } - - topicErr, ok := rsp.TopicErrors[topic] - if !ok { - return ErrIncompleteResponse - } - - if topicErr.Err != ErrNoError { - return topicErr.Err - } - - return nil -} - -func (ca *clusterAdmin) DeleteTopic(topic string) error { - - if topic == "" { - return ErrInvalidTopic - } - - request := &DeleteTopicsRequest{ - Topics: []string{topic}, - Timeout: ca.conf.Admin.Timeout, - } - - if ca.conf.Version.IsAtLeast(V0_11_0_0) { - request.Version = 1 - } - - b, err := ca.Controller() - if err != nil { - return err - } - - rsp, err := b.DeleteTopics(request) - if err != nil { - return err - } - - topicErr, ok := rsp.TopicErrorCodes[topic] - if !ok { - return ErrIncompleteResponse - } - - if topicErr != ErrNoError { - return topicErr - } - return nil -} - -func (ca *clusterAdmin) CreatePartitions(topic string, count int32, assignment [][]int32, validateOnly bool) error { - if topic == "" { - return ErrInvalidTopic - } - - topicPartitions := make(map[string]*TopicPartition) - topicPartitions[topic] = &TopicPartition{Count: count, Assignment: assignment} - - request := &CreatePartitionsRequest{ - TopicPartitions: topicPartitions, - Timeout: ca.conf.Admin.Timeout, - } - - b, err := ca.Controller() - if err != nil { - return err - } - - rsp, err := b.CreatePartitions(request) - if err != nil { - return err - } - - topicErr, ok := rsp.TopicPartitionErrors[topic] - if !ok { - return ErrIncompleteResponse - } - - if topicErr.Err != ErrNoError { - return topicErr.Err - } - - return nil -} - -func (ca *clusterAdmin) DeleteRecords(topic string, partitionOffsets map[int32]int64) error { - - if topic == "" { - return ErrInvalidTopic - } - - topics := make(map[string]*DeleteRecordsRequestTopic) - topics[topic] = &DeleteRecordsRequestTopic{PartitionOffsets: partitionOffsets} - request := &DeleteRecordsRequest{ - Topics: topics, - Timeout: ca.conf.Admin.Timeout, - } - - b, err := ca.Controller() - if err != nil { - return err - } - - rsp, err := b.DeleteRecords(request) - if err != nil { - return err - } - - _, ok := rsp.Topics[topic] - if !ok { - return ErrIncompleteResponse - } - - //todo since we are dealing with couple of partitions it would be good if we return slice of errors - //for each partition instead of one error - return nil -} - -func (ca *clusterAdmin) DescribeConfig(resource ConfigResource) ([]ConfigEntry, error) { - - var entries []ConfigEntry - var resources []*ConfigResource - resources = append(resources, &resource) - - request := &DescribeConfigsRequest{ - Resources: resources, - } - - b, err := ca.Controller() - if err != nil { - return nil, err - } - - rsp, err := b.DescribeConfigs(request) - if err != nil { - return nil, err - } - - for _, rspResource := range rsp.Resources { - if rspResource.Name == resource.Name { - if rspResource.ErrorMsg != "" { - return nil, errors.New(rspResource.ErrorMsg) - } - for _, cfgEntry := range rspResource.Configs { - entries = append(entries, *cfgEntry) - } - } - } - return entries, nil -} - -func (ca *clusterAdmin) AlterConfig(resourceType ConfigResourceType, name string, entries map[string]*string, validateOnly bool) error { - - var resources []*AlterConfigsResource - resources = append(resources, &AlterConfigsResource{ - Type: resourceType, - Name: name, - ConfigEntries: entries, - }) - - request := &AlterConfigsRequest{ - Resources: resources, - ValidateOnly: validateOnly, - } - - b, err := ca.Controller() - if err != nil { - return err - } - - rsp, err := b.AlterConfigs(request) - if err != nil { - return err - } - - for _, rspResource := range rsp.Resources { - if rspResource.Name == name { - if rspResource.ErrorMsg != "" { - return errors.New(rspResource.ErrorMsg) - } - } - } - return nil -} - -func (ca *clusterAdmin) CreateACL(resource Resource, acl Acl) error { - var acls []*AclCreation - acls = append(acls, &AclCreation{resource, acl}) - request := &CreateAclsRequest{AclCreations: acls} - - b, err := ca.Controller() - if err != nil { - return err - } - - _, err = b.CreateAcls(request) - return err -} - -func (ca *clusterAdmin) ListAcls(filter AclFilter) ([]ResourceAcls, error) { - - request := &DescribeAclsRequest{AclFilter: filter} - - b, err := ca.Controller() - if err != nil { - return nil, err - } - - rsp, err := b.DescribeAcls(request) - if err != nil { - return nil, err - } - - var lAcls []ResourceAcls - for _, rAcl := range rsp.ResourceAcls { - lAcls = append(lAcls, *rAcl) - } - return lAcls, nil -} - -func (ca *clusterAdmin) DeleteACL(filter AclFilter, validateOnly bool) ([]MatchingAcl, error) { - var filters []*AclFilter - filters = append(filters, &filter) - request := &DeleteAclsRequest{Filters: filters} - - b, err := ca.Controller() - if err != nil { - return nil, err - } - - rsp, err := b.DeleteAcls(request) - if err != nil { - return nil, err - } - - var mAcls []MatchingAcl - for _, fr := range rsp.FilterResponses { - for _, mACL := range fr.MatchingAcls { - mAcls = append(mAcls, *mACL) - } - - } - return mAcls, nil -} diff --git a/third/github.com/Shopify/sarama/admin_test.go b/third/github.com/Shopify/sarama/admin_test.go deleted file mode 100644 index 9d3cc3170..000000000 --- a/third/github.com/Shopify/sarama/admin_test.go +++ /dev/null @@ -1,501 +0,0 @@ -package sarama - -import ( - "errors" - "testing" -) - -func TestClusterAdmin(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - defer seedBroker.Close() - - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetController(seedBroker.BrokerID()). - SetBroker(seedBroker.Addr(), seedBroker.BrokerID()), - }) - - config := NewConfig() - config.Version = V1_0_0_0 - admin, err := NewClusterAdmin([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - err = admin.Close() - if err != nil { - t.Fatal(err) - } -} - -func TestClusterAdminInvalidController(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - defer seedBroker.Close() - - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetBroker(seedBroker.Addr(), seedBroker.BrokerID()), - }) - - config := NewConfig() - config.Version = V1_0_0_0 - _, err := NewClusterAdmin([]string{seedBroker.Addr()}, config) - if err == nil { - t.Fatal(errors.New("controller not set still cluster admin was created")) - } - - if err != ErrControllerNotAvailable { - t.Fatal(err) - } -} - -func TestClusterAdminCreateTopic(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - defer seedBroker.Close() - - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetController(seedBroker.BrokerID()). - SetBroker(seedBroker.Addr(), seedBroker.BrokerID()), - "CreateTopicsRequest": NewMockCreateTopicsResponse(t), - }) - - config := NewConfig() - config.Version = V0_10_2_0 - admin, err := NewClusterAdmin([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - err = admin.CreateTopic("my_topic", &TopicDetail{NumPartitions: 1, ReplicationFactor: 1}, false) - if err != nil { - t.Fatal(err) - } - - err = admin.Close() - if err != nil { - t.Fatal(err) - } -} - -func TestClusterAdminCreateTopicWithInvalidTopicDetail(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - defer seedBroker.Close() - - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetController(seedBroker.BrokerID()). - SetBroker(seedBroker.Addr(), seedBroker.BrokerID()), - "CreateTopicsRequest": NewMockCreateTopicsResponse(t), - }) - - config := NewConfig() - config.Version = V0_10_2_0 - admin, err := NewClusterAdmin([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - err = admin.CreateTopic("my_topic", nil, false) - if err.Error() != "You must specify topic details" { - t.Fatal(err) - } - err = admin.Close() - if err != nil { - t.Fatal(err) - } -} - -func TestClusterAdminCreateTopicWithDiffVersion(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - defer seedBroker.Close() - - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetController(seedBroker.BrokerID()). - SetBroker(seedBroker.Addr(), seedBroker.BrokerID()), - "CreateTopicsRequest": NewMockCreateTopicsResponse(t), - }) - - config := NewConfig() - config.Version = V0_11_0_0 - admin, err := NewClusterAdmin([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - err = admin.CreateTopic("my_topic", &TopicDetail{NumPartitions: 1, ReplicationFactor: 1}, false) - if err != ErrInsufficientData { - t.Fatal(err) - } - - err = admin.Close() - if err != nil { - t.Fatal(err) - } -} - -func TestClusterAdminDeleteTopic(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - defer seedBroker.Close() - - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetController(seedBroker.BrokerID()). - SetBroker(seedBroker.Addr(), seedBroker.BrokerID()), - "DeleteTopicsRequest": NewMockDeleteTopicsResponse(t), - }) - - config := NewConfig() - config.Version = V0_10_2_0 - admin, err := NewClusterAdmin([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - err = admin.DeleteTopic("my_topic") - if err != nil { - t.Fatal(err) - } - - err = admin.Close() - if err != nil { - t.Fatal(err) - } -} - -func TestClusterAdminDeleteEmptyTopic(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - defer seedBroker.Close() - - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetController(seedBroker.BrokerID()). - SetBroker(seedBroker.Addr(), seedBroker.BrokerID()), - "DeleteTopicsRequest": NewMockDeleteTopicsResponse(t), - }) - - config := NewConfig() - config.Version = V0_10_2_0 - admin, err := NewClusterAdmin([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - err = admin.DeleteTopic("") - if err != ErrInvalidTopic { - t.Fatal(err) - } - - err = admin.Close() - if err != nil { - t.Fatal(err) - } -} - -func TestClusterAdminCreatePartitions(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - defer seedBroker.Close() - - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetController(seedBroker.BrokerID()). - SetBroker(seedBroker.Addr(), seedBroker.BrokerID()), - "CreatePartitionsRequest": NewMockCreatePartitionsResponse(t), - }) - - config := NewConfig() - config.Version = V1_0_0_0 - admin, err := NewClusterAdmin([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - err = admin.CreatePartitions("my_topic", 3, nil, false) - if err != nil { - t.Fatal(err) - } - - err = admin.Close() - if err != nil { - t.Fatal(err) - } -} - -func TestClusterAdminCreatePartitionsWithDiffVersion(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - defer seedBroker.Close() - - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetController(seedBroker.BrokerID()). - SetBroker(seedBroker.Addr(), seedBroker.BrokerID()), - "CreatePartitionsRequest": NewMockCreatePartitionsResponse(t), - }) - - config := NewConfig() - config.Version = V0_10_2_0 - admin, err := NewClusterAdmin([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - err = admin.CreatePartitions("my_topic", 3, nil, false) - if err != ErrUnsupportedVersion { - t.Fatal(err) - } - - err = admin.Close() - if err != nil { - t.Fatal(err) - } -} - -func TestClusterAdminDeleteRecords(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - defer seedBroker.Close() - - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetController(seedBroker.BrokerID()). - SetBroker(seedBroker.Addr(), seedBroker.BrokerID()), - "DeleteRecordsRequest": NewMockDeleteRecordsResponse(t), - }) - - config := NewConfig() - config.Version = V1_0_0_0 - admin, err := NewClusterAdmin([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - partitionOffset := make(map[int32]int64) - partitionOffset[1] = 1000 - partitionOffset[2] = 1000 - partitionOffset[3] = 1000 - - err = admin.DeleteRecords("my_topic", partitionOffset) - if err != nil { - t.Fatal(err) - } - - err = admin.Close() - if err != nil { - t.Fatal(err) - } -} - -func TestClusterAdminDeleteRecordsWithDiffVersion(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - defer seedBroker.Close() - - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetController(seedBroker.BrokerID()). - SetBroker(seedBroker.Addr(), seedBroker.BrokerID()), - "DeleteRecordsRequest": NewMockDeleteRecordsResponse(t), - }) - - config := NewConfig() - config.Version = V0_10_2_0 - admin, err := NewClusterAdmin([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - partitionOffset := make(map[int32]int64) - partitionOffset[1] = 1000 - partitionOffset[2] = 1000 - partitionOffset[3] = 1000 - - err = admin.DeleteRecords("my_topic", partitionOffset) - if err != ErrUnsupportedVersion { - t.Fatal(err) - } - - err = admin.Close() - if err != nil { - t.Fatal(err) - } -} - -func TestClusterAdminDescribeConfig(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - defer seedBroker.Close() - - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetController(seedBroker.BrokerID()). - SetBroker(seedBroker.Addr(), seedBroker.BrokerID()), - "DescribeConfigsRequest": NewMockDescribeConfigsResponse(t), - }) - - config := NewConfig() - config.Version = V1_0_0_0 - admin, err := NewClusterAdmin([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - resource := ConfigResource{Name: "r1", Type: TopicResource, ConfigNames: []string{"my_topic"}} - entries, err := admin.DescribeConfig(resource) - if err != nil { - t.Fatal(err) - } - - if len(entries) <= 0 { - t.Fatal(errors.New("no resource present")) - } - - err = admin.Close() - if err != nil { - t.Fatal(err) - } -} - -func TestClusterAdminAlterConfig(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - defer seedBroker.Close() - - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetController(seedBroker.BrokerID()). - SetBroker(seedBroker.Addr(), seedBroker.BrokerID()), - "AlterConfigsRequest": NewMockAlterConfigsResponse(t), - }) - - config := NewConfig() - config.Version = V1_0_0_0 - admin, err := NewClusterAdmin([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - var value string - entries := make(map[string]*string) - value = "3" - entries["ReplicationFactor"] = &value - err = admin.AlterConfig(TopicResource, "my_topic", entries, false) - if err != nil { - t.Fatal(err) - } - - err = admin.Close() - if err != nil { - t.Fatal(err) - } -} - -func TestClusterAdminCreateAcl(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - defer seedBroker.Close() - - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetController(seedBroker.BrokerID()). - SetBroker(seedBroker.Addr(), seedBroker.BrokerID()), - "CreateAclsRequest": NewMockCreateAclsResponse(t), - }) - - config := NewConfig() - config.Version = V1_0_0_0 - admin, err := NewClusterAdmin([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - r := Resource{ResourceType: AclResourceTopic, ResourceName: "my_topic"} - a := Acl{Host: "localhost", Operation: AclOperationAlter, PermissionType: AclPermissionAny} - - err = admin.CreateACL(r, a) - if err != nil { - t.Fatal(err) - } - - err = admin.Close() - if err != nil { - t.Fatal(err) - } -} - -func TestClusterAdminListAcls(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - defer seedBroker.Close() - - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetController(seedBroker.BrokerID()). - SetBroker(seedBroker.Addr(), seedBroker.BrokerID()), - "DescribeAclsRequest": NewMockListAclsResponse(t), - "CreateAclsRequest": NewMockCreateAclsResponse(t), - }) - - config := NewConfig() - config.Version = V1_0_0_0 - admin, err := NewClusterAdmin([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - r := Resource{ResourceType: AclResourceTopic, ResourceName: "my_topic"} - a := Acl{Host: "localhost", Operation: AclOperationAlter, PermissionType: AclPermissionAny} - - err = admin.CreateACL(r, a) - if err != nil { - t.Fatal(err) - } - resourceName := "my_topic" - filter := AclFilter{ - ResourceType: AclResourceTopic, - Operation: AclOperationRead, - ResourceName: &resourceName, - } - - rAcls, err := admin.ListAcls(filter) - if err != nil { - t.Fatal(err) - } - if len(rAcls) <= 0 { - t.Fatal("no acls present") - } - - err = admin.Close() - if err != nil { - t.Fatal(err) - } -} - -func TestClusterAdminDeleteAcl(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - defer seedBroker.Close() - - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetController(seedBroker.BrokerID()). - SetBroker(seedBroker.Addr(), seedBroker.BrokerID()), - "DeleteAclsRequest": NewMockDeleteAclsResponse(t), - }) - - config := NewConfig() - config.Version = V1_0_0_0 - admin, err := NewClusterAdmin([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - resourceName := "my_topic" - filter := AclFilter{ - ResourceType: AclResourceTopic, - Operation: AclOperationAlter, - ResourceName: &resourceName, - } - - _, err = admin.DeleteACL(filter, false) - if err != nil { - t.Fatal(err) - } - - err = admin.Close() - if err != nil { - t.Fatal(err) - } -} diff --git a/third/github.com/Shopify/sarama/alter_configs_request.go b/third/github.com/Shopify/sarama/alter_configs_request.go deleted file mode 100644 index 48c44ead6..000000000 --- a/third/github.com/Shopify/sarama/alter_configs_request.go +++ /dev/null @@ -1,120 +0,0 @@ -package sarama - -type AlterConfigsRequest struct { - Resources []*AlterConfigsResource - ValidateOnly bool -} - -type AlterConfigsResource struct { - Type ConfigResourceType - Name string - ConfigEntries map[string]*string -} - -func (acr *AlterConfigsRequest) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(acr.Resources)); err != nil { - return err - } - - for _, r := range acr.Resources { - if err := r.encode(pe); err != nil { - return err - } - } - - pe.putBool(acr.ValidateOnly) - return nil -} - -func (acr *AlterConfigsRequest) decode(pd packetDecoder, version int16) error { - resourceCount, err := pd.getArrayLength() - if err != nil { - return err - } - - acr.Resources = make([]*AlterConfigsResource, resourceCount) - for i := range acr.Resources { - r := &AlterConfigsResource{} - err = r.decode(pd, version) - if err != nil { - return err - } - acr.Resources[i] = r - } - - validateOnly, err := pd.getBool() - if err != nil { - return err - } - - acr.ValidateOnly = validateOnly - - return nil -} - -func (ac *AlterConfigsResource) encode(pe packetEncoder) error { - pe.putInt8(int8(ac.Type)) - - if err := pe.putString(ac.Name); err != nil { - return err - } - - if err := pe.putArrayLength(len(ac.ConfigEntries)); err != nil { - return err - } - for configKey, configValue := range ac.ConfigEntries { - if err := pe.putString(configKey); err != nil { - return err - } - if err := pe.putNullableString(configValue); err != nil { - return err - } - } - - return nil -} - -func (ac *AlterConfigsResource) decode(pd packetDecoder, version int16) error { - t, err := pd.getInt8() - if err != nil { - return err - } - ac.Type = ConfigResourceType(t) - - name, err := pd.getString() - if err != nil { - return err - } - ac.Name = name - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - if n > 0 { - ac.ConfigEntries = make(map[string]*string, n) - for i := 0; i < n; i++ { - configKey, err := pd.getString() - if err != nil { - return err - } - if ac.ConfigEntries[configKey], err = pd.getNullableString(); err != nil { - return err - } - } - } - return err -} - -func (acr *AlterConfigsRequest) key() int16 { - return 33 -} - -func (acr *AlterConfigsRequest) version() int16 { - return 0 -} - -func (acr *AlterConfigsRequest) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/third/github.com/Shopify/sarama/alter_configs_request_test.go b/third/github.com/Shopify/sarama/alter_configs_request_test.go deleted file mode 100644 index b9407ca7e..000000000 --- a/third/github.com/Shopify/sarama/alter_configs_request_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package sarama - -import "testing" - -var ( - emptyAlterConfigsRequest = []byte{ - 0, 0, 0, 0, // 0 configs - 0, // don't Validate - } - - singleAlterConfigsRequest = []byte{ - 0, 0, 0, 1, // 1 config - 2, // a topic - 0, 3, 'f', 'o', 'o', // topic name: foo - 0, 0, 0, 1, //1 config name - 0, 10, // 10 chars - 's', 'e', 'g', 'm', 'e', 'n', 't', '.', 'm', 's', - 0, 4, - '1', '0', '0', '0', - 0, // don't validate - } - - doubleAlterConfigsRequest = []byte{ - 0, 0, 0, 2, // 2 config - 2, // a topic - 0, 3, 'f', 'o', 'o', // topic name: foo - 0, 0, 0, 1, //1 config name - 0, 10, // 10 chars - 's', 'e', 'g', 'm', 'e', 'n', 't', '.', 'm', 's', - 0, 4, - '1', '0', '0', '0', - 2, // a topic - 0, 3, 'b', 'a', 'r', // topic name: foo - 0, 0, 0, 1, //2 config - 0, 12, // 12 chars - 'r', 'e', 't', 'e', 'n', 't', 'i', 'o', 'n', '.', 'm', 's', - 0, 4, - '1', '0', '0', '0', - 0, // don't validate - } -) - -func TestAlterConfigsRequest(t *testing.T) { - var request *AlterConfigsRequest - - request = &AlterConfigsRequest{ - Resources: []*AlterConfigsResource{}, - } - testRequest(t, "no requests", request, emptyAlterConfigsRequest) - - configValue := "1000" - request = &AlterConfigsRequest{ - Resources: []*AlterConfigsResource{ - &AlterConfigsResource{ - Type: TopicResource, - Name: "foo", - ConfigEntries: map[string]*string{ - "segment.ms": &configValue, - }, - }, - }, - } - - testRequest(t, "one config", request, singleAlterConfigsRequest) - - request = &AlterConfigsRequest{ - Resources: []*AlterConfigsResource{ - &AlterConfigsResource{ - Type: TopicResource, - Name: "foo", - ConfigEntries: map[string]*string{ - "segment.ms": &configValue, - }, - }, - &AlterConfigsResource{ - Type: TopicResource, - Name: "bar", - ConfigEntries: map[string]*string{ - "retention.ms": &configValue, - }, - }, - }, - } - - testRequest(t, "two configs", request, doubleAlterConfigsRequest) -} diff --git a/third/github.com/Shopify/sarama/alter_configs_response.go b/third/github.com/Shopify/sarama/alter_configs_response.go deleted file mode 100644 index 29b09e1ff..000000000 --- a/third/github.com/Shopify/sarama/alter_configs_response.go +++ /dev/null @@ -1,95 +0,0 @@ -package sarama - -import "time" - -type AlterConfigsResponse struct { - ThrottleTime time.Duration - Resources []*AlterConfigsResourceResponse -} - -type AlterConfigsResourceResponse struct { - ErrorCode int16 - ErrorMsg string - Type ConfigResourceType - Name string -} - -func (ct *AlterConfigsResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(ct.ThrottleTime / time.Millisecond)) - - if err := pe.putArrayLength(len(ct.Resources)); err != nil { - return err - } - - for i := range ct.Resources { - pe.putInt16(ct.Resources[i].ErrorCode) - err := pe.putString(ct.Resources[i].ErrorMsg) - if err != nil { - return nil - } - pe.putInt8(int8(ct.Resources[i].Type)) - err = pe.putString(ct.Resources[i].Name) - if err != nil { - return nil - } - } - - return nil -} - -func (acr *AlterConfigsResponse) decode(pd packetDecoder, version int16) error { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - acr.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - responseCount, err := pd.getArrayLength() - if err != nil { - return err - } - - acr.Resources = make([]*AlterConfigsResourceResponse, responseCount) - - for i := range acr.Resources { - acr.Resources[i] = new(AlterConfigsResourceResponse) - - errCode, err := pd.getInt16() - if err != nil { - return err - } - acr.Resources[i].ErrorCode = errCode - - e, err := pd.getString() - if err != nil { - return err - } - acr.Resources[i].ErrorMsg = e - - t, err := pd.getInt8() - if err != nil { - return err - } - acr.Resources[i].Type = ConfigResourceType(t) - - name, err := pd.getString() - if err != nil { - return err - } - acr.Resources[i].Name = name - } - - return nil -} - -func (r *AlterConfigsResponse) key() int16 { - return 32 -} - -func (r *AlterConfigsResponse) version() int16 { - return 0 -} - -func (r *AlterConfigsResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/third/github.com/Shopify/sarama/alter_configs_response_test.go b/third/github.com/Shopify/sarama/alter_configs_response_test.go deleted file mode 100644 index 459202870..000000000 --- a/third/github.com/Shopify/sarama/alter_configs_response_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package sarama - -import ( - "testing" -) - -var ( - alterResponseEmpty = []byte{ - 0, 0, 0, 0, //throttle - 0, 0, 0, 0, // no configs - } - - alterResponsePopulated = []byte{ - 0, 0, 0, 0, //throttle - 0, 0, 0, 1, // response - 0, 0, //errorcode - 0, 0, //string - 2, // topic - 0, 3, 'f', 'o', 'o', - } -) - -func TestAlterConfigsResponse(t *testing.T) { - var response *AlterConfigsResponse - - response = &AlterConfigsResponse{ - Resources: []*AlterConfigsResourceResponse{}, - } - testVersionDecodable(t, "empty", response, alterResponseEmpty, 0) - if len(response.Resources) != 0 { - t.Error("Expected no groups") - } - - response = &AlterConfigsResponse{ - Resources: []*AlterConfigsResourceResponse{ - &AlterConfigsResourceResponse{ - ErrorCode: 0, - ErrorMsg: "", - Type: TopicResource, - Name: "foo", - }, - }, - } - testResponse(t, "response with error", response, alterResponsePopulated) -} diff --git a/third/github.com/Shopify/sarama/api_versions_request.go b/third/github.com/Shopify/sarama/api_versions_request.go deleted file mode 100644 index ab65f01cc..000000000 --- a/third/github.com/Shopify/sarama/api_versions_request.go +++ /dev/null @@ -1,24 +0,0 @@ -package sarama - -type ApiVersionsRequest struct { -} - -func (r *ApiVersionsRequest) encode(pe packetEncoder) error { - return nil -} - -func (r *ApiVersionsRequest) decode(pd packetDecoder, version int16) (err error) { - return nil -} - -func (r *ApiVersionsRequest) key() int16 { - return 18 -} - -func (r *ApiVersionsRequest) version() int16 { - return 0 -} - -func (r *ApiVersionsRequest) requiredVersion() KafkaVersion { - return V0_10_0_0 -} diff --git a/third/github.com/Shopify/sarama/api_versions_request_test.go b/third/github.com/Shopify/sarama/api_versions_request_test.go deleted file mode 100644 index 5ab4fa71c..000000000 --- a/third/github.com/Shopify/sarama/api_versions_request_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package sarama - -import "testing" - -var ( - apiVersionRequest = []byte{} -) - -func TestApiVersionsRequest(t *testing.T) { - var request *ApiVersionsRequest - - request = new(ApiVersionsRequest) - testRequest(t, "basic", request, apiVersionRequest) -} diff --git a/third/github.com/Shopify/sarama/api_versions_response.go b/third/github.com/Shopify/sarama/api_versions_response.go deleted file mode 100644 index 23bc326e1..000000000 --- a/third/github.com/Shopify/sarama/api_versions_response.go +++ /dev/null @@ -1,87 +0,0 @@ -package sarama - -type ApiVersionsResponseBlock struct { - ApiKey int16 - MinVersion int16 - MaxVersion int16 -} - -func (b *ApiVersionsResponseBlock) encode(pe packetEncoder) error { - pe.putInt16(b.ApiKey) - pe.putInt16(b.MinVersion) - pe.putInt16(b.MaxVersion) - return nil -} - -func (b *ApiVersionsResponseBlock) decode(pd packetDecoder) error { - var err error - - if b.ApiKey, err = pd.getInt16(); err != nil { - return err - } - - if b.MinVersion, err = pd.getInt16(); err != nil { - return err - } - - if b.MaxVersion, err = pd.getInt16(); err != nil { - return err - } - - return nil -} - -type ApiVersionsResponse struct { - Err KError - ApiVersions []*ApiVersionsResponseBlock -} - -func (r *ApiVersionsResponse) encode(pe packetEncoder) error { - pe.putInt16(int16(r.Err)) - if err := pe.putArrayLength(len(r.ApiVersions)); err != nil { - return err - } - for _, apiVersion := range r.ApiVersions { - if err := apiVersion.encode(pe); err != nil { - return err - } - } - return nil -} - -func (r *ApiVersionsResponse) decode(pd packetDecoder, version int16) error { - kerr, err := pd.getInt16() - if err != nil { - return err - } - - r.Err = KError(kerr) - - numBlocks, err := pd.getArrayLength() - if err != nil { - return err - } - - r.ApiVersions = make([]*ApiVersionsResponseBlock, numBlocks) - for i := 0; i < numBlocks; i++ { - block := new(ApiVersionsResponseBlock) - if err := block.decode(pd); err != nil { - return err - } - r.ApiVersions[i] = block - } - - return nil -} - -func (r *ApiVersionsResponse) key() int16 { - return 18 -} - -func (r *ApiVersionsResponse) version() int16 { - return 0 -} - -func (r *ApiVersionsResponse) requiredVersion() KafkaVersion { - return V0_10_0_0 -} diff --git a/third/github.com/Shopify/sarama/api_versions_response_test.go b/third/github.com/Shopify/sarama/api_versions_response_test.go deleted file mode 100644 index 675a65a7d..000000000 --- a/third/github.com/Shopify/sarama/api_versions_response_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package sarama - -import "testing" - -var ( - apiVersionResponse = []byte{ - 0x00, 0x00, - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x03, - 0x00, 0x02, - 0x00, 0x01, - } -) - -func TestApiVersionsResponse(t *testing.T) { - var response *ApiVersionsResponse - - response = new(ApiVersionsResponse) - testVersionDecodable(t, "no error", response, apiVersionResponse, 0) - if response.Err != ErrNoError { - t.Error("Decoding error failed: no error expected but found", response.Err) - } - if response.ApiVersions[0].ApiKey != 0x03 { - t.Error("Decoding error: expected 0x03 but got", response.ApiVersions[0].ApiKey) - } - if response.ApiVersions[0].MinVersion != 0x02 { - t.Error("Decoding error: expected 0x02 but got", response.ApiVersions[0].MinVersion) - } - if response.ApiVersions[0].MaxVersion != 0x01 { - t.Error("Decoding error: expected 0x01 but got", response.ApiVersions[0].MaxVersion) - } -} diff --git a/third/github.com/Shopify/sarama/async_producer.go b/third/github.com/Shopify/sarama/async_producer.go deleted file mode 100644 index 39b5c6e28..000000000 --- a/third/github.com/Shopify/sarama/async_producer.go +++ /dev/null @@ -1,1052 +0,0 @@ -package sarama - -import ( - "encoding/binary" - "fmt" - "sync" - "time" - - "github.com/gogf/gf/third/github.com/eapache/go-resiliency/breaker" - "github.com/gogf/gf/third/github.com/eapache/queue" -) - -// AsyncProducer publishes Kafka messages using a non-blocking API. It routes messages -// to the correct broker for the provided topic-partition, refreshing metadata as appropriate, -// and parses responses for errors. You must read from the Errors() channel or the -// producer will deadlock. You must call Close() or AsyncClose() on a producer to avoid -// leaks: it will not be garbage-collected automatically when it passes out of -// scope. -type AsyncProducer interface { - - // AsyncClose triggers a shutdown of the producer. The shutdown has completed - // when both the Errors and Successes channels have been closed. When calling - // AsyncClose, you *must* continue to read from those channels in order to - // drain the results of any messages in flight. - AsyncClose() - - // Close shuts down the producer and waits for any buffered messages to be - // flushed. You must call this function before a producer object passes out of - // scope, as it may otherwise leak memory. You must call this before calling - // Close on the underlying client. - Close() error - - // Input is the input channel for the user to write messages to that they - // wish to send. - Input() chan<- *ProducerMessage - - // Successes is the success output channel back to the user when Return.Successes is - // enabled. If Return.Successes is true, you MUST read from this channel or the - // Producer will deadlock. It is suggested that you send and read messages - // together in a single select statement. - Successes() <-chan *ProducerMessage - - // Errors is the error output channel back to the user. You MUST read from this - // channel or the Producer will deadlock when the channel is full. Alternatively, - // you can set Producer.Return.Errors in your config to false, which prevents - // errors to be returned. - Errors() <-chan *ProducerError -} - -// transactionManager keeps the state necessary to ensure idempotent production -type transactionManager struct { - producerID int64 - producerEpoch int16 - sequenceNumbers map[string]int32 - mutex sync.Mutex -} - -const ( - noProducerID = -1 - noProducerEpoch = -1 -) - -func (t *transactionManager) getAndIncrementSequenceNumber(topic string, partition int32) int32 { - key := fmt.Sprintf("%s-%d", topic, partition) - t.mutex.Lock() - defer t.mutex.Unlock() - sequence := t.sequenceNumbers[key] - t.sequenceNumbers[key] = sequence + 1 - return sequence -} - -func newTransactionManager(conf *Config, client Client) (*transactionManager, error) { - txnmgr := &transactionManager{ - producerID: noProducerID, - producerEpoch: noProducerEpoch, - } - - if conf.Producer.Idempotent { - initProducerIDResponse, err := client.InitProducerID() - if err != nil { - return nil, err - } - txnmgr.producerID = initProducerIDResponse.ProducerID - txnmgr.producerEpoch = initProducerIDResponse.ProducerEpoch - txnmgr.sequenceNumbers = make(map[string]int32) - txnmgr.mutex = sync.Mutex{} - - Logger.Printf("Obtained a ProducerId: %d and ProducerEpoch: %d\n", txnmgr.producerID, txnmgr.producerEpoch) - } - - return txnmgr, nil -} - -type asyncProducer struct { - client Client - conf *Config - ownClient bool - - errors chan *ProducerError - input, successes, retries chan *ProducerMessage - inFlight sync.WaitGroup - - brokers map[*Broker]*brokerProducer - brokerRefs map[*brokerProducer]int - brokerLock sync.Mutex - - txnmgr *transactionManager -} - -// NewAsyncProducer creates a new AsyncProducer using the given broker addresses and configuration. -func NewAsyncProducer(addrs []string, conf *Config) (AsyncProducer, error) { - client, err := NewClient(addrs, conf) - if err != nil { - return nil, err - } - - p, err := NewAsyncProducerFromClient(client) - if err != nil { - return nil, err - } - p.(*asyncProducer).ownClient = true - return p, nil -} - -// NewAsyncProducerFromClient creates a new Producer using the given client. It is still -// necessary to call Close() on the underlying client when shutting down this producer. -func NewAsyncProducerFromClient(client Client) (AsyncProducer, error) { - // Check that we are not dealing with a closed Client before processing any other arguments - if client.Closed() { - return nil, ErrClosedClient - } - - txnmgr, err := newTransactionManager(client.Config(), client) - if err != nil { - return nil, err - } - - p := &asyncProducer{ - client: client, - conf: client.Config(), - errors: make(chan *ProducerError), - input: make(chan *ProducerMessage), - successes: make(chan *ProducerMessage), - retries: make(chan *ProducerMessage), - brokers: make(map[*Broker]*brokerProducer), - brokerRefs: make(map[*brokerProducer]int), - txnmgr: txnmgr, - } - - // launch our singleton dispatchers - go withRecover(p.dispatcher) - go withRecover(p.retryHandler) - - return p, nil -} - -type flagSet int8 - -const ( - syn flagSet = 1 << iota // first message from partitionProducer to brokerProducer - fin // final message from partitionProducer to brokerProducer and back - shutdown // start the shutdown process -) - -// ProducerMessage is the collection of elements passed to the Producer in order to send a message. -type ProducerMessage struct { - Topic string // The Kafka topic for this message. - // The partitioning key for this message. Pre-existing Encoders include - // StringEncoder and ByteEncoder. - Key Encoder - // The actual message to store in Kafka. Pre-existing Encoders include - // StringEncoder and ByteEncoder. - Value Encoder - - // The headers are key-value pairs that are transparently passed - // by Kafka between producers and consumers. - Headers []RecordHeader - - // This field is used to hold arbitrary data you wish to include so it - // will be available when receiving on the Successes and Errors channels. - // Sarama completely ignores this field and is only to be used for - // pass-through data. - Metadata interface{} - - // Below this point are filled in by the producer as the message is processed - - // Offset is the offset of the message stored on the broker. This is only - // guaranteed to be defined if the message was successfully delivered and - // RequiredAcks is not NoResponse. - Offset int64 - // Partition is the partition that the message was sent to. This is only - // guaranteed to be defined if the message was successfully delivered. - Partition int32 - // Timestamp is the timestamp assigned to the message by the broker. This - // is only guaranteed to be defined if the message was successfully - // delivered, RequiredAcks is not NoResponse, and the Kafka broker is at - // least version 0.10.0. - Timestamp time.Time - - retries int - flags flagSet - expectation chan *ProducerError - sequenceNumber int32 -} - -const producerMessageOverhead = 26 // the metadata overhead of CRC, flags, etc. - -func (m *ProducerMessage) byteSize(version int) int { - var size int - if version >= 2 { - size = maximumRecordOverhead - for _, h := range m.Headers { - size += len(h.Key) + len(h.Value) + 2*binary.MaxVarintLen32 - } - } else { - size = producerMessageOverhead - } - if m.Key != nil { - size += m.Key.Length() - } - if m.Value != nil { - size += m.Value.Length() - } - return size -} - -func (m *ProducerMessage) clear() { - m.flags = 0 - m.retries = 0 -} - -// ProducerError is the type of error generated when the producer fails to deliver a message. -// It contains the original ProducerMessage as well as the actual error value. -type ProducerError struct { - Msg *ProducerMessage - Err error -} - -func (pe ProducerError) Error() string { - return fmt.Sprintf("kafka: Failed to produce message to topic %s: %s", pe.Msg.Topic, pe.Err) -} - -// ProducerErrors is a type that wraps a batch of "ProducerError"s and implements the Error interface. -// It can be returned from the Producer's Close method to avoid the need to manually drain the Errors channel -// when closing a producer. -type ProducerErrors []*ProducerError - -func (pe ProducerErrors) Error() string { - return fmt.Sprintf("kafka: Failed to deliver %d messages.", len(pe)) -} - -func (p *asyncProducer) Errors() <-chan *ProducerError { - return p.errors -} - -func (p *asyncProducer) Successes() <-chan *ProducerMessage { - return p.successes -} - -func (p *asyncProducer) Input() chan<- *ProducerMessage { - return p.input -} - -func (p *asyncProducer) Close() error { - p.AsyncClose() - - if p.conf.Producer.Return.Successes { - go withRecover(func() { - for range p.successes { - } - }) - } - - var errors ProducerErrors - if p.conf.Producer.Return.Errors { - for event := range p.errors { - errors = append(errors, event) - } - } else { - <-p.errors - } - - if len(errors) > 0 { - return errors - } - return nil -} - -func (p *asyncProducer) AsyncClose() { - go withRecover(p.shutdown) -} - -// singleton -// dispatches messages by topic -func (p *asyncProducer) dispatcher() { - handlers := make(map[string]chan<- *ProducerMessage) - shuttingDown := false - - for msg := range p.input { - if msg == nil { - Logger.Println("Something tried to send a nil message, it was ignored.") - continue - } - - if msg.flags&shutdown != 0 { - shuttingDown = true - p.inFlight.Done() - continue - } else if msg.retries == 0 { - if shuttingDown { - // we can't just call returnError here because that decrements the wait group, - // which hasn't been incremented yet for this message, and shouldn't be - pErr := &ProducerError{Msg: msg, Err: ErrShuttingDown} - if p.conf.Producer.Return.Errors { - p.errors <- pErr - } else { - Logger.Println(pErr) - } - continue - } - p.inFlight.Add(1) - } - - version := 1 - if p.conf.Version.IsAtLeast(V0_11_0_0) { - version = 2 - } else if msg.Headers != nil { - p.returnError(msg, ConfigurationError("Producing headers requires Kafka at least v0.11")) - continue - } - if msg.byteSize(version) > p.conf.Producer.MaxMessageBytes { - p.returnError(msg, ErrMessageSizeTooLarge) - continue - } - - handler := handlers[msg.Topic] - if handler == nil { - handler = p.newTopicProducer(msg.Topic) - handlers[msg.Topic] = handler - } - - handler <- msg - } - - for _, handler := range handlers { - close(handler) - } -} - -// one per topic -// partitions messages, then dispatches them by partition -type topicProducer struct { - parent *asyncProducer - topic string - input <-chan *ProducerMessage - - breaker *breaker.Breaker - handlers map[int32]chan<- *ProducerMessage - partitioner Partitioner -} - -func (p *asyncProducer) newTopicProducer(topic string) chan<- *ProducerMessage { - input := make(chan *ProducerMessage, p.conf.ChannelBufferSize) - tp := &topicProducer{ - parent: p, - topic: topic, - input: input, - breaker: breaker.New(3, 1, 10*time.Second), - handlers: make(map[int32]chan<- *ProducerMessage), - partitioner: p.conf.Producer.Partitioner(topic), - } - go withRecover(tp.dispatch) - return input -} - -func (tp *topicProducer) dispatch() { - for msg := range tp.input { - if msg.retries == 0 { - if err := tp.partitionMessage(msg); err != nil { - tp.parent.returnError(msg, err) - continue - } - } - // All messages being retried (sent or not) have already had their retry count updated - if tp.parent.conf.Producer.Idempotent && msg.retries == 0 { - msg.sequenceNumber = tp.parent.txnmgr.getAndIncrementSequenceNumber(msg.Topic, msg.Partition) - } - - handler := tp.handlers[msg.Partition] - if handler == nil { - handler = tp.parent.newPartitionProducer(msg.Topic, msg.Partition) - tp.handlers[msg.Partition] = handler - } - - handler <- msg - } - - for _, handler := range tp.handlers { - close(handler) - } -} - -func (tp *topicProducer) partitionMessage(msg *ProducerMessage) error { - var partitions []int32 - - err := tp.breaker.Run(func() (err error) { - var requiresConsistency = false - if ep, ok := tp.partitioner.(DynamicConsistencyPartitioner); ok { - requiresConsistency = ep.MessageRequiresConsistency(msg) - } else { - requiresConsistency = tp.partitioner.RequiresConsistency() - } - - if requiresConsistency { - partitions, err = tp.parent.client.Partitions(msg.Topic) - } else { - partitions, err = tp.parent.client.WritablePartitions(msg.Topic) - } - return - }) - - if err != nil { - return err - } - - numPartitions := int32(len(partitions)) - - if numPartitions == 0 { - return ErrLeaderNotAvailable - } - - choice, err := tp.partitioner.Partition(msg, numPartitions) - - if err != nil { - return err - } else if choice < 0 || choice >= numPartitions { - return ErrInvalidPartition - } - - msg.Partition = partitions[choice] - - return nil -} - -// one per partition per topic -// dispatches messages to the appropriate broker -// also responsible for maintaining message order during retries -type partitionProducer struct { - parent *asyncProducer - topic string - partition int32 - input <-chan *ProducerMessage - - leader *Broker - breaker *breaker.Breaker - brokerProducer *brokerProducer - - // highWatermark tracks the "current" retry level, which is the only one where we actually let messages through, - // all other messages get buffered in retryState[msg.retries].buf to preserve ordering - // retryState[msg.retries].expectChaser simply tracks whether we've seen a fin message for a given level (and - // therefore whether our buffer is complete and safe to flush) - highWatermark int - retryState []partitionRetryState -} - -type partitionRetryState struct { - buf []*ProducerMessage - expectChaser bool -} - -func (p *asyncProducer) newPartitionProducer(topic string, partition int32) chan<- *ProducerMessage { - input := make(chan *ProducerMessage, p.conf.ChannelBufferSize) - pp := &partitionProducer{ - parent: p, - topic: topic, - partition: partition, - input: input, - - breaker: breaker.New(3, 1, 10*time.Second), - retryState: make([]partitionRetryState, p.conf.Producer.Retry.Max+1), - } - go withRecover(pp.dispatch) - return input -} - -func (pp *partitionProducer) dispatch() { - // try to prefetch the leader; if this doesn't work, we'll do a proper call to `updateLeader` - // on the first message - pp.leader, _ = pp.parent.client.Leader(pp.topic, pp.partition) - if pp.leader != nil { - pp.brokerProducer = pp.parent.getBrokerProducer(pp.leader) - pp.parent.inFlight.Add(1) // we're generating a syn message; track it so we don't shut down while it's still inflight - pp.brokerProducer.input <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: syn} - } - - for msg := range pp.input { - if msg.retries > pp.highWatermark { - // a new, higher, retry level; handle it and then back off - pp.newHighWatermark(msg.retries) - time.Sleep(pp.parent.conf.Producer.Retry.Backoff) - } else if pp.highWatermark > 0 { - // we are retrying something (else highWatermark would be 0) but this message is not a *new* retry level - if msg.retries < pp.highWatermark { - // in fact this message is not even the current retry level, so buffer it for now (unless it's a just a fin) - if msg.flags&fin == fin { - pp.retryState[msg.retries].expectChaser = false - pp.parent.inFlight.Done() // this fin is now handled and will be garbage collected - } else { - pp.retryState[msg.retries].buf = append(pp.retryState[msg.retries].buf, msg) - } - continue - } else if msg.flags&fin == fin { - // this message is of the current retry level (msg.retries == highWatermark) and the fin flag is set, - // meaning this retry level is done and we can go down (at least) one level and flush that - pp.retryState[pp.highWatermark].expectChaser = false - pp.flushRetryBuffers() - pp.parent.inFlight.Done() // this fin is now handled and will be garbage collected - continue - } - } - - // if we made it this far then the current msg contains real data, and can be sent to the next goroutine - // without breaking any of our ordering guarantees - - if pp.brokerProducer == nil { - if err := pp.updateLeader(); err != nil { - pp.parent.returnError(msg, err) - time.Sleep(pp.parent.conf.Producer.Retry.Backoff) - continue - } - Logger.Printf("producer/leader/%s/%d selected broker %d\n", pp.topic, pp.partition, pp.leader.ID()) - } - - pp.brokerProducer.input <- msg - } - - if pp.brokerProducer != nil { - pp.parent.unrefBrokerProducer(pp.leader, pp.brokerProducer) - } -} - -func (pp *partitionProducer) newHighWatermark(hwm int) { - Logger.Printf("producer/leader/%s/%d state change to [retrying-%d]\n", pp.topic, pp.partition, hwm) - pp.highWatermark = hwm - - // send off a fin so that we know when everything "in between" has made it - // back to us and we can safely flush the backlog (otherwise we risk re-ordering messages) - pp.retryState[pp.highWatermark].expectChaser = true - pp.parent.inFlight.Add(1) // we're generating a fin message; track it so we don't shut down while it's still inflight - pp.brokerProducer.input <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: fin, retries: pp.highWatermark - 1} - - // a new HWM means that our current broker selection is out of date - Logger.Printf("producer/leader/%s/%d abandoning broker %d\n", pp.topic, pp.partition, pp.leader.ID()) - pp.parent.unrefBrokerProducer(pp.leader, pp.brokerProducer) - pp.brokerProducer = nil -} - -func (pp *partitionProducer) flushRetryBuffers() { - Logger.Printf("producer/leader/%s/%d state change to [flushing-%d]\n", pp.topic, pp.partition, pp.highWatermark) - for { - pp.highWatermark-- - - if pp.brokerProducer == nil { - if err := pp.updateLeader(); err != nil { - pp.parent.returnErrors(pp.retryState[pp.highWatermark].buf, err) - goto flushDone - } - Logger.Printf("producer/leader/%s/%d selected broker %d\n", pp.topic, pp.partition, pp.leader.ID()) - } - - for _, msg := range pp.retryState[pp.highWatermark].buf { - pp.brokerProducer.input <- msg - } - - flushDone: - pp.retryState[pp.highWatermark].buf = nil - if pp.retryState[pp.highWatermark].expectChaser { - Logger.Printf("producer/leader/%s/%d state change to [retrying-%d]\n", pp.topic, pp.partition, pp.highWatermark) - break - } else if pp.highWatermark == 0 { - Logger.Printf("producer/leader/%s/%d state change to [normal]\n", pp.topic, pp.partition) - break - } - } -} - -func (pp *partitionProducer) updateLeader() error { - return pp.breaker.Run(func() (err error) { - if err = pp.parent.client.RefreshMetadata(pp.topic); err != nil { - return err - } - - if pp.leader, err = pp.parent.client.Leader(pp.topic, pp.partition); err != nil { - return err - } - - pp.brokerProducer = pp.parent.getBrokerProducer(pp.leader) - pp.parent.inFlight.Add(1) // we're generating a syn message; track it so we don't shut down while it's still inflight - pp.brokerProducer.input <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: syn} - - return nil - }) -} - -// one per broker; also constructs an associated flusher -func (p *asyncProducer) newBrokerProducer(broker *Broker) *brokerProducer { - var ( - input = make(chan *ProducerMessage) - bridge = make(chan *produceSet) - responses = make(chan *brokerProducerResponse) - ) - - bp := &brokerProducer{ - parent: p, - broker: broker, - input: input, - output: bridge, - responses: responses, - buffer: newProduceSet(p), - currentRetries: make(map[string]map[int32]error), - } - go withRecover(bp.run) - - // minimal bridge to make the network response `select`able - go withRecover(func() { - for set := range bridge { - request := set.buildRequest() - - response, err := broker.Produce(request) - - responses <- &brokerProducerResponse{ - set: set, - err: err, - res: response, - } - } - close(responses) - }) - - return bp -} - -type brokerProducerResponse struct { - set *produceSet - err error - res *ProduceResponse -} - -// groups messages together into appropriately-sized batches for sending to the broker -// handles state related to retries etc -type brokerProducer struct { - parent *asyncProducer - broker *Broker - - input chan *ProducerMessage - output chan<- *produceSet - responses <-chan *brokerProducerResponse - - buffer *produceSet - timer <-chan time.Time - timerFired bool - - closing error - currentRetries map[string]map[int32]error -} - -func (bp *brokerProducer) run() { - var output chan<- *produceSet - Logger.Printf("producer/broker/%d starting up\n", bp.broker.ID()) - - for { - select { - case msg := <-bp.input: - if msg == nil { - bp.shutdown() - return - } - - if msg.flags&syn == syn { - Logger.Printf("producer/broker/%d state change to [open] on %s/%d\n", - bp.broker.ID(), msg.Topic, msg.Partition) - if bp.currentRetries[msg.Topic] == nil { - bp.currentRetries[msg.Topic] = make(map[int32]error) - } - bp.currentRetries[msg.Topic][msg.Partition] = nil - bp.parent.inFlight.Done() - continue - } - - if reason := bp.needsRetry(msg); reason != nil { - bp.parent.retryMessage(msg, reason) - - if bp.closing == nil && msg.flags&fin == fin { - // we were retrying this partition but we can start processing again - delete(bp.currentRetries[msg.Topic], msg.Partition) - Logger.Printf("producer/broker/%d state change to [closed] on %s/%d\n", - bp.broker.ID(), msg.Topic, msg.Partition) - } - - continue - } - - if bp.buffer.wouldOverflow(msg) { - if err := bp.waitForSpace(msg); err != nil { - bp.parent.retryMessage(msg, err) - continue - } - } - - if err := bp.buffer.add(msg); err != nil { - bp.parent.returnError(msg, err) - continue - } - - if bp.parent.conf.Producer.Flush.Frequency > 0 && bp.timer == nil { - bp.timer = time.After(bp.parent.conf.Producer.Flush.Frequency) - } - case <-bp.timer: - bp.timerFired = true - case output <- bp.buffer: - bp.rollOver() - case response := <-bp.responses: - bp.handleResponse(response) - } - - if bp.timerFired || bp.buffer.readyToFlush() { - output = bp.output - } else { - output = nil - } - } -} - -func (bp *brokerProducer) shutdown() { - for !bp.buffer.empty() { - select { - case response := <-bp.responses: - bp.handleResponse(response) - case bp.output <- bp.buffer: - bp.rollOver() - } - } - close(bp.output) - for response := range bp.responses { - bp.handleResponse(response) - } - - Logger.Printf("producer/broker/%d shut down\n", bp.broker.ID()) -} - -func (bp *brokerProducer) needsRetry(msg *ProducerMessage) error { - if bp.closing != nil { - return bp.closing - } - - return bp.currentRetries[msg.Topic][msg.Partition] -} - -func (bp *brokerProducer) waitForSpace(msg *ProducerMessage) error { - Logger.Printf("producer/broker/%d maximum request accumulated, waiting for space\n", bp.broker.ID()) - - for { - select { - case response := <-bp.responses: - bp.handleResponse(response) - // handling a response can change our state, so re-check some things - if reason := bp.needsRetry(msg); reason != nil { - return reason - } else if !bp.buffer.wouldOverflow(msg) { - return nil - } - case bp.output <- bp.buffer: - bp.rollOver() - return nil - } - } -} - -func (bp *brokerProducer) rollOver() { - bp.timer = nil - bp.timerFired = false - bp.buffer = newProduceSet(bp.parent) -} - -func (bp *brokerProducer) handleResponse(response *brokerProducerResponse) { - if response.err != nil { - bp.handleError(response.set, response.err) - } else { - bp.handleSuccess(response.set, response.res) - } - - if bp.buffer.empty() { - bp.rollOver() // this can happen if the response invalidated our buffer - } -} - -func (bp *brokerProducer) handleSuccess(sent *produceSet, response *ProduceResponse) { - // we iterate through the blocks in the request set, not the response, so that we notice - // if the response is missing a block completely - var retryTopics []string - sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) { - if response == nil { - // this only happens when RequiredAcks is NoResponse, so we have to assume success - bp.parent.returnSuccesses(pSet.msgs) - return - } - - block := response.GetBlock(topic, partition) - if block == nil { - bp.parent.returnErrors(pSet.msgs, ErrIncompleteResponse) - return - } - - switch block.Err { - // Success - case ErrNoError: - if bp.parent.conf.Version.IsAtLeast(V0_10_0_0) && !block.Timestamp.IsZero() { - for _, msg := range pSet.msgs { - msg.Timestamp = block.Timestamp - } - } - for i, msg := range pSet.msgs { - msg.Offset = block.Offset + int64(i) - } - bp.parent.returnSuccesses(pSet.msgs) - // Duplicate - case ErrDuplicateSequenceNumber: - bp.parent.returnSuccesses(pSet.msgs) - // Retriable errors - case ErrInvalidMessage, ErrUnknownTopicOrPartition, ErrLeaderNotAvailable, ErrNotLeaderForPartition, - ErrRequestTimedOut, ErrNotEnoughReplicas, ErrNotEnoughReplicasAfterAppend: - retryTopics = append(retryTopics, topic) - // Other non-retriable errors - default: - bp.parent.returnErrors(pSet.msgs, block.Err) - } - }) - - if len(retryTopics) > 0 { - if bp.parent.conf.Producer.Idempotent { - err := bp.parent.client.RefreshMetadata(retryTopics...) - if err != nil { - Logger.Printf("Failed refreshing metadata because of %v\n", err) - } - } - - sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) { - block := response.GetBlock(topic, partition) - if block == nil { - // handled in the previous "eachPartition" loop - return - } - - switch block.Err { - case ErrInvalidMessage, ErrUnknownTopicOrPartition, ErrLeaderNotAvailable, ErrNotLeaderForPartition, - ErrRequestTimedOut, ErrNotEnoughReplicas, ErrNotEnoughReplicasAfterAppend: - Logger.Printf("producer/broker/%d state change to [retrying] on %s/%d because %v\n", - bp.broker.ID(), topic, partition, block.Err) - if bp.currentRetries[topic] == nil { - bp.currentRetries[topic] = make(map[int32]error) - } - bp.currentRetries[topic][partition] = block.Err - if bp.parent.conf.Producer.Idempotent { - go bp.parent.retryBatch(topic, partition, pSet, block.Err) - } else { - bp.parent.retryMessages(pSet.msgs, block.Err) - } - // dropping the following messages has the side effect of incrementing their retry count - bp.parent.retryMessages(bp.buffer.dropPartition(topic, partition), block.Err) - } - }) - } -} - -func (p *asyncProducer) retryBatch(topic string, partition int32, pSet *partitionSet, kerr KError) { - Logger.Printf("Retrying batch for %v-%d because of %s\n", topic, partition, kerr) - produceSet := newProduceSet(p) - produceSet.msgs[topic] = make(map[int32]*partitionSet) - produceSet.msgs[topic][partition] = pSet - produceSet.bufferBytes += pSet.bufferBytes - produceSet.bufferCount += len(pSet.msgs) - for _, msg := range pSet.msgs { - if msg.retries >= p.conf.Producer.Retry.Max { - p.returnError(msg, kerr) - return - } - msg.retries++ - } - - // it's expected that a metadata refresh has been requested prior to calling retryBatch - leader, err := p.client.Leader(topic, partition) - if err != nil { - Logger.Printf("Failed retrying batch for %v-%d because of %v while looking up for new leader\n", topic, partition, err) - for _, msg := range pSet.msgs { - p.returnError(msg, kerr) - } - return - } - bp := p.getBrokerProducer(leader) - bp.output <- produceSet -} - -func (bp *brokerProducer) handleError(sent *produceSet, err error) { - switch err.(type) { - case PacketEncodingError: - sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) { - bp.parent.returnErrors(pSet.msgs, err) - }) - default: - Logger.Printf("producer/broker/%d state change to [closing] because %s\n", bp.broker.ID(), err) - bp.parent.abandonBrokerConnection(bp.broker) - _ = bp.broker.Close() - bp.closing = err - sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) { - bp.parent.retryMessages(pSet.msgs, err) - }) - bp.buffer.eachPartition(func(topic string, partition int32, pSet *partitionSet) { - bp.parent.retryMessages(pSet.msgs, err) - }) - bp.rollOver() - } -} - -// singleton -// effectively a "bridge" between the flushers and the dispatcher in order to avoid deadlock -// based on https://godoc.org/github.com/eapache/channels#InfiniteChannel -func (p *asyncProducer) retryHandler() { - var msg *ProducerMessage - buf := queue.New() - - for { - if buf.Length() == 0 { - msg = <-p.retries - } else { - select { - case msg = <-p.retries: - case p.input <- buf.Peek().(*ProducerMessage): - buf.Remove() - continue - } - } - - if msg == nil { - return - } - - buf.Add(msg) - } -} - -// utility functions - -func (p *asyncProducer) shutdown() { - Logger.Println("Producer shutting down.") - p.inFlight.Add(1) - p.input <- &ProducerMessage{flags: shutdown} - - p.inFlight.Wait() - - if p.ownClient { - err := p.client.Close() - if err != nil { - Logger.Println("producer/shutdown failed to close the embedded client:", err) - } - } - - close(p.input) - close(p.retries) - close(p.errors) - close(p.successes) -} - -func (p *asyncProducer) returnError(msg *ProducerMessage, err error) { - msg.clear() - pErr := &ProducerError{Msg: msg, Err: err} - if p.conf.Producer.Return.Errors { - p.errors <- pErr - } else { - Logger.Println(pErr) - } - p.inFlight.Done() -} - -func (p *asyncProducer) returnErrors(batch []*ProducerMessage, err error) { - for _, msg := range batch { - p.returnError(msg, err) - } -} - -func (p *asyncProducer) returnSuccesses(batch []*ProducerMessage) { - for _, msg := range batch { - if p.conf.Producer.Return.Successes { - msg.clear() - p.successes <- msg - } - p.inFlight.Done() - } -} - -func (p *asyncProducer) retryMessage(msg *ProducerMessage, err error) { - if msg.retries >= p.conf.Producer.Retry.Max { - p.returnError(msg, err) - } else { - msg.retries++ - p.retries <- msg - } -} - -func (p *asyncProducer) retryMessages(batch []*ProducerMessage, err error) { - for _, msg := range batch { - p.retryMessage(msg, err) - } -} - -func (p *asyncProducer) getBrokerProducer(broker *Broker) *brokerProducer { - p.brokerLock.Lock() - defer p.brokerLock.Unlock() - - bp := p.brokers[broker] - - if bp == nil { - bp = p.newBrokerProducer(broker) - p.brokers[broker] = bp - p.brokerRefs[bp] = 0 - } - - p.brokerRefs[bp]++ - - return bp -} - -func (p *asyncProducer) unrefBrokerProducer(broker *Broker, bp *brokerProducer) { - p.brokerLock.Lock() - defer p.brokerLock.Unlock() - - p.brokerRefs[bp]-- - if p.brokerRefs[bp] == 0 { - close(bp.input) - delete(p.brokerRefs, bp) - - if p.brokers[broker] == bp { - delete(p.brokers, broker) - } - } -} - -func (p *asyncProducer) abandonBrokerConnection(broker *Broker) { - p.brokerLock.Lock() - defer p.brokerLock.Unlock() - - delete(p.brokers, broker) -} diff --git a/third/github.com/Shopify/sarama/async_producer_test.go b/third/github.com/Shopify/sarama/async_producer_test.go deleted file mode 100644 index 038aa0b0c..000000000 --- a/third/github.com/Shopify/sarama/async_producer_test.go +++ /dev/null @@ -1,1095 +0,0 @@ -package sarama - -import ( - "errors" - "log" - "os" - "os/signal" - "sync" - "testing" - "time" -) - -const TestMessage = "ABC THE MESSAGE" - -func closeProducer(t *testing.T, p AsyncProducer) { - var wg sync.WaitGroup - p.AsyncClose() - - wg.Add(2) - go func() { - for range p.Successes() { - t.Error("Unexpected message on Successes()") - } - wg.Done() - }() - go func() { - for msg := range p.Errors() { - t.Error(msg.Err) - } - wg.Done() - }() - wg.Wait() -} - -func expectResults(t *testing.T, p AsyncProducer, successes, errors int) { - expect := successes + errors - for expect > 0 { - select { - case msg := <-p.Errors(): - if msg.Msg.flags != 0 { - t.Error("Message had flags set") - } - errors-- - expect-- - if errors < 0 { - t.Error(msg.Err) - } - case msg := <-p.Successes(): - if msg.flags != 0 { - t.Error("Message had flags set") - } - successes-- - expect-- - if successes < 0 { - t.Error("Too many successes") - } - } - } - if successes != 0 || errors != 0 { - t.Error("Unexpected successes", successes, "or errors", errors) - } -} - -type testPartitioner chan *int32 - -func (p testPartitioner) Partition(msg *ProducerMessage, numPartitions int32) (int32, error) { - part := <-p - if part == nil { - return 0, errors.New("BOOM") - } - - return *part, nil -} - -func (p testPartitioner) RequiresConsistency() bool { - return true -} - -func (p testPartitioner) feed(partition int32) { - p <- &partition -} - -type flakyEncoder bool - -func (f flakyEncoder) Length() int { - return len(TestMessage) -} - -func (f flakyEncoder) Encode() ([]byte, error) { - if !bool(f) { - return nil, errors.New("flaky encoding error") - } - return []byte(TestMessage), nil -} - -func TestAsyncProducer(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader := NewMockBroker(t, 2) - - metadataResponse := new(MetadataResponse) - metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) - metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataResponse) - - prodSuccess := new(ProduceResponse) - prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError) - leader.Returns(prodSuccess) - - config := NewConfig() - config.Producer.Flush.Messages = 10 - config.Producer.Return.Successes = true - producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < 10; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage), Metadata: i} - } - for i := 0; i < 10; i++ { - select { - case msg := <-producer.Errors(): - t.Error(msg.Err) - if msg.Msg.flags != 0 { - t.Error("Message had flags set") - } - case msg := <-producer.Successes(): - if msg.flags != 0 { - t.Error("Message had flags set") - } - if msg.Metadata.(int) != i { - t.Error("Message metadata did not match") - } - case <-time.After(time.Second): - t.Errorf("Timeout waiting for msg #%d", i) - goto done - } - } -done: - closeProducer(t, producer) - leader.Close() - seedBroker.Close() -} - -func TestAsyncProducerMultipleFlushes(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader := NewMockBroker(t, 2) - - metadataResponse := new(MetadataResponse) - metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) - metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataResponse) - - prodSuccess := new(ProduceResponse) - prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError) - leader.Returns(prodSuccess) - leader.Returns(prodSuccess) - leader.Returns(prodSuccess) - - config := NewConfig() - config.Producer.Flush.Messages = 5 - config.Producer.Return.Successes = true - producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - for flush := 0; flush < 3; flush++ { - for i := 0; i < 5; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - } - expectResults(t, producer, 5, 0) - } - - closeProducer(t, producer) - leader.Close() - seedBroker.Close() -} - -func TestAsyncProducerMultipleBrokers(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader0 := NewMockBroker(t, 2) - leader1 := NewMockBroker(t, 3) - - metadataResponse := new(MetadataResponse) - metadataResponse.AddBroker(leader0.Addr(), leader0.BrokerID()) - metadataResponse.AddBroker(leader1.Addr(), leader1.BrokerID()) - metadataResponse.AddTopicPartition("my_topic", 0, leader0.BrokerID(), nil, nil, ErrNoError) - metadataResponse.AddTopicPartition("my_topic", 1, leader1.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataResponse) - - prodResponse0 := new(ProduceResponse) - prodResponse0.AddTopicPartition("my_topic", 0, ErrNoError) - leader0.Returns(prodResponse0) - - prodResponse1 := new(ProduceResponse) - prodResponse1.AddTopicPartition("my_topic", 1, ErrNoError) - leader1.Returns(prodResponse1) - - config := NewConfig() - config.Producer.Flush.Messages = 5 - config.Producer.Return.Successes = true - config.Producer.Partitioner = NewRoundRobinPartitioner - producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < 10; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - } - expectResults(t, producer, 10, 0) - - closeProducer(t, producer) - leader1.Close() - leader0.Close() - seedBroker.Close() -} - -func TestAsyncProducerCustomPartitioner(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader := NewMockBroker(t, 2) - - metadataResponse := new(MetadataResponse) - metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) - metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataResponse) - - prodResponse := new(ProduceResponse) - prodResponse.AddTopicPartition("my_topic", 0, ErrNoError) - leader.Returns(prodResponse) - - config := NewConfig() - config.Producer.Flush.Messages = 2 - config.Producer.Return.Successes = true - config.Producer.Partitioner = func(topic string) Partitioner { - p := make(testPartitioner) - go func() { - p.feed(0) - p <- nil - p <- nil - p <- nil - p.feed(0) - }() - return p - } - producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < 5; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - } - expectResults(t, producer, 2, 3) - - closeProducer(t, producer) - leader.Close() - seedBroker.Close() -} - -func TestAsyncProducerFailureRetry(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader1 := NewMockBroker(t, 2) - leader2 := NewMockBroker(t, 3) - - metadataLeader1 := new(MetadataResponse) - metadataLeader1.AddBroker(leader1.Addr(), leader1.BrokerID()) - metadataLeader1.AddTopicPartition("my_topic", 0, leader1.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataLeader1) - - config := NewConfig() - config.Producer.Flush.Messages = 10 - config.Producer.Return.Successes = true - config.Producer.Retry.Backoff = 0 - producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - seedBroker.Close() - - for i := 0; i < 10; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - } - prodNotLeader := new(ProduceResponse) - prodNotLeader.AddTopicPartition("my_topic", 0, ErrNotLeaderForPartition) - leader1.Returns(prodNotLeader) - - metadataLeader2 := new(MetadataResponse) - metadataLeader2.AddBroker(leader2.Addr(), leader2.BrokerID()) - metadataLeader2.AddTopicPartition("my_topic", 0, leader2.BrokerID(), nil, nil, ErrNoError) - leader1.Returns(metadataLeader2) - - prodSuccess := new(ProduceResponse) - prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError) - leader2.Returns(prodSuccess) - expectResults(t, producer, 10, 0) - leader1.Close() - - for i := 0; i < 10; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - } - leader2.Returns(prodSuccess) - expectResults(t, producer, 10, 0) - - leader2.Close() - closeProducer(t, producer) -} - -func TestAsyncProducerEncoderFailures(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader := NewMockBroker(t, 2) - - metadataResponse := new(MetadataResponse) - metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) - metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataResponse) - - prodSuccess := new(ProduceResponse) - prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError) - leader.Returns(prodSuccess) - leader.Returns(prodSuccess) - leader.Returns(prodSuccess) - - config := NewConfig() - config.Producer.Flush.Messages = 1 - config.Producer.Return.Successes = true - config.Producer.Partitioner = NewManualPartitioner - producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - for flush := 0; flush < 3; flush++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: flakyEncoder(true), Value: flakyEncoder(false)} - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: flakyEncoder(false), Value: flakyEncoder(true)} - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: flakyEncoder(true), Value: flakyEncoder(true)} - expectResults(t, producer, 1, 2) - } - - closeProducer(t, producer) - leader.Close() - seedBroker.Close() -} - -// If a Kafka broker becomes unavailable and then returns back in service, then -// producer reconnects to it and continues sending messages. -func TestAsyncProducerBrokerBounce(t *testing.T) { - // Given - seedBroker := NewMockBroker(t, 1) - leader := NewMockBroker(t, 2) - leaderAddr := leader.Addr() - - metadataResponse := new(MetadataResponse) - metadataResponse.AddBroker(leaderAddr, leader.BrokerID()) - metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataResponse) - - prodSuccess := new(ProduceResponse) - prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError) - - config := NewConfig() - config.Producer.Flush.Messages = 1 - config.Producer.Return.Successes = true - config.Producer.Retry.Backoff = 0 - producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - leader.Returns(prodSuccess) - expectResults(t, producer, 1, 0) - - // When: a broker connection gets reset by a broker (network glitch, restart, you name it). - leader.Close() // producer should get EOF - leader = NewMockBrokerAddr(t, 2, leaderAddr) // start it up again right away for giggles - seedBroker.Returns(metadataResponse) // tell it to go to broker 2 again - - // Then: a produced message goes through the new broker connection. - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - leader.Returns(prodSuccess) - expectResults(t, producer, 1, 0) - - closeProducer(t, producer) - seedBroker.Close() - leader.Close() -} - -func TestAsyncProducerBrokerBounceWithStaleMetadata(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader1 := NewMockBroker(t, 2) - leader2 := NewMockBroker(t, 3) - - metadataLeader1 := new(MetadataResponse) - metadataLeader1.AddBroker(leader1.Addr(), leader1.BrokerID()) - metadataLeader1.AddTopicPartition("my_topic", 0, leader1.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataLeader1) - - config := NewConfig() - config.Producer.Flush.Messages = 10 - config.Producer.Return.Successes = true - config.Producer.Retry.Max = 3 - config.Producer.Retry.Backoff = 0 - producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < 10; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - } - leader1.Close() // producer should get EOF - seedBroker.Returns(metadataLeader1) // tell it to go to leader1 again even though it's still down - seedBroker.Returns(metadataLeader1) // tell it to go to leader1 again even though it's still down - - // ok fine, tell it to go to leader2 finally - metadataLeader2 := new(MetadataResponse) - metadataLeader2.AddBroker(leader2.Addr(), leader2.BrokerID()) - metadataLeader2.AddTopicPartition("my_topic", 0, leader2.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataLeader2) - - prodSuccess := new(ProduceResponse) - prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError) - leader2.Returns(prodSuccess) - expectResults(t, producer, 10, 0) - seedBroker.Close() - leader2.Close() - - closeProducer(t, producer) -} - -func TestAsyncProducerMultipleRetries(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader1 := NewMockBroker(t, 2) - leader2 := NewMockBroker(t, 3) - - metadataLeader1 := new(MetadataResponse) - metadataLeader1.AddBroker(leader1.Addr(), leader1.BrokerID()) - metadataLeader1.AddTopicPartition("my_topic", 0, leader1.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataLeader1) - - config := NewConfig() - config.Producer.Flush.Messages = 10 - config.Producer.Return.Successes = true - config.Producer.Retry.Max = 4 - config.Producer.Retry.Backoff = 0 - producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < 10; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - } - prodNotLeader := new(ProduceResponse) - prodNotLeader.AddTopicPartition("my_topic", 0, ErrNotLeaderForPartition) - leader1.Returns(prodNotLeader) - - metadataLeader2 := new(MetadataResponse) - metadataLeader2.AddBroker(leader2.Addr(), leader2.BrokerID()) - metadataLeader2.AddTopicPartition("my_topic", 0, leader2.BrokerID(), nil, nil, ErrNoError) - - seedBroker.Returns(metadataLeader2) - leader2.Returns(prodNotLeader) - seedBroker.Returns(metadataLeader1) - leader1.Returns(prodNotLeader) - seedBroker.Returns(metadataLeader1) - leader1.Returns(prodNotLeader) - seedBroker.Returns(metadataLeader2) - - prodSuccess := new(ProduceResponse) - prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError) - leader2.Returns(prodSuccess) - expectResults(t, producer, 10, 0) - - for i := 0; i < 10; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - } - leader2.Returns(prodSuccess) - expectResults(t, producer, 10, 0) - - seedBroker.Close() - leader1.Close() - leader2.Close() - closeProducer(t, producer) -} - -func TestAsyncProducerOutOfRetries(t *testing.T) { - t.Skip("Enable once bug #294 is fixed.") - - seedBroker := NewMockBroker(t, 1) - leader := NewMockBroker(t, 2) - - metadataResponse := new(MetadataResponse) - metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) - metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataResponse) - - config := NewConfig() - config.Producer.Flush.Messages = 10 - config.Producer.Return.Successes = true - config.Producer.Retry.Backoff = 0 - config.Producer.Retry.Max = 0 - producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < 10; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - } - - prodNotLeader := new(ProduceResponse) - prodNotLeader.AddTopicPartition("my_topic", 0, ErrNotLeaderForPartition) - leader.Returns(prodNotLeader) - - for i := 0; i < 10; i++ { - select { - case msg := <-producer.Errors(): - if msg.Err != ErrNotLeaderForPartition { - t.Error(msg.Err) - } - case <-producer.Successes(): - t.Error("Unexpected success") - } - } - - seedBroker.Returns(metadataResponse) - - for i := 0; i < 10; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - } - - prodSuccess := new(ProduceResponse) - prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError) - leader.Returns(prodSuccess) - - expectResults(t, producer, 10, 0) - - leader.Close() - seedBroker.Close() - safeClose(t, producer) -} - -func TestAsyncProducerRetryWithReferenceOpen(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader := NewMockBroker(t, 2) - leaderAddr := leader.Addr() - - metadataResponse := new(MetadataResponse) - metadataResponse.AddBroker(leaderAddr, leader.BrokerID()) - metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError) - metadataResponse.AddTopicPartition("my_topic", 1, leader.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataResponse) - - config := NewConfig() - config.Producer.Return.Successes = true - config.Producer.Retry.Backoff = 0 - config.Producer.Retry.Max = 1 - config.Producer.Partitioner = NewRoundRobinPartitioner - producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - // prime partition 0 - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - prodSuccess := new(ProduceResponse) - prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError) - leader.Returns(prodSuccess) - expectResults(t, producer, 1, 0) - - // prime partition 1 - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - prodSuccess = new(ProduceResponse) - prodSuccess.AddTopicPartition("my_topic", 1, ErrNoError) - leader.Returns(prodSuccess) - expectResults(t, producer, 1, 0) - - // reboot the broker (the producer will get EOF on its existing connection) - leader.Close() - leader = NewMockBrokerAddr(t, 2, leaderAddr) - - // send another message on partition 0 to trigger the EOF and retry - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - - // tell partition 0 to go to that broker again - seedBroker.Returns(metadataResponse) - - // succeed this time - prodSuccess = new(ProduceResponse) - prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError) - leader.Returns(prodSuccess) - expectResults(t, producer, 1, 0) - - // shutdown - closeProducer(t, producer) - seedBroker.Close() - leader.Close() -} - -func TestAsyncProducerFlusherRetryCondition(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader := NewMockBroker(t, 2) - - metadataResponse := new(MetadataResponse) - metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) - metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError) - metadataResponse.AddTopicPartition("my_topic", 1, leader.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataResponse) - - config := NewConfig() - config.Producer.Flush.Messages = 5 - config.Producer.Return.Successes = true - config.Producer.Retry.Backoff = 0 - config.Producer.Retry.Max = 1 - config.Producer.Partitioner = NewManualPartitioner - producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - // prime partitions - for p := int32(0); p < 2; p++ { - for i := 0; i < 5; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage), Partition: p} - } - prodSuccess := new(ProduceResponse) - prodSuccess.AddTopicPartition("my_topic", p, ErrNoError) - leader.Returns(prodSuccess) - expectResults(t, producer, 5, 0) - } - - // send more messages on partition 0 - for i := 0; i < 5; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage), Partition: 0} - } - prodNotLeader := new(ProduceResponse) - prodNotLeader.AddTopicPartition("my_topic", 0, ErrNotLeaderForPartition) - leader.Returns(prodNotLeader) - - time.Sleep(50 * time.Millisecond) - - leader.SetHandlerByMap(map[string]MockResponse{ - "ProduceRequest": NewMockProduceResponse(t). - SetVersion(0). - SetError("my_topic", 0, ErrNoError), - }) - - // tell partition 0 to go to that broker again - seedBroker.Returns(metadataResponse) - - // succeed this time - expectResults(t, producer, 5, 0) - - // put five more through - for i := 0; i < 5; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage), Partition: 0} - } - expectResults(t, producer, 5, 0) - - // shutdown - closeProducer(t, producer) - seedBroker.Close() - leader.Close() -} - -func TestAsyncProducerRetryShutdown(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader := NewMockBroker(t, 2) - - metadataLeader := new(MetadataResponse) - metadataLeader.AddBroker(leader.Addr(), leader.BrokerID()) - metadataLeader.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataLeader) - - config := NewConfig() - config.Producer.Flush.Messages = 10 - config.Producer.Return.Successes = true - config.Producer.Retry.Backoff = 0 - producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < 10; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - } - producer.AsyncClose() - time.Sleep(5 * time.Millisecond) // let the shutdown goroutine kick in - - producer.Input() <- &ProducerMessage{Topic: "FOO"} - if err := <-producer.Errors(); err.Err != ErrShuttingDown { - t.Error(err) - } - - prodNotLeader := new(ProduceResponse) - prodNotLeader.AddTopicPartition("my_topic", 0, ErrNotLeaderForPartition) - leader.Returns(prodNotLeader) - - seedBroker.Returns(metadataLeader) - - prodSuccess := new(ProduceResponse) - prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError) - leader.Returns(prodSuccess) - expectResults(t, producer, 10, 0) - - seedBroker.Close() - leader.Close() - - // wait for the async-closed producer to shut down fully - for err := range producer.Errors() { - t.Error(err) - } -} - -func TestAsyncProducerNoReturns(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader := NewMockBroker(t, 2) - - metadataLeader := new(MetadataResponse) - metadataLeader.AddBroker(leader.Addr(), leader.BrokerID()) - metadataLeader.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataLeader) - - config := NewConfig() - config.Producer.Flush.Messages = 10 - config.Producer.Return.Successes = false - config.Producer.Return.Errors = false - config.Producer.Retry.Backoff = 0 - producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < 10; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - } - - wait := make(chan bool) - go func() { - if err := producer.Close(); err != nil { - t.Error(err) - } - close(wait) - }() - - prodSuccess := new(ProduceResponse) - prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError) - leader.Returns(prodSuccess) - - <-wait - seedBroker.Close() - leader.Close() -} - -func TestAsyncProducerIdempotentGoldenPath(t *testing.T) { - broker := NewMockBroker(t, 1) - - metadataResponse := &MetadataResponse{ - Version: 1, - ControllerID: 1, - } - metadataResponse.AddBroker(broker.Addr(), broker.BrokerID()) - metadataResponse.AddTopicPartition("my_topic", 0, broker.BrokerID(), nil, nil, ErrNoError) - broker.Returns(metadataResponse) - - initProducerID := &InitProducerIDResponse{ - ThrottleTime: 0, - ProducerID: 1000, - ProducerEpoch: 1, - } - broker.Returns(initProducerID) - - config := NewConfig() - config.Producer.Flush.Messages = 10 - config.Producer.Return.Successes = true - config.Producer.Retry.Max = 4 - config.Producer.RequiredAcks = WaitForAll - config.Producer.Retry.Backoff = 0 - config.Producer.Idempotent = true - config.Net.MaxOpenRequests = 1 - config.Version = V0_11_0_0 - producer, err := NewAsyncProducer([]string{broker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < 10; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - } - - prodSuccess := &ProduceResponse{ - Version: 3, - ThrottleTime: 0, - } - prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError) - broker.Returns(prodSuccess) - expectResults(t, producer, 10, 0) - - broker.Close() - closeProducer(t, producer) -} - -func TestAsyncProducerIdempotentRetryCheckBatch(t *testing.T) { - //Logger = log.New(os.Stderr, "", log.LstdFlags) - tests := []struct { - name string - failAfterWrite bool - }{ - {"FailAfterWrite", true}, - {"FailBeforeWrite", false}, - } - - for _, test := range tests { - broker := NewMockBroker(t, 1) - - metadataResponse := &MetadataResponse{ - Version: 1, - ControllerID: 1, - } - metadataResponse.AddBroker(broker.Addr(), broker.BrokerID()) - metadataResponse.AddTopicPartition("my_topic", 0, broker.BrokerID(), nil, nil, ErrNoError) - - initProducerIDResponse := &InitProducerIDResponse{ - ThrottleTime: 0, - ProducerID: 1000, - ProducerEpoch: 1, - } - - prodNotLeaderResponse := &ProduceResponse{ - Version: 3, - ThrottleTime: 0, - } - prodNotLeaderResponse.AddTopicPartition("my_topic", 0, ErrNotEnoughReplicas) - - prodDuplicate := &ProduceResponse{ - Version: 3, - ThrottleTime: 0, - } - prodDuplicate.AddTopicPartition("my_topic", 0, ErrDuplicateSequenceNumber) - - prodOutOfSeq := &ProduceResponse{ - Version: 3, - ThrottleTime: 0, - } - prodOutOfSeq.AddTopicPartition("my_topic", 0, ErrOutOfOrderSequenceNumber) - - prodSuccessResponse := &ProduceResponse{ - Version: 3, - ThrottleTime: 0, - } - prodSuccessResponse.AddTopicPartition("my_topic", 0, ErrNoError) - - prodCounter := 0 - lastBatchFirstSeq := -1 - lastBatchSize := -1 - lastSequenceWrittenToDisk := -1 - handlerFailBeforeWrite := func(req *request) (res encoder) { - switch req.body.key() { - case 3: - return metadataResponse - case 22: - return initProducerIDResponse - case 0: - prodCounter++ - - preq := req.body.(*ProduceRequest) - batch := preq.records["my_topic"][0].RecordBatch - batchFirstSeq := int(batch.FirstSequence) - batchSize := len(batch.Records) - - if lastSequenceWrittenToDisk == batchFirstSeq-1 { //in sequence append - - if lastBatchFirstSeq == batchFirstSeq { //is a batch retry - if lastBatchSize == batchSize { //good retry - // mock write to disk - lastSequenceWrittenToDisk = batchFirstSeq + batchSize - 1 - return prodSuccessResponse - } - t.Errorf("[%s] Retried Batch firstSeq=%d with different size old=%d new=%d", test.name, batchFirstSeq, lastBatchSize, batchSize) - return prodOutOfSeq - } else { // not a retry - // save batch just received for future check - lastBatchFirstSeq = batchFirstSeq - lastBatchSize = batchSize - - if prodCounter%2 == 1 { - if test.failAfterWrite { - // mock write to disk - lastSequenceWrittenToDisk = batchFirstSeq + batchSize - 1 - } - return prodNotLeaderResponse - } - // mock write to disk - lastSequenceWrittenToDisk = batchFirstSeq + batchSize - 1 - return prodSuccessResponse - } - } else { - if lastBatchFirstSeq == batchFirstSeq && lastBatchSize == batchSize { // is a good batch retry - if lastSequenceWrittenToDisk == (batchFirstSeq + batchSize - 1) { // we already have the messages - return prodDuplicate - } - // mock write to disk - lastSequenceWrittenToDisk = batchFirstSeq + batchSize - 1 - return prodSuccessResponse - } else { //out of sequence / bad retried batch - if lastBatchFirstSeq == batchFirstSeq && lastBatchSize != batchSize { - t.Errorf("[%s] Retried Batch firstSeq=%d with different size old=%d new=%d", test.name, batchFirstSeq, lastBatchSize, batchSize) - } else if lastSequenceWrittenToDisk+1 != batchFirstSeq { - t.Errorf("[%s] Out of sequence message lastSequence=%d new batch starts at=%d", test.name, lastSequenceWrittenToDisk, batchFirstSeq) - } else { - t.Errorf("[%s] Unexpected error", test.name) - } - - return prodOutOfSeq - } - } - - } - return nil - } - - config := NewConfig() - config.Version = V0_11_0_0 - config.Producer.Idempotent = true - config.Net.MaxOpenRequests = 1 - config.Producer.RequiredAcks = WaitForAll - config.Producer.Return.Successes = true - config.Producer.Flush.Frequency = 50 * time.Millisecond - config.Producer.Retry.Backoff = 100 * time.Millisecond - - broker.setHandler(handlerFailBeforeWrite) - producer, err := NewAsyncProducer([]string{broker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < 3; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - } - - go func() { - for i := 0; i < 7; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder("goroutine")} - time.Sleep(100 * time.Millisecond) - } - }() - - expectResults(t, producer, 10, 0) - - broker.Close() - closeProducer(t, producer) - } -} - -func TestAsyncProducerIdempotentErrorOnOutOfSeq(t *testing.T) { - broker := NewMockBroker(t, 1) - - metadataResponse := &MetadataResponse{ - Version: 1, - ControllerID: 1, - } - metadataResponse.AddBroker(broker.Addr(), broker.BrokerID()) - metadataResponse.AddTopicPartition("my_topic", 0, broker.BrokerID(), nil, nil, ErrNoError) - broker.Returns(metadataResponse) - - initProducerID := &InitProducerIDResponse{ - ThrottleTime: 0, - ProducerID: 1000, - ProducerEpoch: 1, - } - broker.Returns(initProducerID) - - config := NewConfig() - config.Producer.Flush.Messages = 10 - config.Producer.Return.Successes = true - config.Producer.Retry.Max = 400000 - config.Producer.RequiredAcks = WaitForAll - config.Producer.Retry.Backoff = 0 - config.Producer.Idempotent = true - config.Net.MaxOpenRequests = 1 - config.Version = V0_11_0_0 - - producer, err := NewAsyncProducer([]string{broker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < 10; i++ { - producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)} - } - - prodOutOfSeq := &ProduceResponse{ - Version: 3, - ThrottleTime: 0, - } - prodOutOfSeq.AddTopicPartition("my_topic", 0, ErrOutOfOrderSequenceNumber) - broker.Returns(prodOutOfSeq) - expectResults(t, producer, 0, 10) - - broker.Close() - closeProducer(t, producer) -} - -// This example shows how to use the producer while simultaneously -// reading the Errors channel to know about any failures. -func ExampleAsyncProducer_select() { - producer, err := NewAsyncProducer([]string{"localhost:9092"}, nil) - if err != nil { - panic(err) - } - - defer func() { - if err := producer.Close(); err != nil { - log.Fatalln(err) - } - }() - - // Trap SIGINT to trigger a shutdown. - signals := make(chan os.Signal, 1) - signal.Notify(signals, os.Interrupt) - - var enqueued, errors int -ProducerLoop: - for { - select { - case producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder("testing 123")}: - enqueued++ - case err := <-producer.Errors(): - log.Println("Failed to produce message", err) - errors++ - case <-signals: - break ProducerLoop - } - } - - log.Printf("Enqueued: %d; errors: %d\n", enqueued, errors) -} - -// This example shows how to use the producer with separate goroutines -// reading from the Successes and Errors channels. Note that in order -// for the Successes channel to be populated, you have to set -// config.Producer.Return.Successes to true. -func ExampleAsyncProducer_goroutines() { - config := NewConfig() - config.Producer.Return.Successes = true - producer, err := NewAsyncProducer([]string{"localhost:9092"}, config) - if err != nil { - panic(err) - } - - // Trap SIGINT to trigger a graceful shutdown. - signals := make(chan os.Signal, 1) - signal.Notify(signals, os.Interrupt) - - var ( - wg sync.WaitGroup - enqueued, successes, errors int - ) - - wg.Add(1) - go func() { - defer wg.Done() - for range producer.Successes() { - successes++ - } - }() - - wg.Add(1) - go func() { - defer wg.Done() - for err := range producer.Errors() { - log.Println(err) - errors++ - } - }() - -ProducerLoop: - for { - message := &ProducerMessage{Topic: "my_topic", Value: StringEncoder("testing 123")} - select { - case producer.Input() <- message: - enqueued++ - - case <-signals: - producer.AsyncClose() // Trigger a shutdown of the producer. - break ProducerLoop - } - } - - wg.Wait() - - log.Printf("Successfully produced: %d; errors: %d\n", successes, errors) -} diff --git a/third/github.com/Shopify/sarama/balance_strategy.go b/third/github.com/Shopify/sarama/balance_strategy.go deleted file mode 100644 index e78988d71..000000000 --- a/third/github.com/Shopify/sarama/balance_strategy.go +++ /dev/null @@ -1,129 +0,0 @@ -package sarama - -import ( - "math" - "sort" -) - -// BalanceStrategyPlan is the results of any BalanceStrategy.Plan attempt. -// It contains an allocation of topic/partitions by memberID in the form of -// a `memberID -> topic -> partitions` map. -type BalanceStrategyPlan map[string]map[string][]int32 - -// Add assigns a topic with a number partitions to a member. -func (p BalanceStrategyPlan) Add(memberID, topic string, partitions ...int32) { - if len(partitions) == 0 { - return - } - if _, ok := p[memberID]; !ok { - p[memberID] = make(map[string][]int32, 1) - } - p[memberID][topic] = append(p[memberID][topic], partitions...) -} - -// -------------------------------------------------------------------- - -// BalanceStrategy is used to balance topics and partitions -// across memebers of a consumer group -type BalanceStrategy interface { - // Name uniquely identifies the strategy. - Name() string - - // Plan accepts a map of `memberID -> metadata` and a map of `topic -> partitions` - // and returns a distribution plan. - Plan(members map[string]ConsumerGroupMemberMetadata, topics map[string][]int32) (BalanceStrategyPlan, error) -} - -// -------------------------------------------------------------------- - -// BalanceStrategyRange is the default and assigns partitions as ranges to consumer group members. -// Example with one topic T with six partitions (0..5) and two members (M1, M2): -// M1: {T: [0, 1, 2]} -// M2: {T: [3, 4, 5]} -var BalanceStrategyRange = &balanceStrategy{ - name: "range", - coreFn: func(plan BalanceStrategyPlan, memberIDs []string, topic string, partitions []int32) { - step := float64(len(partitions)) / float64(len(memberIDs)) - - for i, memberID := range memberIDs { - pos := float64(i) - min := int(math.Floor(pos*step + 0.5)) - max := int(math.Floor((pos+1)*step + 0.5)) - plan.Add(memberID, topic, partitions[min:max]...) - } - }, -} - -// BalanceStrategyRoundRobin assigns partitions to members in alternating order. -// Example with topic T with six partitions (0..5) and two members (M1, M2): -// M1: {T: [0, 2, 4]} -// M2: {T: [1, 3, 5]} -var BalanceStrategyRoundRobin = &balanceStrategy{ - name: "roundrobin", - coreFn: func(plan BalanceStrategyPlan, memberIDs []string, topic string, partitions []int32) { - for i, part := range partitions { - memberID := memberIDs[i%len(memberIDs)] - plan.Add(memberID, topic, part) - } - }, -} - -// -------------------------------------------------------------------- - -type balanceStrategy struct { - name string - coreFn func(plan BalanceStrategyPlan, memberIDs []string, topic string, partitions []int32) -} - -// Name implements BalanceStrategy. -func (s *balanceStrategy) Name() string { return s.name } - -// Balance implements BalanceStrategy. -func (s *balanceStrategy) Plan(members map[string]ConsumerGroupMemberMetadata, topics map[string][]int32) (BalanceStrategyPlan, error) { - // Build members by topic map - mbt := make(map[string][]string) - for memberID, meta := range members { - for _, topic := range meta.Topics { - mbt[topic] = append(mbt[topic], memberID) - } - } - - // Sort members for each topic - for topic, memberIDs := range mbt { - sort.Sort(&balanceStrategySortable{ - topic: topic, - memberIDs: memberIDs, - }) - } - - // Assemble plan - plan := make(BalanceStrategyPlan, len(members)) - for topic, memberIDs := range mbt { - s.coreFn(plan, memberIDs, topic, topics[topic]) - } - return plan, nil -} - -type balanceStrategySortable struct { - topic string - memberIDs []string -} - -func (p balanceStrategySortable) Len() int { return len(p.memberIDs) } -func (p balanceStrategySortable) Swap(i, j int) { - p.memberIDs[i], p.memberIDs[j] = p.memberIDs[j], p.memberIDs[i] -} -func (p balanceStrategySortable) Less(i, j int) bool { - return balanceStrategyHashValue(p.topic, p.memberIDs[i]) < balanceStrategyHashValue(p.topic, p.memberIDs[j]) -} - -func balanceStrategyHashValue(vv ...string) uint32 { - h := uint32(2166136261) - for _, s := range vv { - for _, c := range s { - h ^= uint32(c) - h *= 16777619 - } - } - return h -} diff --git a/third/github.com/Shopify/sarama/balance_strategy_test.go b/third/github.com/Shopify/sarama/balance_strategy_test.go deleted file mode 100644 index 047157f37..000000000 --- a/third/github.com/Shopify/sarama/balance_strategy_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package sarama - -import ( - "reflect" - "testing" -) - -func TestBalanceStrategyRange(t *testing.T) { - tests := []struct { - members map[string][]string - topics map[string][]int32 - expected BalanceStrategyPlan - }{ - { - members: map[string][]string{"M1": {"T1", "T2"}, "M2": {"T1", "T2"}}, - topics: map[string][]int32{"T1": {0, 1, 2, 3}, "T2": {0, 1, 2, 3}}, - expected: BalanceStrategyPlan{ - "M1": map[string][]int32{"T1": {0, 1}, "T2": {2, 3}}, - "M2": map[string][]int32{"T1": {2, 3}, "T2": {0, 1}}, - }, - }, - { - members: map[string][]string{"M1": {"T1", "T2"}, "M2": {"T1", "T2"}}, - topics: map[string][]int32{"T1": {0, 1, 2}, "T2": {0, 1, 2}}, - expected: BalanceStrategyPlan{ - "M1": map[string][]int32{"T1": {0, 1}, "T2": {2}}, - "M2": map[string][]int32{"T1": {2}, "T2": {0, 1}}, - }, - }, - { - members: map[string][]string{"M1": {"T1"}, "M2": {"T1", "T2"}}, - topics: map[string][]int32{"T1": {0, 1}, "T2": {0, 1}}, - expected: BalanceStrategyPlan{ - "M1": map[string][]int32{"T1": {0}}, - "M2": map[string][]int32{"T1": {1}, "T2": {0, 1}}, - }, - }, - } - - strategy := BalanceStrategyRange - if strategy.Name() != "range" { - t.Errorf("Unexpected stategy name\nexpected: range\nactual: %v", strategy.Name()) - } - - for _, test := range tests { - members := make(map[string]ConsumerGroupMemberMetadata) - for memberID, topics := range test.members { - members[memberID] = ConsumerGroupMemberMetadata{Topics: topics} - } - - actual, err := strategy.Plan(members, test.topics) - if err != nil { - t.Errorf("Unexpected error %v", err) - } else if !reflect.DeepEqual(actual, test.expected) { - t.Errorf("Plan does not match expectation\nexpected: %#v\nactual: %#v", test.expected, actual) - } - } -} - -func TestBalanceStrategyRoundRobin(t *testing.T) { - tests := []struct { - members map[string][]string - topics map[string][]int32 - expected BalanceStrategyPlan - }{ - { - members: map[string][]string{"M1": {"T1", "T2"}, "M2": {"T1", "T2"}}, - topics: map[string][]int32{"T1": {0, 1, 2, 3}, "T2": {0, 1, 2, 3}}, - expected: BalanceStrategyPlan{ - "M1": map[string][]int32{"T1": {0, 2}, "T2": {1, 3}}, - "M2": map[string][]int32{"T1": {1, 3}, "T2": {0, 2}}, - }, - }, - { - members: map[string][]string{"M1": {"T1", "T2"}, "M2": {"T1", "T2"}}, - topics: map[string][]int32{"T1": {0, 1, 2}, "T2": {0, 1, 2}}, - expected: BalanceStrategyPlan{ - "M1": map[string][]int32{"T1": {0, 2}, "T2": {1}}, - "M2": map[string][]int32{"T1": {1}, "T2": {0, 2}}, - }, - }, - } - - strategy := BalanceStrategyRoundRobin - if strategy.Name() != "roundrobin" { - t.Errorf("Unexpected stategy name\nexpected: range\nactual: %v", strategy.Name()) - } - - for _, test := range tests { - members := make(map[string]ConsumerGroupMemberMetadata) - for memberID, topics := range test.members { - members[memberID] = ConsumerGroupMemberMetadata{Topics: topics} - } - - actual, err := strategy.Plan(members, test.topics) - if err != nil { - t.Errorf("Unexpected error %v", err) - } else if !reflect.DeepEqual(actual, test.expected) { - t.Errorf("Plan does not match expectation\nexpected: %#v\nactual: %#v", test.expected, actual) - } - } -} diff --git a/third/github.com/Shopify/sarama/broker.go b/third/github.com/Shopify/sarama/broker.go deleted file mode 100644 index b9034aa3e..000000000 --- a/third/github.com/Shopify/sarama/broker.go +++ /dev/null @@ -1,895 +0,0 @@ -package sarama - -import ( - "crypto/tls" - "encoding/binary" - "fmt" - "io" - "net" - "strconv" - "sync" - "sync/atomic" - "time" - - "github.com/gogf/gf/third/github.com/rcrowley/go-metrics" -) - -// Broker represents a single Kafka broker connection. All operations on this object are entirely concurrency-safe. -type Broker struct { - id int32 - addr string - rack *string - - conf *Config - correlationID int32 - conn net.Conn - connErr error - lock sync.Mutex - opened int32 - - responses chan responsePromise - done chan bool - - incomingByteRate metrics.Meter - requestRate metrics.Meter - requestSize metrics.Histogram - requestLatency metrics.Histogram - outgoingByteRate metrics.Meter - responseRate metrics.Meter - responseSize metrics.Histogram - brokerIncomingByteRate metrics.Meter - brokerRequestRate metrics.Meter - brokerRequestSize metrics.Histogram - brokerRequestLatency metrics.Histogram - brokerOutgoingByteRate metrics.Meter - brokerResponseRate metrics.Meter - brokerResponseSize metrics.Histogram -} - -type responsePromise struct { - requestTime time.Time - correlationID int32 - packets chan []byte - errors chan error -} - -// NewBroker creates and returns a Broker targeting the given host:port address. -// This does not attempt to actually connect, you have to call Open() for that. -func NewBroker(addr string) *Broker { - return &Broker{id: -1, addr: addr} -} - -// Open tries to connect to the Broker if it is not already connected or connecting, but does not block -// waiting for the connection to complete. This means that any subsequent operations on the broker will -// block waiting for the connection to succeed or fail. To get the effect of a fully synchronous Open call, -// follow it by a call to Connected(). The only errors Open will return directly are ConfigurationError or -// AlreadyConnected. If conf is nil, the result of NewConfig() is used. -func (b *Broker) Open(conf *Config) error { - if !atomic.CompareAndSwapInt32(&b.opened, 0, 1) { - return ErrAlreadyConnected - } - - if conf == nil { - conf = NewConfig() - } - - err := conf.Validate() - if err != nil { - return err - } - - b.lock.Lock() - - go withRecover(func() { - defer b.lock.Unlock() - - dialer := net.Dialer{ - Timeout: conf.Net.DialTimeout, - KeepAlive: conf.Net.KeepAlive, - LocalAddr: conf.Net.LocalAddr, - } - - if conf.Net.TLS.Enable { - b.conn, b.connErr = tls.DialWithDialer(&dialer, "tcp", b.addr, conf.Net.TLS.Config) - } else { - b.conn, b.connErr = dialer.Dial("tcp", b.addr) - } - if b.connErr != nil { - Logger.Printf("Failed to connect to broker %s: %s\n", b.addr, b.connErr) - b.conn = nil - atomic.StoreInt32(&b.opened, 0) - return - } - b.conn = newBufConn(b.conn) - - b.conf = conf - - // Create or reuse the global metrics shared between brokers - b.incomingByteRate = metrics.GetOrRegisterMeter("incoming-byte-rate", conf.MetricRegistry) - b.requestRate = metrics.GetOrRegisterMeter("request-rate", conf.MetricRegistry) - b.requestSize = getOrRegisterHistogram("request-size", conf.MetricRegistry) - b.requestLatency = getOrRegisterHistogram("request-latency-in-ms", conf.MetricRegistry) - b.outgoingByteRate = metrics.GetOrRegisterMeter("outgoing-byte-rate", conf.MetricRegistry) - b.responseRate = metrics.GetOrRegisterMeter("response-rate", conf.MetricRegistry) - b.responseSize = getOrRegisterHistogram("response-size", conf.MetricRegistry) - // Do not gather metrics for seeded broker (only used during bootstrap) because they share - // the same id (-1) and are already exposed through the global metrics above - if b.id >= 0 { - b.brokerIncomingByteRate = getOrRegisterBrokerMeter("incoming-byte-rate", b, conf.MetricRegistry) - b.brokerRequestRate = getOrRegisterBrokerMeter("request-rate", b, conf.MetricRegistry) - b.brokerRequestSize = getOrRegisterBrokerHistogram("request-size", b, conf.MetricRegistry) - b.brokerRequestLatency = getOrRegisterBrokerHistogram("request-latency-in-ms", b, conf.MetricRegistry) - b.brokerOutgoingByteRate = getOrRegisterBrokerMeter("outgoing-byte-rate", b, conf.MetricRegistry) - b.brokerResponseRate = getOrRegisterBrokerMeter("response-rate", b, conf.MetricRegistry) - b.brokerResponseSize = getOrRegisterBrokerHistogram("response-size", b, conf.MetricRegistry) - } - - if conf.Net.SASL.Enable { - b.connErr = b.sendAndReceiveSASLPlainAuth() - if b.connErr != nil { - err = b.conn.Close() - if err == nil { - Logger.Printf("Closed connection to broker %s\n", b.addr) - } else { - Logger.Printf("Error while closing connection to broker %s: %s\n", b.addr, err) - } - b.conn = nil - atomic.StoreInt32(&b.opened, 0) - return - } - } - - b.done = make(chan bool) - b.responses = make(chan responsePromise, b.conf.Net.MaxOpenRequests-1) - - if b.id >= 0 { - Logger.Printf("Connected to broker at %s (registered as #%d)\n", b.addr, b.id) - } else { - Logger.Printf("Connected to broker at %s (unregistered)\n", b.addr) - } - go withRecover(b.responseReceiver) - }) - - return nil -} - -// Connected returns true if the broker is connected and false otherwise. If the broker is not -// connected but it had tried to connect, the error from that connection attempt is also returned. -func (b *Broker) Connected() (bool, error) { - b.lock.Lock() - defer b.lock.Unlock() - - return b.conn != nil, b.connErr -} - -func (b *Broker) Close() error { - b.lock.Lock() - defer b.lock.Unlock() - - if b.conn == nil { - return ErrNotConnected - } - - close(b.responses) - <-b.done - - err := b.conn.Close() - - b.conn = nil - b.connErr = nil - b.done = nil - b.responses = nil - - if b.id >= 0 { - b.conf.MetricRegistry.Unregister(getMetricNameForBroker("incoming-byte-rate", b)) - b.conf.MetricRegistry.Unregister(getMetricNameForBroker("request-rate", b)) - b.conf.MetricRegistry.Unregister(getMetricNameForBroker("outgoing-byte-rate", b)) - b.conf.MetricRegistry.Unregister(getMetricNameForBroker("response-rate", b)) - } - - if err == nil { - Logger.Printf("Closed connection to broker %s\n", b.addr) - } else { - Logger.Printf("Error while closing connection to broker %s: %s\n", b.addr, err) - } - - atomic.StoreInt32(&b.opened, 0) - - return err -} - -// ID returns the broker ID retrieved from Kafka's metadata, or -1 if that is not known. -func (b *Broker) ID() int32 { - return b.id -} - -// Addr returns the broker address as either retrieved from Kafka's metadata or passed to NewBroker. -func (b *Broker) Addr() string { - return b.addr -} - -// Rack returns the broker's rack as retrieved from Kafka's metadata or the -// empty string if it is not known. The returned value corresponds to the -// broker's broker.rack configuration setting. Requires protocol version to be -// at least v0.10.0.0. -func (b *Broker) Rack() string { - if b.rack == nil { - return "" - } - return *b.rack -} - -func (b *Broker) GetMetadata(request *MetadataRequest) (*MetadataResponse, error) { - response := new(MetadataResponse) - - err := b.sendAndReceive(request, response) - - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) GetConsumerMetadata(request *ConsumerMetadataRequest) (*ConsumerMetadataResponse, error) { - response := new(ConsumerMetadataResponse) - - err := b.sendAndReceive(request, response) - - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) FindCoordinator(request *FindCoordinatorRequest) (*FindCoordinatorResponse, error) { - response := new(FindCoordinatorResponse) - - err := b.sendAndReceive(request, response) - - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) GetAvailableOffsets(request *OffsetRequest) (*OffsetResponse, error) { - response := new(OffsetResponse) - - err := b.sendAndReceive(request, response) - - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) Produce(request *ProduceRequest) (*ProduceResponse, error) { - var response *ProduceResponse - var err error - - if request.RequiredAcks == NoResponse { - err = b.sendAndReceive(request, nil) - } else { - response = new(ProduceResponse) - err = b.sendAndReceive(request, response) - } - - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) Fetch(request *FetchRequest) (*FetchResponse, error) { - response := new(FetchResponse) - - err := b.sendAndReceive(request, response) - - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) CommitOffset(request *OffsetCommitRequest) (*OffsetCommitResponse, error) { - response := new(OffsetCommitResponse) - - err := b.sendAndReceive(request, response) - - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) FetchOffset(request *OffsetFetchRequest) (*OffsetFetchResponse, error) { - response := new(OffsetFetchResponse) - - err := b.sendAndReceive(request, response) - - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) JoinGroup(request *JoinGroupRequest) (*JoinGroupResponse, error) { - response := new(JoinGroupResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) SyncGroup(request *SyncGroupRequest) (*SyncGroupResponse, error) { - response := new(SyncGroupResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) LeaveGroup(request *LeaveGroupRequest) (*LeaveGroupResponse, error) { - response := new(LeaveGroupResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) Heartbeat(request *HeartbeatRequest) (*HeartbeatResponse, error) { - response := new(HeartbeatResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) ListGroups(request *ListGroupsRequest) (*ListGroupsResponse, error) { - response := new(ListGroupsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) DescribeGroups(request *DescribeGroupsRequest) (*DescribeGroupsResponse, error) { - response := new(DescribeGroupsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) ApiVersions(request *ApiVersionsRequest) (*ApiVersionsResponse, error) { - response := new(ApiVersionsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) CreateTopics(request *CreateTopicsRequest) (*CreateTopicsResponse, error) { - response := new(CreateTopicsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) DeleteTopics(request *DeleteTopicsRequest) (*DeleteTopicsResponse, error) { - response := new(DeleteTopicsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) CreatePartitions(request *CreatePartitionsRequest) (*CreatePartitionsResponse, error) { - response := new(CreatePartitionsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) DeleteRecords(request *DeleteRecordsRequest) (*DeleteRecordsResponse, error) { - response := new(DeleteRecordsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) DescribeAcls(request *DescribeAclsRequest) (*DescribeAclsResponse, error) { - response := new(DescribeAclsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) CreateAcls(request *CreateAclsRequest) (*CreateAclsResponse, error) { - response := new(CreateAclsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) DeleteAcls(request *DeleteAclsRequest) (*DeleteAclsResponse, error) { - response := new(DeleteAclsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) InitProducerID(request *InitProducerIDRequest) (*InitProducerIDResponse, error) { - response := new(InitProducerIDResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) AddPartitionsToTxn(request *AddPartitionsToTxnRequest) (*AddPartitionsToTxnResponse, error) { - response := new(AddPartitionsToTxnResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) AddOffsetsToTxn(request *AddOffsetsToTxnRequest) (*AddOffsetsToTxnResponse, error) { - response := new(AddOffsetsToTxnResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) EndTxn(request *EndTxnRequest) (*EndTxnResponse, error) { - response := new(EndTxnResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) TxnOffsetCommit(request *TxnOffsetCommitRequest) (*TxnOffsetCommitResponse, error) { - response := new(TxnOffsetCommitResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) DescribeConfigs(request *DescribeConfigsRequest) (*DescribeConfigsResponse, error) { - response := new(DescribeConfigsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) AlterConfigs(request *AlterConfigsRequest) (*AlterConfigsResponse, error) { - response := new(AlterConfigsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) DeleteGroups(request *DeleteGroupsRequest) (*DeleteGroupsResponse, error) { - response := new(DeleteGroupsResponse) - - if err := b.sendAndReceive(request, response); err != nil { - return nil, err - } - - return response, nil -} - -func (b *Broker) send(rb protocolBody, promiseResponse bool) (*responsePromise, error) { - b.lock.Lock() - defer b.lock.Unlock() - - if b.conn == nil { - if b.connErr != nil { - return nil, b.connErr - } - return nil, ErrNotConnected - } - - if !b.conf.Version.IsAtLeast(rb.requiredVersion()) { - return nil, ErrUnsupportedVersion - } - - req := &request{correlationID: b.correlationID, clientID: b.conf.ClientID, body: rb} - buf, err := encode(req, b.conf.MetricRegistry) - if err != nil { - return nil, err - } - - err = b.conn.SetWriteDeadline(time.Now().Add(b.conf.Net.WriteTimeout)) - if err != nil { - return nil, err - } - - requestTime := time.Now() - bytes, err := b.conn.Write(buf) - b.updateOutgoingCommunicationMetrics(bytes) - if err != nil { - return nil, err - } - b.correlationID++ - - if !promiseResponse { - // Record request latency without the response - b.updateRequestLatencyMetrics(time.Since(requestTime)) - return nil, nil - } - - promise := responsePromise{requestTime, req.correlationID, make(chan []byte), make(chan error)} - b.responses <- promise - - return &promise, nil -} - -func (b *Broker) sendAndReceive(req protocolBody, res versionedDecoder) error { - promise, err := b.send(req, res != nil) - - if err != nil { - return err - } - - if promise == nil { - return nil - } - - select { - case buf := <-promise.packets: - return versionedDecode(buf, res, req.version()) - case err = <-promise.errors: - return err - } -} - -func (b *Broker) decode(pd packetDecoder, version int16) (err error) { - b.id, err = pd.getInt32() - if err != nil { - return err - } - - host, err := pd.getString() - if err != nil { - return err - } - - port, err := pd.getInt32() - if err != nil { - return err - } - - if version >= 1 { - b.rack, err = pd.getNullableString() - if err != nil { - return err - } - } - - b.addr = net.JoinHostPort(host, fmt.Sprint(port)) - if _, _, err := net.SplitHostPort(b.addr); err != nil { - return err - } - - return nil -} - -func (b *Broker) encode(pe packetEncoder, version int16) (err error) { - - host, portstr, err := net.SplitHostPort(b.addr) - if err != nil { - return err - } - port, err := strconv.Atoi(portstr) - if err != nil { - return err - } - - pe.putInt32(b.id) - - err = pe.putString(host) - if err != nil { - return err - } - - pe.putInt32(int32(port)) - - if version >= 1 { - err = pe.putNullableString(b.rack) - if err != nil { - return err - } - } - - return nil -} - -func (b *Broker) responseReceiver() { - var dead error - header := make([]byte, 8) - for response := range b.responses { - if dead != nil { - response.errors <- dead - continue - } - - err := b.conn.SetReadDeadline(time.Now().Add(b.conf.Net.ReadTimeout)) - if err != nil { - dead = err - response.errors <- err - continue - } - - bytesReadHeader, err := io.ReadFull(b.conn, header) - requestLatency := time.Since(response.requestTime) - if err != nil { - b.updateIncomingCommunicationMetrics(bytesReadHeader, requestLatency) - dead = err - response.errors <- err - continue - } - - decodedHeader := responseHeader{} - err = decode(header, &decodedHeader) - if err != nil { - b.updateIncomingCommunicationMetrics(bytesReadHeader, requestLatency) - dead = err - response.errors <- err - continue - } - if decodedHeader.correlationID != response.correlationID { - b.updateIncomingCommunicationMetrics(bytesReadHeader, requestLatency) - // TODO if decoded ID < cur ID, discard until we catch up - // TODO if decoded ID > cur ID, save it so when cur ID catches up we have a response - dead = PacketDecodingError{fmt.Sprintf("correlation ID didn't match, wanted %d, got %d", response.correlationID, decodedHeader.correlationID)} - response.errors <- dead - continue - } - - buf := make([]byte, decodedHeader.length-4) - bytesReadBody, err := io.ReadFull(b.conn, buf) - b.updateIncomingCommunicationMetrics(bytesReadHeader+bytesReadBody, requestLatency) - if err != nil { - dead = err - response.errors <- err - continue - } - - response.packets <- buf - } - close(b.done) -} - -func (b *Broker) sendAndReceiveSASLPlainHandshake() error { - rb := &SaslHandshakeRequest{"PLAIN"} - req := &request{correlationID: b.correlationID, clientID: b.conf.ClientID, body: rb} - buf, err := encode(req, b.conf.MetricRegistry) - if err != nil { - return err - } - - err = b.conn.SetWriteDeadline(time.Now().Add(b.conf.Net.WriteTimeout)) - if err != nil { - return err - } - - requestTime := time.Now() - bytes, err := b.conn.Write(buf) - b.updateOutgoingCommunicationMetrics(bytes) - if err != nil { - Logger.Printf("Failed to send SASL handshake %s: %s\n", b.addr, err.Error()) - return err - } - b.correlationID++ - //wait for the response - header := make([]byte, 8) // response header - _, err = io.ReadFull(b.conn, header) - if err != nil { - Logger.Printf("Failed to read SASL handshake header : %s\n", err.Error()) - return err - } - length := binary.BigEndian.Uint32(header[:4]) - payload := make([]byte, length-4) - n, err := io.ReadFull(b.conn, payload) - if err != nil { - Logger.Printf("Failed to read SASL handshake payload : %s\n", err.Error()) - return err - } - b.updateIncomingCommunicationMetrics(n+8, time.Since(requestTime)) - res := &SaslHandshakeResponse{} - err = versionedDecode(payload, res, 0) - if err != nil { - Logger.Printf("Failed to parse SASL handshake : %s\n", err.Error()) - return err - } - if res.Err != ErrNoError { - Logger.Printf("Invalid SASL Mechanism : %s\n", res.Err.Error()) - return res.Err - } - Logger.Print("Successful SASL handshake") - return nil -} - -// Kafka 0.10.0 plans to support SASL Plain and Kerberos as per PR #812 (KIP-43)/(JIRA KAFKA-3149) -// Some hosted kafka services such as IBM Message Hub already offer SASL/PLAIN auth with Kafka 0.9 -// -// In SASL Plain, Kafka expects the auth header to be in the following format -// Message format (from https://tools.ietf.org/html/rfc4616): -// -// message = [authzid] UTF8NUL authcid UTF8NUL passwd -// authcid = 1*SAFE ; MUST accept up to 255 octets -// authzid = 1*SAFE ; MUST accept up to 255 octets -// passwd = 1*SAFE ; MUST accept up to 255 octets -// UTF8NUL = %x00 ; UTF-8 encoded NUL character -// -// SAFE = UTF1 / UTF2 / UTF3 / UTF4 -// ;; any UTF-8 encoded Unicode character except NUL -// -// When credentials are valid, Kafka returns a 4 byte array of null characters. -// When credentials are invalid, Kafka closes the connection. This does not seem to be the ideal way -// of responding to bad credentials but thats how its being done today. -func (b *Broker) sendAndReceiveSASLPlainAuth() error { - if b.conf.Net.SASL.Handshake { - handshakeErr := b.sendAndReceiveSASLPlainHandshake() - if handshakeErr != nil { - Logger.Printf("Error while performing SASL handshake %s\n", b.addr) - return handshakeErr - } - } - length := 1 + len(b.conf.Net.SASL.User) + 1 + len(b.conf.Net.SASL.Password) - authBytes := make([]byte, length+4) //4 byte length header + auth data - binary.BigEndian.PutUint32(authBytes, uint32(length)) - copy(authBytes[4:], []byte("\x00"+b.conf.Net.SASL.User+"\x00"+b.conf.Net.SASL.Password)) - - err := b.conn.SetWriteDeadline(time.Now().Add(b.conf.Net.WriteTimeout)) - if err != nil { - Logger.Printf("Failed to set write deadline when doing SASL auth with broker %s: %s\n", b.addr, err.Error()) - return err - } - - requestTime := time.Now() - bytesWritten, err := b.conn.Write(authBytes) - b.updateOutgoingCommunicationMetrics(bytesWritten) - if err != nil { - Logger.Printf("Failed to write SASL auth header to broker %s: %s\n", b.addr, err.Error()) - return err - } - - header := make([]byte, 4) - n, err := io.ReadFull(b.conn, header) - b.updateIncomingCommunicationMetrics(n, time.Since(requestTime)) - // If the credentials are valid, we would get a 4 byte response filled with null characters. - // Otherwise, the broker closes the connection and we get an EOF - if err != nil { - Logger.Printf("Failed to read response while authenticating with SASL to broker %s: %s\n", b.addr, err.Error()) - return err - } - - Logger.Printf("SASL authentication successful with broker %s:%v - %v\n", b.addr, n, header) - return nil -} - -func (b *Broker) updateIncomingCommunicationMetrics(bytes int, requestLatency time.Duration) { - b.updateRequestLatencyMetrics(requestLatency) - b.responseRate.Mark(1) - if b.brokerResponseRate != nil { - b.brokerResponseRate.Mark(1) - } - responseSize := int64(bytes) - b.incomingByteRate.Mark(responseSize) - if b.brokerIncomingByteRate != nil { - b.brokerIncomingByteRate.Mark(responseSize) - } - b.responseSize.Update(responseSize) - if b.brokerResponseSize != nil { - b.brokerResponseSize.Update(responseSize) - } -} - -func (b *Broker) updateRequestLatencyMetrics(requestLatency time.Duration) { - requestLatencyInMs := int64(requestLatency / time.Millisecond) - b.requestLatency.Update(requestLatencyInMs) - if b.brokerRequestLatency != nil { - b.brokerRequestLatency.Update(requestLatencyInMs) - } -} - -func (b *Broker) updateOutgoingCommunicationMetrics(bytes int) { - b.requestRate.Mark(1) - if b.brokerRequestRate != nil { - b.brokerRequestRate.Mark(1) - } - requestSize := int64(bytes) - b.outgoingByteRate.Mark(requestSize) - if b.brokerOutgoingByteRate != nil { - b.brokerOutgoingByteRate.Mark(requestSize) - } - b.requestSize.Update(requestSize) - if b.brokerRequestSize != nil { - b.brokerRequestSize.Update(requestSize) - } -} diff --git a/third/github.com/Shopify/sarama/broker_test.go b/third/github.com/Shopify/sarama/broker_test.go deleted file mode 100644 index daf126b8b..000000000 --- a/third/github.com/Shopify/sarama/broker_test.go +++ /dev/null @@ -1,368 +0,0 @@ -package sarama - -import ( - "fmt" - "testing" - "time" -) - -func ExampleBroker() { - broker := NewBroker("localhost:9092") - err := broker.Open(nil) - if err != nil { - panic(err) - } - - request := MetadataRequest{Topics: []string{"myTopic"}} - response, err := broker.GetMetadata(&request) - if err != nil { - _ = broker.Close() - panic(err) - } - - fmt.Println("There are", len(response.Topics), "topics active in the cluster.") - - if err = broker.Close(); err != nil { - panic(err) - } -} - -type mockEncoder struct { - bytes []byte -} - -func (m mockEncoder) encode(pe packetEncoder) error { - return pe.putRawBytes(m.bytes) -} - -type brokerMetrics struct { - bytesRead int - bytesWritten int -} - -func TestBrokerAccessors(t *testing.T) { - broker := NewBroker("abc:123") - - if broker.ID() != -1 { - t.Error("New broker didn't have an ID of -1.") - } - - if broker.Addr() != "abc:123" { - t.Error("New broker didn't have the correct address") - } - - if broker.Rack() != "" { - t.Error("New broker didn't have an unknown rack.") - } - - broker.id = 34 - if broker.ID() != 34 { - t.Error("Manually setting broker ID did not take effect.") - } - - rack := "dc1" - broker.rack = &rack - if broker.Rack() != rack { - t.Error("Manually setting broker rack did not take effect.") - } -} - -func TestSimpleBrokerCommunication(t *testing.T) { - for _, tt := range brokerTestTable { - Logger.Printf("Testing broker communication for %s", tt.name) - mb := NewMockBroker(t, 0) - mb.Returns(&mockEncoder{tt.response}) - pendingNotify := make(chan brokerMetrics) - // Register a callback to be notified about successful requests - mb.SetNotifier(func(bytesRead, bytesWritten int) { - pendingNotify <- brokerMetrics{bytesRead, bytesWritten} - }) - broker := NewBroker(mb.Addr()) - // Set the broker id in order to validate local broker metrics - broker.id = 0 - conf := NewConfig() - conf.Version = tt.version - err := broker.Open(conf) - if err != nil { - t.Fatal(err) - } - tt.runner(t, broker) - // Wait up to 500 ms for the remote broker to process the request and - // notify us about the metrics - timeout := 500 * time.Millisecond - select { - case mockBrokerMetrics := <-pendingNotify: - validateBrokerMetrics(t, broker, mockBrokerMetrics) - case <-time.After(timeout): - t.Errorf("No request received for: %s after waiting for %v", tt.name, timeout) - } - mb.Close() - err = broker.Close() - if err != nil { - t.Error(err) - } - } - -} - -// We're not testing encoding/decoding here, so most of the requests/responses will be empty for simplicity's sake -var brokerTestTable = []struct { - version KafkaVersion - name string - response []byte - runner func(*testing.T, *Broker) -}{ - {V0_10_0_0, - "MetadataRequest", - []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - func(t *testing.T, broker *Broker) { - request := MetadataRequest{} - response, err := broker.GetMetadata(&request) - if err != nil { - t.Error(err) - } - if response == nil { - t.Error("Metadata request got no response!") - } - }}, - - {V0_10_0_0, - "ConsumerMetadataRequest", - []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 't', 0x00, 0x00, 0x00, 0x00}, - func(t *testing.T, broker *Broker) { - request := ConsumerMetadataRequest{} - response, err := broker.GetConsumerMetadata(&request) - if err != nil { - t.Error(err) - } - if response == nil { - t.Error("Consumer Metadata request got no response!") - } - }}, - - {V0_10_0_0, - "ProduceRequest (NoResponse)", - []byte{}, - func(t *testing.T, broker *Broker) { - request := ProduceRequest{} - request.RequiredAcks = NoResponse - response, err := broker.Produce(&request) - if err != nil { - t.Error(err) - } - if response != nil { - t.Error("Produce request with NoResponse got a response!") - } - }}, - - {V0_10_0_0, - "ProduceRequest (WaitForLocal)", - []byte{0x00, 0x00, 0x00, 0x00}, - func(t *testing.T, broker *Broker) { - request := ProduceRequest{} - request.RequiredAcks = WaitForLocal - response, err := broker.Produce(&request) - if err != nil { - t.Error(err) - } - if response == nil { - t.Error("Produce request without NoResponse got no response!") - } - }}, - - {V0_10_0_0, - "FetchRequest", - []byte{0x00, 0x00, 0x00, 0x00}, - func(t *testing.T, broker *Broker) { - request := FetchRequest{} - response, err := broker.Fetch(&request) - if err != nil { - t.Error(err) - } - if response == nil { - t.Error("Fetch request got no response!") - } - }}, - - {V0_10_0_0, - "OffsetFetchRequest", - []byte{0x00, 0x00, 0x00, 0x00}, - func(t *testing.T, broker *Broker) { - request := OffsetFetchRequest{} - response, err := broker.FetchOffset(&request) - if err != nil { - t.Error(err) - } - if response == nil { - t.Error("OffsetFetch request got no response!") - } - }}, - - {V0_10_0_0, - "OffsetCommitRequest", - []byte{0x00, 0x00, 0x00, 0x00}, - func(t *testing.T, broker *Broker) { - request := OffsetCommitRequest{} - response, err := broker.CommitOffset(&request) - if err != nil { - t.Error(err) - } - if response == nil { - t.Error("OffsetCommit request got no response!") - } - }}, - - {V0_10_0_0, - "OffsetRequest", - []byte{0x00, 0x00, 0x00, 0x00}, - func(t *testing.T, broker *Broker) { - request := OffsetRequest{} - response, err := broker.GetAvailableOffsets(&request) - if err != nil { - t.Error(err) - } - if response == nil { - t.Error("Offset request got no response!") - } - }}, - - {V0_10_0_0, - "JoinGroupRequest", - []byte{0x00, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - func(t *testing.T, broker *Broker) { - request := JoinGroupRequest{} - response, err := broker.JoinGroup(&request) - if err != nil { - t.Error(err) - } - if response == nil { - t.Error("JoinGroup request got no response!") - } - }}, - - {V0_10_0_0, - "SyncGroupRequest", - []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - func(t *testing.T, broker *Broker) { - request := SyncGroupRequest{} - response, err := broker.SyncGroup(&request) - if err != nil { - t.Error(err) - } - if response == nil { - t.Error("SyncGroup request got no response!") - } - }}, - - {V0_10_0_0, - "LeaveGroupRequest", - []byte{0x00, 0x00}, - func(t *testing.T, broker *Broker) { - request := LeaveGroupRequest{} - response, err := broker.LeaveGroup(&request) - if err != nil { - t.Error(err) - } - if response == nil { - t.Error("LeaveGroup request got no response!") - } - }}, - - {V0_10_0_0, - "HeartbeatRequest", - []byte{0x00, 0x00}, - func(t *testing.T, broker *Broker) { - request := HeartbeatRequest{} - response, err := broker.Heartbeat(&request) - if err != nil { - t.Error(err) - } - if response == nil { - t.Error("Heartbeat request got no response!") - } - }}, - - {V0_10_0_0, - "ListGroupsRequest", - []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - func(t *testing.T, broker *Broker) { - request := ListGroupsRequest{} - response, err := broker.ListGroups(&request) - if err != nil { - t.Error(err) - } - if response == nil { - t.Error("ListGroups request got no response!") - } - }}, - - {V0_10_0_0, - "DescribeGroupsRequest", - []byte{0x00, 0x00, 0x00, 0x00}, - func(t *testing.T, broker *Broker) { - request := DescribeGroupsRequest{} - response, err := broker.DescribeGroups(&request) - if err != nil { - t.Error(err) - } - if response == nil { - t.Error("DescribeGroups request got no response!") - } - }}, - - {V0_10_0_0, - "ApiVersionsRequest", - []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - func(t *testing.T, broker *Broker) { - request := ApiVersionsRequest{} - response, err := broker.ApiVersions(&request) - if err != nil { - t.Error(err) - } - if response == nil { - t.Error("ApiVersions request got no response!") - } - }}, - - {V1_1_0_0, - "DeleteGroupsRequest", - []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - func(t *testing.T, broker *Broker) { - request := DeleteGroupsRequest{} - response, err := broker.DeleteGroups(&request) - if err != nil { - t.Error(err) - } - if response == nil { - t.Error("DeleteGroups request got no response!") - } - }}, -} - -func validateBrokerMetrics(t *testing.T, broker *Broker, mockBrokerMetrics brokerMetrics) { - metricValidators := newMetricValidators() - mockBrokerBytesRead := mockBrokerMetrics.bytesRead - mockBrokerBytesWritten := mockBrokerMetrics.bytesWritten - - // Check that the number of bytes sent corresponds to what the mock broker received - metricValidators.registerForAllBrokers(broker, countMeterValidator("incoming-byte-rate", mockBrokerBytesWritten)) - if mockBrokerBytesWritten == 0 { - // This a ProduceRequest with NoResponse - metricValidators.registerForAllBrokers(broker, countMeterValidator("response-rate", 0)) - metricValidators.registerForAllBrokers(broker, countHistogramValidator("response-size", 0)) - metricValidators.registerForAllBrokers(broker, minMaxHistogramValidator("response-size", 0, 0)) - } else { - metricValidators.registerForAllBrokers(broker, countMeterValidator("response-rate", 1)) - metricValidators.registerForAllBrokers(broker, countHistogramValidator("response-size", 1)) - metricValidators.registerForAllBrokers(broker, minMaxHistogramValidator("response-size", mockBrokerBytesWritten, mockBrokerBytesWritten)) - } - - // Check that the number of bytes received corresponds to what the mock broker sent - metricValidators.registerForAllBrokers(broker, countMeterValidator("outgoing-byte-rate", mockBrokerBytesRead)) - metricValidators.registerForAllBrokers(broker, countMeterValidator("request-rate", 1)) - metricValidators.registerForAllBrokers(broker, countHistogramValidator("request-size", 1)) - metricValidators.registerForAllBrokers(broker, minMaxHistogramValidator("request-size", mockBrokerBytesRead, mockBrokerBytesRead)) - - // Run the validators - metricValidators.run(t, broker.conf.MetricRegistry) -} diff --git a/third/github.com/Shopify/sarama/client.go b/third/github.com/Shopify/sarama/client.go deleted file mode 100644 index 79be5ce53..000000000 --- a/third/github.com/Shopify/sarama/client.go +++ /dev/null @@ -1,899 +0,0 @@ -package sarama - -import ( - "math/rand" - "sort" - "sync" - "time" -) - -// Client is a generic Kafka client. It manages connections to one or more Kafka brokers. -// You MUST call Close() on a client to avoid leaks, it will not be garbage-collected -// automatically when it passes out of scope. It is safe to share a client amongst many -// users, however Kafka will process requests from a single client strictly in serial, -// so it is generally more efficient to use the default one client per producer/consumer. -type Client interface { - // Config returns the Config struct of the client. This struct should not be - // altered after it has been created. - Config() *Config - - // Controller returns the cluster controller broker. Requires Kafka 0.10 or higher. - Controller() (*Broker, error) - - // Brokers returns the current set of active brokers as retrieved from cluster metadata. - Brokers() []*Broker - - // Topics returns the set of available topics as retrieved from cluster metadata. - Topics() ([]string, error) - - // Partitions returns the sorted list of all partition IDs for the given topic. - Partitions(topic string) ([]int32, error) - - // WritablePartitions returns the sorted list of all writable partition IDs for - // the given topic, where "writable" means "having a valid leader accepting - // writes". - WritablePartitions(topic string) ([]int32, error) - - // Leader returns the broker object that is the leader of the current - // topic/partition, as determined by querying the cluster metadata. - Leader(topic string, partitionID int32) (*Broker, error) - - // Replicas returns the set of all replica IDs for the given partition. - Replicas(topic string, partitionID int32) ([]int32, error) - - // InSyncReplicas returns the set of all in-sync replica IDs for the given - // partition. In-sync replicas are replicas which are fully caught up with - // the partition leader. - InSyncReplicas(topic string, partitionID int32) ([]int32, error) - - // RefreshMetadata takes a list of topics and queries the cluster to refresh the - // available metadata for those topics. If no topics are provided, it will refresh - // metadata for all topics. - RefreshMetadata(topics ...string) error - - // GetOffset queries the cluster to get the most recent available offset at the - // given time (in milliseconds) on the topic/partition combination. - // Time should be OffsetOldest for the earliest available offset, - // OffsetNewest for the offset of the message that will be produced next, or a time. - GetOffset(topic string, partitionID int32, time int64) (int64, error) - - // Coordinator returns the coordinating broker for a consumer group. It will - // return a locally cached value if it's available. You can call - // RefreshCoordinator to update the cached value. This function only works on - // Kafka 0.8.2 and higher. - Coordinator(consumerGroup string) (*Broker, error) - - // RefreshCoordinator retrieves the coordinator for a consumer group and stores it - // in local cache. This function only works on Kafka 0.8.2 and higher. - RefreshCoordinator(consumerGroup string) error - - // InitProducerID retrieves information required for Idempotent Producer - InitProducerID() (*InitProducerIDResponse, error) - - // Close shuts down all broker connections managed by this client. It is required - // to call this function before a client object passes out of scope, as it will - // otherwise leak memory. You must close any Producers or Consumers using a client - // before you close the client. - Close() error - - // Closed returns true if the client has already had Close called on it - Closed() bool -} - -const ( - // OffsetNewest stands for the log head offset, i.e. the offset that will be - // assigned to the next message that will be produced to the partition. You - // can send this to a client's GetOffset method to get this offset, or when - // calling ConsumePartition to start consuming new messages. - OffsetNewest int64 = -1 - // OffsetOldest stands for the oldest offset available on the broker for a - // partition. You can send this to a client's GetOffset method to get this - // offset, or when calling ConsumePartition to start consuming from the - // oldest offset that is still available on the broker. - OffsetOldest int64 = -2 -) - -type client struct { - conf *Config - closer, closed chan none // for shutting down background metadata updater - - // the broker addresses given to us through the constructor are not guaranteed to be returned in - // the cluster metadata (I *think* it only returns brokers who are currently leading partitions?) - // so we store them separately - seedBrokers []*Broker - deadSeeds []*Broker - - controllerID int32 // cluster controller broker id - brokers map[int32]*Broker // maps broker ids to brokers - metadata map[string]map[int32]*PartitionMetadata // maps topics to partition ids to metadata - metadataTopics map[string]none // topics that need to collect metadata - coordinators map[string]int32 // Maps consumer group names to coordinating broker IDs - - // If the number of partitions is large, we can get some churn calling cachedPartitions, - // so the result is cached. It is important to update this value whenever metadata is changed - cachedPartitionsResults map[string][maxPartitionIndex][]int32 - - lock sync.RWMutex // protects access to the maps that hold cluster state. -} - -// NewClient creates a new Client. It connects to one of the given broker addresses -// and uses that broker to automatically fetch metadata on the rest of the kafka cluster. If metadata cannot -// be retrieved from any of the given broker addresses, the client is not created. -func NewClient(addrs []string, conf *Config) (Client, error) { - Logger.Println("Initializing new client") - - if conf == nil { - conf = NewConfig() - } - - if err := conf.Validate(); err != nil { - return nil, err - } - - if len(addrs) < 1 { - return nil, ConfigurationError("You must provide at least one broker address") - } - - client := &client{ - conf: conf, - closer: make(chan none), - closed: make(chan none), - brokers: make(map[int32]*Broker), - metadata: make(map[string]map[int32]*PartitionMetadata), - metadataTopics: make(map[string]none), - cachedPartitionsResults: make(map[string][maxPartitionIndex][]int32), - coordinators: make(map[string]int32), - } - - random := rand.New(rand.NewSource(time.Now().UnixNano())) - for _, index := range random.Perm(len(addrs)) { - client.seedBrokers = append(client.seedBrokers, NewBroker(addrs[index])) - } - - if conf.Metadata.Full { - // do an initial fetch of all cluster metadata by specifying an empty list of topics - err := client.RefreshMetadata() - switch err { - case nil: - break - case ErrLeaderNotAvailable, ErrReplicaNotAvailable, ErrTopicAuthorizationFailed, ErrClusterAuthorizationFailed: - // indicates that maybe part of the cluster is down, but is not fatal to creating the client - Logger.Println(err) - default: - close(client.closed) // we haven't started the background updater yet, so we have to do this manually - _ = client.Close() - return nil, err - } - } - go withRecover(client.backgroundMetadataUpdater) - - Logger.Println("Successfully initialized new client") - - return client, nil -} - -func (client *client) Config() *Config { - return client.conf -} - -func (client *client) Brokers() []*Broker { - client.lock.RLock() - defer client.lock.RUnlock() - brokers := make([]*Broker, 0, len(client.brokers)) - for _, broker := range client.brokers { - brokers = append(brokers, broker) - } - return brokers -} - -func (client *client) InitProducerID() (*InitProducerIDResponse, error) { - var err error - for broker := client.any(); broker != nil; broker = client.any() { - - req := &InitProducerIDRequest{} - - response, err := broker.InitProducerID(req) - switch err.(type) { - case nil: - return response, nil - default: - // some error, remove that broker and try again - Logger.Printf("Client got error from broker %d when issuing InitProducerID : %v\n", broker.ID(), err) - _ = broker.Close() - client.deregisterBroker(broker) - } - } - return nil, err -} - -func (client *client) Close() error { - if client.Closed() { - // Chances are this is being called from a defer() and the error will go unobserved - // so we go ahead and log the event in this case. - Logger.Printf("Close() called on already closed client") - return ErrClosedClient - } - - // shutdown and wait for the background thread before we take the lock, to avoid races - close(client.closer) - <-client.closed - - client.lock.Lock() - defer client.lock.Unlock() - Logger.Println("Closing Client") - - for _, broker := range client.brokers { - safeAsyncClose(broker) - } - - for _, broker := range client.seedBrokers { - safeAsyncClose(broker) - } - - client.brokers = nil - client.metadata = nil - client.metadataTopics = nil - - return nil -} - -func (client *client) Closed() bool { - return client.brokers == nil -} - -func (client *client) Topics() ([]string, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - client.lock.RLock() - defer client.lock.RUnlock() - - ret := make([]string, 0, len(client.metadata)) - for topic := range client.metadata { - ret = append(ret, topic) - } - - return ret, nil -} - -func (client *client) MetadataTopics() ([]string, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - client.lock.RLock() - defer client.lock.RUnlock() - - ret := make([]string, 0, len(client.metadataTopics)) - for topic := range client.metadataTopics { - ret = append(ret, topic) - } - - return ret, nil -} - -func (client *client) Partitions(topic string) ([]int32, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - partitions := client.cachedPartitions(topic, allPartitions) - - if len(partitions) == 0 { - err := client.RefreshMetadata(topic) - if err != nil { - return nil, err - } - partitions = client.cachedPartitions(topic, allPartitions) - } - - if partitions == nil { - return nil, ErrUnknownTopicOrPartition - } - - return partitions, nil -} - -func (client *client) WritablePartitions(topic string) ([]int32, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - partitions := client.cachedPartitions(topic, writablePartitions) - - // len==0 catches when it's nil (no such topic) and the odd case when every single - // partition is undergoing leader election simultaneously. Callers have to be able to handle - // this function returning an empty slice (which is a valid return value) but catching it - // here the first time (note we *don't* catch it below where we return ErrUnknownTopicOrPartition) triggers - // a metadata refresh as a nicety so callers can just try again and don't have to manually - // trigger a refresh (otherwise they'd just keep getting a stale cached copy). - if len(partitions) == 0 { - err := client.RefreshMetadata(topic) - if err != nil { - return nil, err - } - partitions = client.cachedPartitions(topic, writablePartitions) - } - - if partitions == nil { - return nil, ErrUnknownTopicOrPartition - } - - return partitions, nil -} - -func (client *client) Replicas(topic string, partitionID int32) ([]int32, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - metadata := client.cachedMetadata(topic, partitionID) - - if metadata == nil { - err := client.RefreshMetadata(topic) - if err != nil { - return nil, err - } - metadata = client.cachedMetadata(topic, partitionID) - } - - if metadata == nil { - return nil, ErrUnknownTopicOrPartition - } - - if metadata.Err == ErrReplicaNotAvailable { - return dupInt32Slice(metadata.Replicas), metadata.Err - } - return dupInt32Slice(metadata.Replicas), nil -} - -func (client *client) InSyncReplicas(topic string, partitionID int32) ([]int32, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - metadata := client.cachedMetadata(topic, partitionID) - - if metadata == nil { - err := client.RefreshMetadata(topic) - if err != nil { - return nil, err - } - metadata = client.cachedMetadata(topic, partitionID) - } - - if metadata == nil { - return nil, ErrUnknownTopicOrPartition - } - - if metadata.Err == ErrReplicaNotAvailable { - return dupInt32Slice(metadata.Isr), metadata.Err - } - return dupInt32Slice(metadata.Isr), nil -} - -func (client *client) Leader(topic string, partitionID int32) (*Broker, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - leader, err := client.cachedLeader(topic, partitionID) - - if leader == nil { - err = client.RefreshMetadata(topic) - if err != nil { - return nil, err - } - leader, err = client.cachedLeader(topic, partitionID) - } - - return leader, err -} - -func (client *client) RefreshMetadata(topics ...string) error { - if client.Closed() { - return ErrClosedClient - } - - // Prior to 0.8.2, Kafka will throw exceptions on an empty topic and not return a proper - // error. This handles the case by returning an error instead of sending it - // off to Kafka. See: https://github.com/Shopify/sarama/pull/38#issuecomment-26362310 - for _, topic := range topics { - if len(topic) == 0 { - return ErrInvalidTopic // this is the error that 0.8.2 and later correctly return - } - } - - return client.tryRefreshMetadata(topics, client.conf.Metadata.Retry.Max) -} - -func (client *client) GetOffset(topic string, partitionID int32, time int64) (int64, error) { - if client.Closed() { - return -1, ErrClosedClient - } - - offset, err := client.getOffset(topic, partitionID, time) - - if err != nil { - if err := client.RefreshMetadata(topic); err != nil { - return -1, err - } - return client.getOffset(topic, partitionID, time) - } - - return offset, err -} - -func (client *client) Controller() (*Broker, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - if !client.conf.Version.IsAtLeast(V0_10_0_0) { - return nil, ErrUnsupportedVersion - } - - controller := client.cachedController() - if controller == nil { - if err := client.refreshMetadata(); err != nil { - return nil, err - } - controller = client.cachedController() - } - - if controller == nil { - return nil, ErrControllerNotAvailable - } - - _ = controller.Open(client.conf) - return controller, nil -} - -func (client *client) Coordinator(consumerGroup string) (*Broker, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - coordinator := client.cachedCoordinator(consumerGroup) - - if coordinator == nil { - if err := client.RefreshCoordinator(consumerGroup); err != nil { - return nil, err - } - coordinator = client.cachedCoordinator(consumerGroup) - } - - if coordinator == nil { - return nil, ErrConsumerCoordinatorNotAvailable - } - - _ = coordinator.Open(client.conf) - return coordinator, nil -} - -func (client *client) RefreshCoordinator(consumerGroup string) error { - if client.Closed() { - return ErrClosedClient - } - - response, err := client.getConsumerMetadata(consumerGroup, client.conf.Metadata.Retry.Max) - if err != nil { - return err - } - - client.lock.Lock() - defer client.lock.Unlock() - client.registerBroker(response.Coordinator) - client.coordinators[consumerGroup] = response.Coordinator.ID() - return nil -} - -// private broker management helpers - -// registerBroker makes sure a broker received by a Metadata or Coordinator request is registered -// in the brokers map. It returns the broker that is registered, which may be the provided broker, -// or a previously registered Broker instance. You must hold the write lock before calling this function. -func (client *client) registerBroker(broker *Broker) { - if client.brokers[broker.ID()] == nil { - client.brokers[broker.ID()] = broker - Logger.Printf("client/brokers registered new broker #%d at %s", broker.ID(), broker.Addr()) - } else if broker.Addr() != client.brokers[broker.ID()].Addr() { - safeAsyncClose(client.brokers[broker.ID()]) - client.brokers[broker.ID()] = broker - Logger.Printf("client/brokers replaced registered broker #%d with %s", broker.ID(), broker.Addr()) - } -} - -// deregisterBroker removes a broker from the seedsBroker list, and if it's -// not the seedbroker, removes it from brokers map completely. -func (client *client) deregisterBroker(broker *Broker) { - client.lock.Lock() - defer client.lock.Unlock() - - if len(client.seedBrokers) > 0 && broker == client.seedBrokers[0] { - client.deadSeeds = append(client.deadSeeds, broker) - client.seedBrokers = client.seedBrokers[1:] - } else { - // we do this so that our loop in `tryRefreshMetadata` doesn't go on forever, - // but we really shouldn't have to; once that loop is made better this case can be - // removed, and the function generally can be renamed from `deregisterBroker` to - // `nextSeedBroker` or something - Logger.Printf("client/brokers deregistered broker #%d at %s", broker.ID(), broker.Addr()) - delete(client.brokers, broker.ID()) - } -} - -func (client *client) resurrectDeadBrokers() { - client.lock.Lock() - defer client.lock.Unlock() - - Logger.Printf("client/brokers resurrecting %d dead seed brokers", len(client.deadSeeds)) - client.seedBrokers = append(client.seedBrokers, client.deadSeeds...) - client.deadSeeds = nil -} - -func (client *client) any() *Broker { - client.lock.RLock() - defer client.lock.RUnlock() - - if len(client.seedBrokers) > 0 { - _ = client.seedBrokers[0].Open(client.conf) - return client.seedBrokers[0] - } - - // not guaranteed to be random *or* deterministic - for _, broker := range client.brokers { - _ = broker.Open(client.conf) - return broker - } - - return nil -} - -// private caching/lazy metadata helpers - -type partitionType int - -const ( - allPartitions partitionType = iota - writablePartitions - // If you add any more types, update the partition cache in update() - - // Ensure this is the last partition type value - maxPartitionIndex -) - -func (client *client) cachedMetadata(topic string, partitionID int32) *PartitionMetadata { - client.lock.RLock() - defer client.lock.RUnlock() - - partitions := client.metadata[topic] - if partitions != nil { - return partitions[partitionID] - } - - return nil -} - -func (client *client) cachedPartitions(topic string, partitionSet partitionType) []int32 { - client.lock.RLock() - defer client.lock.RUnlock() - - partitions, exists := client.cachedPartitionsResults[topic] - - if !exists { - return nil - } - return partitions[partitionSet] -} - -func (client *client) setPartitionCache(topic string, partitionSet partitionType) []int32 { - partitions := client.metadata[topic] - - if partitions == nil { - return nil - } - - ret := make([]int32, 0, len(partitions)) - for _, partition := range partitions { - if partitionSet == writablePartitions && partition.Err == ErrLeaderNotAvailable { - continue - } - ret = append(ret, partition.ID) - } - - sort.Sort(int32Slice(ret)) - return ret -} - -func (client *client) cachedLeader(topic string, partitionID int32) (*Broker, error) { - client.lock.RLock() - defer client.lock.RUnlock() - - partitions := client.metadata[topic] - if partitions != nil { - metadata, ok := partitions[partitionID] - if ok { - if metadata.Err == ErrLeaderNotAvailable { - return nil, ErrLeaderNotAvailable - } - b := client.brokers[metadata.Leader] - if b == nil { - return nil, ErrLeaderNotAvailable - } - _ = b.Open(client.conf) - return b, nil - } - } - - return nil, ErrUnknownTopicOrPartition -} - -func (client *client) getOffset(topic string, partitionID int32, time int64) (int64, error) { - broker, err := client.Leader(topic, partitionID) - if err != nil { - return -1, err - } - - request := &OffsetRequest{} - if client.conf.Version.IsAtLeast(V0_10_1_0) { - request.Version = 1 - } - request.AddBlock(topic, partitionID, time, 1) - - response, err := broker.GetAvailableOffsets(request) - if err != nil { - _ = broker.Close() - return -1, err - } - - block := response.GetBlock(topic, partitionID) - if block == nil { - _ = broker.Close() - return -1, ErrIncompleteResponse - } - if block.Err != ErrNoError { - return -1, block.Err - } - if len(block.Offsets) != 1 { - return -1, ErrOffsetOutOfRange - } - - return block.Offsets[0], nil -} - -// core metadata update logic - -func (client *client) backgroundMetadataUpdater() { - defer close(client.closed) - - if client.conf.Metadata.RefreshFrequency == time.Duration(0) { - return - } - - ticker := time.NewTicker(client.conf.Metadata.RefreshFrequency) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - if err := client.refreshMetadata(); err != nil { - Logger.Println("Client background metadata update:", err) - } - case <-client.closer: - return - } - } -} - -func (client *client) refreshMetadata() error { - topics := []string{} - - if !client.conf.Metadata.Full { - if specificTopics, err := client.MetadataTopics(); err != nil { - return err - } else if len(specificTopics) == 0 { - return ErrNoTopicsToUpdateMetadata - } else { - topics = specificTopics - } - } - - if err := client.RefreshMetadata(topics...); err != nil { - return err - } - - return nil -} - -func (client *client) tryRefreshMetadata(topics []string, attemptsRemaining int) error { - retry := func(err error) error { - if attemptsRemaining > 0 { - Logger.Printf("client/metadata retrying after %dms... (%d attempts remaining)\n", client.conf.Metadata.Retry.Backoff/time.Millisecond, attemptsRemaining) - time.Sleep(client.conf.Metadata.Retry.Backoff) - return client.tryRefreshMetadata(topics, attemptsRemaining-1) - } - return err - } - - for broker := client.any(); broker != nil; broker = client.any() { - if len(topics) > 0 { - Logger.Printf("client/metadata fetching metadata for %v from broker %s\n", topics, broker.addr) - } else { - Logger.Printf("client/metadata fetching metadata for all topics from broker %s\n", broker.addr) - } - - req := &MetadataRequest{Topics: topics} - if client.conf.Version.IsAtLeast(V0_10_0_0) { - req.Version = 1 - } - response, err := broker.GetMetadata(req) - - switch err.(type) { - case nil: - allKnownMetaData := len(topics) == 0 - // valid response, use it - shouldRetry, err := client.updateMetadata(response, allKnownMetaData) - if shouldRetry { - Logger.Println("client/metadata found some partitions to be leaderless") - return retry(err) // note: err can be nil - } - return err - - case PacketEncodingError: - // didn't even send, return the error - return err - default: - // some other error, remove that broker and try again - Logger.Printf("client/metadata got error from broker %d while fetching metadata: %v\n", broker.ID(), err) - _ = broker.Close() - client.deregisterBroker(broker) - } - } - - Logger.Println("client/metadata no available broker to send metadata request to") - client.resurrectDeadBrokers() - return retry(ErrOutOfBrokers) -} - -// if no fatal error, returns a list of topics that need retrying due to ErrLeaderNotAvailable -func (client *client) updateMetadata(data *MetadataResponse, allKnownMetaData bool) (retry bool, err error) { - client.lock.Lock() - defer client.lock.Unlock() - - // For all the brokers we received: - // - if it is a new ID, save it - // - if it is an existing ID, but the address we have is stale, discard the old one and save it - // - otherwise ignore it, replacing our existing one would just bounce the connection - for _, broker := range data.Brokers { - client.registerBroker(broker) - } - - client.controllerID = data.ControllerID - - if allKnownMetaData { - client.metadata = make(map[string]map[int32]*PartitionMetadata) - client.metadataTopics = make(map[string]none) - client.cachedPartitionsResults = make(map[string][maxPartitionIndex][]int32) - } - for _, topic := range data.Topics { - // topics must be added firstly to `metadataTopics` to guarantee that all - // requested topics must be recorded to keep them trackable for periodically - // metadata refresh. - if _, exists := client.metadataTopics[topic.Name]; !exists { - client.metadataTopics[topic.Name] = none{} - } - delete(client.metadata, topic.Name) - delete(client.cachedPartitionsResults, topic.Name) - - switch topic.Err { - case ErrNoError: - break - case ErrInvalidTopic, ErrTopicAuthorizationFailed: // don't retry, don't store partial results - err = topic.Err - continue - case ErrUnknownTopicOrPartition: // retry, do not store partial partition results - err = topic.Err - retry = true - continue - case ErrLeaderNotAvailable: // retry, but store partial partition results - retry = true - break - default: // don't retry, don't store partial results - Logger.Printf("Unexpected topic-level metadata error: %s", topic.Err) - err = topic.Err - continue - } - - client.metadata[topic.Name] = make(map[int32]*PartitionMetadata, len(topic.Partitions)) - for _, partition := range topic.Partitions { - client.metadata[topic.Name][partition.ID] = partition - if partition.Err == ErrLeaderNotAvailable { - retry = true - } - } - - var partitionCache [maxPartitionIndex][]int32 - partitionCache[allPartitions] = client.setPartitionCache(topic.Name, allPartitions) - partitionCache[writablePartitions] = client.setPartitionCache(topic.Name, writablePartitions) - client.cachedPartitionsResults[topic.Name] = partitionCache - } - - return -} - -func (client *client) cachedCoordinator(consumerGroup string) *Broker { - client.lock.RLock() - defer client.lock.RUnlock() - if coordinatorID, ok := client.coordinators[consumerGroup]; ok { - return client.brokers[coordinatorID] - } - return nil -} - -func (client *client) cachedController() *Broker { - client.lock.RLock() - defer client.lock.RUnlock() - - return client.brokers[client.controllerID] -} - -func (client *client) getConsumerMetadata(consumerGroup string, attemptsRemaining int) (*FindCoordinatorResponse, error) { - retry := func(err error) (*FindCoordinatorResponse, error) { - if attemptsRemaining > 0 { - Logger.Printf("client/coordinator retrying after %dms... (%d attempts remaining)\n", client.conf.Metadata.Retry.Backoff/time.Millisecond, attemptsRemaining) - time.Sleep(client.conf.Metadata.Retry.Backoff) - return client.getConsumerMetadata(consumerGroup, attemptsRemaining-1) - } - return nil, err - } - - for broker := client.any(); broker != nil; broker = client.any() { - Logger.Printf("client/coordinator requesting coordinator for consumergroup %s from %s\n", consumerGroup, broker.Addr()) - - request := new(FindCoordinatorRequest) - request.CoordinatorKey = consumerGroup - request.CoordinatorType = CoordinatorGroup - - response, err := broker.FindCoordinator(request) - - if err != nil { - Logger.Printf("client/coordinator request to broker %s failed: %s\n", broker.Addr(), err) - - switch err.(type) { - case PacketEncodingError: - return nil, err - default: - _ = broker.Close() - client.deregisterBroker(broker) - continue - } - } - - switch response.Err { - case ErrNoError: - Logger.Printf("client/coordinator coordinator for consumergroup %s is #%d (%s)\n", consumerGroup, response.Coordinator.ID(), response.Coordinator.Addr()) - return response, nil - - case ErrConsumerCoordinatorNotAvailable: - Logger.Printf("client/coordinator coordinator for consumer group %s is not available\n", consumerGroup) - - // This is very ugly, but this scenario will only happen once per cluster. - // The __consumer_offsets topic only has to be created one time. - // The number of partitions not configurable, but partition 0 should always exist. - if _, err := client.Leader("__consumer_offsets", 0); err != nil { - Logger.Printf("client/coordinator the __consumer_offsets topic is not initialized completely yet. Waiting 2 seconds...\n") - time.Sleep(2 * time.Second) - } - - return retry(ErrConsumerCoordinatorNotAvailable) - default: - return nil, response.Err - } - } - - Logger.Println("client/coordinator no available broker to send consumer metadata request to") - client.resurrectDeadBrokers() - return retry(ErrOutOfBrokers) -} diff --git a/third/github.com/Shopify/sarama/client_test.go b/third/github.com/Shopify/sarama/client_test.go deleted file mode 100644 index 1d0924d05..000000000 --- a/third/github.com/Shopify/sarama/client_test.go +++ /dev/null @@ -1,661 +0,0 @@ -package sarama - -import ( - "io" - "sync" - "testing" - "time" -) - -func safeClose(t testing.TB, c io.Closer) { - err := c.Close() - if err != nil { - t.Error(err) - } -} - -func TestSimpleClient(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - - seedBroker.Returns(new(MetadataResponse)) - - client, err := NewClient([]string{seedBroker.Addr()}, nil) - if err != nil { - t.Fatal(err) - } - - seedBroker.Close() - safeClose(t, client) -} - -func TestCachedPartitions(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - - replicas := []int32{3, 1, 5} - isr := []int32{5, 1} - - metadataResponse := new(MetadataResponse) - metadataResponse.AddBroker("localhost:12345", 2) - metadataResponse.AddTopicPartition("my_topic", 0, 2, replicas, isr, ErrNoError) - metadataResponse.AddTopicPartition("my_topic", 1, 2, replicas, isr, ErrLeaderNotAvailable) - seedBroker.Returns(metadataResponse) - - config := NewConfig() - config.Metadata.Retry.Max = 0 - c, err := NewClient([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - client := c.(*client) - - // Verify they aren't cached the same - allP := client.cachedPartitionsResults["my_topic"][allPartitions] - writeP := client.cachedPartitionsResults["my_topic"][writablePartitions] - if len(allP) == len(writeP) { - t.Fatal("Invalid lengths!") - } - - tmp := client.cachedPartitionsResults["my_topic"] - // Verify we actually use the cache at all! - tmp[allPartitions] = []int32{1, 2, 3, 4} - client.cachedPartitionsResults["my_topic"] = tmp - if 4 != len(client.cachedPartitions("my_topic", allPartitions)) { - t.Fatal("Not using the cache!") - } - - seedBroker.Close() - safeClose(t, client) -} - -func TestClientDoesntCachePartitionsForTopicsWithErrors(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - - replicas := []int32{seedBroker.BrokerID()} - - metadataResponse := new(MetadataResponse) - metadataResponse.AddBroker(seedBroker.Addr(), seedBroker.BrokerID()) - metadataResponse.AddTopicPartition("my_topic", 1, replicas[0], replicas, replicas, ErrNoError) - metadataResponse.AddTopicPartition("my_topic", 2, replicas[0], replicas, replicas, ErrNoError) - seedBroker.Returns(metadataResponse) - - config := NewConfig() - config.Metadata.Retry.Max = 0 - client, err := NewClient([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - metadataResponse = new(MetadataResponse) - metadataResponse.AddTopic("unknown", ErrUnknownTopicOrPartition) - seedBroker.Returns(metadataResponse) - - partitions, err := client.Partitions("unknown") - - if err != ErrUnknownTopicOrPartition { - t.Error("Expected ErrUnknownTopicOrPartition, found", err) - } - if partitions != nil { - t.Errorf("Should return nil as partition list, found %v", partitions) - } - - // Should still use the cache of a known topic - partitions, err = client.Partitions("my_topic") - if err != nil { - t.Errorf("Expected no error, found %v", err) - } - - metadataResponse = new(MetadataResponse) - metadataResponse.AddTopic("unknown", ErrUnknownTopicOrPartition) - seedBroker.Returns(metadataResponse) - - // Should not use cache for unknown topic - partitions, err = client.Partitions("unknown") - if err != ErrUnknownTopicOrPartition { - t.Error("Expected ErrUnknownTopicOrPartition, found", err) - } - if partitions != nil { - t.Errorf("Should return nil as partition list, found %v", partitions) - } - - seedBroker.Close() - safeClose(t, client) -} - -func TestClientSeedBrokers(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - - metadataResponse := new(MetadataResponse) - metadataResponse.AddBroker("localhost:12345", 2) - seedBroker.Returns(metadataResponse) - - client, err := NewClient([]string{seedBroker.Addr()}, nil) - if err != nil { - t.Fatal(err) - } - - seedBroker.Close() - safeClose(t, client) -} - -func TestClientMetadata(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader := NewMockBroker(t, 5) - - replicas := []int32{3, 1, 5} - isr := []int32{5, 1} - - metadataResponse := new(MetadataResponse) - metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) - metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), replicas, isr, ErrNoError) - metadataResponse.AddTopicPartition("my_topic", 1, leader.BrokerID(), replicas, isr, ErrLeaderNotAvailable) - seedBroker.Returns(metadataResponse) - - config := NewConfig() - config.Metadata.Retry.Max = 0 - client, err := NewClient([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - topics, err := client.Topics() - if err != nil { - t.Error(err) - } else if len(topics) != 1 || topics[0] != "my_topic" { - t.Error("Client returned incorrect topics:", topics) - } - - parts, err := client.Partitions("my_topic") - if err != nil { - t.Error(err) - } else if len(parts) != 2 || parts[0] != 0 || parts[1] != 1 { - t.Error("Client returned incorrect partitions for my_topic:", parts) - } - - parts, err = client.WritablePartitions("my_topic") - if err != nil { - t.Error(err) - } else if len(parts) != 1 || parts[0] != 0 { - t.Error("Client returned incorrect writable partitions for my_topic:", parts) - } - - tst, err := client.Leader("my_topic", 0) - if err != nil { - t.Error(err) - } else if tst.ID() != 5 { - t.Error("Leader for my_topic had incorrect ID.") - } - - replicas, err = client.Replicas("my_topic", 0) - if err != nil { - t.Error(err) - } else if replicas[0] != 3 { - t.Error("Incorrect (or sorted) replica") - } else if replicas[1] != 1 { - t.Error("Incorrect (or sorted) replica") - } else if replicas[2] != 5 { - t.Error("Incorrect (or sorted) replica") - } - - isr, err = client.InSyncReplicas("my_topic", 0) - if err != nil { - t.Error(err) - } else if len(isr) != 2 { - t.Error("Client returned incorrect ISRs for partition:", isr) - } else if isr[0] != 5 { - t.Error("Incorrect (or sorted) ISR:", isr) - } else if isr[1] != 1 { - t.Error("Incorrect (or sorted) ISR:", isr) - } - - leader.Close() - seedBroker.Close() - safeClose(t, client) -} - -func TestClientGetOffset(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader := NewMockBroker(t, 2) - leaderAddr := leader.Addr() - - metadata := new(MetadataResponse) - metadata.AddTopicPartition("foo", 0, leader.BrokerID(), nil, nil, ErrNoError) - metadata.AddBroker(leaderAddr, leader.BrokerID()) - seedBroker.Returns(metadata) - - client, err := NewClient([]string{seedBroker.Addr()}, nil) - if err != nil { - t.Fatal(err) - } - - offsetResponse := new(OffsetResponse) - offsetResponse.AddTopicPartition("foo", 0, 123) - leader.Returns(offsetResponse) - - offset, err := client.GetOffset("foo", 0, OffsetNewest) - if err != nil { - t.Error(err) - } - if offset != 123 { - t.Error("Unexpected offset, got ", offset) - } - - leader.Close() - seedBroker.Returns(metadata) - - leader = NewMockBrokerAddr(t, 2, leaderAddr) - offsetResponse = new(OffsetResponse) - offsetResponse.AddTopicPartition("foo", 0, 456) - leader.Returns(offsetResponse) - - offset, err = client.GetOffset("foo", 0, OffsetNewest) - if err != nil { - t.Error(err) - } - if offset != 456 { - t.Error("Unexpected offset, got ", offset) - } - - seedBroker.Close() - leader.Close() - safeClose(t, client) -} - -func TestClientReceivingUnknownTopic(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - - metadataResponse1 := new(MetadataResponse) - seedBroker.Returns(metadataResponse1) - - config := NewConfig() - config.Metadata.Retry.Max = 1 - config.Metadata.Retry.Backoff = 0 - client, err := NewClient([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - metadataUnknownTopic := new(MetadataResponse) - metadataUnknownTopic.AddTopic("new_topic", ErrUnknownTopicOrPartition) - seedBroker.Returns(metadataUnknownTopic) - seedBroker.Returns(metadataUnknownTopic) - - if err := client.RefreshMetadata("new_topic"); err != ErrUnknownTopicOrPartition { - t.Error("ErrUnknownTopicOrPartition expected, got", err) - } - - // If we are asking for the leader of a partition of the non-existing topic. - // we will request metadata again. - seedBroker.Returns(metadataUnknownTopic) - seedBroker.Returns(metadataUnknownTopic) - - if _, err = client.Leader("new_topic", 1); err != ErrUnknownTopicOrPartition { - t.Error("Expected ErrUnknownTopicOrPartition, got", err) - } - - safeClose(t, client) - seedBroker.Close() -} - -func TestClientReceivingPartialMetadata(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader := NewMockBroker(t, 5) - - metadataResponse1 := new(MetadataResponse) - metadataResponse1.AddBroker(leader.Addr(), leader.BrokerID()) - seedBroker.Returns(metadataResponse1) - - config := NewConfig() - config.Metadata.Retry.Max = 0 - client, err := NewClient([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - replicas := []int32{leader.BrokerID(), seedBroker.BrokerID()} - - metadataPartial := new(MetadataResponse) - metadataPartial.AddTopic("new_topic", ErrLeaderNotAvailable) - metadataPartial.AddTopicPartition("new_topic", 0, leader.BrokerID(), replicas, replicas, ErrNoError) - metadataPartial.AddTopicPartition("new_topic", 1, -1, replicas, []int32{}, ErrLeaderNotAvailable) - seedBroker.Returns(metadataPartial) - - if err := client.RefreshMetadata("new_topic"); err != nil { - t.Error("ErrLeaderNotAvailable should not make RefreshMetadata respond with an error") - } - - // Even though the metadata was incomplete, we should be able to get the leader of a partition - // for which we did get a useful response, without doing additional requests. - - partition0Leader, err := client.Leader("new_topic", 0) - if err != nil { - t.Error(err) - } else if partition0Leader.Addr() != leader.Addr() { - t.Error("Unexpected leader returned", partition0Leader.Addr()) - } - - // If we are asking for the leader of a partition that didn't have a leader before, - // we will do another metadata request. - - seedBroker.Returns(metadataPartial) - - // Still no leader for the partition, so asking for it should return an error. - _, err = client.Leader("new_topic", 1) - if err != ErrLeaderNotAvailable { - t.Error("Expected ErrLeaderNotAvailable, got", err) - } - - safeClose(t, client) - seedBroker.Close() - leader.Close() -} - -func TestClientRefreshBehaviour(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader := NewMockBroker(t, 5) - - metadataResponse1 := new(MetadataResponse) - metadataResponse1.AddBroker(leader.Addr(), leader.BrokerID()) - seedBroker.Returns(metadataResponse1) - - metadataResponse2 := new(MetadataResponse) - metadataResponse2.AddTopicPartition("my_topic", 0xb, leader.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataResponse2) - - client, err := NewClient([]string{seedBroker.Addr()}, nil) - if err != nil { - t.Fatal(err) - } - - parts, err := client.Partitions("my_topic") - if err != nil { - t.Error(err) - } else if len(parts) != 1 || parts[0] != 0xb { - t.Error("Client returned incorrect partitions for my_topic:", parts) - } - - tst, err := client.Leader("my_topic", 0xb) - if err != nil { - t.Error(err) - } else if tst.ID() != 5 { - t.Error("Leader for my_topic had incorrect ID.") - } - - leader.Close() - seedBroker.Close() - safeClose(t, client) -} - -func TestClientResurrectDeadSeeds(t *testing.T) { - initialSeed := NewMockBroker(t, 0) - emptyMetadata := new(MetadataResponse) - initialSeed.Returns(emptyMetadata) - - conf := NewConfig() - conf.Metadata.Retry.Backoff = 0 - conf.Metadata.RefreshFrequency = 0 - c, err := NewClient([]string{initialSeed.Addr()}, conf) - if err != nil { - t.Fatal(err) - } - initialSeed.Close() - - client := c.(*client) - - seed1 := NewMockBroker(t, 1) - seed2 := NewMockBroker(t, 2) - seed3 := NewMockBroker(t, 3) - addr1 := seed1.Addr() - addr2 := seed2.Addr() - addr3 := seed3.Addr() - - // Overwrite the seed brokers with a fixed ordering to make this test deterministic. - safeClose(t, client.seedBrokers[0]) - client.seedBrokers = []*Broker{NewBroker(addr1), NewBroker(addr2), NewBroker(addr3)} - client.deadSeeds = []*Broker{} - - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - if err := client.RefreshMetadata(); err != nil { - t.Error(err) - } - wg.Done() - }() - seed1.Close() - seed2.Close() - - seed1 = NewMockBrokerAddr(t, 1, addr1) - seed2 = NewMockBrokerAddr(t, 2, addr2) - - seed3.Close() - - seed1.Close() - seed2.Returns(emptyMetadata) - - wg.Wait() - - if len(client.seedBrokers) != 2 { - t.Error("incorrect number of live seeds") - } - if len(client.deadSeeds) != 1 { - t.Error("incorrect number of dead seeds") - } - - safeClose(t, c) -} - -func TestClientController(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - defer seedBroker.Close() - controllerBroker := NewMockBroker(t, 2) - defer controllerBroker.Close() - - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetController(controllerBroker.BrokerID()). - SetBroker(seedBroker.Addr(), seedBroker.BrokerID()). - SetBroker(controllerBroker.Addr(), controllerBroker.BrokerID()), - }) - - cfg := NewConfig() - - // test kafka version greater than 0.10.0.0 - cfg.Version = V0_10_0_0 - client1, err := NewClient([]string{seedBroker.Addr()}, cfg) - if err != nil { - t.Fatal(err) - } - defer safeClose(t, client1) - broker, err := client1.Controller() - if err != nil { - t.Fatal(err) - } - if broker.Addr() != controllerBroker.Addr() { - t.Errorf("Expected controller to have address %s, found %s", controllerBroker.Addr(), broker.Addr()) - } - - // test kafka version earlier than 0.10.0.0 - cfg.Version = V0_9_0_1 - client2, err := NewClient([]string{seedBroker.Addr()}, cfg) - if err != nil { - t.Fatal(err) - } - defer safeClose(t, client2) - if _, err = client2.Controller(); err != ErrUnsupportedVersion { - t.Errorf("Expected Contoller() to return %s, found %s", ErrUnsupportedVersion, err) - } -} - -func TestClientCoordinatorWithConsumerOffsetsTopic(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - staleCoordinator := NewMockBroker(t, 2) - freshCoordinator := NewMockBroker(t, 3) - - replicas := []int32{staleCoordinator.BrokerID(), freshCoordinator.BrokerID()} - metadataResponse1 := new(MetadataResponse) - metadataResponse1.AddBroker(staleCoordinator.Addr(), staleCoordinator.BrokerID()) - metadataResponse1.AddBroker(freshCoordinator.Addr(), freshCoordinator.BrokerID()) - metadataResponse1.AddTopicPartition("__consumer_offsets", 0, replicas[0], replicas, replicas, ErrNoError) - seedBroker.Returns(metadataResponse1) - - client, err := NewClient([]string{seedBroker.Addr()}, nil) - if err != nil { - t.Fatal(err) - } - - coordinatorResponse1 := new(ConsumerMetadataResponse) - coordinatorResponse1.Err = ErrConsumerCoordinatorNotAvailable - seedBroker.Returns(coordinatorResponse1) - - coordinatorResponse2 := new(ConsumerMetadataResponse) - coordinatorResponse2.CoordinatorID = staleCoordinator.BrokerID() - coordinatorResponse2.CoordinatorHost = "127.0.0.1" - coordinatorResponse2.CoordinatorPort = staleCoordinator.Port() - - seedBroker.Returns(coordinatorResponse2) - - broker, err := client.Coordinator("my_group") - if err != nil { - t.Error(err) - } - - if staleCoordinator.Addr() != broker.Addr() { - t.Errorf("Expected coordinator to have address %s, found %s", staleCoordinator.Addr(), broker.Addr()) - } - - if staleCoordinator.BrokerID() != broker.ID() { - t.Errorf("Expected coordinator to have ID %d, found %d", staleCoordinator.BrokerID(), broker.ID()) - } - - // Grab the cached value - broker2, err := client.Coordinator("my_group") - if err != nil { - t.Error(err) - } - - if broker2.Addr() != broker.Addr() { - t.Errorf("Expected the coordinator to be the same, but found %s vs. %s", broker2.Addr(), broker.Addr()) - } - - coordinatorResponse3 := new(ConsumerMetadataResponse) - coordinatorResponse3.CoordinatorID = freshCoordinator.BrokerID() - coordinatorResponse3.CoordinatorHost = "127.0.0.1" - coordinatorResponse3.CoordinatorPort = freshCoordinator.Port() - - seedBroker.Returns(coordinatorResponse3) - - // Refresh the locally cahced value because it's stale - if err := client.RefreshCoordinator("my_group"); err != nil { - t.Error(err) - } - - // Grab the fresh value - broker3, err := client.Coordinator("my_group") - if err != nil { - t.Error(err) - } - - if broker3.Addr() != freshCoordinator.Addr() { - t.Errorf("Expected the freshCoordinator to be returned, but found %s.", broker3.Addr()) - } - - freshCoordinator.Close() - staleCoordinator.Close() - seedBroker.Close() - safeClose(t, client) -} - -func TestClientCoordinatorWithoutConsumerOffsetsTopic(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - coordinator := NewMockBroker(t, 2) - - metadataResponse1 := new(MetadataResponse) - seedBroker.Returns(metadataResponse1) - - config := NewConfig() - config.Metadata.Retry.Max = 1 - config.Metadata.Retry.Backoff = 0 - client, err := NewClient([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - coordinatorResponse1 := new(ConsumerMetadataResponse) - coordinatorResponse1.Err = ErrConsumerCoordinatorNotAvailable - seedBroker.Returns(coordinatorResponse1) - - metadataResponse2 := new(MetadataResponse) - metadataResponse2.AddTopic("__consumer_offsets", ErrUnknownTopicOrPartition) - seedBroker.Returns(metadataResponse2) - - replicas := []int32{coordinator.BrokerID()} - metadataResponse3 := new(MetadataResponse) - metadataResponse3.AddTopicPartition("__consumer_offsets", 0, replicas[0], replicas, replicas, ErrNoError) - seedBroker.Returns(metadataResponse3) - - coordinatorResponse2 := new(ConsumerMetadataResponse) - coordinatorResponse2.CoordinatorID = coordinator.BrokerID() - coordinatorResponse2.CoordinatorHost = "127.0.0.1" - coordinatorResponse2.CoordinatorPort = coordinator.Port() - - seedBroker.Returns(coordinatorResponse2) - - broker, err := client.Coordinator("my_group") - if err != nil { - t.Error(err) - } - - if coordinator.Addr() != broker.Addr() { - t.Errorf("Expected coordinator to have address %s, found %s", coordinator.Addr(), broker.Addr()) - } - - if coordinator.BrokerID() != broker.ID() { - t.Errorf("Expected coordinator to have ID %d, found %d", coordinator.BrokerID(), broker.ID()) - } - - coordinator.Close() - seedBroker.Close() - safeClose(t, client) -} - -func TestClientAutorefreshShutdownRace(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - - metadataResponse := new(MetadataResponse) - seedBroker.Returns(metadataResponse) - - conf := NewConfig() - conf.Metadata.RefreshFrequency = 100 * time.Millisecond - client, err := NewClient([]string{seedBroker.Addr()}, conf) - if err != nil { - t.Fatal(err) - } - - // Wait for the background refresh to kick in - time.Sleep(110 * time.Millisecond) - - done := make(chan none) - go func() { - // Close the client - if err := client.Close(); err != nil { - t.Fatal(err) - } - close(done) - }() - - // Wait for the Close to kick in - time.Sleep(10 * time.Millisecond) - - // Then return some metadata to the still-running background thread - leader := NewMockBroker(t, 2) - metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) - metadataResponse.AddTopicPartition("foo", 0, leader.BrokerID(), []int32{2}, []int32{2}, ErrNoError) - seedBroker.Returns(metadataResponse) - - <-done - - seedBroker.Close() - - // give the update time to happen so we get a panic if it's still running (which it shouldn't) - time.Sleep(10 * time.Millisecond) -} diff --git a/third/github.com/Shopify/sarama/client_tls_test.go b/third/github.com/Shopify/sarama/client_tls_test.go deleted file mode 100644 index eff84c771..000000000 --- a/third/github.com/Shopify/sarama/client_tls_test.go +++ /dev/null @@ -1,206 +0,0 @@ -package sarama - -import ( - "math/big" - "net" - "testing" - "time" - - "crypto/rand" - "crypto/rsa" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" -) - -func TestTLS(t *testing.T) { - cakey, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - t.Fatal(err) - } - - clientkey, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - t.Fatal(err) - } - - hostkey, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - t.Fatal(err) - } - - nvb := time.Now().Add(-1 * time.Hour) - nva := time.Now().Add(1 * time.Hour) - - caTemplate := &x509.Certificate{ - Subject: pkix.Name{CommonName: "ca"}, - Issuer: pkix.Name{CommonName: "ca"}, - SerialNumber: big.NewInt(0), - NotAfter: nva, - NotBefore: nvb, - IsCA: true, - BasicConstraintsValid: true, - KeyUsage: x509.KeyUsageCertSign, - } - caDer, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &cakey.PublicKey, cakey) - if err != nil { - t.Fatal(err) - } - caFinalCert, err := x509.ParseCertificate(caDer) - if err != nil { - t.Fatal(err) - } - - hostDer, err := x509.CreateCertificate(rand.Reader, &x509.Certificate{ - Subject: pkix.Name{CommonName: "host"}, - Issuer: pkix.Name{CommonName: "ca"}, - IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}, - SerialNumber: big.NewInt(0), - NotAfter: nva, - NotBefore: nvb, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - }, caFinalCert, &hostkey.PublicKey, cakey) - if err != nil { - t.Fatal(err) - } - - clientDer, err := x509.CreateCertificate(rand.Reader, &x509.Certificate{ - Subject: pkix.Name{CommonName: "client"}, - Issuer: pkix.Name{CommonName: "ca"}, - SerialNumber: big.NewInt(0), - NotAfter: nva, - NotBefore: nvb, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - }, caFinalCert, &clientkey.PublicKey, cakey) - if err != nil { - t.Fatal(err) - } - - pool := x509.NewCertPool() - pool.AddCert(caFinalCert) - - systemCerts, err := x509.SystemCertPool() - if err != nil { - t.Fatal(err) - } - - // Keep server the same - it's the client that we're testing - serverTLSConfig := &tls.Config{ - Certificates: []tls.Certificate{tls.Certificate{ - Certificate: [][]byte{hostDer}, - PrivateKey: hostkey, - }}, - ClientAuth: tls.RequireAndVerifyClientCert, - ClientCAs: pool, - } - - for _, tc := range []struct { - Succeed bool - Server, Client *tls.Config - }{ - { // Verify client fails if wrong CA cert pool is specified - Succeed: false, - Server: serverTLSConfig, - Client: &tls.Config{ - RootCAs: systemCerts, - Certificates: []tls.Certificate{tls.Certificate{ - Certificate: [][]byte{clientDer}, - PrivateKey: clientkey, - }}, - }, - }, - { // Verify client fails if wrong key is specified - Succeed: false, - Server: serverTLSConfig, - Client: &tls.Config{ - RootCAs: pool, - Certificates: []tls.Certificate{tls.Certificate{ - Certificate: [][]byte{clientDer}, - PrivateKey: hostkey, - }}, - }, - }, - { // Verify client fails if wrong cert is specified - Succeed: false, - Server: serverTLSConfig, - Client: &tls.Config{ - RootCAs: pool, - Certificates: []tls.Certificate{tls.Certificate{ - Certificate: [][]byte{hostDer}, - PrivateKey: clientkey, - }}, - }, - }, - { // Verify client fails if no CAs are specified - Succeed: false, - Server: serverTLSConfig, - Client: &tls.Config{ - Certificates: []tls.Certificate{tls.Certificate{ - Certificate: [][]byte{clientDer}, - PrivateKey: clientkey, - }}, - }, - }, - { // Verify client fails if no keys are specified - Succeed: false, - Server: serverTLSConfig, - Client: &tls.Config{ - RootCAs: pool, - }, - }, - { // Finally, verify it all works happily with client and server cert in place - Succeed: true, - Server: serverTLSConfig, - Client: &tls.Config{ - RootCAs: pool, - Certificates: []tls.Certificate{tls.Certificate{ - Certificate: [][]byte{clientDer}, - PrivateKey: clientkey, - }}, - }, - }, - } { - doListenerTLSTest(t, tc.Succeed, tc.Server, tc.Client) - } -} - -func doListenerTLSTest(t *testing.T, expectSuccess bool, serverConfig, clientConfig *tls.Config) { - serverConfig.BuildNameToCertificate() - clientConfig.BuildNameToCertificate() - - seedListener, err := tls.Listen("tcp", "127.0.0.1:0", serverConfig) - if err != nil { - t.Fatal("cannot open listener", err) - } - - var childT *testing.T - if expectSuccess { - childT = t - } else { - childT = &testing.T{} // we want to swallow errors - } - - seedBroker := NewMockBrokerListener(childT, 1, seedListener) - defer seedBroker.Close() - - seedBroker.Returns(new(MetadataResponse)) - - config := NewConfig() - config.Net.TLS.Enable = true - config.Net.TLS.Config = clientConfig - - client, err := NewClient([]string{seedBroker.Addr()}, config) - if err == nil { - safeClose(t, client) - } - - if expectSuccess { - if err != nil { - t.Fatal(err) - } - } else { - if err == nil { - t.Fatal("expected failure") - } - } -} diff --git a/third/github.com/Shopify/sarama/compress.go b/third/github.com/Shopify/sarama/compress.go deleted file mode 100644 index ef6a63294..000000000 --- a/third/github.com/Shopify/sarama/compress.go +++ /dev/null @@ -1,75 +0,0 @@ -package sarama - -import ( - "bytes" - "compress/gzip" - "fmt" - "sync" - - "github.com/gogf/gf/third/github.com/eapache/go-xerial-snappy" - "github.com/gogf/gf/third/github.com/pierrec/lz4" -) - -var ( - lz4WriterPool = sync.Pool{ - New: func() interface{} { - return lz4.NewWriter(nil) - }, - } - - gzipWriterPool = sync.Pool{ - New: func() interface{} { - return gzip.NewWriter(nil) - }, - } -) - -func compress(cc CompressionCodec, level int, data []byte) ([]byte, error) { - switch cc { - case CompressionNone: - return data, nil - case CompressionGZIP: - var ( - err error - buf bytes.Buffer - writer *gzip.Writer - ) - if level != CompressionLevelDefault { - writer, err = gzip.NewWriterLevel(&buf, level) - if err != nil { - return nil, err - } - } else { - writer = gzipWriterPool.Get().(*gzip.Writer) - defer gzipWriterPool.Put(writer) - writer.Reset(&buf) - } - if _, err := writer.Write(data); err != nil { - return nil, err - } - if err := writer.Close(); err != nil { - return nil, err - } - return buf.Bytes(), nil - case CompressionSnappy: - return snappy.Encode(data), nil - case CompressionLZ4: - writer := lz4WriterPool.Get().(*lz4.Writer) - defer lz4WriterPool.Put(writer) - - var buf bytes.Buffer - writer.Reset(&buf) - - if _, err := writer.Write(data); err != nil { - return nil, err - } - if err := writer.Close(); err != nil { - return nil, err - } - return buf.Bytes(), nil - case CompressionZSTD: - return zstdCompressLevel(nil, data, level) - default: - return nil, PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", cc)} - } -} diff --git a/third/github.com/Shopify/sarama/config.go b/third/github.com/Shopify/sarama/config.go deleted file mode 100644 index f7f6619f4..000000000 --- a/third/github.com/Shopify/sarama/config.go +++ /dev/null @@ -1,581 +0,0 @@ -package sarama - -import ( - "compress/gzip" - "crypto/tls" - "fmt" - "io/ioutil" - "net" - "regexp" - "time" - - "github.com/gogf/gf/third/github.com/rcrowley/go-metrics" -) - -const defaultClientID = "sarama" - -var validID = regexp.MustCompile(`\A[A-Za-z0-9._-]+\z`) - -// Config is used to pass multiple configuration options to Sarama's constructors. -type Config struct { - // Admin is the namespace for ClusterAdmin properties used by the administrative Kafka client. - Admin struct { - // The maximum duration the administrative Kafka client will wait for ClusterAdmin operations, - // including topics, brokers, configurations and ACLs (defaults to 3 seconds). - Timeout time.Duration - } - - // Net is the namespace for network-level properties used by the Broker, and - // shared by the Client/Producer/Consumer. - Net struct { - // How many outstanding requests a connection is allowed to have before - // sending on it blocks (default 5). - MaxOpenRequests int - - // All three of the below configurations are similar to the - // `socket.timeout.ms` setting in JVM kafka. All of them default - // to 30 seconds. - DialTimeout time.Duration // How long to wait for the initial connection. - ReadTimeout time.Duration // How long to wait for a response. - WriteTimeout time.Duration // How long to wait for a transmit. - - TLS struct { - // Whether or not to use TLS when connecting to the broker - // (defaults to false). - Enable bool - // The TLS configuration to use for secure connections if - // enabled (defaults to nil). - Config *tls.Config - } - - // SASL based authentication with broker. While there are multiple SASL authentication methods - // the current implementation is limited to plaintext (SASL/PLAIN) authentication - SASL struct { - // Whether or not to use SASL authentication when connecting to the broker - // (defaults to false). - Enable bool - // Whether or not to send the Kafka SASL handshake first if enabled - // (defaults to true). You should only set this to false if you're using - // a non-Kafka SASL proxy. - Handshake bool - //username and password for SASL/PLAIN authentication - User string - Password string - } - - // KeepAlive specifies the keep-alive period for an active network connection. - // If zero, keep-alives are disabled. (default is 0: disabled). - KeepAlive time.Duration - - // LocalAddr is the local address to use when dialing an - // address. The address must be of a compatible type for the - // network being dialed. - // If nil, a local address is automatically chosen. - LocalAddr net.Addr - } - - // Metadata is the namespace for metadata management properties used by the - // Client, and shared by the Producer/Consumer. - Metadata struct { - Retry struct { - // The total number of times to retry a metadata request when the - // cluster is in the middle of a leader election (default 3). - Max int - // How long to wait for leader election to occur before retrying - // (default 250ms). Similar to the JVM's `retry.backoff.ms`. - Backoff time.Duration - } - // How frequently to refresh the cluster metadata in the background. - // Defaults to 10 minutes. Set to 0 to disable. Similar to - // `topic.metadata.refresh.interval.ms` in the JVM version. - RefreshFrequency time.Duration - - // Whether to maintain a full set of metadata for all topics, or just - // the minimal set that has been necessary so far. The full set is simpler - // and usually more convenient, but can take up a substantial amount of - // memory if you have many topics and partitions. Defaults to true. - Full bool - } - - // Producer is the namespace for configuration related to producing messages, - // used by the Producer. - Producer struct { - // The maximum permitted size of a message (defaults to 1000000). Should be - // set equal to or smaller than the broker's `message.max.bytes`. - MaxMessageBytes int - // The level of acknowledgement reliability needed from the broker (defaults - // to WaitForLocal). Equivalent to the `request.required.acks` setting of the - // JVM producer. - RequiredAcks RequiredAcks - // The maximum duration the broker will wait the receipt of the number of - // RequiredAcks (defaults to 10 seconds). This is only relevant when - // RequiredAcks is set to WaitForAll or a number > 1. Only supports - // millisecond resolution, nanoseconds will be truncated. Equivalent to - // the JVM producer's `request.timeout.ms` setting. - Timeout time.Duration - // The type of compression to use on messages (defaults to no compression). - // Similar to `compression.codec` setting of the JVM producer. - Compression CompressionCodec - // The level of compression to use on messages. The meaning depends - // on the actual compression type used and defaults to default compression - // level for the codec. - CompressionLevel int - // Generates partitioners for choosing the partition to send messages to - // (defaults to hashing the message key). Similar to the `partitioner.class` - // setting for the JVM producer. - Partitioner PartitionerConstructor - // If enabled, the producer will ensure that exactly one copy of each message is - // written. - Idempotent bool - - // Return specifies what channels will be populated. If they are set to true, - // you must read from the respective channels to prevent deadlock. If, - // however, this config is used to create a `SyncProducer`, both must be set - // to true and you shall not read from the channels since the producer does - // this internally. - Return struct { - // If enabled, successfully delivered messages will be returned on the - // Successes channel (default disabled). - Successes bool - - // If enabled, messages that failed to deliver will be returned on the - // Errors channel, including error (default enabled). - Errors bool - } - - // The following config options control how often messages are batched up and - // sent to the broker. By default, messages are sent as fast as possible, and - // all messages received while the current batch is in-flight are placed - // into the subsequent batch. - Flush struct { - // The best-effort number of bytes needed to trigger a flush. Use the - // global sarama.MaxRequestSize to set a hard upper limit. - Bytes int - // The best-effort number of messages needed to trigger a flush. Use - // `MaxMessages` to set a hard upper limit. - Messages int - // The best-effort frequency of flushes. Equivalent to - // `queue.buffering.max.ms` setting of JVM producer. - Frequency time.Duration - // The maximum number of messages the producer will send in a single - // broker request. Defaults to 0 for unlimited. Similar to - // `queue.buffering.max.messages` in the JVM producer. - MaxMessages int - } - - Retry struct { - // The total number of times to retry sending a message (default 3). - // Similar to the `message.send.max.retries` setting of the JVM producer. - Max int - // How long to wait for the cluster to settle between retries - // (default 100ms). Similar to the `retry.backoff.ms` setting of the - // JVM producer. - Backoff time.Duration - } - } - - // Consumer is the namespace for configuration related to consuming messages, - // used by the Consumer. - Consumer struct { - - // Group is the namespace for configuring consumer group. - Group struct { - Session struct { - // The timeout used to detect consumer failures when using Kafka's group management facility. - // The consumer sends periodic heartbeats to indicate its liveness to the broker. - // If no heartbeats are received by the broker before the expiration of this session timeout, - // then the broker will remove this consumer from the group and initiate a rebalance. - // Note that the value must be in the allowable range as configured in the broker configuration - // by `group.min.session.timeout.ms` and `group.max.session.timeout.ms` (default 10s) - Timeout time.Duration - } - Heartbeat struct { - // The expected time between heartbeats to the consumer coordinator when using Kafka's group - // management facilities. Heartbeats are used to ensure that the consumer's session stays active and - // to facilitate rebalancing when new consumers join or leave the group. - // The value must be set lower than Consumer.Group.Session.Timeout, but typically should be set no - // higher than 1/3 of that value. - // It can be adjusted even lower to control the expected time for normal rebalances (default 3s) - Interval time.Duration - } - Rebalance struct { - // Strategy for allocating topic partitions to members (default BalanceStrategyRange) - Strategy BalanceStrategy - // The maximum allowed time for each worker to join the group once a rebalance has begun. - // This is basically a limit on the amount of time needed for all tasks to flush any pending - // data and commit offsets. If the timeout is exceeded, then the worker will be removed from - // the group, which will cause offset commit failures (default 60s). - Timeout time.Duration - - Retry struct { - // When a new consumer joins a consumer group the set of consumers attempt to "rebalance" - // the load to assign partitions to each consumer. If the set of consumers changes while - // this assignment is taking place the rebalance will fail and retry. This setting controls - // the maximum number of attempts before giving up (default 4). - Max int - // Backoff time between retries during rebalance (default 2s) - Backoff time.Duration - } - } - Member struct { - // Custom metadata to include when joining the group. The user data for all joined members - // can be retrieved by sending a DescribeGroupRequest to the broker that is the - // coordinator for the group. - UserData []byte - } - } - - Retry struct { - // How long to wait after a failing to read from a partition before - // trying again (default 2s). - Backoff time.Duration - } - - // Fetch is the namespace for controlling how many bytes are retrieved by any - // given request. - Fetch struct { - // The minimum number of message bytes to fetch in a request - the broker - // will wait until at least this many are available. The default is 1, - // as 0 causes the consumer to spin when no messages are available. - // Equivalent to the JVM's `fetch.min.bytes`. - Min int32 - // The default number of message bytes to fetch from the broker in each - // request (default 1MB). This should be larger than the majority of - // your messages, or else the consumer will spend a lot of time - // negotiating sizes and not actually consuming. Similar to the JVM's - // `fetch.message.max.bytes`. - Default int32 - // The maximum number of message bytes to fetch from the broker in a - // single request. Messages larger than this will return - // ErrMessageTooLarge and will not be consumable, so you must be sure - // this is at least as large as your largest message. Defaults to 0 - // (no limit). Similar to the JVM's `fetch.message.max.bytes`. The - // global `sarama.MaxResponseSize` still applies. - Max int32 - } - // The maximum amount of time the broker will wait for Consumer.Fetch.Min - // bytes to become available before it returns fewer than that anyways. The - // default is 250ms, since 0 causes the consumer to spin when no events are - // available. 100-500ms is a reasonable range for most cases. Kafka only - // supports precision up to milliseconds; nanoseconds will be truncated. - // Equivalent to the JVM's `fetch.wait.max.ms`. - MaxWaitTime time.Duration - - // The maximum amount of time the consumer expects a message takes to - // process for the user. If writing to the Messages channel takes longer - // than this, that partition will stop fetching more messages until it - // can proceed again. - // Note that, since the Messages channel is buffered, the actual grace time is - // (MaxProcessingTime * ChanneBufferSize). Defaults to 100ms. - // If a message is not written to the Messages channel between two ticks - // of the expiryTicker then a timeout is detected. - // Using a ticker instead of a timer to detect timeouts should typically - // result in many fewer calls to Timer functions which may result in a - // significant performance improvement if many messages are being sent - // and timeouts are infrequent. - // The disadvantage of using a ticker instead of a timer is that - // timeouts will be less accurate. That is, the effective timeout could - // be between `MaxProcessingTime` and `2 * MaxProcessingTime`. For - // example, if `MaxProcessingTime` is 100ms then a delay of 180ms - // between two messages being sent may not be recognized as a timeout. - MaxProcessingTime time.Duration - - // Return specifies what channels will be populated. If they are set to true, - // you must read from them to prevent deadlock. - Return struct { - // If enabled, any errors that occurred while consuming are returned on - // the Errors channel (default disabled). - Errors bool - } - - // Offsets specifies configuration for how and when to commit consumed - // offsets. This currently requires the manual use of an OffsetManager - // but will eventually be automated. - Offsets struct { - // How frequently to commit updated offsets. Defaults to 1s. - CommitInterval time.Duration - - // The initial offset to use if no offset was previously committed. - // Should be OffsetNewest or OffsetOldest. Defaults to OffsetNewest. - Initial int64 - - // The retention duration for committed offsets. If zero, disabled - // (in which case the `offsets.retention.minutes` option on the - // broker will be used). Kafka only supports precision up to - // milliseconds; nanoseconds will be truncated. Requires Kafka - // broker version 0.9.0 or later. - // (default is 0: disabled). - Retention time.Duration - - Retry struct { - // The total number of times to retry failing commit - // requests during OffsetManager shutdown (default 3). - Max int - } - } - } - - // A user-provided string sent with every request to the brokers for logging, - // debugging, and auditing purposes. Defaults to "sarama", but you should - // probably set it to something specific to your application. - ClientID string - // The number of events to buffer in internal and external channels. This - // permits the producer and consumer to continue processing some messages - // in the background while user code is working, greatly improving throughput. - // Defaults to 256. - ChannelBufferSize int - // The version of Kafka that Sarama will assume it is running against. - // Defaults to the oldest supported stable version. Since Kafka provides - // backwards-compatibility, setting it to a version older than you have - // will not break anything, although it may prevent you from using the - // latest features. Setting it to a version greater than you are actually - // running may lead to random breakage. - Version KafkaVersion - // The registry to define metrics into. - // Defaults to a local registry. - // If you want to disable metrics gathering, set "metrics.UseNilMetrics" to "true" - // prior to starting Sarama. - // See Examples on how to use the metrics registry - MetricRegistry metrics.Registry -} - -// NewConfig returns a new configuration instance with sane defaults. -func NewConfig() *Config { - c := &Config{} - - c.Admin.Timeout = 3 * time.Second - - c.Net.MaxOpenRequests = 5 - c.Net.DialTimeout = 30 * time.Second - c.Net.ReadTimeout = 30 * time.Second - c.Net.WriteTimeout = 30 * time.Second - c.Net.SASL.Handshake = true - - c.Metadata.Retry.Max = 3 - c.Metadata.Retry.Backoff = 250 * time.Millisecond - c.Metadata.RefreshFrequency = 10 * time.Minute - c.Metadata.Full = true - - c.Producer.MaxMessageBytes = 1000000 - c.Producer.RequiredAcks = WaitForLocal - c.Producer.Timeout = 10 * time.Second - c.Producer.Partitioner = NewHashPartitioner - c.Producer.Retry.Max = 3 - c.Producer.Retry.Backoff = 100 * time.Millisecond - c.Producer.Return.Errors = true - c.Producer.CompressionLevel = CompressionLevelDefault - - c.Consumer.Fetch.Min = 1 - c.Consumer.Fetch.Default = 1024 * 1024 - c.Consumer.Retry.Backoff = 2 * time.Second - c.Consumer.MaxWaitTime = 250 * time.Millisecond - c.Consumer.MaxProcessingTime = 100 * time.Millisecond - c.Consumer.Return.Errors = false - c.Consumer.Offsets.CommitInterval = 1 * time.Second - c.Consumer.Offsets.Initial = OffsetNewest - c.Consumer.Offsets.Retry.Max = 3 - - c.Consumer.Group.Session.Timeout = 10 * time.Second - c.Consumer.Group.Heartbeat.Interval = 3 * time.Second - c.Consumer.Group.Rebalance.Strategy = BalanceStrategyRange - c.Consumer.Group.Rebalance.Timeout = 60 * time.Second - c.Consumer.Group.Rebalance.Retry.Max = 4 - c.Consumer.Group.Rebalance.Retry.Backoff = 2 * time.Second - - c.ClientID = defaultClientID - c.ChannelBufferSize = 256 - c.Version = MinVersion - c.MetricRegistry = metrics.NewRegistry() - - return c -} - -// Validate checks a Config instance. It will return a -// ConfigurationError if the specified values don't make sense. -func (c *Config) Validate() error { - // some configuration values should be warned on but not fail completely, do those first - if c.Net.TLS.Enable == false && c.Net.TLS.Config != nil { - Logger.Println("Net.TLS is disabled but a non-nil configuration was provided.") - } - if c.Net.SASL.Enable == false { - if c.Net.SASL.User != "" { - Logger.Println("Net.SASL is disabled but a non-empty username was provided.") - } - if c.Net.SASL.Password != "" { - Logger.Println("Net.SASL is disabled but a non-empty password was provided.") - } - } - if c.Producer.RequiredAcks > 1 { - Logger.Println("Producer.RequiredAcks > 1 is deprecated and will raise an exception with kafka >= 0.8.2.0.") - } - if c.Producer.MaxMessageBytes >= int(MaxRequestSize) { - Logger.Println("Producer.MaxMessageBytes must be smaller than MaxRequestSize; it will be ignored.") - } - if c.Producer.Flush.Bytes >= int(MaxRequestSize) { - Logger.Println("Producer.Flush.Bytes must be smaller than MaxRequestSize; it will be ignored.") - } - if (c.Producer.Flush.Bytes > 0 || c.Producer.Flush.Messages > 0) && c.Producer.Flush.Frequency == 0 { - Logger.Println("Producer.Flush: Bytes or Messages are set, but Frequency is not; messages may not get flushed.") - } - if c.Producer.Timeout%time.Millisecond != 0 { - Logger.Println("Producer.Timeout only supports millisecond resolution; nanoseconds will be truncated.") - } - if c.Consumer.MaxWaitTime < 100*time.Millisecond { - Logger.Println("Consumer.MaxWaitTime is very low, which can cause high CPU and network usage. See documentation for details.") - } - if c.Consumer.MaxWaitTime%time.Millisecond != 0 { - Logger.Println("Consumer.MaxWaitTime only supports millisecond precision; nanoseconds will be truncated.") - } - if c.Consumer.Offsets.Retention%time.Millisecond != 0 { - Logger.Println("Consumer.Offsets.Retention only supports millisecond precision; nanoseconds will be truncated.") - } - if c.Consumer.Group.Session.Timeout%time.Millisecond != 0 { - Logger.Println("Consumer.Group.Session.Timeout only supports millisecond precision; nanoseconds will be truncated.") - } - if c.Consumer.Group.Heartbeat.Interval%time.Millisecond != 0 { - Logger.Println("Consumer.Group.Heartbeat.Interval only supports millisecond precision; nanoseconds will be truncated.") - } - if c.Consumer.Group.Rebalance.Timeout%time.Millisecond != 0 { - Logger.Println("Consumer.Group.Rebalance.Timeout only supports millisecond precision; nanoseconds will be truncated.") - } - if c.ClientID == defaultClientID { - Logger.Println("ClientID is the default of 'sarama', you should consider setting it to something application-specific.") - } - - // validate Net values - switch { - case c.Net.MaxOpenRequests <= 0: - return ConfigurationError("Net.MaxOpenRequests must be > 0") - case c.Net.DialTimeout <= 0: - return ConfigurationError("Net.DialTimeout must be > 0") - case c.Net.ReadTimeout <= 0: - return ConfigurationError("Net.ReadTimeout must be > 0") - case c.Net.WriteTimeout <= 0: - return ConfigurationError("Net.WriteTimeout must be > 0") - case c.Net.KeepAlive < 0: - return ConfigurationError("Net.KeepAlive must be >= 0") - case c.Net.SASL.Enable == true && c.Net.SASL.User == "": - return ConfigurationError("Net.SASL.User must not be empty when SASL is enabled") - case c.Net.SASL.Enable == true && c.Net.SASL.Password == "": - return ConfigurationError("Net.SASL.Password must not be empty when SASL is enabled") - } - - // validate the Admin values - switch { - case c.Admin.Timeout <= 0: - return ConfigurationError("Admin.Timeout must be > 0") - } - - // validate the Metadata values - switch { - case c.Metadata.Retry.Max < 0: - return ConfigurationError("Metadata.Retry.Max must be >= 0") - case c.Metadata.Retry.Backoff < 0: - return ConfigurationError("Metadata.Retry.Backoff must be >= 0") - case c.Metadata.RefreshFrequency < 0: - return ConfigurationError("Metadata.RefreshFrequency must be >= 0") - } - - // validate the Producer values - switch { - case c.Producer.MaxMessageBytes <= 0: - return ConfigurationError("Producer.MaxMessageBytes must be > 0") - case c.Producer.RequiredAcks < -1: - return ConfigurationError("Producer.RequiredAcks must be >= -1") - case c.Producer.Timeout <= 0: - return ConfigurationError("Producer.Timeout must be > 0") - case c.Producer.Partitioner == nil: - return ConfigurationError("Producer.Partitioner must not be nil") - case c.Producer.Flush.Bytes < 0: - return ConfigurationError("Producer.Flush.Bytes must be >= 0") - case c.Producer.Flush.Messages < 0: - return ConfigurationError("Producer.Flush.Messages must be >= 0") - case c.Producer.Flush.Frequency < 0: - return ConfigurationError("Producer.Flush.Frequency must be >= 0") - case c.Producer.Flush.MaxMessages < 0: - return ConfigurationError("Producer.Flush.MaxMessages must be >= 0") - case c.Producer.Flush.MaxMessages > 0 && c.Producer.Flush.MaxMessages < c.Producer.Flush.Messages: - return ConfigurationError("Producer.Flush.MaxMessages must be >= Producer.Flush.Messages when set") - case c.Producer.Retry.Max < 0: - return ConfigurationError("Producer.Retry.Max must be >= 0") - case c.Producer.Retry.Backoff < 0: - return ConfigurationError("Producer.Retry.Backoff must be >= 0") - } - - if c.Producer.Compression == CompressionLZ4 && !c.Version.IsAtLeast(V0_10_0_0) { - return ConfigurationError("lz4 compression requires Version >= V0_10_0_0") - } - - if c.Producer.Compression == CompressionGZIP { - if c.Producer.CompressionLevel != CompressionLevelDefault { - if _, err := gzip.NewWriterLevel(ioutil.Discard, c.Producer.CompressionLevel); err != nil { - return ConfigurationError(fmt.Sprintf("gzip compression does not work with level %d: %v", c.Producer.CompressionLevel, err)) - } - } - } - - if c.Producer.Idempotent { - if !c.Version.IsAtLeast(V0_11_0_0) { - return ConfigurationError("Idempotent producer requires Version >= V0_11_0_0") - } - if c.Producer.Retry.Max == 0 { - return ConfigurationError("Idempotent producer requires Producer.Retry.Max >= 1") - } - if c.Producer.RequiredAcks != WaitForAll { - return ConfigurationError("Idempotent producer requires Producer.RequiredAcks to be WaitForAll") - } - if c.Net.MaxOpenRequests > 1 { - return ConfigurationError("Idempotent producer requires Net.MaxOpenRequests to be 1") - } - } - - // validate the Consumer values - switch { - case c.Consumer.Fetch.Min <= 0: - return ConfigurationError("Consumer.Fetch.Min must be > 0") - case c.Consumer.Fetch.Default <= 0: - return ConfigurationError("Consumer.Fetch.Default must be > 0") - case c.Consumer.Fetch.Max < 0: - return ConfigurationError("Consumer.Fetch.Max must be >= 0") - case c.Consumer.MaxWaitTime < 1*time.Millisecond: - return ConfigurationError("Consumer.MaxWaitTime must be >= 1ms") - case c.Consumer.MaxProcessingTime <= 0: - return ConfigurationError("Consumer.MaxProcessingTime must be > 0") - case c.Consumer.Retry.Backoff < 0: - return ConfigurationError("Consumer.Retry.Backoff must be >= 0") - case c.Consumer.Offsets.CommitInterval <= 0: - return ConfigurationError("Consumer.Offsets.CommitInterval must be > 0") - case c.Consumer.Offsets.Initial != OffsetOldest && c.Consumer.Offsets.Initial != OffsetNewest: - return ConfigurationError("Consumer.Offsets.Initial must be OffsetOldest or OffsetNewest") - case c.Consumer.Offsets.Retry.Max < 0: - return ConfigurationError("Consumer.Offsets.Retry.Max must be >= 0") - } - - // validate the Consumer Group values - switch { - case c.Consumer.Group.Session.Timeout <= 2*time.Millisecond: - return ConfigurationError("Consumer.Group.Session.Timeout must be >= 2ms") - case c.Consumer.Group.Heartbeat.Interval < 1*time.Millisecond: - return ConfigurationError("Consumer.Group.Heartbeat.Interval must be >= 1ms") - case c.Consumer.Group.Heartbeat.Interval >= c.Consumer.Group.Session.Timeout: - return ConfigurationError("Consumer.Group.Heartbeat.Interval must be < Consumer.Group.Session.Timeout") - case c.Consumer.Group.Rebalance.Strategy == nil: - return ConfigurationError("Consumer.Group.Rebalance.Strategy must not be empty") - case c.Consumer.Group.Rebalance.Timeout <= time.Millisecond: - return ConfigurationError("Consumer.Group.Rebalance.Timeout must be >= 1ms") - case c.Consumer.Group.Rebalance.Retry.Max < 0: - return ConfigurationError("Consumer.Group.Rebalance.Retry.Max must be >= 0") - case c.Consumer.Group.Rebalance.Retry.Backoff < 0: - return ConfigurationError("Consumer.Group.Rebalance.Retry.Backoff must be >= 0") - } - - // validate misc shared values - switch { - case c.ChannelBufferSize < 0: - return ConfigurationError("ChannelBufferSize must be >= 0") - case !validID.MatchString(c.ClientID): - return ConfigurationError("ClientID is invalid") - } - - return nil -} diff --git a/third/github.com/Shopify/sarama/config_resource_type.go b/third/github.com/Shopify/sarama/config_resource_type.go deleted file mode 100644 index 848cc9c90..000000000 --- a/third/github.com/Shopify/sarama/config_resource_type.go +++ /dev/null @@ -1,15 +0,0 @@ -package sarama - -type ConfigResourceType int8 - -// Taken from : -// https://cwiki.apache.org/confluence/display/KAFKA/KIP-133%3A+Describe+and+Alter+Configs+Admin+APIs#KIP-133:DescribeandAlterConfigsAdminAPIs-WireFormattypes - -const ( - UnknownResource ConfigResourceType = 0 - AnyResource ConfigResourceType = 1 - TopicResource ConfigResourceType = 2 - GroupResource ConfigResourceType = 3 - ClusterResource ConfigResourceType = 4 - BrokerResource ConfigResourceType = 5 -) diff --git a/third/github.com/Shopify/sarama/config_test.go b/third/github.com/Shopify/sarama/config_test.go deleted file mode 100644 index 58f05a184..000000000 --- a/third/github.com/Shopify/sarama/config_test.go +++ /dev/null @@ -1,281 +0,0 @@ -package sarama - -import ( - "os" - "testing" - - "github.com/gogf/gf/third/github.com/rcrowley/go-metrics" -) - -func TestDefaultConfigValidates(t *testing.T) { - config := NewConfig() - if err := config.Validate(); err != nil { - t.Error(err) - } - if config.MetricRegistry == nil { - t.Error("Expected non nil metrics.MetricRegistry, got nil") - } -} - -func TestInvalidClientIDConfigValidates(t *testing.T) { - config := NewConfig() - config.ClientID = "foo:bar" - if err := config.Validate(); string(err.(ConfigurationError)) != "ClientID is invalid" { - t.Error("Expected invalid ClientID, got ", err) - } -} - -func TestEmptyClientIDConfigValidates(t *testing.T) { - config := NewConfig() - config.ClientID = "" - if err := config.Validate(); string(err.(ConfigurationError)) != "ClientID is invalid" { - t.Error("Expected invalid ClientID, got ", err) - } -} - -func TestNetConfigValidates(t *testing.T) { - tests := []struct { - name string - cfg func(*Config) // resorting to using a function as a param because of internal composite structs - err string - }{ - { - "OpenRequests", - func(cfg *Config) { - cfg.Net.MaxOpenRequests = 0 - }, - "Net.MaxOpenRequests must be > 0"}, - {"DialTimeout", - func(cfg *Config) { - cfg.Net.DialTimeout = 0 - }, - "Net.DialTimeout must be > 0"}, - {"ReadTimeout", - func(cfg *Config) { - cfg.Net.ReadTimeout = 0 - }, - "Net.ReadTimeout must be > 0"}, - {"WriteTimeout", - func(cfg *Config) { - cfg.Net.WriteTimeout = 0 - }, - "Net.WriteTimeout must be > 0"}, - {"KeepAlive", - func(cfg *Config) { - cfg.Net.KeepAlive = -1 - }, - "Net.KeepAlive must be >= 0"}, - {"SASL.User", - func(cfg *Config) { - cfg.Net.SASL.Enable = true - cfg.Net.SASL.User = "" - }, - "Net.SASL.User must not be empty when SASL is enabled"}, - {"SASL.Password", - func(cfg *Config) { - cfg.Net.SASL.Enable = true - cfg.Net.SASL.User = "user" - cfg.Net.SASL.Password = "" - }, - "Net.SASL.Password must not be empty when SASL is enabled"}, - } - - for i, test := range tests { - c := NewConfig() - test.cfg(c) - if err := c.Validate(); string(err.(ConfigurationError)) != test.err { - t.Errorf("[%d]:[%s] Expected %s, Got %s\n", i, test.name, test.err, err) - } - } -} - -func TestMetadataConfigValidates(t *testing.T) { - tests := []struct { - name string - cfg func(*Config) // resorting to using a function as a param because of internal composite structs - err string - }{ - { - "Retry.Max", - func(cfg *Config) { - cfg.Metadata.Retry.Max = -1 - }, - "Metadata.Retry.Max must be >= 0"}, - {"Retry.Backoff", - func(cfg *Config) { - cfg.Metadata.Retry.Backoff = -1 - }, - "Metadata.Retry.Backoff must be >= 0"}, - {"RefreshFrequency", - func(cfg *Config) { - cfg.Metadata.RefreshFrequency = -1 - }, - "Metadata.RefreshFrequency must be >= 0"}, - } - - for i, test := range tests { - c := NewConfig() - test.cfg(c) - if err := c.Validate(); string(err.(ConfigurationError)) != test.err { - t.Errorf("[%d]:[%s] Expected %s, Got %s\n", i, test.name, test.err, err) - } - } -} - -func TestAdminConfigValidates(t *testing.T) { - tests := []struct { - name string - cfg func(*Config) // resorting to using a function as a param because of internal composite structs - err string - }{ - {"Timeout", - func(cfg *Config) { - cfg.Admin.Timeout = 0 - }, - "Admin.Timeout must be > 0"}, - } - - for i, test := range tests { - c := NewConfig() - test.cfg(c) - if err := c.Validate(); string(err.(ConfigurationError)) != test.err { - t.Errorf("[%d]:[%s] Expected %s, Got %s\n", i, test.name, test.err, err) - } - } -} - -func TestProducerConfigValidates(t *testing.T) { - tests := []struct { - name string - cfg func(*Config) // resorting to using a function as a param because of internal composite structs - err string - }{ - { - "MaxMessageBytes", - func(cfg *Config) { - cfg.Producer.MaxMessageBytes = 0 - }, - "Producer.MaxMessageBytes must be > 0"}, - {"RequiredAcks", - func(cfg *Config) { - cfg.Producer.RequiredAcks = -2 - }, - "Producer.RequiredAcks must be >= -1"}, - {"Timeout", - func(cfg *Config) { - cfg.Producer.Timeout = 0 - }, - "Producer.Timeout must be > 0"}, - {"Partitioner", - func(cfg *Config) { - cfg.Producer.Partitioner = nil - }, - "Producer.Partitioner must not be nil"}, - {"Flush.Bytes", - func(cfg *Config) { - cfg.Producer.Flush.Bytes = -1 - }, - "Producer.Flush.Bytes must be >= 0"}, - {"Flush.Messages", - func(cfg *Config) { - cfg.Producer.Flush.Messages = -1 - }, - "Producer.Flush.Messages must be >= 0"}, - {"Flush.Frequency", - func(cfg *Config) { - cfg.Producer.Flush.Frequency = -1 - }, - "Producer.Flush.Frequency must be >= 0"}, - {"Flush.MaxMessages", - func(cfg *Config) { - cfg.Producer.Flush.MaxMessages = -1 - }, - "Producer.Flush.MaxMessages must be >= 0"}, - {"Flush.MaxMessages with Producer.Flush.Messages", - func(cfg *Config) { - cfg.Producer.Flush.MaxMessages = 1 - cfg.Producer.Flush.Messages = 2 - }, - "Producer.Flush.MaxMessages must be >= Producer.Flush.Messages when set"}, - {"Flush.Retry.Max", - func(cfg *Config) { - cfg.Producer.Retry.Max = -1 - }, - "Producer.Retry.Max must be >= 0"}, - {"Flush.Retry.Backoff", - func(cfg *Config) { - cfg.Producer.Retry.Backoff = -1 - }, - "Producer.Retry.Backoff must be >= 0"}, - {"Idempotent Version", - func(cfg *Config) { - cfg.Producer.Idempotent = true - cfg.Version = V0_10_0_0 - }, - "Idempotent producer requires Version >= V0_11_0_0"}, - {"Idempotent with Producer.Retry.Max", - func(cfg *Config) { - cfg.Version = V0_11_0_0 - cfg.Producer.Idempotent = true - cfg.Producer.Retry.Max = 0 - }, - "Idempotent producer requires Producer.Retry.Max >= 1"}, - {"Idempotent with Producer.RequiredAcks", - func(cfg *Config) { - cfg.Version = V0_11_0_0 - cfg.Producer.Idempotent = true - }, - "Idempotent producer requires Producer.RequiredAcks to be WaitForAll"}, - {"Idempotent with Net.MaxOpenRequests", - func(cfg *Config) { - cfg.Version = V0_11_0_0 - cfg.Producer.Idempotent = true - cfg.Producer.RequiredAcks = WaitForAll - }, - "Idempotent producer requires Net.MaxOpenRequests to be 1"}, - } - - for i, test := range tests { - c := NewConfig() - test.cfg(c) - if err := c.Validate(); string(err.(ConfigurationError)) != test.err { - t.Errorf("[%d]:[%s] Expected %s, Got %s\n", i, test.name, test.err, err) - } - } -} - -func TestLZ4ConfigValidation(t *testing.T) { - config := NewConfig() - config.Producer.Compression = CompressionLZ4 - if err := config.Validate(); string(err.(ConfigurationError)) != "lz4 compression requires Version >= V0_10_0_0" { - t.Error("Expected invalid lz4/kafka version error, got ", err) - } - config.Version = V0_10_0_0 - if err := config.Validate(); err != nil { - t.Error("Expected lz4 to work, got ", err) - } -} - -// This example shows how to integrate with an existing registry as well as publishing metrics -// on the standard output -func ExampleConfig_metrics() { - // Our application registry - appMetricRegistry := metrics.NewRegistry() - appGauge := metrics.GetOrRegisterGauge("m1", appMetricRegistry) - appGauge.Update(1) - - config := NewConfig() - // Use a prefix registry instead of the default local one - config.MetricRegistry = metrics.NewPrefixedChildRegistry(appMetricRegistry, "sarama.") - - // Simulate a metric created by sarama without starting a broker - saramaGauge := metrics.GetOrRegisterGauge("m2", config.MetricRegistry) - saramaGauge.Update(2) - - metrics.WriteOnce(appMetricRegistry, os.Stdout) - // Output: - // gauge m1 - // value: 1 - // gauge sarama.m2 - // value: 2 -} diff --git a/third/github.com/Shopify/sarama/consumer.go b/third/github.com/Shopify/sarama/consumer.go deleted file mode 100644 index 33d9d143f..000000000 --- a/third/github.com/Shopify/sarama/consumer.go +++ /dev/null @@ -1,807 +0,0 @@ -package sarama - -import ( - "errors" - "fmt" - "sync" - "sync/atomic" - "time" -) - -// ConsumerMessage encapsulates a Kafka message returned by the consumer. -type ConsumerMessage struct { - Key, Value []byte - Topic string - Partition int32 - Offset int64 - Timestamp time.Time // only set if kafka is version 0.10+, inner message timestamp - BlockTimestamp time.Time // only set if kafka is version 0.10+, outer (compressed) block timestamp - Headers []*RecordHeader // only set if kafka is version 0.11+ -} - -// ConsumerError is what is provided to the user when an error occurs. -// It wraps an error and includes the topic and partition. -type ConsumerError struct { - Topic string - Partition int32 - Err error -} - -func (ce ConsumerError) Error() string { - return fmt.Sprintf("kafka: error while consuming %s/%d: %s", ce.Topic, ce.Partition, ce.Err) -} - -// ConsumerErrors is a type that wraps a batch of errors and implements the Error interface. -// It can be returned from the PartitionConsumer's Close methods to avoid the need to manually drain errors -// when stopping. -type ConsumerErrors []*ConsumerError - -func (ce ConsumerErrors) Error() string { - return fmt.Sprintf("kafka: %d errors while consuming", len(ce)) -} - -// Consumer manages PartitionConsumers which process Kafka messages from brokers. You MUST call Close() -// on a consumer to avoid leaks, it will not be garbage-collected automatically when it passes out of -// scope. -// -// Sarama's Consumer type does not currently support automatic consumer-group rebalancing and offset tracking. -// For Zookeeper-based tracking (Kafka 0.8.2 and earlier), the https://github.com/wvanbergen/kafka library -// builds on Sarama to add this support. For Kafka-based tracking (Kafka 0.9 and later), the -// https://github.com/bsm/sarama-cluster library builds on Sarama to add this support. -type Consumer interface { - - // Topics returns the set of available topics as retrieved from the cluster - // metadata. This method is the same as Client.Topics(), and is provided for - // convenience. - Topics() ([]string, error) - - // Partitions returns the sorted list of all partition IDs for the given topic. - // This method is the same as Client.Partitions(), and is provided for convenience. - Partitions(topic string) ([]int32, error) - - // ConsumePartition creates a PartitionConsumer on the given topic/partition with - // the given offset. It will return an error if this Consumer is already consuming - // on the given topic/partition. Offset can be a literal offset, or OffsetNewest - // or OffsetOldest - ConsumePartition(topic string, partition int32, offset int64) (PartitionConsumer, error) - - // HighWaterMarks returns the current high water marks for each topic and partition. - // Consistency between partitions is not guaranteed since high water marks are updated separately. - HighWaterMarks() map[string]map[int32]int64 - - // Close shuts down the consumer. It must be called after all child - // PartitionConsumers have already been closed. - Close() error -} - -type consumer struct { - client Client - conf *Config - ownClient bool - - lock sync.Mutex - children map[string]map[int32]*partitionConsumer - brokerConsumers map[*Broker]*brokerConsumer -} - -// NewConsumer creates a new consumer using the given broker addresses and configuration. -func NewConsumer(addrs []string, config *Config) (Consumer, error) { - client, err := NewClient(addrs, config) - if err != nil { - return nil, err - } - - c, err := NewConsumerFromClient(client) - if err != nil { - return nil, err - } - c.(*consumer).ownClient = true - return c, nil -} - -// NewConsumerFromClient creates a new consumer using the given client. It is still -// necessary to call Close() on the underlying client when shutting down this consumer. -func NewConsumerFromClient(client Client) (Consumer, error) { - // Check that we are not dealing with a closed Client before processing any other arguments - if client.Closed() { - return nil, ErrClosedClient - } - - c := &consumer{ - client: client, - conf: client.Config(), - children: make(map[string]map[int32]*partitionConsumer), - brokerConsumers: make(map[*Broker]*brokerConsumer), - } - - return c, nil -} - -func (c *consumer) Close() error { - if c.ownClient { - return c.client.Close() - } - return nil -} - -func (c *consumer) Topics() ([]string, error) { - return c.client.Topics() -} - -func (c *consumer) Partitions(topic string) ([]int32, error) { - return c.client.Partitions(topic) -} - -func (c *consumer) ConsumePartition(topic string, partition int32, offset int64) (PartitionConsumer, error) { - child := &partitionConsumer{ - consumer: c, - conf: c.conf, - topic: topic, - partition: partition, - messages: make(chan *ConsumerMessage, c.conf.ChannelBufferSize), - errors: make(chan *ConsumerError, c.conf.ChannelBufferSize), - feeder: make(chan *FetchResponse, 1), - trigger: make(chan none, 1), - dying: make(chan none), - fetchSize: c.conf.Consumer.Fetch.Default, - } - - if err := child.chooseStartingOffset(offset); err != nil { - return nil, err - } - - var leader *Broker - var err error - if leader, err = c.client.Leader(child.topic, child.partition); err != nil { - return nil, err - } - - if err := c.addChild(child); err != nil { - return nil, err - } - - go withRecover(child.dispatcher) - go withRecover(child.responseFeeder) - - child.broker = c.refBrokerConsumer(leader) - child.broker.input <- child - - return child, nil -} - -func (c *consumer) HighWaterMarks() map[string]map[int32]int64 { - c.lock.Lock() - defer c.lock.Unlock() - - hwms := make(map[string]map[int32]int64) - for topic, p := range c.children { - hwm := make(map[int32]int64, len(p)) - for partition, pc := range p { - hwm[partition] = pc.HighWaterMarkOffset() - } - hwms[topic] = hwm - } - - return hwms -} - -func (c *consumer) addChild(child *partitionConsumer) error { - c.lock.Lock() - defer c.lock.Unlock() - - topicChildren := c.children[child.topic] - if topicChildren == nil { - topicChildren = make(map[int32]*partitionConsumer) - c.children[child.topic] = topicChildren - } - - if topicChildren[child.partition] != nil { - return ConfigurationError("That topic/partition is already being consumed") - } - - topicChildren[child.partition] = child - return nil -} - -func (c *consumer) removeChild(child *partitionConsumer) { - c.lock.Lock() - defer c.lock.Unlock() - - delete(c.children[child.topic], child.partition) -} - -func (c *consumer) refBrokerConsumer(broker *Broker) *brokerConsumer { - c.lock.Lock() - defer c.lock.Unlock() - - bc := c.brokerConsumers[broker] - if bc == nil { - bc = c.newBrokerConsumer(broker) - c.brokerConsumers[broker] = bc - } - - bc.refs++ - - return bc -} - -func (c *consumer) unrefBrokerConsumer(brokerWorker *brokerConsumer) { - c.lock.Lock() - defer c.lock.Unlock() - - brokerWorker.refs-- - - if brokerWorker.refs == 0 { - close(brokerWorker.input) - if c.brokerConsumers[brokerWorker.broker] == brokerWorker { - delete(c.brokerConsumers, brokerWorker.broker) - } - } -} - -func (c *consumer) abandonBrokerConsumer(brokerWorker *brokerConsumer) { - c.lock.Lock() - defer c.lock.Unlock() - - delete(c.brokerConsumers, brokerWorker.broker) -} - -// PartitionConsumer - -// PartitionConsumer processes Kafka messages from a given topic and partition. You MUST call one of Close() or -// AsyncClose() on a PartitionConsumer to avoid leaks; it will not be garbage-collected automatically when it passes out -// of scope. -// -// The simplest way of using a PartitionConsumer is to loop over its Messages channel using a for/range -// loop. The PartitionConsumer will only stop itself in one case: when the offset being consumed is reported -// as out of range by the brokers. In this case you should decide what you want to do (try a different offset, -// notify a human, etc) and handle it appropriately. For all other error cases, it will just keep retrying. -// By default, it logs these errors to sarama.Logger; if you want to be notified directly of all errors, set -// your config's Consumer.Return.Errors to true and read from the Errors channel, using a select statement -// or a separate goroutine. Check out the Consumer examples to see implementations of these different approaches. -// -// To terminate such a for/range loop while the loop is executing, call AsyncClose. This will kick off the process of -// consumer tear-down & return imediately. Continue to loop, servicing the Messages channel until the teardown process -// AsyncClose initiated closes it (thus terminating the for/range loop). If you've already ceased reading Messages, call -// Close; this will signal the PartitionConsumer's goroutines to begin shutting down (just like AsyncClose), but will -// also drain the Messages channel, harvest all errors & return them once cleanup has completed. -type PartitionConsumer interface { - - // AsyncClose initiates a shutdown of the PartitionConsumer. This method will return immediately, after which you - // should continue to service the 'Messages' and 'Errors' channels until they are empty. It is required to call this - // function, or Close before a consumer object passes out of scope, as it will otherwise leak memory. You must call - // this before calling Close on the underlying client. - AsyncClose() - - // Close stops the PartitionConsumer from fetching messages. It will initiate a shutdown just like AsyncClose, drain - // the Messages channel, harvest any errors & return them to the caller. Note that if you are continuing to service - // the Messages channel when this function is called, you will be competing with Close for messages; consider - // calling AsyncClose, instead. It is required to call this function (or AsyncClose) before a consumer object passes - // out of scope, as it will otherwise leak memory. You must call this before calling Close on the underlying client. - Close() error - - // Messages returns the read channel for the messages that are returned by - // the broker. - Messages() <-chan *ConsumerMessage - - // Errors returns a read channel of errors that occurred during consuming, if - // enabled. By default, errors are logged and not returned over this channel. - // If you want to implement any custom error handling, set your config's - // Consumer.Return.Errors setting to true, and read from this channel. - Errors() <-chan *ConsumerError - - // HighWaterMarkOffset returns the high water mark offset of the partition, - // i.e. the offset that will be used for the next message that will be produced. - // You can use this to determine how far behind the processing is. - HighWaterMarkOffset() int64 -} - -type partitionConsumer struct { - highWaterMarkOffset int64 // must be at the top of the struct because https://golang.org/pkg/sync/atomic/#pkg-note-BUG - consumer *consumer - conf *Config - topic string - partition int32 - - broker *brokerConsumer - messages chan *ConsumerMessage - errors chan *ConsumerError - feeder chan *FetchResponse - - trigger, dying chan none - responseResult error - closeOnce sync.Once - - fetchSize int32 - offset int64 -} - -var errTimedOut = errors.New("timed out feeding messages to the user") // not user-facing - -func (child *partitionConsumer) sendError(err error) { - cErr := &ConsumerError{ - Topic: child.topic, - Partition: child.partition, - Err: err, - } - - if child.conf.Consumer.Return.Errors { - child.errors <- cErr - } else { - Logger.Println(cErr) - } -} - -func (child *partitionConsumer) dispatcher() { - for range child.trigger { - select { - case <-child.dying: - close(child.trigger) - case <-time.After(child.conf.Consumer.Retry.Backoff): - if child.broker != nil { - child.consumer.unrefBrokerConsumer(child.broker) - child.broker = nil - } - - Logger.Printf("consumer/%s/%d finding new broker\n", child.topic, child.partition) - if err := child.dispatch(); err != nil { - child.sendError(err) - child.trigger <- none{} - } - } - } - - if child.broker != nil { - child.consumer.unrefBrokerConsumer(child.broker) - } - child.consumer.removeChild(child) - close(child.feeder) -} - -func (child *partitionConsumer) dispatch() error { - if err := child.consumer.client.RefreshMetadata(child.topic); err != nil { - return err - } - - var leader *Broker - var err error - if leader, err = child.consumer.client.Leader(child.topic, child.partition); err != nil { - return err - } - - child.broker = child.consumer.refBrokerConsumer(leader) - - child.broker.input <- child - - return nil -} - -func (child *partitionConsumer) chooseStartingOffset(offset int64) error { - newestOffset, err := child.consumer.client.GetOffset(child.topic, child.partition, OffsetNewest) - if err != nil { - return err - } - oldestOffset, err := child.consumer.client.GetOffset(child.topic, child.partition, OffsetOldest) - if err != nil { - return err - } - - switch { - case offset == OffsetNewest: - child.offset = newestOffset - case offset == OffsetOldest: - child.offset = oldestOffset - case offset >= oldestOffset && offset <= newestOffset: - child.offset = offset - default: - return ErrOffsetOutOfRange - } - - return nil -} - -func (child *partitionConsumer) Messages() <-chan *ConsumerMessage { - return child.messages -} - -func (child *partitionConsumer) Errors() <-chan *ConsumerError { - return child.errors -} - -func (child *partitionConsumer) AsyncClose() { - // this triggers whatever broker owns this child to abandon it and close its trigger channel, which causes - // the dispatcher to exit its loop, which removes it from the consumer then closes its 'messages' and - // 'errors' channel (alternatively, if the child is already at the dispatcher for some reason, that will - // also just close itself) - child.closeOnce.Do(func() { - close(child.dying) - }) -} - -func (child *partitionConsumer) Close() error { - child.AsyncClose() - - go withRecover(func() { - for range child.messages { - // drain - } - }) - - var errors ConsumerErrors - for err := range child.errors { - errors = append(errors, err) - } - - if len(errors) > 0 { - return errors - } - return nil -} - -func (child *partitionConsumer) HighWaterMarkOffset() int64 { - return atomic.LoadInt64(&child.highWaterMarkOffset) -} - -func (child *partitionConsumer) responseFeeder() { - var msgs []*ConsumerMessage - expiryTicker := time.NewTicker(child.conf.Consumer.MaxProcessingTime) - firstAttempt := true - -feederLoop: - for response := range child.feeder { - msgs, child.responseResult = child.parseResponse(response) - - for i, msg := range msgs { - messageSelect: - select { - case child.messages <- msg: - firstAttempt = true - case <-expiryTicker.C: - if !firstAttempt { - child.responseResult = errTimedOut - child.broker.acks.Done() - for _, msg = range msgs[i:] { - child.messages <- msg - } - child.broker.input <- child - continue feederLoop - } else { - // current message has not been sent, return to select - // statement - firstAttempt = false - goto messageSelect - } - } - } - - child.broker.acks.Done() - } - - expiryTicker.Stop() - close(child.messages) - close(child.errors) -} - -func (child *partitionConsumer) parseMessages(msgSet *MessageSet) ([]*ConsumerMessage, error) { - var messages []*ConsumerMessage - for _, msgBlock := range msgSet.Messages { - for _, msg := range msgBlock.Messages() { - offset := msg.Offset - if msg.Msg.Version >= 1 { - baseOffset := msgBlock.Offset - msgBlock.Messages()[len(msgBlock.Messages())-1].Offset - offset += baseOffset - } - if offset < child.offset { - continue - } - messages = append(messages, &ConsumerMessage{ - Topic: child.topic, - Partition: child.partition, - Key: msg.Msg.Key, - Value: msg.Msg.Value, - Offset: offset, - Timestamp: msg.Msg.Timestamp, - BlockTimestamp: msgBlock.Msg.Timestamp, - }) - child.offset = offset + 1 - } - } - if len(messages) == 0 { - return nil, ErrIncompleteResponse - } - return messages, nil -} - -func (child *partitionConsumer) parseRecords(batch *RecordBatch) ([]*ConsumerMessage, error) { - var messages []*ConsumerMessage - for _, rec := range batch.Records { - offset := batch.FirstOffset + rec.OffsetDelta - if offset < child.offset { - continue - } - messages = append(messages, &ConsumerMessage{ - Topic: child.topic, - Partition: child.partition, - Key: rec.Key, - Value: rec.Value, - Offset: offset, - Timestamp: batch.FirstTimestamp.Add(rec.TimestampDelta), - Headers: rec.Headers, - }) - child.offset = offset + 1 - } - if len(messages) == 0 { - child.offset += 1 - } - return messages, nil -} - -func (child *partitionConsumer) parseResponse(response *FetchResponse) ([]*ConsumerMessage, error) { - block := response.GetBlock(child.topic, child.partition) - if block == nil { - return nil, ErrIncompleteResponse - } - - if block.Err != ErrNoError { - return nil, block.Err - } - - nRecs, err := block.numRecords() - if err != nil { - return nil, err - } - if nRecs == 0 { - partialTrailingMessage, err := block.isPartial() - if err != nil { - return nil, err - } - // We got no messages. If we got a trailing one then we need to ask for more data. - // Otherwise we just poll again and wait for one to be produced... - if partialTrailingMessage { - if child.conf.Consumer.Fetch.Max > 0 && child.fetchSize == child.conf.Consumer.Fetch.Max { - // we can't ask for more data, we've hit the configured limit - child.sendError(ErrMessageTooLarge) - child.offset++ // skip this one so we can keep processing future messages - } else { - child.fetchSize *= 2 - if child.conf.Consumer.Fetch.Max > 0 && child.fetchSize > child.conf.Consumer.Fetch.Max { - child.fetchSize = child.conf.Consumer.Fetch.Max - } - } - } - - return nil, nil - } - - // we got messages, reset our fetch size in case it was increased for a previous request - child.fetchSize = child.conf.Consumer.Fetch.Default - atomic.StoreInt64(&child.highWaterMarkOffset, block.HighWaterMarkOffset) - - messages := []*ConsumerMessage{} - for _, records := range block.RecordsSet { - switch records.recordsType { - case legacyRecords: - messageSetMessages, err := child.parseMessages(records.MsgSet) - if err != nil { - return nil, err - } - - messages = append(messages, messageSetMessages...) - case defaultRecords: - recordBatchMessages, err := child.parseRecords(records.RecordBatch) - if err != nil { - return nil, err - } - if control, err := records.isControl(); err != nil || control { - continue - } - - messages = append(messages, recordBatchMessages...) - default: - return nil, fmt.Errorf("unknown records type: %v", records.recordsType) - } - } - - return messages, nil -} - -// brokerConsumer - -type brokerConsumer struct { - consumer *consumer - broker *Broker - input chan *partitionConsumer - newSubscriptions chan []*partitionConsumer - wait chan none - subscriptions map[*partitionConsumer]none - acks sync.WaitGroup - refs int -} - -func (c *consumer) newBrokerConsumer(broker *Broker) *brokerConsumer { - bc := &brokerConsumer{ - consumer: c, - broker: broker, - input: make(chan *partitionConsumer), - newSubscriptions: make(chan []*partitionConsumer), - wait: make(chan none), - subscriptions: make(map[*partitionConsumer]none), - refs: 0, - } - - go withRecover(bc.subscriptionManager) - go withRecover(bc.subscriptionConsumer) - - return bc -} - -func (bc *brokerConsumer) subscriptionManager() { - var buffer []*partitionConsumer - - // The subscriptionManager constantly accepts new subscriptions on `input` (even when the main subscriptionConsumer - // goroutine is in the middle of a network request) and batches it up. The main worker goroutine picks - // up a batch of new subscriptions between every network request by reading from `newSubscriptions`, so we give - // it nil if no new subscriptions are available. We also write to `wait` only when new subscriptions is available, - // so the main goroutine can block waiting for work if it has none. - for { - if len(buffer) > 0 { - select { - case event, ok := <-bc.input: - if !ok { - goto done - } - buffer = append(buffer, event) - case bc.newSubscriptions <- buffer: - buffer = nil - case bc.wait <- none{}: - } - } else { - select { - case event, ok := <-bc.input: - if !ok { - goto done - } - buffer = append(buffer, event) - case bc.newSubscriptions <- nil: - } - } - } - -done: - close(bc.wait) - if len(buffer) > 0 { - bc.newSubscriptions <- buffer - } - close(bc.newSubscriptions) -} - -func (bc *brokerConsumer) subscriptionConsumer() { - <-bc.wait // wait for our first piece of work - - // the subscriptionConsumer ensures we will get nil right away if no new subscriptions is available - for newSubscriptions := range bc.newSubscriptions { - bc.updateSubscriptions(newSubscriptions) - - if len(bc.subscriptions) == 0 { - // We're about to be shut down or we're about to receive more subscriptions. - // Either way, the signal just hasn't propagated to our goroutine yet. - <-bc.wait - continue - } - - response, err := bc.fetchNewMessages() - - if err != nil { - Logger.Printf("consumer/broker/%d disconnecting due to error processing FetchRequest: %s\n", bc.broker.ID(), err) - bc.abort(err) - return - } - - bc.acks.Add(len(bc.subscriptions)) - for child := range bc.subscriptions { - child.feeder <- response - } - bc.acks.Wait() - bc.handleResponses() - } -} - -func (bc *brokerConsumer) updateSubscriptions(newSubscriptions []*partitionConsumer) { - for _, child := range newSubscriptions { - bc.subscriptions[child] = none{} - Logger.Printf("consumer/broker/%d added subscription to %s/%d\n", bc.broker.ID(), child.topic, child.partition) - } - - for child := range bc.subscriptions { - select { - case <-child.dying: - Logger.Printf("consumer/broker/%d closed dead subscription to %s/%d\n", bc.broker.ID(), child.topic, child.partition) - close(child.trigger) - delete(bc.subscriptions, child) - default: - break - } - } -} - -func (bc *brokerConsumer) handleResponses() { - // handles the response codes left for us by our subscriptions, and abandons ones that have been closed - for child := range bc.subscriptions { - result := child.responseResult - child.responseResult = nil - - switch result { - case nil: - break - case errTimedOut: - Logger.Printf("consumer/broker/%d abandoned subscription to %s/%d because consuming was taking too long\n", - bc.broker.ID(), child.topic, child.partition) - delete(bc.subscriptions, child) - case ErrOffsetOutOfRange: - // there's no point in retrying this it will just fail the same way again - // shut it down and force the user to choose what to do - child.sendError(result) - Logger.Printf("consumer/%s/%d shutting down because %s\n", child.topic, child.partition, result) - close(child.trigger) - delete(bc.subscriptions, child) - case ErrUnknownTopicOrPartition, ErrNotLeaderForPartition, ErrLeaderNotAvailable, ErrReplicaNotAvailable: - // not an error, but does need redispatching - Logger.Printf("consumer/broker/%d abandoned subscription to %s/%d because %s\n", - bc.broker.ID(), child.topic, child.partition, result) - child.trigger <- none{} - delete(bc.subscriptions, child) - default: - // dunno, tell the user and try redispatching - child.sendError(result) - Logger.Printf("consumer/broker/%d abandoned subscription to %s/%d because %s\n", - bc.broker.ID(), child.topic, child.partition, result) - child.trigger <- none{} - delete(bc.subscriptions, child) - } - } -} - -func (bc *brokerConsumer) abort(err error) { - bc.consumer.abandonBrokerConsumer(bc) - _ = bc.broker.Close() // we don't care about the error this might return, we already have one - - for child := range bc.subscriptions { - child.sendError(err) - child.trigger <- none{} - } - - for newSubscriptions := range bc.newSubscriptions { - if len(newSubscriptions) == 0 { - <-bc.wait - continue - } - for _, child := range newSubscriptions { - child.sendError(err) - child.trigger <- none{} - } - } -} - -func (bc *brokerConsumer) fetchNewMessages() (*FetchResponse, error) { - request := &FetchRequest{ - MinBytes: bc.consumer.conf.Consumer.Fetch.Min, - MaxWaitTime: int32(bc.consumer.conf.Consumer.MaxWaitTime / time.Millisecond), - } - if bc.consumer.conf.Version.IsAtLeast(V0_10_0_0) { - request.Version = 2 - } - if bc.consumer.conf.Version.IsAtLeast(V0_10_1_0) { - request.Version = 3 - request.MaxBytes = MaxResponseSize - } - if bc.consumer.conf.Version.IsAtLeast(V0_11_0_0) { - request.Version = 4 - request.Isolation = ReadUncommitted // We don't support yet transactions. - } - - for child := range bc.subscriptions { - request.AddBlock(child.topic, child.partition, child.offset, child.fetchSize) - } - - return bc.broker.Fetch(request) -} diff --git a/third/github.com/Shopify/sarama/consumer_group.go b/third/github.com/Shopify/sarama/consumer_group.go deleted file mode 100644 index bb6a2c2b9..000000000 --- a/third/github.com/Shopify/sarama/consumer_group.go +++ /dev/null @@ -1,774 +0,0 @@ -package sarama - -import ( - "context" - "errors" - "fmt" - "sort" - "sync" - "time" -) - -// ErrClosedConsumerGroup is the error returned when a method is called on a consumer group that has been closed. -var ErrClosedConsumerGroup = errors.New("kafka: tried to use a consumer group that was closed") - -// ConsumerGroup is responsible for dividing up processing of topics and partitions -// over a collection of processes (the members of the consumer group). -type ConsumerGroup interface { - // Consume joins a cluster of consumers for a given list of topics and - // starts a blocking ConsumerGroupSession through the ConsumerGroupHandler. - // - // The life-cycle of a session is represented by the following steps: - // - // 1. The consumers join the group (as explained in https://kafka.apache.org/documentation/#intro_consumers) - // and is assigned their "fair share" of partitions, aka 'claims'. - // 2. Before processing starts, the handler's Setup() hook is called to notify the user - // of the claims and allow any necessary preparation or alteration of state. - // 3. For each of the assigned claims the handler's ConsumeClaim() function is then called - // in a separate goroutine which requires it to be thread-safe. Any state must be carefully protected - // from concurrent reads/writes. - // 4. The session will persist until one of the ConsumeClaim() functions exits. This can be either when the - // parent context is cancelled or when a server-side rebalance cycle is initiated. - // 5. Once all the ConsumeClaim() loops have exited, the handler's Cleanup() hook is called - // to allow the user to perform any final tasks before a rebalance. - // 6. Finally, marked offsets are committed one last time before claims are released. - // - // Please note, that once a rebalance is triggered, sessions must be completed within - // Config.Consumer.Group.Rebalance.Timeout. This means that ConsumeClaim() functions must exit - // as quickly as possible to allow time for Cleanup() and the final offset commit. If the timeout - // is exceeded, the consumer will be removed from the group by Kafka, which will cause offset - // commit failures. - Consume(ctx context.Context, topics []string, handler ConsumerGroupHandler) error - - // Errors returns a read channel of errors that occurred during the consumer life-cycle. - // By default, errors are logged and not returned over this channel. - // If you want to implement any custom error handling, set your config's - // Consumer.Return.Errors setting to true, and read from this channel. - Errors() <-chan error - - // Close stops the ConsumerGroup and detaches any running sessions. It is required to call - // this function before the object passes out of scope, as it will otherwise leak memory. - Close() error -} - -type consumerGroup struct { - client Client - ownClient bool - - config *Config - consumer Consumer - groupID string - memberID string - errors chan error - - lock sync.Mutex - closed chan none - closeOnce sync.Once -} - -// NewConsumerGroup creates a new consumer group the given broker addresses and configuration. -func NewConsumerGroup(addrs []string, groupID string, config *Config) (ConsumerGroup, error) { - client, err := NewClient(addrs, config) - if err != nil { - return nil, err - } - - c, err := NewConsumerGroupFromClient(groupID, client) - if err != nil { - _ = client.Close() - return nil, err - } - - c.(*consumerGroup).ownClient = true - return c, nil -} - -// NewConsumerGroupFromClient creates a new consumer group using the given client. It is still -// necessary to call Close() on the underlying client when shutting down this consumer. -// PLEASE NOTE: consumer groups can only re-use but not share clients. -func NewConsumerGroupFromClient(groupID string, client Client) (ConsumerGroup, error) { - config := client.Config() - if !config.Version.IsAtLeast(V0_10_2_0) { - return nil, ConfigurationError("consumer groups require Version to be >= V0_10_2_0") - } - - consumer, err := NewConsumerFromClient(client) - if err != nil { - return nil, err - } - - return &consumerGroup{ - client: client, - consumer: consumer, - config: config, - groupID: groupID, - errors: make(chan error, config.ChannelBufferSize), - closed: make(chan none), - }, nil -} - -// Errors implements ConsumerGroup. -func (c *consumerGroup) Errors() <-chan error { return c.errors } - -// Close implements ConsumerGroup. -func (c *consumerGroup) Close() (err error) { - c.closeOnce.Do(func() { - close(c.closed) - - c.lock.Lock() - defer c.lock.Unlock() - - // leave group - if e := c.leave(); e != nil { - err = e - } - - // drain errors - go func() { - close(c.errors) - }() - for e := range c.errors { - err = e - } - - if c.ownClient { - if e := c.client.Close(); e != nil { - err = e - } - } - }) - return -} - -// Consume implements ConsumerGroup. -func (c *consumerGroup) Consume(ctx context.Context, topics []string, handler ConsumerGroupHandler) error { - // Ensure group is not closed - select { - case <-c.closed: - return ErrClosedConsumerGroup - default: - } - - c.lock.Lock() - defer c.lock.Unlock() - - // Quick exit when no topics are provided - if len(topics) == 0 { - return fmt.Errorf("no topics provided") - } - - // Refresh metadata for requested topics - if err := c.client.RefreshMetadata(topics...); err != nil { - return err - } - - // Get coordinator - coordinator, err := c.client.Coordinator(c.groupID) - if err != nil { - return err - } - - // Init session - sess, err := c.newSession(ctx, coordinator, topics, handler, c.config.Consumer.Group.Rebalance.Retry.Max) - if err == ErrClosedClient { - return ErrClosedConsumerGroup - } else if err != nil { - return err - } - - // Wait for session exit signal - <-sess.ctx.Done() - - // Gracefully release session claims - return sess.release(true) -} - -func (c *consumerGroup) newSession(ctx context.Context, coordinator *Broker, topics []string, handler ConsumerGroupHandler, retries int) (*consumerGroupSession, error) { - // Join consumer group - join, err := c.joinGroupRequest(coordinator, topics) - if err != nil { - _ = coordinator.Close() - return nil, err - } - switch join.Err { - case ErrNoError: - c.memberID = join.MemberId - case ErrUnknownMemberId, ErrIllegalGeneration: // reset member ID and retry immediately - c.memberID = "" - return c.newSession(ctx, coordinator, topics, handler, retries) - case ErrRebalanceInProgress: // retry after backoff - if retries <= 0 { - return nil, join.Err - } - - select { - case <-c.closed: - return nil, ErrClosedConsumerGroup - case <-time.After(c.config.Consumer.Group.Rebalance.Retry.Backoff): - } - - return c.newSession(ctx, coordinator, topics, handler, retries-1) - default: - return nil, join.Err - } - - // Prepare distribution plan if we joined as the leader - var plan BalanceStrategyPlan - if join.LeaderId == join.MemberId { - members, err := join.GetMembers() - if err != nil { - return nil, err - } - - plan, err = c.balance(members) - if err != nil { - return nil, err - } - } - - // Sync consumer group - sync, err := c.syncGroupRequest(coordinator, plan, join.GenerationId) - if err != nil { - _ = coordinator.Close() - return nil, err - } - switch sync.Err { - case ErrNoError: - case ErrUnknownMemberId, ErrIllegalGeneration: // reset member ID and retry immediately - c.memberID = "" - return c.newSession(ctx, coordinator, topics, handler, retries) - case ErrRebalanceInProgress: // retry after backoff - if retries <= 0 { - return nil, sync.Err - } - - select { - case <-c.closed: - return nil, ErrClosedConsumerGroup - case <-time.After(c.config.Consumer.Group.Rebalance.Retry.Backoff): - } - - return c.newSession(ctx, coordinator, topics, handler, retries-1) - default: - return nil, sync.Err - } - - // Retrieve and sort claims - var claims map[string][]int32 - if len(sync.MemberAssignment) > 0 { - members, err := sync.GetMemberAssignment() - if err != nil { - return nil, err - } - claims = members.Topics - - for _, partitions := range claims { - sort.Sort(int32Slice(partitions)) - } - } - - return newConsumerGroupSession(ctx, c, claims, join.MemberId, join.GenerationId, handler) -} - -func (c *consumerGroup) joinGroupRequest(coordinator *Broker, topics []string) (*JoinGroupResponse, error) { - req := &JoinGroupRequest{ - GroupId: c.groupID, - MemberId: c.memberID, - SessionTimeout: int32(c.config.Consumer.Group.Session.Timeout / time.Millisecond), - ProtocolType: "consumer", - } - if c.config.Version.IsAtLeast(V0_10_1_0) { - req.Version = 1 - req.RebalanceTimeout = int32(c.config.Consumer.Group.Rebalance.Timeout / time.Millisecond) - } - - meta := &ConsumerGroupMemberMetadata{ - Topics: topics, - UserData: c.config.Consumer.Group.Member.UserData, - } - strategy := c.config.Consumer.Group.Rebalance.Strategy - if err := req.AddGroupProtocolMetadata(strategy.Name(), meta); err != nil { - return nil, err - } - - return coordinator.JoinGroup(req) -} - -func (c *consumerGroup) syncGroupRequest(coordinator *Broker, plan BalanceStrategyPlan, generationID int32) (*SyncGroupResponse, error) { - req := &SyncGroupRequest{ - GroupId: c.groupID, - MemberId: c.memberID, - GenerationId: generationID, - } - for memberID, topics := range plan { - err := req.AddGroupAssignmentMember(memberID, &ConsumerGroupMemberAssignment{ - Topics: topics, - }) - if err != nil { - return nil, err - } - } - return coordinator.SyncGroup(req) -} - -func (c *consumerGroup) heartbeatRequest(coordinator *Broker, memberID string, generationID int32) (*HeartbeatResponse, error) { - req := &HeartbeatRequest{ - GroupId: c.groupID, - MemberId: memberID, - GenerationId: generationID, - } - - return coordinator.Heartbeat(req) -} - -func (c *consumerGroup) balance(members map[string]ConsumerGroupMemberMetadata) (BalanceStrategyPlan, error) { - topics := make(map[string][]int32) - for _, meta := range members { - for _, topic := range meta.Topics { - topics[topic] = nil - } - } - - for topic := range topics { - partitions, err := c.client.Partitions(topic) - if err != nil { - return nil, err - } - topics[topic] = partitions - } - - strategy := c.config.Consumer.Group.Rebalance.Strategy - return strategy.Plan(members, topics) -} - -// Leaves the cluster, called by Close, protected by lock. -func (c *consumerGroup) leave() error { - if c.memberID == "" { - return nil - } - - coordinator, err := c.client.Coordinator(c.groupID) - if err != nil { - return err - } - - resp, err := coordinator.LeaveGroup(&LeaveGroupRequest{ - GroupId: c.groupID, - MemberId: c.memberID, - }) - if err != nil { - _ = coordinator.Close() - return err - } - - // Unset memberID - c.memberID = "" - - // Check response - switch resp.Err { - case ErrRebalanceInProgress, ErrUnknownMemberId, ErrNoError: - return nil - default: - return resp.Err - } -} - -func (c *consumerGroup) handleError(err error, topic string, partition int32) { - select { - case <-c.closed: - return - default: - } - - if _, ok := err.(*ConsumerError); !ok && topic != "" && partition > -1 { - err = &ConsumerError{ - Topic: topic, - Partition: partition, - Err: err, - } - } - - if c.config.Consumer.Return.Errors { - select { - case c.errors <- err: - default: - } - } else { - Logger.Println(err) - } -} - -// -------------------------------------------------------------------- - -// ConsumerGroupSession represents a consumer group member session. -type ConsumerGroupSession interface { - // Claims returns information about the claimed partitions by topic. - Claims() map[string][]int32 - - // MemberID returns the cluster member ID. - MemberID() string - - // GenerationID returns the current generation ID. - GenerationID() int32 - - // MarkOffset marks the provided offset, alongside a metadata string - // that represents the state of the partition consumer at that point in time. The - // metadata string can be used by another consumer to restore that state, so it - // can resume consumption. - // - // To follow upstream conventions, you are expected to mark the offset of the - // next message to read, not the last message read. Thus, when calling `MarkOffset` - // you should typically add one to the offset of the last consumed message. - // - // Note: calling MarkOffset does not necessarily commit the offset to the backend - // store immediately for efficiency reasons, and it may never be committed if - // your application crashes. This means that you may end up processing the same - // message twice, and your processing should ideally be idempotent. - MarkOffset(topic string, partition int32, offset int64, metadata string) - - // ResetOffset resets to the provided offset, alongside a metadata string that - // represents the state of the partition consumer at that point in time. Reset - // acts as a counterpart to MarkOffset, the difference being that it allows to - // reset an offset to an earlier or smaller value, where MarkOffset only - // allows incrementing the offset. cf MarkOffset for more details. - ResetOffset(topic string, partition int32, offset int64, metadata string) - - // MarkMessage marks a message as consumed. - MarkMessage(msg *ConsumerMessage, metadata string) - - // Context returns the session context. - Context() context.Context -} - -type consumerGroupSession struct { - parent *consumerGroup - memberID string - generationID int32 - handler ConsumerGroupHandler - - claims map[string][]int32 - offsets *offsetManager - ctx context.Context - cancel func() - - waitGroup sync.WaitGroup - releaseOnce sync.Once - hbDying, hbDead chan none -} - -func newConsumerGroupSession(ctx context.Context, parent *consumerGroup, claims map[string][]int32, memberID string, generationID int32, handler ConsumerGroupHandler) (*consumerGroupSession, error) { - // init offset manager - offsets, err := newOffsetManagerFromClient(parent.groupID, memberID, generationID, parent.client) - if err != nil { - return nil, err - } - - // init context - ctx, cancel := context.WithCancel(ctx) - - // init session - sess := &consumerGroupSession{ - parent: parent, - memberID: memberID, - generationID: generationID, - handler: handler, - offsets: offsets, - claims: claims, - ctx: ctx, - cancel: cancel, - hbDying: make(chan none), - hbDead: make(chan none), - } - - // start heartbeat loop - go sess.heartbeatLoop() - - // create a POM for each claim - for topic, partitions := range claims { - for _, partition := range partitions { - pom, err := offsets.ManagePartition(topic, partition) - if err != nil { - _ = sess.release(false) - return nil, err - } - - // handle POM errors - go func(topic string, partition int32) { - for err := range pom.Errors() { - sess.parent.handleError(err, topic, partition) - } - }(topic, partition) - } - } - - // perform setup - if err := handler.Setup(sess); err != nil { - _ = sess.release(true) - return nil, err - } - - // start consuming - for topic, partitions := range claims { - for _, partition := range partitions { - sess.waitGroup.Add(1) - - go func(topic string, partition int32) { - defer sess.waitGroup.Done() - - // cancel the as session as soon as the first - // goroutine exits - defer sess.cancel() - - // consume a single topic/partition, blocking - sess.consume(topic, partition) - }(topic, partition) - } - } - return sess, nil -} - -func (s *consumerGroupSession) Claims() map[string][]int32 { return s.claims } -func (s *consumerGroupSession) MemberID() string { return s.memberID } -func (s *consumerGroupSession) GenerationID() int32 { return s.generationID } - -func (s *consumerGroupSession) MarkOffset(topic string, partition int32, offset int64, metadata string) { - if pom := s.offsets.findPOM(topic, partition); pom != nil { - pom.MarkOffset(offset, metadata) - } -} - -func (s *consumerGroupSession) ResetOffset(topic string, partition int32, offset int64, metadata string) { - if pom := s.offsets.findPOM(topic, partition); pom != nil { - pom.ResetOffset(offset, metadata) - } -} - -func (s *consumerGroupSession) MarkMessage(msg *ConsumerMessage, metadata string) { - s.MarkOffset(msg.Topic, msg.Partition, msg.Offset+1, metadata) -} - -func (s *consumerGroupSession) Context() context.Context { - return s.ctx -} - -func (s *consumerGroupSession) consume(topic string, partition int32) { - // quick exit if rebalance is due - select { - case <-s.ctx.Done(): - return - case <-s.parent.closed: - return - default: - } - - // get next offset - offset := s.parent.config.Consumer.Offsets.Initial - if pom := s.offsets.findPOM(topic, partition); pom != nil { - offset, _ = pom.NextOffset() - } - - // create new claim - claim, err := newConsumerGroupClaim(s, topic, partition, offset) - if err != nil { - s.parent.handleError(err, topic, partition) - return - } - - // handle errors - go func() { - for err := range claim.Errors() { - s.parent.handleError(err, topic, partition) - } - }() - - // trigger close when session is done - go func() { - select { - case <-s.ctx.Done(): - case <-s.parent.closed: - } - claim.AsyncClose() - }() - - // start processing - if err := s.handler.ConsumeClaim(s, claim); err != nil { - s.parent.handleError(err, topic, partition) - } - - // ensure consumer is closed & drained - claim.AsyncClose() - for _, err := range claim.waitClosed() { - s.parent.handleError(err, topic, partition) - } -} - -func (s *consumerGroupSession) release(withCleanup bool) (err error) { - // signal release, stop heartbeat - s.cancel() - - // wait for consumers to exit - s.waitGroup.Wait() - - // perform release - s.releaseOnce.Do(func() { - if withCleanup { - if e := s.handler.Cleanup(s); e != nil { - s.parent.handleError(err, "", -1) - err = e - } - } - - if e := s.offsets.Close(); e != nil { - err = e - } - - close(s.hbDying) - <-s.hbDead - }) - - return -} - -func (s *consumerGroupSession) heartbeatLoop() { - defer close(s.hbDead) - defer s.cancel() // trigger the end of the session on exit - - pause := time.NewTicker(s.parent.config.Consumer.Group.Heartbeat.Interval) - defer pause.Stop() - - retries := s.parent.config.Metadata.Retry.Max - for { - coordinator, err := s.parent.client.Coordinator(s.parent.groupID) - if err != nil { - if retries <= 0 { - s.parent.handleError(err, "", -1) - return - } - - select { - case <-s.hbDying: - return - case <-time.After(s.parent.config.Metadata.Retry.Backoff): - retries-- - } - continue - } - - resp, err := s.parent.heartbeatRequest(coordinator, s.memberID, s.generationID) - if err != nil { - _ = coordinator.Close() - retries-- - continue - } - - switch resp.Err { - case ErrNoError: - retries = s.parent.config.Metadata.Retry.Max - case ErrRebalanceInProgress, ErrUnknownMemberId, ErrIllegalGeneration: - return - default: - s.parent.handleError(err, "", -1) - return - } - - select { - case <-pause.C: - case <-s.hbDying: - return - } - } -} - -// -------------------------------------------------------------------- - -// ConsumerGroupHandler instances are used to handle individual topic/partition claims. -// It also provides hooks for your consumer group session life-cycle and allow you to -// trigger logic before or after the consume loop(s). -// -// PLEASE NOTE that handlers are likely be called from several goroutines concurrently, -// ensure that all state is safely protected against race conditions. -type ConsumerGroupHandler interface { - // Setup is run at the beginning of a new session, before ConsumeClaim. - Setup(ConsumerGroupSession) error - - // Cleanup is run at the end of a session, once all ConsumeClaim goroutines have exited - // but before the offsets are committed for the very last time. - Cleanup(ConsumerGroupSession) error - - // ConsumeClaim must start a consumer loop of ConsumerGroupClaim's Messages(). - // Once the Messages() channel is closed, the Handler must finish its processing - // loop and exit. - ConsumeClaim(ConsumerGroupSession, ConsumerGroupClaim) error -} - -// ConsumerGroupClaim processes Kafka messages from a given topic and partition within a consumer group. -type ConsumerGroupClaim interface { - // Topic returns the consumed topic name. - Topic() string - - // Partition returns the consumed partition. - Partition() int32 - - // InitialOffset returns the initial offset that was used as a starting point for this claim. - InitialOffset() int64 - - // HighWaterMarkOffset returns the high water mark offset of the partition, - // i.e. the offset that will be used for the next message that will be produced. - // You can use this to determine how far behind the processing is. - HighWaterMarkOffset() int64 - - // Messages returns the read channel for the messages that are returned by - // the broker. The messages channel will be closed when a new rebalance cycle - // is due. You must finish processing and mark offsets within - // Config.Consumer.Group.Session.Timeout before the topic/partition is eventually - // re-assigned to another group member. - Messages() <-chan *ConsumerMessage -} - -type consumerGroupClaim struct { - topic string - partition int32 - offset int64 - PartitionConsumer -} - -func newConsumerGroupClaim(sess *consumerGroupSession, topic string, partition int32, offset int64) (*consumerGroupClaim, error) { - pcm, err := sess.parent.consumer.ConsumePartition(topic, partition, offset) - if err == ErrOffsetOutOfRange { - offset = sess.parent.config.Consumer.Offsets.Initial - pcm, err = sess.parent.consumer.ConsumePartition(topic, partition, offset) - } - if err != nil { - return nil, err - } - - go func() { - for err := range pcm.Errors() { - sess.parent.handleError(err, topic, partition) - } - }() - - return &consumerGroupClaim{ - topic: topic, - partition: partition, - offset: offset, - PartitionConsumer: pcm, - }, nil -} - -func (c *consumerGroupClaim) Topic() string { return c.topic } -func (c *consumerGroupClaim) Partition() int32 { return c.partition } -func (c *consumerGroupClaim) InitialOffset() int64 { return c.offset } - -// Drains messages and errors, ensures the claim is fully closed. -func (c *consumerGroupClaim) waitClosed() (errs ConsumerErrors) { - go func() { - for range c.Messages() { - } - }() - - for err := range c.Errors() { - errs = append(errs, err) - } - return -} diff --git a/third/github.com/Shopify/sarama/consumer_group_members.go b/third/github.com/Shopify/sarama/consumer_group_members.go deleted file mode 100644 index 2d02cc386..000000000 --- a/third/github.com/Shopify/sarama/consumer_group_members.go +++ /dev/null @@ -1,96 +0,0 @@ -package sarama - -//ConsumerGroupMemberMetadata holds the metadata for consumer group -type ConsumerGroupMemberMetadata struct { - Version int16 - Topics []string - UserData []byte -} - -func (m *ConsumerGroupMemberMetadata) encode(pe packetEncoder) error { - pe.putInt16(m.Version) - - if err := pe.putStringArray(m.Topics); err != nil { - return err - } - - if err := pe.putBytes(m.UserData); err != nil { - return err - } - - return nil -} - -func (m *ConsumerGroupMemberMetadata) decode(pd packetDecoder) (err error) { - if m.Version, err = pd.getInt16(); err != nil { - return - } - - if m.Topics, err = pd.getStringArray(); err != nil { - return - } - - if m.UserData, err = pd.getBytes(); err != nil { - return - } - - return nil -} - -//ConsumerGroupMemberAssignment holds the member assignment for a consume group -type ConsumerGroupMemberAssignment struct { - Version int16 - Topics map[string][]int32 - UserData []byte -} - -func (m *ConsumerGroupMemberAssignment) encode(pe packetEncoder) error { - pe.putInt16(m.Version) - - if err := pe.putArrayLength(len(m.Topics)); err != nil { - return err - } - - for topic, partitions := range m.Topics { - if err := pe.putString(topic); err != nil { - return err - } - if err := pe.putInt32Array(partitions); err != nil { - return err - } - } - - if err := pe.putBytes(m.UserData); err != nil { - return err - } - - return nil -} - -func (m *ConsumerGroupMemberAssignment) decode(pd packetDecoder) (err error) { - if m.Version, err = pd.getInt16(); err != nil { - return - } - - var topicLen int - if topicLen, err = pd.getArrayLength(); err != nil { - return - } - - m.Topics = make(map[string][]int32, topicLen) - for i := 0; i < topicLen; i++ { - var topic string - if topic, err = pd.getString(); err != nil { - return - } - if m.Topics[topic], err = pd.getInt32Array(); err != nil { - return - } - } - - if m.UserData, err = pd.getBytes(); err != nil { - return - } - - return nil -} diff --git a/third/github.com/Shopify/sarama/consumer_group_members_test.go b/third/github.com/Shopify/sarama/consumer_group_members_test.go deleted file mode 100644 index d65e8adc4..000000000 --- a/third/github.com/Shopify/sarama/consumer_group_members_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package sarama - -import ( - "bytes" - "reflect" - "testing" -) - -var ( - groupMemberMetadata = []byte{ - 0, 1, // Version - 0, 0, 0, 2, // Topic array length - 0, 3, 'o', 'n', 'e', // Topic one - 0, 3, 't', 'w', 'o', // Topic two - 0, 0, 0, 3, 0x01, 0x02, 0x03, // Userdata - } - groupMemberAssignment = []byte{ - 0, 1, // Version - 0, 0, 0, 1, // Topic array length - 0, 3, 'o', 'n', 'e', // Topic one - 0, 0, 0, 3, // Topic one, partition array length - 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, // 0, 2, 4 - 0, 0, 0, 3, 0x01, 0x02, 0x03, // Userdata - } -) - -func TestConsumerGroupMemberMetadata(t *testing.T) { - meta := &ConsumerGroupMemberMetadata{ - Version: 1, - Topics: []string{"one", "two"}, - UserData: []byte{0x01, 0x02, 0x03}, - } - - buf, err := encode(meta, nil) - if err != nil { - t.Error("Failed to encode data", err) - } else if !bytes.Equal(groupMemberMetadata, buf) { - t.Errorf("Encoded data does not match expectation\nexpected: %v\nactual: %v", groupMemberMetadata, buf) - } - - meta2 := new(ConsumerGroupMemberMetadata) - err = decode(buf, meta2) - if err != nil { - t.Error("Failed to decode data", err) - } else if !reflect.DeepEqual(meta, meta2) { - t.Errorf("Encoded data does not match expectation\nexpected: %v\nactual: %v", meta, meta2) - } -} - -func TestConsumerGroupMemberAssignment(t *testing.T) { - amt := &ConsumerGroupMemberAssignment{ - Version: 1, - Topics: map[string][]int32{ - "one": {0, 2, 4}, - }, - UserData: []byte{0x01, 0x02, 0x03}, - } - - buf, err := encode(amt, nil) - if err != nil { - t.Error("Failed to encode data", err) - } else if !bytes.Equal(groupMemberAssignment, buf) { - t.Errorf("Encoded data does not match expectation\nexpected: %v\nactual: %v", groupMemberAssignment, buf) - } - - amt2 := new(ConsumerGroupMemberAssignment) - err = decode(buf, amt2) - if err != nil { - t.Error("Failed to decode data", err) - } else if !reflect.DeepEqual(amt, amt2) { - t.Errorf("Encoded data does not match expectation\nexpected: %v\nactual: %v", amt, amt2) - } -} diff --git a/third/github.com/Shopify/sarama/consumer_group_test.go b/third/github.com/Shopify/sarama/consumer_group_test.go deleted file mode 100644 index 8bf44e661..000000000 --- a/third/github.com/Shopify/sarama/consumer_group_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package sarama - -import ( - "context" - "fmt" -) - -type exampleConsumerGroupHandler struct{} - -func (exampleConsumerGroupHandler) Setup(_ ConsumerGroupSession) error { return nil } -func (exampleConsumerGroupHandler) Cleanup(_ ConsumerGroupSession) error { return nil } -func (h exampleConsumerGroupHandler) ConsumeClaim(sess ConsumerGroupSession, claim ConsumerGroupClaim) error { - for msg := range claim.Messages() { - fmt.Printf("Message topic:%q partition:%d offset:%d\n", msg.Topic, msg.Partition, msg.Offset) - sess.MarkMessage(msg, "") - } - return nil -} - -func ExampleConsumerGroup() { - // Init config, specify appropriate version - config := NewConfig() - config.Version = V1_0_0_0 - config.Consumer.Return.Errors = true - - // Start with a client - client, err := NewClient([]string{"localhost:9092"}, config) - if err != nil { - panic(err) - } - defer func() { _ = client.Close() }() - - // Start a new consumer group - group, err := NewConsumerGroupFromClient("my-group", client) - if err != nil { - panic(err) - } - defer func() { _ = group.Close() }() - - // Track errors - go func() { - for err := range group.Errors() { - fmt.Println("ERROR", err) - } - }() - - // Iterate over consumer sessions. - ctx := context.Background() - for { - topics := []string{"my-topic"} - handler := exampleConsumerGroupHandler{} - - err := group.Consume(ctx, topics, handler) - if err != nil { - panic(err) - } - } -} diff --git a/third/github.com/Shopify/sarama/consumer_metadata_request.go b/third/github.com/Shopify/sarama/consumer_metadata_request.go deleted file mode 100644 index a8dcaefe8..000000000 --- a/third/github.com/Shopify/sarama/consumer_metadata_request.go +++ /dev/null @@ -1,34 +0,0 @@ -package sarama - -//ConsumerMetadataRequest is used for metadata requests -type ConsumerMetadataRequest struct { - ConsumerGroup string -} - -func (r *ConsumerMetadataRequest) encode(pe packetEncoder) error { - tmp := new(FindCoordinatorRequest) - tmp.CoordinatorKey = r.ConsumerGroup - tmp.CoordinatorType = CoordinatorGroup - return tmp.encode(pe) -} - -func (r *ConsumerMetadataRequest) decode(pd packetDecoder, version int16) (err error) { - tmp := new(FindCoordinatorRequest) - if err := tmp.decode(pd, version); err != nil { - return err - } - r.ConsumerGroup = tmp.CoordinatorKey - return nil -} - -func (r *ConsumerMetadataRequest) key() int16 { - return 10 -} - -func (r *ConsumerMetadataRequest) version() int16 { - return 0 -} - -func (r *ConsumerMetadataRequest) requiredVersion() KafkaVersion { - return V0_8_2_0 -} diff --git a/third/github.com/Shopify/sarama/consumer_metadata_request_test.go b/third/github.com/Shopify/sarama/consumer_metadata_request_test.go deleted file mode 100644 index 24e5f0a43..000000000 --- a/third/github.com/Shopify/sarama/consumer_metadata_request_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package sarama - -import ( - "testing" -) - -var ( - consumerMetadataRequestEmpty = []byte{ - 0x00, 0x00} - - consumerMetadataRequestString = []byte{ - 0x00, 0x06, 'f', 'o', 'o', 'b', 'a', 'r'} -) - -func TestConsumerMetadataRequest(t *testing.T) { - request := new(ConsumerMetadataRequest) - testEncodable(t, "empty string", request, consumerMetadataRequestEmpty) - testVersionDecodable(t, "empty string", request, consumerMetadataRequestEmpty, 0) - - request.ConsumerGroup = "foobar" - testEncodable(t, "with string", request, consumerMetadataRequestString) - testVersionDecodable(t, "with string", request, consumerMetadataRequestString, 0) -} diff --git a/third/github.com/Shopify/sarama/consumer_metadata_response.go b/third/github.com/Shopify/sarama/consumer_metadata_response.go deleted file mode 100644 index 4d86e9303..000000000 --- a/third/github.com/Shopify/sarama/consumer_metadata_response.go +++ /dev/null @@ -1,78 +0,0 @@ -package sarama - -import ( - "net" - "strconv" -) - -//ConsumerMetadataResponse holds the reponse for a consumer gorup meta data request -type ConsumerMetadataResponse struct { - Err KError - Coordinator *Broker - CoordinatorID int32 // deprecated: use Coordinator.ID() - CoordinatorHost string // deprecated: use Coordinator.Addr() - CoordinatorPort int32 // deprecated: use Coordinator.Addr() -} - -func (r *ConsumerMetadataResponse) decode(pd packetDecoder, version int16) (err error) { - tmp := new(FindCoordinatorResponse) - - if err := tmp.decode(pd, version); err != nil { - return err - } - - r.Err = tmp.Err - - r.Coordinator = tmp.Coordinator - if tmp.Coordinator == nil { - return nil - } - - // this can all go away in 2.0, but we have to fill in deprecated fields to maintain - // backwards compatibility - host, portstr, err := net.SplitHostPort(r.Coordinator.Addr()) - if err != nil { - return err - } - port, err := strconv.ParseInt(portstr, 10, 32) - if err != nil { - return err - } - r.CoordinatorID = r.Coordinator.ID() - r.CoordinatorHost = host - r.CoordinatorPort = int32(port) - - return nil -} - -func (r *ConsumerMetadataResponse) encode(pe packetEncoder) error { - if r.Coordinator == nil { - r.Coordinator = new(Broker) - r.Coordinator.id = r.CoordinatorID - r.Coordinator.addr = net.JoinHostPort(r.CoordinatorHost, strconv.Itoa(int(r.CoordinatorPort))) - } - - tmp := &FindCoordinatorResponse{ - Version: 0, - Err: r.Err, - Coordinator: r.Coordinator, - } - - if err := tmp.encode(pe); err != nil { - return err - } - - return nil -} - -func (r *ConsumerMetadataResponse) key() int16 { - return 10 -} - -func (r *ConsumerMetadataResponse) version() int16 { - return 0 -} - -func (r *ConsumerMetadataResponse) requiredVersion() KafkaVersion { - return V0_8_2_0 -} diff --git a/third/github.com/Shopify/sarama/consumer_metadata_response_test.go b/third/github.com/Shopify/sarama/consumer_metadata_response_test.go deleted file mode 100644 index 8482f6ff1..000000000 --- a/third/github.com/Shopify/sarama/consumer_metadata_response_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package sarama - -import "testing" - -var ( - consumerMetadataResponseError = []byte{ - 0x00, 0x0E, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00} - - consumerMetadataResponseSuccess = []byte{ - 0x00, 0x00, - 0x00, 0x00, 0x00, 0xAB, - 0x00, 0x03, 'f', 'o', 'o', - 0x00, 0x00, 0xCC, 0xDD} -) - -func TestConsumerMetadataResponseError(t *testing.T) { - response := &ConsumerMetadataResponse{Err: ErrOffsetsLoadInProgress} - testEncodable(t, "", response, consumerMetadataResponseError) - - decodedResp := &ConsumerMetadataResponse{} - if err := versionedDecode(consumerMetadataResponseError, decodedResp, 0); err != nil { - t.Error("could not decode: ", err) - } - - if decodedResp.Err != ErrOffsetsLoadInProgress { - t.Errorf("got %s, want %s", decodedResp.Err, ErrOffsetsLoadInProgress) - } -} - -func TestConsumerMetadataResponseSuccess(t *testing.T) { - broker := NewBroker("foo:52445") - broker.id = 0xAB - response := ConsumerMetadataResponse{ - Coordinator: broker, - CoordinatorID: 0xAB, - CoordinatorHost: "foo", - CoordinatorPort: 0xCCDD, - Err: ErrNoError, - } - testResponse(t, "success", &response, consumerMetadataResponseSuccess) -} diff --git a/third/github.com/Shopify/sarama/consumer_test.go b/third/github.com/Shopify/sarama/consumer_test.go deleted file mode 100644 index 4bd662908..000000000 --- a/third/github.com/Shopify/sarama/consumer_test.go +++ /dev/null @@ -1,1036 +0,0 @@ -package sarama - -import ( - "log" - "os" - "os/signal" - "sync" - "testing" - "time" -) - -var testMsg = StringEncoder("Foo") - -// If a particular offset is provided then messages are consumed starting from -// that offset. -func TestConsumerOffsetManual(t *testing.T) { - // Given - broker0 := NewMockBroker(t, 0) - - mockFetchResponse := NewMockFetchResponse(t, 1) - for i := 0; i < 10; i++ { - mockFetchResponse.SetMessage("my_topic", 0, int64(i+1234), testMsg) - } - - broker0.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetBroker(broker0.Addr(), broker0.BrokerID()). - SetLeader("my_topic", 0, broker0.BrokerID()), - "OffsetRequest": NewMockOffsetResponse(t). - SetOffset("my_topic", 0, OffsetOldest, 0). - SetOffset("my_topic", 0, OffsetNewest, 2345), - "FetchRequest": mockFetchResponse, - }) - - // When - master, err := NewConsumer([]string{broker0.Addr()}, nil) - if err != nil { - t.Fatal(err) - } - - consumer, err := master.ConsumePartition("my_topic", 0, 1234) - if err != nil { - t.Fatal(err) - } - - // Then: messages starting from offset 1234 are consumed. - for i := 0; i < 10; i++ { - select { - case message := <-consumer.Messages(): - assertMessageOffset(t, message, int64(i+1234)) - case err := <-consumer.Errors(): - t.Error(err) - } - } - - safeClose(t, consumer) - safeClose(t, master) - broker0.Close() -} - -// If `OffsetNewest` is passed as the initial offset then the first consumed -// message is indeed corresponds to the offset that broker claims to be the -// newest in its metadata response. -func TestConsumerOffsetNewest(t *testing.T) { - // Given - broker0 := NewMockBroker(t, 0) - broker0.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetBroker(broker0.Addr(), broker0.BrokerID()). - SetLeader("my_topic", 0, broker0.BrokerID()), - "OffsetRequest": NewMockOffsetResponse(t). - SetOffset("my_topic", 0, OffsetNewest, 10). - SetOffset("my_topic", 0, OffsetOldest, 7), - "FetchRequest": NewMockFetchResponse(t, 1). - SetMessage("my_topic", 0, 9, testMsg). - SetMessage("my_topic", 0, 10, testMsg). - SetMessage("my_topic", 0, 11, testMsg). - SetHighWaterMark("my_topic", 0, 14), - }) - - master, err := NewConsumer([]string{broker0.Addr()}, nil) - if err != nil { - t.Fatal(err) - } - - // When - consumer, err := master.ConsumePartition("my_topic", 0, OffsetNewest) - if err != nil { - t.Fatal(err) - } - - // Then - assertMessageOffset(t, <-consumer.Messages(), 10) - if hwmo := consumer.HighWaterMarkOffset(); hwmo != 14 { - t.Errorf("Expected high water mark offset 14, found %d", hwmo) - } - - safeClose(t, consumer) - safeClose(t, master) - broker0.Close() -} - -// It is possible to close a partition consumer and create the same anew. -func TestConsumerRecreate(t *testing.T) { - // Given - broker0 := NewMockBroker(t, 0) - broker0.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetBroker(broker0.Addr(), broker0.BrokerID()). - SetLeader("my_topic", 0, broker0.BrokerID()), - "OffsetRequest": NewMockOffsetResponse(t). - SetOffset("my_topic", 0, OffsetOldest, 0). - SetOffset("my_topic", 0, OffsetNewest, 1000), - "FetchRequest": NewMockFetchResponse(t, 1). - SetMessage("my_topic", 0, 10, testMsg), - }) - - c, err := NewConsumer([]string{broker0.Addr()}, nil) - if err != nil { - t.Fatal(err) - } - - pc, err := c.ConsumePartition("my_topic", 0, 10) - if err != nil { - t.Fatal(err) - } - assertMessageOffset(t, <-pc.Messages(), 10) - - // When - safeClose(t, pc) - pc, err = c.ConsumePartition("my_topic", 0, 10) - if err != nil { - t.Fatal(err) - } - - // Then - assertMessageOffset(t, <-pc.Messages(), 10) - - safeClose(t, pc) - safeClose(t, c) - broker0.Close() -} - -// An attempt to consume the same partition twice should fail. -func TestConsumerDuplicate(t *testing.T) { - // Given - broker0 := NewMockBroker(t, 0) - broker0.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetBroker(broker0.Addr(), broker0.BrokerID()). - SetLeader("my_topic", 0, broker0.BrokerID()), - "OffsetRequest": NewMockOffsetResponse(t). - SetOffset("my_topic", 0, OffsetOldest, 0). - SetOffset("my_topic", 0, OffsetNewest, 1000), - "FetchRequest": NewMockFetchResponse(t, 1), - }) - - config := NewConfig() - config.ChannelBufferSize = 0 - c, err := NewConsumer([]string{broker0.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - pc1, err := c.ConsumePartition("my_topic", 0, 0) - if err != nil { - t.Fatal(err) - } - - // When - pc2, err := c.ConsumePartition("my_topic", 0, 0) - - // Then - if pc2 != nil || err != ConfigurationError("That topic/partition is already being consumed") { - t.Fatal("A partition cannot be consumed twice at the same time") - } - - safeClose(t, pc1) - safeClose(t, c) - broker0.Close() -} - -// If consumer fails to refresh metadata it keeps retrying with frequency -// specified by `Config.Consumer.Retry.Backoff`. -func TestConsumerLeaderRefreshError(t *testing.T) { - // Given - broker0 := NewMockBroker(t, 100) - - // Stage 1: my_topic/0 served by broker0 - Logger.Printf(" STAGE 1") - - broker0.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetBroker(broker0.Addr(), broker0.BrokerID()). - SetLeader("my_topic", 0, broker0.BrokerID()), - "OffsetRequest": NewMockOffsetResponse(t). - SetOffset("my_topic", 0, OffsetOldest, 123). - SetOffset("my_topic", 0, OffsetNewest, 1000), - "FetchRequest": NewMockFetchResponse(t, 1). - SetMessage("my_topic", 0, 123, testMsg), - }) - - config := NewConfig() - config.Net.ReadTimeout = 100 * time.Millisecond - config.Consumer.Retry.Backoff = 200 * time.Millisecond - config.Consumer.Return.Errors = true - config.Metadata.Retry.Max = 0 - c, err := NewConsumer([]string{broker0.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - pc, err := c.ConsumePartition("my_topic", 0, OffsetOldest) - if err != nil { - t.Fatal(err) - } - - assertMessageOffset(t, <-pc.Messages(), 123) - - // Stage 2: broker0 says that it is no longer the leader for my_topic/0, - // but the requests to retrieve metadata fail with network timeout. - Logger.Printf(" STAGE 2") - - fetchResponse2 := &FetchResponse{} - fetchResponse2.AddError("my_topic", 0, ErrNotLeaderForPartition) - - broker0.SetHandlerByMap(map[string]MockResponse{ - "FetchRequest": NewMockWrapper(fetchResponse2), - }) - - if consErr := <-pc.Errors(); consErr.Err != ErrOutOfBrokers { - t.Errorf("Unexpected error: %v", consErr.Err) - } - - // Stage 3: finally the metadata returned by broker0 tells that broker1 is - // a new leader for my_topic/0. Consumption resumes. - - Logger.Printf(" STAGE 3") - - broker1 := NewMockBroker(t, 101) - - broker1.SetHandlerByMap(map[string]MockResponse{ - "FetchRequest": NewMockFetchResponse(t, 1). - SetMessage("my_topic", 0, 124, testMsg), - }) - broker0.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetBroker(broker0.Addr(), broker0.BrokerID()). - SetBroker(broker1.Addr(), broker1.BrokerID()). - SetLeader("my_topic", 0, broker1.BrokerID()), - }) - - assertMessageOffset(t, <-pc.Messages(), 124) - - safeClose(t, pc) - safeClose(t, c) - broker1.Close() - broker0.Close() -} - -func TestConsumerInvalidTopic(t *testing.T) { - // Given - broker0 := NewMockBroker(t, 100) - broker0.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetBroker(broker0.Addr(), broker0.BrokerID()), - }) - - c, err := NewConsumer([]string{broker0.Addr()}, nil) - if err != nil { - t.Fatal(err) - } - - // When - pc, err := c.ConsumePartition("my_topic", 0, OffsetOldest) - - // Then - if pc != nil || err != ErrUnknownTopicOrPartition { - t.Errorf("Should fail with, err=%v", err) - } - - safeClose(t, c) - broker0.Close() -} - -// Nothing bad happens if a partition consumer that has no leader assigned at -// the moment is closed. -func TestConsumerClosePartitionWithoutLeader(t *testing.T) { - // Given - broker0 := NewMockBroker(t, 100) - broker0.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetBroker(broker0.Addr(), broker0.BrokerID()). - SetLeader("my_topic", 0, broker0.BrokerID()), - "OffsetRequest": NewMockOffsetResponse(t). - SetOffset("my_topic", 0, OffsetOldest, 123). - SetOffset("my_topic", 0, OffsetNewest, 1000), - "FetchRequest": NewMockFetchResponse(t, 1). - SetMessage("my_topic", 0, 123, testMsg), - }) - - config := NewConfig() - config.Net.ReadTimeout = 100 * time.Millisecond - config.Consumer.Retry.Backoff = 100 * time.Millisecond - config.Consumer.Return.Errors = true - config.Metadata.Retry.Max = 0 - c, err := NewConsumer([]string{broker0.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - pc, err := c.ConsumePartition("my_topic", 0, OffsetOldest) - if err != nil { - t.Fatal(err) - } - - assertMessageOffset(t, <-pc.Messages(), 123) - - // broker0 says that it is no longer the leader for my_topic/0, but the - // requests to retrieve metadata fail with network timeout. - fetchResponse2 := &FetchResponse{} - fetchResponse2.AddError("my_topic", 0, ErrNotLeaderForPartition) - - broker0.SetHandlerByMap(map[string]MockResponse{ - "FetchRequest": NewMockWrapper(fetchResponse2), - }) - - // When - if consErr := <-pc.Errors(); consErr.Err != ErrOutOfBrokers { - t.Errorf("Unexpected error: %v", consErr.Err) - } - - // Then: the partition consumer can be closed without any problem. - safeClose(t, pc) - safeClose(t, c) - broker0.Close() -} - -// If the initial offset passed on partition consumer creation is out of the -// actual offset range for the partition, then the partition consumer stops -// immediately closing its output channels. -func TestConsumerShutsDownOutOfRange(t *testing.T) { - // Given - broker0 := NewMockBroker(t, 0) - fetchResponse := new(FetchResponse) - fetchResponse.AddError("my_topic", 0, ErrOffsetOutOfRange) - broker0.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetBroker(broker0.Addr(), broker0.BrokerID()). - SetLeader("my_topic", 0, broker0.BrokerID()), - "OffsetRequest": NewMockOffsetResponse(t). - SetOffset("my_topic", 0, OffsetNewest, 1234). - SetOffset("my_topic", 0, OffsetOldest, 7), - "FetchRequest": NewMockWrapper(fetchResponse), - }) - - master, err := NewConsumer([]string{broker0.Addr()}, nil) - if err != nil { - t.Fatal(err) - } - - // When - consumer, err := master.ConsumePartition("my_topic", 0, 101) - if err != nil { - t.Fatal(err) - } - - // Then: consumer should shut down closing its messages and errors channels. - if _, ok := <-consumer.Messages(); ok { - t.Error("Expected the consumer to shut down") - } - safeClose(t, consumer) - - safeClose(t, master) - broker0.Close() -} - -// If a fetch response contains messages with offsets that are smaller then -// requested, then such messages are ignored. -func TestConsumerExtraOffsets(t *testing.T) { - // Given - legacyFetchResponse := &FetchResponse{} - legacyFetchResponse.AddMessage("my_topic", 0, nil, testMsg, 1) - legacyFetchResponse.AddMessage("my_topic", 0, nil, testMsg, 2) - legacyFetchResponse.AddMessage("my_topic", 0, nil, testMsg, 3) - legacyFetchResponse.AddMessage("my_topic", 0, nil, testMsg, 4) - newFetchResponse := &FetchResponse{Version: 4} - newFetchResponse.AddRecord("my_topic", 0, nil, testMsg, 1) - newFetchResponse.AddRecord("my_topic", 0, nil, testMsg, 2) - newFetchResponse.AddRecord("my_topic", 0, nil, testMsg, 3) - newFetchResponse.AddRecord("my_topic", 0, nil, testMsg, 4) - newFetchResponse.SetLastOffsetDelta("my_topic", 0, 4) - newFetchResponse.SetLastStableOffset("my_topic", 0, 4) - for _, fetchResponse1 := range []*FetchResponse{legacyFetchResponse, newFetchResponse} { - var offsetResponseVersion int16 - cfg := NewConfig() - cfg.Consumer.Return.Errors = true - if fetchResponse1.Version >= 4 { - cfg.Version = V0_11_0_0 - offsetResponseVersion = 1 - } - - broker0 := NewMockBroker(t, 0) - fetchResponse2 := &FetchResponse{} - fetchResponse2.Version = fetchResponse1.Version - fetchResponse2.AddError("my_topic", 0, ErrNoError) - broker0.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetBroker(broker0.Addr(), broker0.BrokerID()). - SetLeader("my_topic", 0, broker0.BrokerID()), - "OffsetRequest": NewMockOffsetResponse(t). - SetVersion(offsetResponseVersion). - SetOffset("my_topic", 0, OffsetNewest, 1234). - SetOffset("my_topic", 0, OffsetOldest, 0), - "FetchRequest": NewMockSequence(fetchResponse1, fetchResponse2), - }) - - master, err := NewConsumer([]string{broker0.Addr()}, cfg) - if err != nil { - t.Fatal(err) - } - - // When - consumer, err := master.ConsumePartition("my_topic", 0, 3) - if err != nil { - t.Fatal(err) - } - - // Then: messages with offsets 1 and 2 are not returned even though they - // are present in the response. - select { - case msg := <-consumer.Messages(): - assertMessageOffset(t, msg, 3) - case err := <-consumer.Errors(): - t.Fatal(err) - } - - select { - case msg := <-consumer.Messages(): - assertMessageOffset(t, msg, 4) - case err := <-consumer.Errors(): - t.Fatal(err) - } - - safeClose(t, consumer) - safeClose(t, master) - broker0.Close() - } -} - -// In some situations broker may return a block containing only -// messages older then requested, even though there would be -// more messages if higher offset was requested. -func TestConsumerReceivingFetchResponseWithTooOldRecords(t *testing.T) { - // Given - fetchResponse1 := &FetchResponse{Version: 4} - fetchResponse1.AddRecord("my_topic", 0, nil, testMsg, 1) - - fetchResponse2 := &FetchResponse{Version: 4} - fetchResponse2.AddRecord("my_topic", 0, nil, testMsg, 1000000) - - cfg := NewConfig() - cfg.Consumer.Return.Errors = true - cfg.Version = V1_1_0_0 - - broker0 := NewMockBroker(t, 0) - - broker0.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetBroker(broker0.Addr(), broker0.BrokerID()). - SetLeader("my_topic", 0, broker0.BrokerID()), - "OffsetRequest": NewMockOffsetResponse(t). - SetVersion(1). - SetOffset("my_topic", 0, OffsetNewest, 1234). - SetOffset("my_topic", 0, OffsetOldest, 0), - "FetchRequest": NewMockSequence(fetchResponse1, fetchResponse2), - }) - - master, err := NewConsumer([]string{broker0.Addr()}, cfg) - if err != nil { - t.Fatal(err) - } - - // When - consumer, err := master.ConsumePartition("my_topic", 0, 2) - if err != nil { - t.Fatal(err) - } - - select { - case msg := <-consumer.Messages(): - assertMessageOffset(t, msg, 1000000) - case err := <-consumer.Errors(): - t.Fatal(err) - } - - safeClose(t, consumer) - safeClose(t, master) - broker0.Close() -} - -func TestConsumeMessageWithNewerFetchAPIVersion(t *testing.T) { - // Given - fetchResponse1 := &FetchResponse{Version: 4} - fetchResponse1.AddMessage("my_topic", 0, nil, testMsg, 1) - fetchResponse1.AddMessage("my_topic", 0, nil, testMsg, 2) - - cfg := NewConfig() - cfg.Version = V0_11_0_0 - - broker0 := NewMockBroker(t, 0) - fetchResponse2 := &FetchResponse{} - fetchResponse2.Version = 4 - fetchResponse2.AddError("my_topic", 0, ErrNoError) - broker0.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetBroker(broker0.Addr(), broker0.BrokerID()). - SetLeader("my_topic", 0, broker0.BrokerID()), - "OffsetRequest": NewMockOffsetResponse(t). - SetVersion(1). - SetOffset("my_topic", 0, OffsetNewest, 1234). - SetOffset("my_topic", 0, OffsetOldest, 0), - "FetchRequest": NewMockSequence(fetchResponse1, fetchResponse2), - }) - - master, err := NewConsumer([]string{broker0.Addr()}, cfg) - if err != nil { - t.Fatal(err) - } - - // When - consumer, err := master.ConsumePartition("my_topic", 0, 1) - if err != nil { - t.Fatal(err) - } - - assertMessageOffset(t, <-consumer.Messages(), 1) - assertMessageOffset(t, <-consumer.Messages(), 2) - - safeClose(t, consumer) - safeClose(t, master) - broker0.Close() -} - -// It is fine if offsets of fetched messages are not sequential (although -// strictly increasing!). -func TestConsumerNonSequentialOffsets(t *testing.T) { - // Given - legacyFetchResponse := &FetchResponse{} - legacyFetchResponse.AddMessage("my_topic", 0, nil, testMsg, 5) - legacyFetchResponse.AddMessage("my_topic", 0, nil, testMsg, 7) - legacyFetchResponse.AddMessage("my_topic", 0, nil, testMsg, 11) - newFetchResponse := &FetchResponse{Version: 4} - newFetchResponse.AddRecord("my_topic", 0, nil, testMsg, 5) - newFetchResponse.AddRecord("my_topic", 0, nil, testMsg, 7) - newFetchResponse.AddRecord("my_topic", 0, nil, testMsg, 11) - newFetchResponse.SetLastOffsetDelta("my_topic", 0, 11) - newFetchResponse.SetLastStableOffset("my_topic", 0, 11) - for _, fetchResponse1 := range []*FetchResponse{legacyFetchResponse, newFetchResponse} { - var offsetResponseVersion int16 - cfg := NewConfig() - if fetchResponse1.Version >= 4 { - cfg.Version = V0_11_0_0 - offsetResponseVersion = 1 - } - - broker0 := NewMockBroker(t, 0) - fetchResponse2 := &FetchResponse{Version: fetchResponse1.Version} - fetchResponse2.AddError("my_topic", 0, ErrNoError) - broker0.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetBroker(broker0.Addr(), broker0.BrokerID()). - SetLeader("my_topic", 0, broker0.BrokerID()), - "OffsetRequest": NewMockOffsetResponse(t). - SetVersion(offsetResponseVersion). - SetOffset("my_topic", 0, OffsetNewest, 1234). - SetOffset("my_topic", 0, OffsetOldest, 0), - "FetchRequest": NewMockSequence(fetchResponse1, fetchResponse2), - }) - - master, err := NewConsumer([]string{broker0.Addr()}, cfg) - if err != nil { - t.Fatal(err) - } - - // When - consumer, err := master.ConsumePartition("my_topic", 0, 3) - if err != nil { - t.Fatal(err) - } - - // Then: messages with offsets 1 and 2 are not returned even though they - // are present in the response. - assertMessageOffset(t, <-consumer.Messages(), 5) - assertMessageOffset(t, <-consumer.Messages(), 7) - assertMessageOffset(t, <-consumer.Messages(), 11) - - safeClose(t, consumer) - safeClose(t, master) - broker0.Close() - } -} - -// If leadership for a partition is changing then consumer resolves the new -// leader and switches to it. -func TestConsumerRebalancingMultiplePartitions(t *testing.T) { - // initial setup - seedBroker := NewMockBroker(t, 10) - leader0 := NewMockBroker(t, 0) - leader1 := NewMockBroker(t, 1) - - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetBroker(leader0.Addr(), leader0.BrokerID()). - SetBroker(leader1.Addr(), leader1.BrokerID()). - SetLeader("my_topic", 0, leader0.BrokerID()). - SetLeader("my_topic", 1, leader1.BrokerID()), - }) - - mockOffsetResponse1 := NewMockOffsetResponse(t). - SetOffset("my_topic", 0, OffsetOldest, 0). - SetOffset("my_topic", 0, OffsetNewest, 1000). - SetOffset("my_topic", 1, OffsetOldest, 0). - SetOffset("my_topic", 1, OffsetNewest, 1000) - leader0.SetHandlerByMap(map[string]MockResponse{ - "OffsetRequest": mockOffsetResponse1, - "FetchRequest": NewMockFetchResponse(t, 1), - }) - leader1.SetHandlerByMap(map[string]MockResponse{ - "OffsetRequest": mockOffsetResponse1, - "FetchRequest": NewMockFetchResponse(t, 1), - }) - - // launch test goroutines - config := NewConfig() - config.Consumer.Retry.Backoff = 50 - master, err := NewConsumer([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - // we expect to end up (eventually) consuming exactly ten messages on each partition - var wg sync.WaitGroup - for i := int32(0); i < 2; i++ { - consumer, err := master.ConsumePartition("my_topic", i, 0) - if err != nil { - t.Error(err) - } - - go func(c PartitionConsumer) { - for err := range c.Errors() { - t.Error(err) - } - }(consumer) - - wg.Add(1) - go func(partition int32, c PartitionConsumer) { - for i := 0; i < 10; i++ { - message := <-consumer.Messages() - if message.Offset != int64(i) { - t.Error("Incorrect message offset!", i, partition, message.Offset) - } - if message.Partition != partition { - t.Error("Incorrect message partition!") - } - } - safeClose(t, consumer) - wg.Done() - }(i, consumer) - } - - time.Sleep(50 * time.Millisecond) - Logger.Printf(" STAGE 1") - // Stage 1: - // * my_topic/0 -> leader0 serves 4 messages - // * my_topic/1 -> leader1 serves 0 messages - - mockFetchResponse := NewMockFetchResponse(t, 1) - for i := 0; i < 4; i++ { - mockFetchResponse.SetMessage("my_topic", 0, int64(i), testMsg) - } - leader0.SetHandlerByMap(map[string]MockResponse{ - "FetchRequest": mockFetchResponse, - }) - - time.Sleep(50 * time.Millisecond) - Logger.Printf(" STAGE 2") - // Stage 2: - // * leader0 says that it is no longer serving my_topic/0 - // * seedBroker tells that leader1 is serving my_topic/0 now - - // seed broker tells that the new partition 0 leader is leader1 - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetLeader("my_topic", 0, leader1.BrokerID()). - SetLeader("my_topic", 1, leader1.BrokerID()), - }) - - // leader0 says no longer leader of partition 0 - fetchResponse := new(FetchResponse) - fetchResponse.AddError("my_topic", 0, ErrNotLeaderForPartition) - leader0.SetHandlerByMap(map[string]MockResponse{ - "FetchRequest": NewMockWrapper(fetchResponse), - }) - - time.Sleep(50 * time.Millisecond) - Logger.Printf(" STAGE 3") - // Stage 3: - // * my_topic/0 -> leader1 serves 3 messages - // * my_topic/1 -> leader1 server 8 messages - - // leader1 provides 3 message on partition 0, and 8 messages on partition 1 - mockFetchResponse2 := NewMockFetchResponse(t, 2) - for i := 4; i < 7; i++ { - mockFetchResponse2.SetMessage("my_topic", 0, int64(i), testMsg) - } - for i := 0; i < 8; i++ { - mockFetchResponse2.SetMessage("my_topic", 1, int64(i), testMsg) - } - leader1.SetHandlerByMap(map[string]MockResponse{ - "FetchRequest": mockFetchResponse2, - }) - - time.Sleep(50 * time.Millisecond) - Logger.Printf(" STAGE 4") - // Stage 4: - // * my_topic/0 -> leader1 serves 3 messages - // * my_topic/1 -> leader1 tells that it is no longer the leader - // * seedBroker tells that leader0 is a new leader for my_topic/1 - - // metadata assigns 0 to leader1 and 1 to leader0 - seedBroker.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetLeader("my_topic", 0, leader1.BrokerID()). - SetLeader("my_topic", 1, leader0.BrokerID()), - }) - - // leader1 provides three more messages on partition0, says no longer leader of partition1 - mockFetchResponse3 := NewMockFetchResponse(t, 3). - SetMessage("my_topic", 0, int64(7), testMsg). - SetMessage("my_topic", 0, int64(8), testMsg). - SetMessage("my_topic", 0, int64(9), testMsg) - fetchResponse4 := new(FetchResponse) - fetchResponse4.AddError("my_topic", 1, ErrNotLeaderForPartition) - leader1.SetHandlerByMap(map[string]MockResponse{ - "FetchRequest": NewMockSequence(mockFetchResponse3, fetchResponse4), - }) - - // leader0 provides two messages on partition 1 - mockFetchResponse4 := NewMockFetchResponse(t, 2) - for i := 8; i < 10; i++ { - mockFetchResponse4.SetMessage("my_topic", 1, int64(i), testMsg) - } - leader0.SetHandlerByMap(map[string]MockResponse{ - "FetchRequest": mockFetchResponse4, - }) - - wg.Wait() - safeClose(t, master) - leader1.Close() - leader0.Close() - seedBroker.Close() -} - -// When two partitions have the same broker as the leader, if one partition -// consumer channel buffer is full then that does not affect the ability to -// read messages by the other consumer. -func TestConsumerInterleavedClose(t *testing.T) { - // Given - broker0 := NewMockBroker(t, 0) - broker0.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetBroker(broker0.Addr(), broker0.BrokerID()). - SetLeader("my_topic", 0, broker0.BrokerID()). - SetLeader("my_topic", 1, broker0.BrokerID()), - "OffsetRequest": NewMockOffsetResponse(t). - SetOffset("my_topic", 0, OffsetOldest, 1000). - SetOffset("my_topic", 0, OffsetNewest, 1100). - SetOffset("my_topic", 1, OffsetOldest, 2000). - SetOffset("my_topic", 1, OffsetNewest, 2100), - "FetchRequest": NewMockFetchResponse(t, 1). - SetMessage("my_topic", 0, 1000, testMsg). - SetMessage("my_topic", 0, 1001, testMsg). - SetMessage("my_topic", 0, 1002, testMsg). - SetMessage("my_topic", 1, 2000, testMsg), - }) - - config := NewConfig() - config.ChannelBufferSize = 0 - master, err := NewConsumer([]string{broker0.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - c0, err := master.ConsumePartition("my_topic", 0, 1000) - if err != nil { - t.Fatal(err) - } - - c1, err := master.ConsumePartition("my_topic", 1, 2000) - if err != nil { - t.Fatal(err) - } - - // When/Then: we can read from partition 0 even if nobody reads from partition 1 - assertMessageOffset(t, <-c0.Messages(), 1000) - assertMessageOffset(t, <-c0.Messages(), 1001) - assertMessageOffset(t, <-c0.Messages(), 1002) - - safeClose(t, c1) - safeClose(t, c0) - safeClose(t, master) - broker0.Close() -} - -func TestConsumerBounceWithReferenceOpen(t *testing.T) { - broker0 := NewMockBroker(t, 0) - broker0Addr := broker0.Addr() - broker1 := NewMockBroker(t, 1) - - mockMetadataResponse := NewMockMetadataResponse(t). - SetBroker(broker0.Addr(), broker0.BrokerID()). - SetBroker(broker1.Addr(), broker1.BrokerID()). - SetLeader("my_topic", 0, broker0.BrokerID()). - SetLeader("my_topic", 1, broker1.BrokerID()) - - mockOffsetResponse := NewMockOffsetResponse(t). - SetOffset("my_topic", 0, OffsetOldest, 1000). - SetOffset("my_topic", 0, OffsetNewest, 1100). - SetOffset("my_topic", 1, OffsetOldest, 2000). - SetOffset("my_topic", 1, OffsetNewest, 2100) - - mockFetchResponse := NewMockFetchResponse(t, 1) - for i := 0; i < 10; i++ { - mockFetchResponse.SetMessage("my_topic", 0, int64(1000+i), testMsg) - mockFetchResponse.SetMessage("my_topic", 1, int64(2000+i), testMsg) - } - - broker0.SetHandlerByMap(map[string]MockResponse{ - "OffsetRequest": mockOffsetResponse, - "FetchRequest": mockFetchResponse, - }) - broker1.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": mockMetadataResponse, - "OffsetRequest": mockOffsetResponse, - "FetchRequest": mockFetchResponse, - }) - - config := NewConfig() - config.Consumer.Return.Errors = true - config.Consumer.Retry.Backoff = 100 * time.Millisecond - config.ChannelBufferSize = 1 - master, err := NewConsumer([]string{broker1.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - c0, err := master.ConsumePartition("my_topic", 0, 1000) - if err != nil { - t.Fatal(err) - } - - c1, err := master.ConsumePartition("my_topic", 1, 2000) - if err != nil { - t.Fatal(err) - } - - // read messages from both partition to make sure that both brokers operate - // normally. - assertMessageOffset(t, <-c0.Messages(), 1000) - assertMessageOffset(t, <-c1.Messages(), 2000) - - // Simulate broker shutdown. Note that metadata response does not change, - // that is the leadership does not move to another broker. So partition - // consumer will keep retrying to restore the connection with the broker. - broker0.Close() - - // Make sure that while the partition/0 leader is down, consumer/partition/1 - // is capable of pulling messages from broker1. - for i := 1; i < 7; i++ { - offset := (<-c1.Messages()).Offset - if offset != int64(2000+i) { - t.Errorf("Expected offset %d from consumer/partition/1", int64(2000+i)) - } - } - - // Bring broker0 back to service. - broker0 = NewMockBrokerAddr(t, 0, broker0Addr) - broker0.SetHandlerByMap(map[string]MockResponse{ - "FetchRequest": mockFetchResponse, - }) - - // Read the rest of messages from both partitions. - for i := 7; i < 10; i++ { - assertMessageOffset(t, <-c1.Messages(), int64(2000+i)) - } - for i := 1; i < 10; i++ { - assertMessageOffset(t, <-c0.Messages(), int64(1000+i)) - } - - select { - case <-c0.Errors(): - default: - t.Errorf("Partition consumer should have detected broker restart") - } - - safeClose(t, c1) - safeClose(t, c0) - safeClose(t, master) - broker0.Close() - broker1.Close() -} - -func TestConsumerOffsetOutOfRange(t *testing.T) { - // Given - broker0 := NewMockBroker(t, 2) - broker0.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetBroker(broker0.Addr(), broker0.BrokerID()). - SetLeader("my_topic", 0, broker0.BrokerID()), - "OffsetRequest": NewMockOffsetResponse(t). - SetOffset("my_topic", 0, OffsetNewest, 1234). - SetOffset("my_topic", 0, OffsetOldest, 2345), - }) - - master, err := NewConsumer([]string{broker0.Addr()}, nil) - if err != nil { - t.Fatal(err) - } - - // When/Then - if _, err := master.ConsumePartition("my_topic", 0, 0); err != ErrOffsetOutOfRange { - t.Fatal("Should return ErrOffsetOutOfRange, got:", err) - } - if _, err := master.ConsumePartition("my_topic", 0, 3456); err != ErrOffsetOutOfRange { - t.Fatal("Should return ErrOffsetOutOfRange, got:", err) - } - if _, err := master.ConsumePartition("my_topic", 0, -3); err != ErrOffsetOutOfRange { - t.Fatal("Should return ErrOffsetOutOfRange, got:", err) - } - - safeClose(t, master) - broker0.Close() -} - -func TestConsumerExpiryTicker(t *testing.T) { - // Given - broker0 := NewMockBroker(t, 0) - fetchResponse1 := &FetchResponse{} - for i := 1; i <= 8; i++ { - fetchResponse1.AddMessage("my_topic", 0, nil, testMsg, int64(i)) - } - broker0.SetHandlerByMap(map[string]MockResponse{ - "MetadataRequest": NewMockMetadataResponse(t). - SetBroker(broker0.Addr(), broker0.BrokerID()). - SetLeader("my_topic", 0, broker0.BrokerID()), - "OffsetRequest": NewMockOffsetResponse(t). - SetOffset("my_topic", 0, OffsetNewest, 1234). - SetOffset("my_topic", 0, OffsetOldest, 1), - "FetchRequest": NewMockSequence(fetchResponse1), - }) - - config := NewConfig() - config.ChannelBufferSize = 0 - config.Consumer.MaxProcessingTime = 10 * time.Millisecond - master, err := NewConsumer([]string{broker0.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - // When - consumer, err := master.ConsumePartition("my_topic", 0, 1) - if err != nil { - t.Fatal(err) - } - - // Then: messages with offsets 1 through 8 are read - for i := 1; i <= 8; i++ { - assertMessageOffset(t, <-consumer.Messages(), int64(i)) - time.Sleep(2 * time.Millisecond) - } - - safeClose(t, consumer) - safeClose(t, master) - broker0.Close() -} - -func assertMessageOffset(t *testing.T, msg *ConsumerMessage, expectedOffset int64) { - if msg.Offset != expectedOffset { - t.Errorf("Incorrect message offset: expected=%d, actual=%d", expectedOffset, msg.Offset) - } -} - -// This example shows how to use the consumer to read messages -// from a single partition. -func ExampleConsumer() { - consumer, err := NewConsumer([]string{"localhost:9092"}, nil) - if err != nil { - panic(err) - } - - defer func() { - if err := consumer.Close(); err != nil { - log.Fatalln(err) - } - }() - - partitionConsumer, err := consumer.ConsumePartition("my_topic", 0, OffsetNewest) - if err != nil { - panic(err) - } - - defer func() { - if err := partitionConsumer.Close(); err != nil { - log.Fatalln(err) - } - }() - - // Trap SIGINT to trigger a shutdown. - signals := make(chan os.Signal, 1) - signal.Notify(signals, os.Interrupt) - - consumed := 0 -ConsumerLoop: - for { - select { - case msg := <-partitionConsumer.Messages(): - log.Printf("Consumed message offset %d\n", msg.Offset) - consumed++ - case <-signals: - break ConsumerLoop - } - } - - log.Printf("Consumed: %d\n", consumed) -} diff --git a/third/github.com/Shopify/sarama/crc32_field.go b/third/github.com/Shopify/sarama/crc32_field.go deleted file mode 100644 index 1f144431a..000000000 --- a/third/github.com/Shopify/sarama/crc32_field.go +++ /dev/null @@ -1,69 +0,0 @@ -package sarama - -import ( - "encoding/binary" - "fmt" - "hash/crc32" -) - -type crcPolynomial int8 - -const ( - crcIEEE crcPolynomial = iota - crcCastagnoli -) - -var castagnoliTable = crc32.MakeTable(crc32.Castagnoli) - -// crc32Field implements the pushEncoder and pushDecoder interfaces for calculating CRC32s. -type crc32Field struct { - startOffset int - polynomial crcPolynomial -} - -func (c *crc32Field) saveOffset(in int) { - c.startOffset = in -} - -func (c *crc32Field) reserveLength() int { - return 4 -} - -func newCRC32Field(polynomial crcPolynomial) *crc32Field { - return &crc32Field{polynomial: polynomial} -} - -func (c *crc32Field) run(curOffset int, buf []byte) error { - crc, err := c.crc(curOffset, buf) - if err != nil { - return err - } - binary.BigEndian.PutUint32(buf[c.startOffset:], crc) - return nil -} - -func (c *crc32Field) check(curOffset int, buf []byte) error { - crc, err := c.crc(curOffset, buf) - if err != nil { - return err - } - - expected := binary.BigEndian.Uint32(buf[c.startOffset:]) - if crc != expected { - return PacketDecodingError{fmt.Sprintf("CRC didn't match expected %#x got %#x", expected, crc)} - } - - return nil -} -func (c *crc32Field) crc(curOffset int, buf []byte) (uint32, error) { - var tab *crc32.Table - switch c.polynomial { - case crcIEEE: - tab = crc32.IEEETable - case crcCastagnoli: - tab = castagnoliTable - default: - return 0, PacketDecodingError{"invalid CRC type"} - } - return crc32.Checksum(buf[c.startOffset+4:curOffset], tab), nil -} diff --git a/third/github.com/Shopify/sarama/create_partitions_request.go b/third/github.com/Shopify/sarama/create_partitions_request.go deleted file mode 100644 index af321e994..000000000 --- a/third/github.com/Shopify/sarama/create_partitions_request.go +++ /dev/null @@ -1,121 +0,0 @@ -package sarama - -import "time" - -type CreatePartitionsRequest struct { - TopicPartitions map[string]*TopicPartition - Timeout time.Duration - ValidateOnly bool -} - -func (c *CreatePartitionsRequest) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(c.TopicPartitions)); err != nil { - return err - } - - for topic, partition := range c.TopicPartitions { - if err := pe.putString(topic); err != nil { - return err - } - if err := partition.encode(pe); err != nil { - return err - } - } - - pe.putInt32(int32(c.Timeout / time.Millisecond)) - - pe.putBool(c.ValidateOnly) - - return nil -} - -func (c *CreatePartitionsRequest) decode(pd packetDecoder, version int16) (err error) { - n, err := pd.getArrayLength() - if err != nil { - return err - } - c.TopicPartitions = make(map[string]*TopicPartition, n) - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - c.TopicPartitions[topic] = new(TopicPartition) - if err := c.TopicPartitions[topic].decode(pd, version); err != nil { - return err - } - } - - timeout, err := pd.getInt32() - if err != nil { - return err - } - c.Timeout = time.Duration(timeout) * time.Millisecond - - if c.ValidateOnly, err = pd.getBool(); err != nil { - return err - } - - return nil -} - -func (r *CreatePartitionsRequest) key() int16 { - return 37 -} - -func (r *CreatePartitionsRequest) version() int16 { - return 0 -} - -func (r *CreatePartitionsRequest) requiredVersion() KafkaVersion { - return V1_0_0_0 -} - -type TopicPartition struct { - Count int32 - Assignment [][]int32 -} - -func (t *TopicPartition) encode(pe packetEncoder) error { - pe.putInt32(t.Count) - - if len(t.Assignment) == 0 { - pe.putInt32(-1) - return nil - } - - if err := pe.putArrayLength(len(t.Assignment)); err != nil { - return err - } - - for _, assign := range t.Assignment { - if err := pe.putInt32Array(assign); err != nil { - return err - } - } - - return nil -} - -func (t *TopicPartition) decode(pd packetDecoder, version int16) (err error) { - if t.Count, err = pd.getInt32(); err != nil { - return err - } - - n, err := pd.getInt32() - if err != nil { - return err - } - if n <= 0 { - return nil - } - t.Assignment = make([][]int32, n) - - for i := 0; i < int(n); i++ { - if t.Assignment[i], err = pd.getInt32Array(); err != nil { - return err - } - } - - return nil -} diff --git a/third/github.com/Shopify/sarama/create_partitions_request_test.go b/third/github.com/Shopify/sarama/create_partitions_request_test.go deleted file mode 100644 index db8cb7329..000000000 --- a/third/github.com/Shopify/sarama/create_partitions_request_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -var ( - createPartitionRequestNoAssignment = []byte{ - 0, 0, 0, 1, // one topic - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 0, 0, 3, // 3 partitions - 255, 255, 255, 255, // no assignments - 0, 0, 0, 100, // timeout - 0, // validate only = false - } - - createPartitionRequestAssignment = []byte{ - 0, 0, 0, 1, - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 0, 0, 3, // 3 partitions - 0, 0, 0, 2, - 0, 0, 0, 2, - 0, 0, 0, 2, 0, 0, 0, 3, - 0, 0, 0, 2, - 0, 0, 0, 3, 0, 0, 0, 1, - 0, 0, 0, 100, - 1, // validate only = true - } -) - -func TestCreatePartitionsRequest(t *testing.T) { - req := &CreatePartitionsRequest{ - TopicPartitions: map[string]*TopicPartition{ - "topic": &TopicPartition{ - Count: 3, - }, - }, - Timeout: 100 * time.Millisecond, - } - - buf := testRequestEncode(t, "no assignment", req, createPartitionRequestNoAssignment) - testRequestDecode(t, "no assignment", req, buf) - - req.ValidateOnly = true - req.TopicPartitions["topic"].Assignment = [][]int32{{2, 3}, {3, 1}} - - buf = testRequestEncode(t, "assignment", req, createPartitionRequestAssignment) - testRequestDecode(t, "assignment", req, buf) -} diff --git a/third/github.com/Shopify/sarama/create_partitions_response.go b/third/github.com/Shopify/sarama/create_partitions_response.go deleted file mode 100644 index abd621c64..000000000 --- a/third/github.com/Shopify/sarama/create_partitions_response.go +++ /dev/null @@ -1,94 +0,0 @@ -package sarama - -import "time" - -type CreatePartitionsResponse struct { - ThrottleTime time.Duration - TopicPartitionErrors map[string]*TopicPartitionError -} - -func (c *CreatePartitionsResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(c.ThrottleTime / time.Millisecond)) - if err := pe.putArrayLength(len(c.TopicPartitionErrors)); err != nil { - return err - } - - for topic, partitionError := range c.TopicPartitionErrors { - if err := pe.putString(topic); err != nil { - return err - } - if err := partitionError.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (c *CreatePartitionsResponse) decode(pd packetDecoder, version int16) (err error) { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - c.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - c.TopicPartitionErrors = make(map[string]*TopicPartitionError, n) - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - c.TopicPartitionErrors[topic] = new(TopicPartitionError) - if err := c.TopicPartitionErrors[topic].decode(pd, version); err != nil { - return err - } - } - - return nil -} - -func (r *CreatePartitionsResponse) key() int16 { - return 37 -} - -func (r *CreatePartitionsResponse) version() int16 { - return 0 -} - -func (r *CreatePartitionsResponse) requiredVersion() KafkaVersion { - return V1_0_0_0 -} - -type TopicPartitionError struct { - Err KError - ErrMsg *string -} - -func (t *TopicPartitionError) encode(pe packetEncoder) error { - pe.putInt16(int16(t.Err)) - - if err := pe.putNullableString(t.ErrMsg); err != nil { - return err - } - - return nil -} - -func (t *TopicPartitionError) decode(pd packetDecoder, version int16) (err error) { - kerr, err := pd.getInt16() - if err != nil { - return err - } - t.Err = KError(kerr) - - if t.ErrMsg, err = pd.getNullableString(); err != nil { - return err - } - - return nil -} diff --git a/third/github.com/Shopify/sarama/create_partitions_response_test.go b/third/github.com/Shopify/sarama/create_partitions_response_test.go deleted file mode 100644 index 3219882ca..000000000 --- a/third/github.com/Shopify/sarama/create_partitions_response_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package sarama - -import ( - "reflect" - "testing" - "time" -) - -var ( - createPartitionResponseSuccess = []byte{ - 0, 0, 0, 100, // throttleTimeMs - 0, 0, 0, 1, - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 0, // no error - 255, 255, // no error message - } - - createPartitionResponseFail = []byte{ - 0, 0, 0, 100, // throttleTimeMs - 0, 0, 0, 1, - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 37, // partition error - 0, 5, 'e', 'r', 'r', 'o', 'r', - } -) - -func TestCreatePartitionsResponse(t *testing.T) { - resp := &CreatePartitionsResponse{ - ThrottleTime: 100 * time.Millisecond, - TopicPartitionErrors: map[string]*TopicPartitionError{ - "topic": &TopicPartitionError{}, - }, - } - - testResponse(t, "success", resp, createPartitionResponseSuccess) - decodedresp := new(CreatePartitionsResponse) - testVersionDecodable(t, "success", decodedresp, createPartitionResponseSuccess, 0) - if !reflect.DeepEqual(decodedresp, resp) { - t.Errorf("Decoding error: expected %v but got %v", decodedresp, resp) - } - - errMsg := "error" - resp.TopicPartitionErrors["topic"].Err = ErrInvalidPartitions - resp.TopicPartitionErrors["topic"].ErrMsg = &errMsg - - testResponse(t, "with errors", resp, createPartitionResponseFail) - decodedresp = new(CreatePartitionsResponse) - testVersionDecodable(t, "with errors", decodedresp, createPartitionResponseFail, 0) - if !reflect.DeepEqual(decodedresp, resp) { - t.Errorf("Decoding error: expected %v but got %v", decodedresp, resp) - } -} diff --git a/third/github.com/Shopify/sarama/create_topics_request.go b/third/github.com/Shopify/sarama/create_topics_request.go deleted file mode 100644 index 709c0a44e..000000000 --- a/third/github.com/Shopify/sarama/create_topics_request.go +++ /dev/null @@ -1,174 +0,0 @@ -package sarama - -import ( - "time" -) - -type CreateTopicsRequest struct { - Version int16 - - TopicDetails map[string]*TopicDetail - Timeout time.Duration - ValidateOnly bool -} - -func (c *CreateTopicsRequest) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(c.TopicDetails)); err != nil { - return err - } - for topic, detail := range c.TopicDetails { - if err := pe.putString(topic); err != nil { - return err - } - if err := detail.encode(pe); err != nil { - return err - } - } - - pe.putInt32(int32(c.Timeout / time.Millisecond)) - - if c.Version >= 1 { - pe.putBool(c.ValidateOnly) - } - - return nil -} - -func (c *CreateTopicsRequest) decode(pd packetDecoder, version int16) (err error) { - n, err := pd.getArrayLength() - if err != nil { - return err - } - - c.TopicDetails = make(map[string]*TopicDetail, n) - - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - c.TopicDetails[topic] = new(TopicDetail) - if err = c.TopicDetails[topic].decode(pd, version); err != nil { - return err - } - } - - timeout, err := pd.getInt32() - if err != nil { - return err - } - c.Timeout = time.Duration(timeout) * time.Millisecond - - if version >= 1 { - c.ValidateOnly, err = pd.getBool() - if err != nil { - return err - } - - c.Version = version - } - - return nil -} - -func (c *CreateTopicsRequest) key() int16 { - return 19 -} - -func (c *CreateTopicsRequest) version() int16 { - return c.Version -} - -func (c *CreateTopicsRequest) requiredVersion() KafkaVersion { - switch c.Version { - case 2: - return V1_0_0_0 - case 1: - return V0_11_0_0 - default: - return V0_10_1_0 - } -} - -type TopicDetail struct { - NumPartitions int32 - ReplicationFactor int16 - ReplicaAssignment map[int32][]int32 - ConfigEntries map[string]*string -} - -func (t *TopicDetail) encode(pe packetEncoder) error { - pe.putInt32(t.NumPartitions) - pe.putInt16(t.ReplicationFactor) - - if err := pe.putArrayLength(len(t.ReplicaAssignment)); err != nil { - return err - } - for partition, assignment := range t.ReplicaAssignment { - pe.putInt32(partition) - if err := pe.putInt32Array(assignment); err != nil { - return err - } - } - - if err := pe.putArrayLength(len(t.ConfigEntries)); err != nil { - return err - } - for configKey, configValue := range t.ConfigEntries { - if err := pe.putString(configKey); err != nil { - return err - } - if err := pe.putNullableString(configValue); err != nil { - return err - } - } - - return nil -} - -func (t *TopicDetail) decode(pd packetDecoder, version int16) (err error) { - if t.NumPartitions, err = pd.getInt32(); err != nil { - return err - } - if t.ReplicationFactor, err = pd.getInt16(); err != nil { - return err - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - if n > 0 { - t.ReplicaAssignment = make(map[int32][]int32, n) - for i := 0; i < n; i++ { - replica, err := pd.getInt32() - if err != nil { - return err - } - if t.ReplicaAssignment[replica], err = pd.getInt32Array(); err != nil { - return err - } - } - } - - n, err = pd.getArrayLength() - if err != nil { - return err - } - - if n > 0 { - t.ConfigEntries = make(map[string]*string, n) - for i := 0; i < n; i++ { - configKey, err := pd.getString() - if err != nil { - return err - } - if t.ConfigEntries[configKey], err = pd.getNullableString(); err != nil { - return err - } - } - } - - return nil -} diff --git a/third/github.com/Shopify/sarama/create_topics_request_test.go b/third/github.com/Shopify/sarama/create_topics_request_test.go deleted file mode 100644 index 56b1b80e5..000000000 --- a/third/github.com/Shopify/sarama/create_topics_request_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -var ( - createTopicsRequestV0 = []byte{ - 0, 0, 0, 1, - 0, 5, 't', 'o', 'p', 'i', 'c', - 255, 255, 255, 255, - 255, 255, - 0, 0, 0, 1, // 1 replica assignment - 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, - 0, 0, 0, 1, // 1 config - 0, 12, 'r', 'e', 't', 'e', 'n', 't', 'i', 'o', 'n', '.', 'm', 's', - 0, 2, '-', '1', - 0, 0, 0, 100, - } - - createTopicsRequestV1 = append(createTopicsRequestV0, byte(1)) -) - -func TestCreateTopicsRequest(t *testing.T) { - retention := "-1" - - req := &CreateTopicsRequest{ - TopicDetails: map[string]*TopicDetail{ - "topic": { - NumPartitions: -1, - ReplicationFactor: -1, - ReplicaAssignment: map[int32][]int32{ - 0: []int32{0, 1, 2}, - }, - ConfigEntries: map[string]*string{ - "retention.ms": &retention, - }, - }, - }, - Timeout: 100 * time.Millisecond, - } - - testRequest(t, "version 0", req, createTopicsRequestV0) - - req.Version = 1 - req.ValidateOnly = true - - testRequest(t, "version 1", req, createTopicsRequestV1) -} diff --git a/third/github.com/Shopify/sarama/create_topics_response.go b/third/github.com/Shopify/sarama/create_topics_response.go deleted file mode 100644 index 66207e00c..000000000 --- a/third/github.com/Shopify/sarama/create_topics_response.go +++ /dev/null @@ -1,112 +0,0 @@ -package sarama - -import "time" - -type CreateTopicsResponse struct { - Version int16 - ThrottleTime time.Duration - TopicErrors map[string]*TopicError -} - -func (c *CreateTopicsResponse) encode(pe packetEncoder) error { - if c.Version >= 2 { - pe.putInt32(int32(c.ThrottleTime / time.Millisecond)) - } - - if err := pe.putArrayLength(len(c.TopicErrors)); err != nil { - return err - } - for topic, topicError := range c.TopicErrors { - if err := pe.putString(topic); err != nil { - return err - } - if err := topicError.encode(pe, c.Version); err != nil { - return err - } - } - - return nil -} - -func (c *CreateTopicsResponse) decode(pd packetDecoder, version int16) (err error) { - c.Version = version - - if version >= 2 { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - c.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - c.TopicErrors = make(map[string]*TopicError, n) - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - c.TopicErrors[topic] = new(TopicError) - if err := c.TopicErrors[topic].decode(pd, version); err != nil { - return err - } - } - - return nil -} - -func (c *CreateTopicsResponse) key() int16 { - return 19 -} - -func (c *CreateTopicsResponse) version() int16 { - return c.Version -} - -func (c *CreateTopicsResponse) requiredVersion() KafkaVersion { - switch c.Version { - case 2: - return V1_0_0_0 - case 1: - return V0_11_0_0 - default: - return V0_10_1_0 - } -} - -type TopicError struct { - Err KError - ErrMsg *string -} - -func (t *TopicError) encode(pe packetEncoder, version int16) error { - pe.putInt16(int16(t.Err)) - - if version >= 1 { - if err := pe.putNullableString(t.ErrMsg); err != nil { - return err - } - } - - return nil -} - -func (t *TopicError) decode(pd packetDecoder, version int16) (err error) { - kErr, err := pd.getInt16() - if err != nil { - return err - } - t.Err = KError(kErr) - - if version >= 1 { - if t.ErrMsg, err = pd.getNullableString(); err != nil { - return err - } - } - - return nil -} diff --git a/third/github.com/Shopify/sarama/create_topics_response_test.go b/third/github.com/Shopify/sarama/create_topics_response_test.go deleted file mode 100644 index 53790064f..000000000 --- a/third/github.com/Shopify/sarama/create_topics_response_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -var ( - createTopicsResponseV0 = []byte{ - 0, 0, 0, 1, - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 42, - } - - createTopicsResponseV1 = []byte{ - 0, 0, 0, 1, - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 42, - 0, 3, 'm', 's', 'g', - } - - createTopicsResponseV2 = []byte{ - 0, 0, 0, 100, - 0, 0, 0, 1, - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 42, - 0, 3, 'm', 's', 'g', - } -) - -func TestCreateTopicsResponse(t *testing.T) { - resp := &CreateTopicsResponse{ - TopicErrors: map[string]*TopicError{ - "topic": &TopicError{ - Err: ErrInvalidRequest, - }, - }, - } - - testResponse(t, "version 0", resp, createTopicsResponseV0) - - resp.Version = 1 - msg := "msg" - resp.TopicErrors["topic"].ErrMsg = &msg - - testResponse(t, "version 1", resp, createTopicsResponseV1) - - resp.Version = 2 - resp.ThrottleTime = 100 * time.Millisecond - - testResponse(t, "version 2", resp, createTopicsResponseV2) -} diff --git a/third/github.com/Shopify/sarama/decompress.go b/third/github.com/Shopify/sarama/decompress.go deleted file mode 100644 index f4cc3b389..000000000 --- a/third/github.com/Shopify/sarama/decompress.go +++ /dev/null @@ -1,63 +0,0 @@ -package sarama - -import ( - "bytes" - "compress/gzip" - "fmt" - "io/ioutil" - "sync" - - "github.com/gogf/gf/third/github.com/eapache/go-xerial-snappy" - "github.com/gogf/gf/third/github.com/pierrec/lz4" -) - -var ( - lz4ReaderPool = sync.Pool{ - New: func() interface{} { - return lz4.NewReader(nil) - }, - } - - gzipReaderPool sync.Pool -) - -func decompress(cc CompressionCodec, data []byte) ([]byte, error) { - switch cc { - case CompressionNone: - return data, nil - case CompressionGZIP: - var ( - err error - reader *gzip.Reader - readerIntf = gzipReaderPool.Get() - ) - if readerIntf != nil { - reader = readerIntf.(*gzip.Reader) - } else { - reader, err = gzip.NewReader(bytes.NewReader(data)) - if err != nil { - return nil, err - } - } - - defer gzipReaderPool.Put(reader) - - if err := reader.Reset(bytes.NewReader(data)); err != nil { - return nil, err - } - - return ioutil.ReadAll(reader) - case CompressionSnappy: - return snappy.Decode(data) - case CompressionLZ4: - reader := lz4ReaderPool.Get().(*lz4.Reader) - defer lz4ReaderPool.Put(reader) - - reader.Reset(bytes.NewReader(data)) - return ioutil.ReadAll(reader) - case CompressionZSTD: - return zstdDecompress(nil, data) - default: - return nil, PacketDecodingError{fmt.Sprintf("invalid compression specified (%d)", cc)} - } -} diff --git a/third/github.com/Shopify/sarama/delete_groups_request.go b/third/github.com/Shopify/sarama/delete_groups_request.go deleted file mode 100644 index 305a324ac..000000000 --- a/third/github.com/Shopify/sarama/delete_groups_request.go +++ /dev/null @@ -1,30 +0,0 @@ -package sarama - -type DeleteGroupsRequest struct { - Groups []string -} - -func (r *DeleteGroupsRequest) encode(pe packetEncoder) error { - return pe.putStringArray(r.Groups) -} - -func (r *DeleteGroupsRequest) decode(pd packetDecoder, version int16) (err error) { - r.Groups, err = pd.getStringArray() - return -} - -func (r *DeleteGroupsRequest) key() int16 { - return 42 -} - -func (r *DeleteGroupsRequest) version() int16 { - return 0 -} - -func (r *DeleteGroupsRequest) requiredVersion() KafkaVersion { - return V1_1_0_0 -} - -func (r *DeleteGroupsRequest) AddGroup(group string) { - r.Groups = append(r.Groups, group) -} diff --git a/third/github.com/Shopify/sarama/delete_groups_request_test.go b/third/github.com/Shopify/sarama/delete_groups_request_test.go deleted file mode 100644 index 908172498..000000000 --- a/third/github.com/Shopify/sarama/delete_groups_request_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package sarama - -import "testing" - -var ( - emptyDeleteGroupsRequest = []byte{0, 0, 0, 0} - - singleDeleteGroupsRequest = []byte{ - 0, 0, 0, 1, // 1 group - 0, 3, 'f', 'o', 'o', // group name: foo - } - - doubleDeleteGroupsRequest = []byte{ - 0, 0, 0, 2, // 2 groups - 0, 3, 'f', 'o', 'o', // group name: foo - 0, 3, 'b', 'a', 'r', // group name: foo - } -) - -func TestDeleteGroupsRequest(t *testing.T) { - var request *DeleteGroupsRequest - - request = new(DeleteGroupsRequest) - testRequest(t, "no groups", request, emptyDeleteGroupsRequest) - - request = new(DeleteGroupsRequest) - request.AddGroup("foo") - testRequest(t, "one group", request, singleDeleteGroupsRequest) - - request = new(DeleteGroupsRequest) - request.AddGroup("foo") - request.AddGroup("bar") - testRequest(t, "two groups", request, doubleDeleteGroupsRequest) -} diff --git a/third/github.com/Shopify/sarama/delete_groups_response.go b/third/github.com/Shopify/sarama/delete_groups_response.go deleted file mode 100644 index c067ebb42..000000000 --- a/third/github.com/Shopify/sarama/delete_groups_response.go +++ /dev/null @@ -1,70 +0,0 @@ -package sarama - -import ( - "time" -) - -type DeleteGroupsResponse struct { - ThrottleTime time.Duration - GroupErrorCodes map[string]KError -} - -func (r *DeleteGroupsResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(r.ThrottleTime / time.Millisecond)) - - if err := pe.putArrayLength(len(r.GroupErrorCodes)); err != nil { - return err - } - for groupID, errorCode := range r.GroupErrorCodes { - if err := pe.putString(groupID); err != nil { - return err - } - pe.putInt16(int16(errorCode)) - } - - return nil -} - -func (r *DeleteGroupsResponse) decode(pd packetDecoder, version int16) error { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - r.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - n, err := pd.getArrayLength() - if err != nil { - return err - } - if n == 0 { - return nil - } - - r.GroupErrorCodes = make(map[string]KError, n) - for i := 0; i < n; i++ { - groupID, err := pd.getString() - if err != nil { - return err - } - errorCode, err := pd.getInt16() - if err != nil { - return err - } - - r.GroupErrorCodes[groupID] = KError(errorCode) - } - - return nil -} - -func (r *DeleteGroupsResponse) key() int16 { - return 42 -} - -func (r *DeleteGroupsResponse) version() int16 { - return 0 -} - -func (r *DeleteGroupsResponse) requiredVersion() KafkaVersion { - return V1_1_0_0 -} diff --git a/third/github.com/Shopify/sarama/delete_groups_response_test.go b/third/github.com/Shopify/sarama/delete_groups_response_test.go deleted file mode 100644 index 6f622b5f0..000000000 --- a/third/github.com/Shopify/sarama/delete_groups_response_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package sarama - -import ( - "testing" -) - -var ( - emptyDeleteGroupsResponse = []byte{ - 0, 0, 0, 0, // does not violate any quota - 0, 0, 0, 0, // no groups - } - - errorDeleteGroupsResponse = []byte{ - 0, 0, 0, 0, // does not violate any quota - 0, 0, 0, 1, // 1 group - 0, 3, 'f', 'o', 'o', // group name - 0, 31, // error ErrClusterAuthorizationFailed - } - - noErrorDeleteGroupsResponse = []byte{ - 0, 0, 0, 0, // does not violate any quota - 0, 0, 0, 1, // 1 group - 0, 3, 'f', 'o', 'o', // group name - 0, 0, // no error - } -) - -func TestDeleteGroupsResponse(t *testing.T) { - var response *DeleteGroupsResponse - - response = new(DeleteGroupsResponse) - testVersionDecodable(t, "empty", response, emptyDeleteGroupsResponse, 0) - if response.ThrottleTime != 0 { - t.Error("Expected no violation") - } - if len(response.GroupErrorCodes) != 0 { - t.Error("Expected no groups") - } - - response = new(DeleteGroupsResponse) - testVersionDecodable(t, "error", response, errorDeleteGroupsResponse, 0) - if response.ThrottleTime != 0 { - t.Error("Expected no violation") - } - if response.GroupErrorCodes["foo"] != ErrClusterAuthorizationFailed { - t.Error("Expected error ErrClusterAuthorizationFailed, found:", response.GroupErrorCodes["foo"]) - } - - response = new(DeleteGroupsResponse) - testVersionDecodable(t, "no error", response, noErrorDeleteGroupsResponse, 0) - if response.ThrottleTime != 0 { - t.Error("Expected no violation") - } - if response.GroupErrorCodes["foo"] != ErrNoError { - t.Error("Expected error ErrClusterAuthorizationFailed, found:", response.GroupErrorCodes["foo"]) - } -} diff --git a/third/github.com/Shopify/sarama/delete_records_request.go b/third/github.com/Shopify/sarama/delete_records_request.go deleted file mode 100644 index 93efafd4d..000000000 --- a/third/github.com/Shopify/sarama/delete_records_request.go +++ /dev/null @@ -1,126 +0,0 @@ -package sarama - -import ( - "sort" - "time" -) - -// request message format is: -// [topic] timeout(int32) -// where topic is: -// name(string) [partition] -// where partition is: -// id(int32) offset(int64) - -type DeleteRecordsRequest struct { - Topics map[string]*DeleteRecordsRequestTopic - Timeout time.Duration -} - -func (d *DeleteRecordsRequest) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(d.Topics)); err != nil { - return err - } - keys := make([]string, 0, len(d.Topics)) - for topic := range d.Topics { - keys = append(keys, topic) - } - sort.Strings(keys) - for _, topic := range keys { - if err := pe.putString(topic); err != nil { - return err - } - if err := d.Topics[topic].encode(pe); err != nil { - return err - } - } - pe.putInt32(int32(d.Timeout / time.Millisecond)) - - return nil -} - -func (d *DeleteRecordsRequest) decode(pd packetDecoder, version int16) error { - n, err := pd.getArrayLength() - if err != nil { - return err - } - - if n > 0 { - d.Topics = make(map[string]*DeleteRecordsRequestTopic, n) - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - details := new(DeleteRecordsRequestTopic) - if err = details.decode(pd, version); err != nil { - return err - } - d.Topics[topic] = details - } - } - - timeout, err := pd.getInt32() - if err != nil { - return err - } - d.Timeout = time.Duration(timeout) * time.Millisecond - - return nil -} - -func (d *DeleteRecordsRequest) key() int16 { - return 21 -} - -func (d *DeleteRecordsRequest) version() int16 { - return 0 -} - -func (d *DeleteRecordsRequest) requiredVersion() KafkaVersion { - return V0_11_0_0 -} - -type DeleteRecordsRequestTopic struct { - PartitionOffsets map[int32]int64 // partition => offset -} - -func (t *DeleteRecordsRequestTopic) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(t.PartitionOffsets)); err != nil { - return err - } - keys := make([]int32, 0, len(t.PartitionOffsets)) - for partition := range t.PartitionOffsets { - keys = append(keys, partition) - } - sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) - for _, partition := range keys { - pe.putInt32(partition) - pe.putInt64(t.PartitionOffsets[partition]) - } - return nil -} - -func (t *DeleteRecordsRequestTopic) decode(pd packetDecoder, version int16) error { - n, err := pd.getArrayLength() - if err != nil { - return err - } - - if n > 0 { - t.PartitionOffsets = make(map[int32]int64, n) - for i := 0; i < n; i++ { - partition, err := pd.getInt32() - if err != nil { - return err - } - offset, err := pd.getInt64() - if err != nil { - return err - } - t.PartitionOffsets[partition] = offset - } - } - - return nil -} diff --git a/third/github.com/Shopify/sarama/delete_records_request_test.go b/third/github.com/Shopify/sarama/delete_records_request_test.go deleted file mode 100644 index c72960cfb..000000000 --- a/third/github.com/Shopify/sarama/delete_records_request_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -var deleteRecordsRequest = []byte{ - 0, 0, 0, 2, - 0, 5, 'o', 't', 'h', 'e', 'r', - 0, 0, 0, 0, - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 0, 0, 2, - 0, 0, 0, 19, - 0, 0, 0, 0, 0, 0, 0, 200, - 0, 0, 0, 20, - 0, 0, 0, 0, 0, 0, 0, 190, - 0, 0, 0, 100, -} - -func TestDeleteRecordsRequest(t *testing.T) { - req := &DeleteRecordsRequest{ - Topics: map[string]*DeleteRecordsRequestTopic{ - "topic": { - PartitionOffsets: map[int32]int64{ - 19: 200, - 20: 190, - }, - }, - "other": {}, - }, - Timeout: 100 * time.Millisecond, - } - - testRequest(t, "", req, deleteRecordsRequest) -} diff --git a/third/github.com/Shopify/sarama/delete_records_response.go b/third/github.com/Shopify/sarama/delete_records_response.go deleted file mode 100644 index 733a58b6b..000000000 --- a/third/github.com/Shopify/sarama/delete_records_response.go +++ /dev/null @@ -1,158 +0,0 @@ -package sarama - -import ( - "sort" - "time" -) - -// response message format is: -// throttleMs(int32) [topic] -// where topic is: -// name(string) [partition] -// where partition is: -// id(int32) low_watermark(int64) error_code(int16) - -type DeleteRecordsResponse struct { - Version int16 - ThrottleTime time.Duration - Topics map[string]*DeleteRecordsResponseTopic -} - -func (d *DeleteRecordsResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(d.ThrottleTime / time.Millisecond)) - - if err := pe.putArrayLength(len(d.Topics)); err != nil { - return err - } - keys := make([]string, 0, len(d.Topics)) - for topic := range d.Topics { - keys = append(keys, topic) - } - sort.Strings(keys) - for _, topic := range keys { - if err := pe.putString(topic); err != nil { - return err - } - if err := d.Topics[topic].encode(pe); err != nil { - return err - } - } - return nil -} - -func (d *DeleteRecordsResponse) decode(pd packetDecoder, version int16) error { - d.Version = version - - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - d.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - if n > 0 { - d.Topics = make(map[string]*DeleteRecordsResponseTopic, n) - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - details := new(DeleteRecordsResponseTopic) - if err = details.decode(pd, version); err != nil { - return err - } - d.Topics[topic] = details - } - } - - return nil -} - -func (d *DeleteRecordsResponse) key() int16 { - return 21 -} - -func (d *DeleteRecordsResponse) version() int16 { - return 0 -} - -func (d *DeleteRecordsResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} - -type DeleteRecordsResponseTopic struct { - Partitions map[int32]*DeleteRecordsResponsePartition -} - -func (t *DeleteRecordsResponseTopic) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(t.Partitions)); err != nil { - return err - } - keys := make([]int32, 0, len(t.Partitions)) - for partition := range t.Partitions { - keys = append(keys, partition) - } - sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) - for _, partition := range keys { - pe.putInt32(partition) - if err := t.Partitions[partition].encode(pe); err != nil { - return err - } - } - return nil -} - -func (t *DeleteRecordsResponseTopic) decode(pd packetDecoder, version int16) error { - n, err := pd.getArrayLength() - if err != nil { - return err - } - - if n > 0 { - t.Partitions = make(map[int32]*DeleteRecordsResponsePartition, n) - for i := 0; i < n; i++ { - partition, err := pd.getInt32() - if err != nil { - return err - } - details := new(DeleteRecordsResponsePartition) - if err = details.decode(pd, version); err != nil { - return err - } - t.Partitions[partition] = details - } - } - - return nil -} - -type DeleteRecordsResponsePartition struct { - LowWatermark int64 - Err KError -} - -func (t *DeleteRecordsResponsePartition) encode(pe packetEncoder) error { - pe.putInt64(t.LowWatermark) - pe.putInt16(int16(t.Err)) - return nil -} - -func (t *DeleteRecordsResponsePartition) decode(pd packetDecoder, version int16) error { - lowWatermark, err := pd.getInt64() - if err != nil { - return err - } - t.LowWatermark = lowWatermark - - kErr, err := pd.getInt16() - if err != nil { - return err - } - t.Err = KError(kErr) - - return nil -} diff --git a/third/github.com/Shopify/sarama/delete_records_response_test.go b/third/github.com/Shopify/sarama/delete_records_response_test.go deleted file mode 100644 index 3653cdc41..000000000 --- a/third/github.com/Shopify/sarama/delete_records_response_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -var deleteRecordsResponse = []byte{ - 0, 0, 0, 100, - 0, 0, 0, 2, - 0, 5, 'o', 't', 'h', 'e', 'r', - 0, 0, 0, 0, - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 0, 0, 2, - 0, 0, 0, 19, - 0, 0, 0, 0, 0, 0, 0, 200, - 0, 0, - 0, 0, 0, 20, - 255, 255, 255, 255, 255, 255, 255, 255, - 0, 3, -} - -func TestDeleteRecordsResponse(t *testing.T) { - resp := &DeleteRecordsResponse{ - Version: 0, - ThrottleTime: 100 * time.Millisecond, - Topics: map[string]*DeleteRecordsResponseTopic{ - "topic": { - Partitions: map[int32]*DeleteRecordsResponsePartition{ - 19: {LowWatermark: 200, Err: 0}, - 20: {LowWatermark: -1, Err: 3}, - }, - }, - "other": {}, - }, - } - - testResponse(t, "", resp, deleteRecordsResponse) -} diff --git a/third/github.com/Shopify/sarama/delete_topics_request.go b/third/github.com/Shopify/sarama/delete_topics_request.go deleted file mode 100644 index 911f67d31..000000000 --- a/third/github.com/Shopify/sarama/delete_topics_request.go +++ /dev/null @@ -1,48 +0,0 @@ -package sarama - -import "time" - -type DeleteTopicsRequest struct { - Version int16 - Topics []string - Timeout time.Duration -} - -func (d *DeleteTopicsRequest) encode(pe packetEncoder) error { - if err := pe.putStringArray(d.Topics); err != nil { - return err - } - pe.putInt32(int32(d.Timeout / time.Millisecond)) - - return nil -} - -func (d *DeleteTopicsRequest) decode(pd packetDecoder, version int16) (err error) { - if d.Topics, err = pd.getStringArray(); err != nil { - return err - } - timeout, err := pd.getInt32() - if err != nil { - return err - } - d.Timeout = time.Duration(timeout) * time.Millisecond - d.Version = version - return nil -} - -func (d *DeleteTopicsRequest) key() int16 { - return 20 -} - -func (d *DeleteTopicsRequest) version() int16 { - return d.Version -} - -func (d *DeleteTopicsRequest) requiredVersion() KafkaVersion { - switch d.Version { - case 1: - return V0_11_0_0 - default: - return V0_10_1_0 - } -} diff --git a/third/github.com/Shopify/sarama/delete_topics_request_test.go b/third/github.com/Shopify/sarama/delete_topics_request_test.go deleted file mode 100644 index c313a2f3b..000000000 --- a/third/github.com/Shopify/sarama/delete_topics_request_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -var deleteTopicsRequest = []byte{ - 0, 0, 0, 2, - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 5, 'o', 't', 'h', 'e', 'r', - 0, 0, 0, 100, -} - -func TestDeleteTopicsRequestV0(t *testing.T) { - req := &DeleteTopicsRequest{ - Version: 0, - Topics: []string{"topic", "other"}, - Timeout: 100 * time.Millisecond, - } - - testRequest(t, "", req, deleteTopicsRequest) -} - -func TestDeleteTopicsRequestV1(t *testing.T) { - req := &DeleteTopicsRequest{ - Version: 1, - Topics: []string{"topic", "other"}, - Timeout: 100 * time.Millisecond, - } - - testRequest(t, "", req, deleteTopicsRequest) -} diff --git a/third/github.com/Shopify/sarama/delete_topics_response.go b/third/github.com/Shopify/sarama/delete_topics_response.go deleted file mode 100644 index 34225460a..000000000 --- a/third/github.com/Shopify/sarama/delete_topics_response.go +++ /dev/null @@ -1,78 +0,0 @@ -package sarama - -import "time" - -type DeleteTopicsResponse struct { - Version int16 - ThrottleTime time.Duration - TopicErrorCodes map[string]KError -} - -func (d *DeleteTopicsResponse) encode(pe packetEncoder) error { - if d.Version >= 1 { - pe.putInt32(int32(d.ThrottleTime / time.Millisecond)) - } - - if err := pe.putArrayLength(len(d.TopicErrorCodes)); err != nil { - return err - } - for topic, errorCode := range d.TopicErrorCodes { - if err := pe.putString(topic); err != nil { - return err - } - pe.putInt16(int16(errorCode)) - } - - return nil -} - -func (d *DeleteTopicsResponse) decode(pd packetDecoder, version int16) (err error) { - if version >= 1 { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - d.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - d.Version = version - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - d.TopicErrorCodes = make(map[string]KError, n) - - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - errorCode, err := pd.getInt16() - if err != nil { - return err - } - - d.TopicErrorCodes[topic] = KError(errorCode) - } - - return nil -} - -func (d *DeleteTopicsResponse) key() int16 { - return 20 -} - -func (d *DeleteTopicsResponse) version() int16 { - return d.Version -} - -func (d *DeleteTopicsResponse) requiredVersion() KafkaVersion { - switch d.Version { - case 1: - return V0_11_0_0 - default: - return V0_10_1_0 - } -} diff --git a/third/github.com/Shopify/sarama/delete_topics_response_test.go b/third/github.com/Shopify/sarama/delete_topics_response_test.go deleted file mode 100644 index 516f1a3bd..000000000 --- a/third/github.com/Shopify/sarama/delete_topics_response_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -var ( - deleteTopicsResponseV0 = []byte{ - 0, 0, 0, 1, - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 0, - } - - deleteTopicsResponseV1 = []byte{ - 0, 0, 0, 100, - 0, 0, 0, 1, - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 0, - } -) - -func TestDeleteTopicsResponse(t *testing.T) { - resp := &DeleteTopicsResponse{ - TopicErrorCodes: map[string]KError{ - "topic": ErrNoError, - }, - } - - testResponse(t, "version 0", resp, deleteTopicsResponseV0) - - resp.Version = 1 - resp.ThrottleTime = 100 * time.Millisecond - - testResponse(t, "version 1", resp, deleteTopicsResponseV1) -} diff --git a/third/github.com/Shopify/sarama/describe_configs_request.go b/third/github.com/Shopify/sarama/describe_configs_request.go deleted file mode 100644 index 416a4fe65..000000000 --- a/third/github.com/Shopify/sarama/describe_configs_request.go +++ /dev/null @@ -1,112 +0,0 @@ -package sarama - -type DescribeConfigsRequest struct { - Version int16 - Resources []*ConfigResource - IncludeSynonyms bool -} - -type ConfigResource struct { - Type ConfigResourceType - Name string - ConfigNames []string -} - -func (r *DescribeConfigsRequest) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(r.Resources)); err != nil { - return err - } - - for _, c := range r.Resources { - pe.putInt8(int8(c.Type)) - if err := pe.putString(c.Name); err != nil { - return err - } - - if len(c.ConfigNames) == 0 { - pe.putInt32(-1) - continue - } - if err := pe.putStringArray(c.ConfigNames); err != nil { - return err - } - } - - if r.Version >= 1 { - pe.putBool(r.IncludeSynonyms) - } - - return nil -} - -func (r *DescribeConfigsRequest) decode(pd packetDecoder, version int16) (err error) { - n, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Resources = make([]*ConfigResource, n) - - for i := 0; i < n; i++ { - r.Resources[i] = &ConfigResource{} - t, err := pd.getInt8() - if err != nil { - return err - } - r.Resources[i].Type = ConfigResourceType(t) - name, err := pd.getString() - if err != nil { - return err - } - r.Resources[i].Name = name - - confLength, err := pd.getArrayLength() - - if err != nil { - return err - } - - if confLength == -1 { - continue - } - - cfnames := make([]string, confLength) - for i := 0; i < confLength; i++ { - s, err := pd.getString() - if err != nil { - return err - } - cfnames[i] = s - } - r.Resources[i].ConfigNames = cfnames - } - r.Version = version - if r.Version >= 1 { - b, err := pd.getBool() - if err != nil { - return err - } - r.IncludeSynonyms = b - } - - return nil -} - -func (r *DescribeConfigsRequest) key() int16 { - return 32 -} - -func (r *DescribeConfigsRequest) version() int16 { - return r.Version -} - -func (r *DescribeConfigsRequest) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V1_0_0_0 - case 2: - return V2_0_0_0 - default: - return V0_11_0_0 - } -} diff --git a/third/github.com/Shopify/sarama/describe_configs_request_test.go b/third/github.com/Shopify/sarama/describe_configs_request_test.go deleted file mode 100644 index fec2f1163..000000000 --- a/third/github.com/Shopify/sarama/describe_configs_request_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package sarama - -import "testing" - -var ( - emptyDescribeConfigsRequest = []byte{ - 0, 0, 0, 0, // 0 configs - } - - singleDescribeConfigsRequest = []byte{ - 0, 0, 0, 1, // 1 config - 2, // a topic - 0, 3, 'f', 'o', 'o', // topic name: foo - 0, 0, 0, 1, //1 config name - 0, 10, // 10 chars - 's', 'e', 'g', 'm', 'e', 'n', 't', '.', 'm', 's', - } - - doubleDescribeConfigsRequest = []byte{ - 0, 0, 0, 2, // 2 configs - 2, // a topic - 0, 3, 'f', 'o', 'o', // topic name: foo - 0, 0, 0, 2, //2 config name - 0, 10, // 10 chars - 's', 'e', 'g', 'm', 'e', 'n', 't', '.', 'm', 's', - 0, 12, // 12 chars - 'r', 'e', 't', 'e', 'n', 't', 'i', 'o', 'n', '.', 'm', 's', - 2, // a topic - 0, 3, 'b', 'a', 'r', // topic name: foo - 0, 0, 0, 1, // 1 config - 0, 10, // 10 chars - 's', 'e', 'g', 'm', 'e', 'n', 't', '.', 'm', 's', - } - - singleDescribeConfigsRequestAllConfigs = []byte{ - 0, 0, 0, 1, // 1 config - 2, // a topic - 0, 3, 'f', 'o', 'o', // topic name: foo - 255, 255, 255, 255, // all configs - } - - singleDescribeConfigsRequestAllConfigsv1 = []byte{ - 0, 0, 0, 1, // 1 config - 2, // a topic - 0, 3, 'f', 'o', 'o', // topic name: foo - 255, 255, 255, 255, // no configs - 1, //synoms - } -) - -func TestDescribeConfigsRequestv0(t *testing.T) { - var request *DescribeConfigsRequest - - request = &DescribeConfigsRequest{ - Version: 0, - Resources: []*ConfigResource{}, - } - testRequest(t, "no requests", request, emptyDescribeConfigsRequest) - - configs := []string{"segment.ms"} - request = &DescribeConfigsRequest{ - Version: 0, - Resources: []*ConfigResource{ - &ConfigResource{ - Type: TopicResource, - Name: "foo", - ConfigNames: configs, - }, - }, - } - - testRequest(t, "one config", request, singleDescribeConfigsRequest) - - request = &DescribeConfigsRequest{ - Version: 0, - Resources: []*ConfigResource{ - &ConfigResource{ - Type: TopicResource, - Name: "foo", - ConfigNames: []string{"segment.ms", "retention.ms"}, - }, - &ConfigResource{ - Type: TopicResource, - Name: "bar", - ConfigNames: []string{"segment.ms"}, - }, - }, - } - testRequest(t, "two configs", request, doubleDescribeConfigsRequest) - - request = &DescribeConfigsRequest{ - Version: 0, - Resources: []*ConfigResource{ - &ConfigResource{ - Type: TopicResource, - Name: "foo", - }, - }, - } - - testRequest(t, "one topic, all configs", request, singleDescribeConfigsRequestAllConfigs) -} - -func TestDescribeConfigsRequestv1(t *testing.T) { - var request *DescribeConfigsRequest - - request = &DescribeConfigsRequest{ - Version: 1, - Resources: []*ConfigResource{ - { - Type: TopicResource, - Name: "foo", - }, - }, - IncludeSynonyms: true, - } - - testRequest(t, "one topic, all configs", request, singleDescribeConfigsRequestAllConfigsv1) -} diff --git a/third/github.com/Shopify/sarama/describe_configs_response.go b/third/github.com/Shopify/sarama/describe_configs_response.go deleted file mode 100644 index 63fb6ea81..000000000 --- a/third/github.com/Shopify/sarama/describe_configs_response.go +++ /dev/null @@ -1,320 +0,0 @@ -package sarama - -import ( - "fmt" - "time" -) - -type ConfigSource int8 - -func (s ConfigSource) String() string { - switch s { - case SourceUnknown: - return "Unknown" - case SourceTopic: - return "Topic" - case SourceDynamicBroker: - return "DynamicBroker" - case SourceDynamicDefaultBroker: - return "DynamicDefaultBroker" - case SourceStaticBroker: - return "StaticBroker" - case SourceDefault: - return "Default" - } - return fmt.Sprintf("Source Invalid: %d", int(s)) -} - -const ( - SourceUnknown ConfigSource = 0 - SourceTopic ConfigSource = 1 - SourceDynamicBroker ConfigSource = 2 - SourceDynamicDefaultBroker ConfigSource = 3 - SourceStaticBroker ConfigSource = 4 - SourceDefault ConfigSource = 5 -) - -type DescribeConfigsResponse struct { - Version int16 - ThrottleTime time.Duration - Resources []*ResourceResponse -} - -type ResourceResponse struct { - ErrorCode int16 - ErrorMsg string - Type ConfigResourceType - Name string - Configs []*ConfigEntry -} - -type ConfigEntry struct { - Name string - Value string - ReadOnly bool - Default bool - Source ConfigSource - Sensitive bool - Synonyms []*ConfigSynonym -} - -type ConfigSynonym struct { - ConfigName string - ConfigValue string - Source ConfigSource -} - -func (r *DescribeConfigsResponse) encode(pe packetEncoder) (err error) { - pe.putInt32(int32(r.ThrottleTime / time.Millisecond)) - if err = pe.putArrayLength(len(r.Resources)); err != nil { - return err - } - - for _, c := range r.Resources { - if err = c.encode(pe, r.Version); err != nil { - return err - } - } - - return nil -} - -func (r *DescribeConfigsResponse) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - r.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Resources = make([]*ResourceResponse, n) - for i := 0; i < n; i++ { - rr := &ResourceResponse{} - if err := rr.decode(pd, version); err != nil { - return err - } - r.Resources[i] = rr - } - - return nil -} - -func (r *DescribeConfigsResponse) key() int16 { - return 32 -} - -func (r *DescribeConfigsResponse) version() int16 { - return r.Version -} - -func (r *DescribeConfigsResponse) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V1_0_0_0 - case 2: - return V2_0_0_0 - default: - return V0_11_0_0 - } -} - -func (r *ResourceResponse) encode(pe packetEncoder, version int16) (err error) { - pe.putInt16(r.ErrorCode) - - if err = pe.putString(r.ErrorMsg); err != nil { - return err - } - - pe.putInt8(int8(r.Type)) - - if err = pe.putString(r.Name); err != nil { - return err - } - - if err = pe.putArrayLength(len(r.Configs)); err != nil { - return err - } - - for _, c := range r.Configs { - if err = c.encode(pe, version); err != nil { - return err - } - } - return nil -} - -func (r *ResourceResponse) decode(pd packetDecoder, version int16) (err error) { - ec, err := pd.getInt16() - if err != nil { - return err - } - r.ErrorCode = ec - - em, err := pd.getString() - if err != nil { - return err - } - r.ErrorMsg = em - - t, err := pd.getInt8() - if err != nil { - return err - } - r.Type = ConfigResourceType(t) - - name, err := pd.getString() - if err != nil { - return err - } - r.Name = name - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Configs = make([]*ConfigEntry, n) - for i := 0; i < n; i++ { - c := &ConfigEntry{} - if err := c.decode(pd, version); err != nil { - return err - } - r.Configs[i] = c - } - return nil -} - -func (r *ConfigEntry) encode(pe packetEncoder, version int16) (err error) { - if err = pe.putString(r.Name); err != nil { - return err - } - - if err = pe.putString(r.Value); err != nil { - return err - } - - pe.putBool(r.ReadOnly) - - if version <= 0 { - pe.putBool(r.Default) - pe.putBool(r.Sensitive) - } else { - pe.putInt8(int8(r.Source)) - pe.putBool(r.Sensitive) - - if err := pe.putArrayLength(len(r.Synonyms)); err != nil { - return err - } - for _, c := range r.Synonyms { - if err = c.encode(pe, version); err != nil { - return err - } - } - } - - return nil -} - -//https://cwiki.apache.org/confluence/display/KAFKA/KIP-226+-+Dynamic+Broker+Configuration -func (r *ConfigEntry) decode(pd packetDecoder, version int16) (err error) { - if version == 0 { - r.Source = SourceUnknown - } - name, err := pd.getString() - if err != nil { - return err - } - r.Name = name - - value, err := pd.getString() - if err != nil { - return err - } - r.Value = value - - read, err := pd.getBool() - if err != nil { - return err - } - r.ReadOnly = read - - if version == 0 { - defaultB, err := pd.getBool() - if err != nil { - return err - } - r.Default = defaultB - } else { - source, err := pd.getInt8() - if err != nil { - return err - } - r.Source = ConfigSource(source) - } - - sensitive, err := pd.getBool() - if err != nil { - return err - } - r.Sensitive = sensitive - - if version > 0 { - n, err := pd.getArrayLength() - if err != nil { - return err - } - r.Synonyms = make([]*ConfigSynonym, n) - - for i := 0; i < n; i++ { - s := &ConfigSynonym{} - if err := s.decode(pd, version); err != nil { - return err - } - r.Synonyms[i] = s - } - - } - return nil -} - -func (c *ConfigSynonym) encode(pe packetEncoder, version int16) (err error) { - err = pe.putString(c.ConfigName) - if err != nil { - return err - } - - err = pe.putString(c.ConfigValue) - if err != nil { - return err - } - - pe.putInt8(int8(c.Source)) - - return nil -} - -func (c *ConfigSynonym) decode(pd packetDecoder, version int16) error { - name, err := pd.getString() - if err != nil { - return nil - } - c.ConfigName = name - - value, err := pd.getString() - if err != nil { - return nil - } - c.ConfigValue = value - - source, err := pd.getInt8() - if err != nil { - return nil - } - c.Source = ConfigSource(source) - return nil -} diff --git a/third/github.com/Shopify/sarama/describe_configs_response_test.go b/third/github.com/Shopify/sarama/describe_configs_response_test.go deleted file mode 100644 index 9584d577c..000000000 --- a/third/github.com/Shopify/sarama/describe_configs_response_test.go +++ /dev/null @@ -1,171 +0,0 @@ -package sarama - -import ( - "testing" -) - -var ( - describeConfigsResponseEmpty = []byte{ - 0, 0, 0, 0, //throttle - 0, 0, 0, 0, // no configs - } - - describeConfigsResponsePopulatedv0 = []byte{ - 0, 0, 0, 0, //throttle - 0, 0, 0, 1, // response - 0, 0, //errorcode - 0, 0, //string - 2, // topic - 0, 3, 'f', 'o', 'o', - 0, 0, 0, 1, //configs - 0, 10, 's', 'e', 'g', 'm', 'e', 'n', 't', '.', 'm', 's', - 0, 4, '1', '0', '0', '0', - 0, // ReadOnly - 0, // Default - 0, // Sensitive - } - - describeConfigsResponsePopulatedv1 = []byte{ - 0, 0, 0, 0, //throttle - 0, 0, 0, 1, // response - 0, 0, //errorcode - 0, 0, //string - 2, // topic - 0, 3, 'f', 'o', 'o', - 0, 0, 0, 1, //configs - 0, 10, 's', 'e', 'g', 'm', 'e', 'n', 't', '.', 'm', 's', - 0, 4, '1', '0', '0', '0', - 0, // ReadOnly - 4, // Source - 0, // Sensitive - 0, 0, 0, 0, // No Synonym - } - - describeConfigsResponseWithSynonymv1 = []byte{ - 0, 0, 0, 0, //throttle - 0, 0, 0, 1, // response - 0, 0, //errorcode - 0, 0, //string - 2, // topic - 0, 3, 'f', 'o', 'o', - 0, 0, 0, 1, //configs - 0, 10, 's', 'e', 'g', 'm', 'e', 'n', 't', '.', 'm', 's', - 0, 4, '1', '0', '0', '0', - 0, // ReadOnly - 4, // Source - 0, // Sensitive - 0, 0, 0, 1, // 1 Synonym - 0, 14, 'l', 'o', 'g', '.', 's', 'e', 'g', 'm', 'e', 'n', 't', '.', 'm', 's', - 0, 4, '1', '0', '0', '0', - 4, // Source - } -) - -func TestDescribeConfigsResponsev0(t *testing.T) { - var response *DescribeConfigsResponse - - response = &DescribeConfigsResponse{ - Resources: []*ResourceResponse{}, - } - testVersionDecodable(t, "empty", response, describeConfigsResponseEmpty, 0) - if len(response.Resources) != 0 { - t.Error("Expected no groups") - } - - response = &DescribeConfigsResponse{ - Version: 0, Resources: []*ResourceResponse{ - &ResourceResponse{ - ErrorCode: 0, - ErrorMsg: "", - Type: TopicResource, - Name: "foo", - Configs: []*ConfigEntry{ - &ConfigEntry{ - Name: "segment.ms", - Value: "1000", - ReadOnly: false, - Default: false, - Sensitive: false, - }, - }, - }, - }, - } - testResponse(t, "response with error", response, describeConfigsResponsePopulatedv0) -} - -func TestDescribeConfigsResponsev1(t *testing.T) { - var response *DescribeConfigsResponse - - response = &DescribeConfigsResponse{ - Resources: []*ResourceResponse{}, - } - testVersionDecodable(t, "empty", response, describeConfigsResponseEmpty, 0) - if len(response.Resources) != 0 { - t.Error("Expected no groups") - } - - response = &DescribeConfigsResponse{ - Version: 1, - Resources: []*ResourceResponse{ - &ResourceResponse{ - ErrorCode: 0, - ErrorMsg: "", - Type: TopicResource, - Name: "foo", - Configs: []*ConfigEntry{ - &ConfigEntry{ - Name: "segment.ms", - Value: "1000", - ReadOnly: false, - Source: SourceStaticBroker, - Sensitive: false, - Synonyms: []*ConfigSynonym{}, - }, - }, - }, - }, - } - testResponse(t, "response with error", response, describeConfigsResponsePopulatedv1) -} - -func TestDescribeConfigsResponseWithSynonym(t *testing.T) { - var response *DescribeConfigsResponse - - response = &DescribeConfigsResponse{ - Resources: []*ResourceResponse{}, - } - testVersionDecodable(t, "empty", response, describeConfigsResponseEmpty, 0) - if len(response.Resources) != 0 { - t.Error("Expected no groups") - } - - response = &DescribeConfigsResponse{ - Version: 1, - Resources: []*ResourceResponse{ - &ResourceResponse{ - ErrorCode: 0, - ErrorMsg: "", - Type: TopicResource, - Name: "foo", - Configs: []*ConfigEntry{ - &ConfigEntry{ - Name: "segment.ms", - Value: "1000", - ReadOnly: false, - Source: SourceStaticBroker, - Sensitive: false, - Synonyms: []*ConfigSynonym{ - { - ConfigName: "log.segment.ms", - ConfigValue: "1000", - Source: SourceStaticBroker, - }, - }, - }, - }, - }, - }, - } - testResponse(t, "response with error", response, describeConfigsResponseWithSynonymv1) -} diff --git a/third/github.com/Shopify/sarama/describe_groups_request.go b/third/github.com/Shopify/sarama/describe_groups_request.go deleted file mode 100644 index 1fb356777..000000000 --- a/third/github.com/Shopify/sarama/describe_groups_request.go +++ /dev/null @@ -1,30 +0,0 @@ -package sarama - -type DescribeGroupsRequest struct { - Groups []string -} - -func (r *DescribeGroupsRequest) encode(pe packetEncoder) error { - return pe.putStringArray(r.Groups) -} - -func (r *DescribeGroupsRequest) decode(pd packetDecoder, version int16) (err error) { - r.Groups, err = pd.getStringArray() - return -} - -func (r *DescribeGroupsRequest) key() int16 { - return 15 -} - -func (r *DescribeGroupsRequest) version() int16 { - return 0 -} - -func (r *DescribeGroupsRequest) requiredVersion() KafkaVersion { - return V0_9_0_0 -} - -func (r *DescribeGroupsRequest) AddGroup(group string) { - r.Groups = append(r.Groups, group) -} diff --git a/third/github.com/Shopify/sarama/describe_groups_request_test.go b/third/github.com/Shopify/sarama/describe_groups_request_test.go deleted file mode 100644 index 7d45f3fee..000000000 --- a/third/github.com/Shopify/sarama/describe_groups_request_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package sarama - -import "testing" - -var ( - emptyDescribeGroupsRequest = []byte{0, 0, 0, 0} - - singleDescribeGroupsRequest = []byte{ - 0, 0, 0, 1, // 1 group - 0, 3, 'f', 'o', 'o', // group name: foo - } - - doubleDescribeGroupsRequest = []byte{ - 0, 0, 0, 2, // 2 groups - 0, 3, 'f', 'o', 'o', // group name: foo - 0, 3, 'b', 'a', 'r', // group name: foo - } -) - -func TestDescribeGroupsRequest(t *testing.T) { - var request *DescribeGroupsRequest - - request = new(DescribeGroupsRequest) - testRequest(t, "no groups", request, emptyDescribeGroupsRequest) - - request = new(DescribeGroupsRequest) - request.AddGroup("foo") - testRequest(t, "one group", request, singleDescribeGroupsRequest) - - request = new(DescribeGroupsRequest) - request.AddGroup("foo") - request.AddGroup("bar") - testRequest(t, "two groups", request, doubleDescribeGroupsRequest) -} diff --git a/third/github.com/Shopify/sarama/describe_groups_response.go b/third/github.com/Shopify/sarama/describe_groups_response.go deleted file mode 100644 index 542b3a971..000000000 --- a/third/github.com/Shopify/sarama/describe_groups_response.go +++ /dev/null @@ -1,187 +0,0 @@ -package sarama - -type DescribeGroupsResponse struct { - Groups []*GroupDescription -} - -func (r *DescribeGroupsResponse) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(r.Groups)); err != nil { - return err - } - - for _, groupDescription := range r.Groups { - if err := groupDescription.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (r *DescribeGroupsResponse) decode(pd packetDecoder, version int16) (err error) { - n, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Groups = make([]*GroupDescription, n) - for i := 0; i < n; i++ { - r.Groups[i] = new(GroupDescription) - if err := r.Groups[i].decode(pd); err != nil { - return err - } - } - - return nil -} - -func (r *DescribeGroupsResponse) key() int16 { - return 15 -} - -func (r *DescribeGroupsResponse) version() int16 { - return 0 -} - -func (r *DescribeGroupsResponse) requiredVersion() KafkaVersion { - return V0_9_0_0 -} - -type GroupDescription struct { - Err KError - GroupId string - State string - ProtocolType string - Protocol string - Members map[string]*GroupMemberDescription -} - -func (gd *GroupDescription) encode(pe packetEncoder) error { - pe.putInt16(int16(gd.Err)) - - if err := pe.putString(gd.GroupId); err != nil { - return err - } - if err := pe.putString(gd.State); err != nil { - return err - } - if err := pe.putString(gd.ProtocolType); err != nil { - return err - } - if err := pe.putString(gd.Protocol); err != nil { - return err - } - - if err := pe.putArrayLength(len(gd.Members)); err != nil { - return err - } - - for memberId, groupMemberDescription := range gd.Members { - if err := pe.putString(memberId); err != nil { - return err - } - if err := groupMemberDescription.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (gd *GroupDescription) decode(pd packetDecoder) (err error) { - kerr, err := pd.getInt16() - if err != nil { - return err - } - - gd.Err = KError(kerr) - - if gd.GroupId, err = pd.getString(); err != nil { - return - } - if gd.State, err = pd.getString(); err != nil { - return - } - if gd.ProtocolType, err = pd.getString(); err != nil { - return - } - if gd.Protocol, err = pd.getString(); err != nil { - return - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - if n == 0 { - return nil - } - - gd.Members = make(map[string]*GroupMemberDescription) - for i := 0; i < n; i++ { - memberId, err := pd.getString() - if err != nil { - return err - } - - gd.Members[memberId] = new(GroupMemberDescription) - if err := gd.Members[memberId].decode(pd); err != nil { - return err - } - } - - return nil -} - -type GroupMemberDescription struct { - ClientId string - ClientHost string - MemberMetadata []byte - MemberAssignment []byte -} - -func (gmd *GroupMemberDescription) encode(pe packetEncoder) error { - if err := pe.putString(gmd.ClientId); err != nil { - return err - } - if err := pe.putString(gmd.ClientHost); err != nil { - return err - } - if err := pe.putBytes(gmd.MemberMetadata); err != nil { - return err - } - if err := pe.putBytes(gmd.MemberAssignment); err != nil { - return err - } - - return nil -} - -func (gmd *GroupMemberDescription) decode(pd packetDecoder) (err error) { - if gmd.ClientId, err = pd.getString(); err != nil { - return - } - if gmd.ClientHost, err = pd.getString(); err != nil { - return - } - if gmd.MemberMetadata, err = pd.getBytes(); err != nil { - return - } - if gmd.MemberAssignment, err = pd.getBytes(); err != nil { - return - } - - return nil -} - -func (gmd *GroupMemberDescription) GetMemberAssignment() (*ConsumerGroupMemberAssignment, error) { - assignment := new(ConsumerGroupMemberAssignment) - err := decode(gmd.MemberAssignment, assignment) - return assignment, err -} - -func (gmd *GroupMemberDescription) GetMemberMetadata() (*ConsumerGroupMemberMetadata, error) { - metadata := new(ConsumerGroupMemberMetadata) - err := decode(gmd.MemberMetadata, metadata) - return metadata, err -} diff --git a/third/github.com/Shopify/sarama/describe_groups_response_test.go b/third/github.com/Shopify/sarama/describe_groups_response_test.go deleted file mode 100644 index dd3973191..000000000 --- a/third/github.com/Shopify/sarama/describe_groups_response_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package sarama - -import ( - "reflect" - "testing" -) - -var ( - describeGroupsResponseEmpty = []byte{ - 0, 0, 0, 0, // no groups - } - - describeGroupsResponsePopulated = []byte{ - 0, 0, 0, 2, // 2 groups - - 0, 0, // no error - 0, 3, 'f', 'o', 'o', // Group ID - 0, 3, 'b', 'a', 'r', // State - 0, 8, 'c', 'o', 'n', 's', 'u', 'm', 'e', 'r', // ConsumerProtocol type - 0, 3, 'b', 'a', 'z', // Protocol name - 0, 0, 0, 1, // 1 member - 0, 2, 'i', 'd', // Member ID - 0, 6, 's', 'a', 'r', 'a', 'm', 'a', // Client ID - 0, 9, 'l', 'o', 'c', 'a', 'l', 'h', 'o', 's', 't', // Client Host - 0, 0, 0, 3, 0x01, 0x02, 0x03, // MemberMetadata - 0, 0, 0, 3, 0x04, 0x05, 0x06, // MemberAssignment - - 0, 30, // ErrGroupAuthorizationFailed - 0, 0, - 0, 0, - 0, 0, - 0, 0, - 0, 0, 0, 0, - } -) - -func TestDescribeGroupsResponse(t *testing.T) { - var response *DescribeGroupsResponse - - response = new(DescribeGroupsResponse) - testVersionDecodable(t, "empty", response, describeGroupsResponseEmpty, 0) - if len(response.Groups) != 0 { - t.Error("Expected no groups") - } - - response = new(DescribeGroupsResponse) - testVersionDecodable(t, "populated", response, describeGroupsResponsePopulated, 0) - if len(response.Groups) != 2 { - t.Error("Expected two groups") - } - - group0 := response.Groups[0] - if group0.Err != ErrNoError { - t.Error("Unxpected groups[0].Err, found", group0.Err) - } - if group0.GroupId != "foo" { - t.Error("Unxpected groups[0].GroupId, found", group0.GroupId) - } - if group0.State != "bar" { - t.Error("Unxpected groups[0].State, found", group0.State) - } - if group0.ProtocolType != "consumer" { - t.Error("Unxpected groups[0].ProtocolType, found", group0.ProtocolType) - } - if group0.Protocol != "baz" { - t.Error("Unxpected groups[0].Protocol, found", group0.Protocol) - } - if len(group0.Members) != 1 { - t.Error("Unxpected groups[0].Members, found", group0.Members) - } - if group0.Members["id"].ClientId != "sarama" { - t.Error("Unxpected groups[0].Members[id].ClientId, found", group0.Members["id"].ClientId) - } - if group0.Members["id"].ClientHost != "localhost" { - t.Error("Unxpected groups[0].Members[id].ClientHost, found", group0.Members["id"].ClientHost) - } - if !reflect.DeepEqual(group0.Members["id"].MemberMetadata, []byte{0x01, 0x02, 0x03}) { - t.Error("Unxpected groups[0].Members[id].MemberMetadata, found", group0.Members["id"].MemberMetadata) - } - if !reflect.DeepEqual(group0.Members["id"].MemberAssignment, []byte{0x04, 0x05, 0x06}) { - t.Error("Unxpected groups[0].Members[id].MemberAssignment, found", group0.Members["id"].MemberAssignment) - } - - group1 := response.Groups[1] - if group1.Err != ErrGroupAuthorizationFailed { - t.Error("Unxpected groups[1].Err, found", group0.Err) - } - if len(group1.Members) != 0 { - t.Error("Unxpected groups[1].Members, found", group0.Members) - } -} diff --git a/third/github.com/Shopify/sarama/dev.yml b/third/github.com/Shopify/sarama/dev.yml deleted file mode 100644 index 97eed3ad9..000000000 --- a/third/github.com/Shopify/sarama/dev.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: sarama - -up: - - go: - version: '1.11' - -commands: - test: - run: make test - desc: 'run unit tests' diff --git a/third/github.com/Shopify/sarama/encoder_decoder.go b/third/github.com/Shopify/sarama/encoder_decoder.go deleted file mode 100644 index 98bf77a6d..000000000 --- a/third/github.com/Shopify/sarama/encoder_decoder.go +++ /dev/null @@ -1,89 +0,0 @@ -package sarama - -import ( - "fmt" - - "github.com/gogf/gf/third/github.com/rcrowley/go-metrics" -) - -// Encoder is the interface that wraps the basic Encode method. -// Anything implementing Encoder can be turned into bytes using Kafka's encoding rules. -type encoder interface { - encode(pe packetEncoder) error -} - -// Encode takes an Encoder and turns it into bytes while potentially recording metrics. -func encode(e encoder, metricRegistry metrics.Registry) ([]byte, error) { - if e == nil { - return nil, nil - } - - var prepEnc prepEncoder - var realEnc realEncoder - - err := e.encode(&prepEnc) - if err != nil { - return nil, err - } - - if prepEnc.length < 0 || prepEnc.length > int(MaxRequestSize) { - return nil, PacketEncodingError{fmt.Sprintf("invalid request size (%d)", prepEnc.length)} - } - - realEnc.raw = make([]byte, prepEnc.length) - realEnc.registry = metricRegistry - err = e.encode(&realEnc) - if err != nil { - return nil, err - } - - return realEnc.raw, nil -} - -// Decoder is the interface that wraps the basic Decode method. -// Anything implementing Decoder can be extracted from bytes using Kafka's encoding rules. -type decoder interface { - decode(pd packetDecoder) error -} - -type versionedDecoder interface { - decode(pd packetDecoder, version int16) error -} - -// Decode takes bytes and a Decoder and fills the fields of the decoder from the bytes, -// interpreted using Kafka's encoding rules. -func decode(buf []byte, in decoder) error { - if buf == nil { - return nil - } - - helper := realDecoder{raw: buf} - err := in.decode(&helper) - if err != nil { - return err - } - - if helper.off != len(buf) { - return PacketDecodingError{"invalid length"} - } - - return nil -} - -func versionedDecode(buf []byte, in versionedDecoder, version int16) error { - if buf == nil { - return nil - } - - helper := realDecoder{raw: buf} - err := in.decode(&helper, version) - if err != nil { - return err - } - - if helper.off != len(buf) { - return PacketDecodingError{"invalid length"} - } - - return nil -} diff --git a/third/github.com/Shopify/sarama/end_txn_request.go b/third/github.com/Shopify/sarama/end_txn_request.go deleted file mode 100644 index 2cd9b506d..000000000 --- a/third/github.com/Shopify/sarama/end_txn_request.go +++ /dev/null @@ -1,50 +0,0 @@ -package sarama - -type EndTxnRequest struct { - TransactionalID string - ProducerID int64 - ProducerEpoch int16 - TransactionResult bool -} - -func (a *EndTxnRequest) encode(pe packetEncoder) error { - if err := pe.putString(a.TransactionalID); err != nil { - return err - } - - pe.putInt64(a.ProducerID) - - pe.putInt16(a.ProducerEpoch) - - pe.putBool(a.TransactionResult) - - return nil -} - -func (a *EndTxnRequest) decode(pd packetDecoder, version int16) (err error) { - if a.TransactionalID, err = pd.getString(); err != nil { - return err - } - if a.ProducerID, err = pd.getInt64(); err != nil { - return err - } - if a.ProducerEpoch, err = pd.getInt16(); err != nil { - return err - } - if a.TransactionResult, err = pd.getBool(); err != nil { - return err - } - return nil -} - -func (a *EndTxnRequest) key() int16 { - return 26 -} - -func (a *EndTxnRequest) version() int16 { - return 0 -} - -func (a *EndTxnRequest) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/third/github.com/Shopify/sarama/end_txn_request_test.go b/third/github.com/Shopify/sarama/end_txn_request_test.go deleted file mode 100644 index 20e404eb8..000000000 --- a/third/github.com/Shopify/sarama/end_txn_request_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package sarama - -import "testing" - -var ( - endTxnRequest = []byte{ - 0, 3, 't', 'x', 'n', - 0, 0, 0, 0, 0, 0, 31, 64, - 0, 1, - 1, - } -) - -func TestEndTxnRequest(t *testing.T) { - req := &EndTxnRequest{ - TransactionalID: "txn", - ProducerID: 8000, - ProducerEpoch: 1, - TransactionResult: true, - } - - testRequest(t, "", req, endTxnRequest) -} diff --git a/third/github.com/Shopify/sarama/end_txn_response.go b/third/github.com/Shopify/sarama/end_txn_response.go deleted file mode 100644 index 33b27e33d..000000000 --- a/third/github.com/Shopify/sarama/end_txn_response.go +++ /dev/null @@ -1,44 +0,0 @@ -package sarama - -import ( - "time" -) - -type EndTxnResponse struct { - ThrottleTime time.Duration - Err KError -} - -func (e *EndTxnResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(e.ThrottleTime / time.Millisecond)) - pe.putInt16(int16(e.Err)) - return nil -} - -func (e *EndTxnResponse) decode(pd packetDecoder, version int16) (err error) { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - e.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - kerr, err := pd.getInt16() - if err != nil { - return err - } - e.Err = KError(kerr) - - return nil -} - -func (e *EndTxnResponse) key() int16 { - return 25 -} - -func (e *EndTxnResponse) version() int16 { - return 0 -} - -func (e *EndTxnResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/third/github.com/Shopify/sarama/end_txn_response_test.go b/third/github.com/Shopify/sarama/end_txn_response_test.go deleted file mode 100644 index 41d730418..000000000 --- a/third/github.com/Shopify/sarama/end_txn_response_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -var ( - endTxnResponse = []byte{ - 0, 0, 0, 100, - 0, 49, - } -) - -func TestEndTxnResponse(t *testing.T) { - resp := &EndTxnResponse{ - ThrottleTime: 100 * time.Millisecond, - Err: ErrInvalidProducerIDMapping, - } - - testResponse(t, "", resp, endTxnResponse) -} diff --git a/third/github.com/Shopify/sarama/errors.go b/third/github.com/Shopify/sarama/errors.go deleted file mode 100644 index c11421d9e..000000000 --- a/third/github.com/Shopify/sarama/errors.go +++ /dev/null @@ -1,317 +0,0 @@ -package sarama - -import ( - "errors" - "fmt" -) - -// ErrOutOfBrokers is the error returned when the client has run out of brokers to talk to because all of them errored -// or otherwise failed to respond. -var ErrOutOfBrokers = errors.New("kafka: client has run out of available brokers to talk to (Is your cluster reachable?)") - -// ErrClosedClient is the error returned when a method is called on a client that has been closed. -var ErrClosedClient = errors.New("kafka: tried to use a client that was closed") - -// ErrIncompleteResponse is the error returned when the server returns a syntactically valid response, but it does -// not contain the expected information. -var ErrIncompleteResponse = errors.New("kafka: response did not contain all the expected topic/partition blocks") - -// ErrInvalidPartition is the error returned when a partitioner returns an invalid partition index -// (meaning one outside of the range [0...numPartitions-1]). -var ErrInvalidPartition = errors.New("kafka: partitioner returned an invalid partition index") - -// ErrAlreadyConnected is the error returned when calling Open() on a Broker that is already connected or connecting. -var ErrAlreadyConnected = errors.New("kafka: broker connection already initiated") - -// ErrNotConnected is the error returned when trying to send or call Close() on a Broker that is not connected. -var ErrNotConnected = errors.New("kafka: broker not connected") - -// ErrInsufficientData is returned when decoding and the packet is truncated. This can be expected -// when requesting messages, since as an optimization the server is allowed to return a partial message at the end -// of the message set. -var ErrInsufficientData = errors.New("kafka: insufficient data to decode packet, more bytes expected") - -// ErrShuttingDown is returned when a producer receives a message during shutdown. -var ErrShuttingDown = errors.New("kafka: message received by producer in process of shutting down") - -// ErrMessageTooLarge is returned when the next message to consume is larger than the configured Consumer.Fetch.Max -var ErrMessageTooLarge = errors.New("kafka: message is larger than Consumer.Fetch.Max") - -// ErrConsumerOffsetNotAdvanced is returned when a partition consumer didn't advance its offset after parsing -// a RecordBatch. -var ErrConsumerOffsetNotAdvanced = errors.New("kafka: consumer offset was not advanced after a RecordBatch") - -// ErrControllerNotAvailable is returned when server didn't give correct controller id. May be kafka server's version -// is lower than 0.10.0.0. -var ErrControllerNotAvailable = errors.New("kafka: controller is not available") - -// ErrNoTopicsToUpdateMetadata is returned when Meta.Full is set to false but no specific topics were found to update -// the metadata. -var ErrNoTopicsToUpdateMetadata = errors.New("kafka: no specific topics to update metadata") - -// PacketEncodingError is returned from a failure while encoding a Kafka packet. This can happen, for example, -// if you try to encode a string over 2^15 characters in length, since Kafka's encoding rules do not permit that. -type PacketEncodingError struct { - Info string -} - -func (err PacketEncodingError) Error() string { - return fmt.Sprintf("kafka: error encoding packet: %s", err.Info) -} - -// PacketDecodingError is returned when there was an error (other than truncated data) decoding the Kafka broker's response. -// This can be a bad CRC or length field, or any other invalid value. -type PacketDecodingError struct { - Info string -} - -func (err PacketDecodingError) Error() string { - return fmt.Sprintf("kafka: error decoding packet: %s", err.Info) -} - -// ConfigurationError is the type of error returned from a constructor (e.g. NewClient, or NewConsumer) -// when the specified configuration is invalid. -type ConfigurationError string - -func (err ConfigurationError) Error() string { - return "kafka: invalid configuration (" + string(err) + ")" -} - -// KError is the type of error that can be returned directly by the Kafka broker. -// See https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ErrorCodes -type KError int16 - -// Numeric error codes returned by the Kafka server. -const ( - ErrNoError KError = 0 - ErrUnknown KError = -1 - ErrOffsetOutOfRange KError = 1 - ErrInvalidMessage KError = 2 - ErrUnknownTopicOrPartition KError = 3 - ErrInvalidMessageSize KError = 4 - ErrLeaderNotAvailable KError = 5 - ErrNotLeaderForPartition KError = 6 - ErrRequestTimedOut KError = 7 - ErrBrokerNotAvailable KError = 8 - ErrReplicaNotAvailable KError = 9 - ErrMessageSizeTooLarge KError = 10 - ErrStaleControllerEpochCode KError = 11 - ErrOffsetMetadataTooLarge KError = 12 - ErrNetworkException KError = 13 - ErrOffsetsLoadInProgress KError = 14 - ErrConsumerCoordinatorNotAvailable KError = 15 - ErrNotCoordinatorForConsumer KError = 16 - ErrInvalidTopic KError = 17 - ErrMessageSetSizeTooLarge KError = 18 - ErrNotEnoughReplicas KError = 19 - ErrNotEnoughReplicasAfterAppend KError = 20 - ErrInvalidRequiredAcks KError = 21 - ErrIllegalGeneration KError = 22 - ErrInconsistentGroupProtocol KError = 23 - ErrInvalidGroupId KError = 24 - ErrUnknownMemberId KError = 25 - ErrInvalidSessionTimeout KError = 26 - ErrRebalanceInProgress KError = 27 - ErrInvalidCommitOffsetSize KError = 28 - ErrTopicAuthorizationFailed KError = 29 - ErrGroupAuthorizationFailed KError = 30 - ErrClusterAuthorizationFailed KError = 31 - ErrInvalidTimestamp KError = 32 - ErrUnsupportedSASLMechanism KError = 33 - ErrIllegalSASLState KError = 34 - ErrUnsupportedVersion KError = 35 - ErrTopicAlreadyExists KError = 36 - ErrInvalidPartitions KError = 37 - ErrInvalidReplicationFactor KError = 38 - ErrInvalidReplicaAssignment KError = 39 - ErrInvalidConfig KError = 40 - ErrNotController KError = 41 - ErrInvalidRequest KError = 42 - ErrUnsupportedForMessageFormat KError = 43 - ErrPolicyViolation KError = 44 - ErrOutOfOrderSequenceNumber KError = 45 - ErrDuplicateSequenceNumber KError = 46 - ErrInvalidProducerEpoch KError = 47 - ErrInvalidTxnState KError = 48 - ErrInvalidProducerIDMapping KError = 49 - ErrInvalidTransactionTimeout KError = 50 - ErrConcurrentTransactions KError = 51 - ErrTransactionCoordinatorFenced KError = 52 - ErrTransactionalIDAuthorizationFailed KError = 53 - ErrSecurityDisabled KError = 54 - ErrOperationNotAttempted KError = 55 - ErrKafkaStorageError KError = 56 - ErrLogDirNotFound KError = 57 - ErrSASLAuthenticationFailed KError = 58 - ErrUnknownProducerID KError = 59 - ErrReassignmentInProgress KError = 60 - ErrDelegationTokenAuthDisabled KError = 61 - ErrDelegationTokenNotFound KError = 62 - ErrDelegationTokenOwnerMismatch KError = 63 - ErrDelegationTokenRequestNotAllowed KError = 64 - ErrDelegationTokenAuthorizationFailed KError = 65 - ErrDelegationTokenExpired KError = 66 - ErrInvalidPrincipalType KError = 67 - ErrNonEmptyGroup KError = 68 - ErrGroupIDNotFound KError = 69 - ErrFetchSessionIDNotFound KError = 70 - ErrInvalidFetchSessionEpoch KError = 71 - ErrListenerNotFound KError = 72 -) - -func (err KError) Error() string { - // Error messages stolen/adapted from - // https://kafka.apache.org/protocol#protocol_error_codes - switch err { - case ErrNoError: - return "kafka server: Not an error, why are you printing me?" - case ErrUnknown: - return "kafka server: Unexpected (unknown?) server error." - case ErrOffsetOutOfRange: - return "kafka server: The requested offset is outside the range of offsets maintained by the server for the given topic/partition." - case ErrInvalidMessage: - return "kafka server: Message contents does not match its CRC." - case ErrUnknownTopicOrPartition: - return "kafka server: Request was for a topic or partition that does not exist on this broker." - case ErrInvalidMessageSize: - return "kafka server: The message has a negative size." - case ErrLeaderNotAvailable: - return "kafka server: In the middle of a leadership election, there is currently no leader for this partition and hence it is unavailable for writes." - case ErrNotLeaderForPartition: - return "kafka server: Tried to send a message to a replica that is not the leader for some partition. Your metadata is out of date." - case ErrRequestTimedOut: - return "kafka server: Request exceeded the user-specified time limit in the request." - case ErrBrokerNotAvailable: - return "kafka server: Broker not available. Not a client facing error, we should never receive this!!!" - case ErrReplicaNotAvailable: - return "kafka server: Replica information not available, one or more brokers are down." - case ErrMessageSizeTooLarge: - return "kafka server: Message was too large, server rejected it to avoid allocation error." - case ErrStaleControllerEpochCode: - return "kafka server: StaleControllerEpochCode (internal error code for broker-to-broker communication)." - case ErrOffsetMetadataTooLarge: - return "kafka server: Specified a string larger than the configured maximum for offset metadata." - case ErrNetworkException: - return "kafka server: The server disconnected before a response was received." - case ErrOffsetsLoadInProgress: - return "kafka server: The broker is still loading offsets after a leader change for that offset's topic partition." - case ErrConsumerCoordinatorNotAvailable: - return "kafka server: Offset's topic has not yet been created." - case ErrNotCoordinatorForConsumer: - return "kafka server: Request was for a consumer group that is not coordinated by this broker." - case ErrInvalidTopic: - return "kafka server: The request attempted to perform an operation on an invalid topic." - case ErrMessageSetSizeTooLarge: - return "kafka server: The request included message batch larger than the configured segment size on the server." - case ErrNotEnoughReplicas: - return "kafka server: Messages are rejected since there are fewer in-sync replicas than required." - case ErrNotEnoughReplicasAfterAppend: - return "kafka server: Messages are written to the log, but to fewer in-sync replicas than required." - case ErrInvalidRequiredAcks: - return "kafka server: The number of required acks is invalid (should be either -1, 0, or 1)." - case ErrIllegalGeneration: - return "kafka server: The provided generation id is not the current generation." - case ErrInconsistentGroupProtocol: - return "kafka server: The provider group protocol type is incompatible with the other members." - case ErrInvalidGroupId: - return "kafka server: The provided group id was empty." - case ErrUnknownMemberId: - return "kafka server: The provided member is not known in the current generation." - case ErrInvalidSessionTimeout: - return "kafka server: The provided session timeout is outside the allowed range." - case ErrRebalanceInProgress: - return "kafka server: A rebalance for the group is in progress. Please re-join the group." - case ErrInvalidCommitOffsetSize: - return "kafka server: The provided commit metadata was too large." - case ErrTopicAuthorizationFailed: - return "kafka server: The client is not authorized to access this topic." - case ErrGroupAuthorizationFailed: - return "kafka server: The client is not authorized to access this group." - case ErrClusterAuthorizationFailed: - return "kafka server: The client is not authorized to send this request type." - case ErrInvalidTimestamp: - return "kafka server: The timestamp of the message is out of acceptable range." - case ErrUnsupportedSASLMechanism: - return "kafka server: The broker does not support the requested SASL mechanism." - case ErrIllegalSASLState: - return "kafka server: Request is not valid given the current SASL state." - case ErrUnsupportedVersion: - return "kafka server: The version of API is not supported." - case ErrTopicAlreadyExists: - return "kafka server: Topic with this name already exists." - case ErrInvalidPartitions: - return "kafka server: Number of partitions is invalid." - case ErrInvalidReplicationFactor: - return "kafka server: Replication-factor is invalid." - case ErrInvalidReplicaAssignment: - return "kafka server: Replica assignment is invalid." - case ErrInvalidConfig: - return "kafka server: Configuration is invalid." - case ErrNotController: - return "kafka server: This is not the correct controller for this cluster." - case ErrInvalidRequest: - return "kafka server: This most likely occurs because of a request being malformed by the client library or the message was sent to an incompatible broker. See the broker logs for more details." - case ErrUnsupportedForMessageFormat: - return "kafka server: The requested operation is not supported by the message format version." - case ErrPolicyViolation: - return "kafka server: Request parameters do not satisfy the configured policy." - case ErrOutOfOrderSequenceNumber: - return "kafka server: The broker received an out of order sequence number." - case ErrDuplicateSequenceNumber: - return "kafka server: The broker received a duplicate sequence number." - case ErrInvalidProducerEpoch: - return "kafka server: Producer attempted an operation with an old epoch." - case ErrInvalidTxnState: - return "kafka server: The producer attempted a transactional operation in an invalid state." - case ErrInvalidProducerIDMapping: - return "kafka server: The producer attempted to use a producer id which is not currently assigned to its transactional id." - case ErrInvalidTransactionTimeout: - return "kafka server: The transaction timeout is larger than the maximum value allowed by the broker (as configured by max.transaction.timeout.ms)." - case ErrConcurrentTransactions: - return "kafka server: The producer attempted to update a transaction while another concurrent operation on the same transaction was ongoing." - case ErrTransactionCoordinatorFenced: - return "kafka server: The transaction coordinator sending a WriteTxnMarker is no longer the current coordinator for a given producer." - case ErrTransactionalIDAuthorizationFailed: - return "kafka server: Transactional ID authorization failed." - case ErrSecurityDisabled: - return "kafka server: Security features are disabled." - case ErrOperationNotAttempted: - return "kafka server: The broker did not attempt to execute this operation." - case ErrKafkaStorageError: - return "kafka server: Disk error when trying to access log file on the disk." - case ErrLogDirNotFound: - return "kafka server: The specified log directory is not found in the broker config." - case ErrSASLAuthenticationFailed: - return "kafka server: SASL Authentication failed." - case ErrUnknownProducerID: - return "kafka server: The broker could not locate the producer metadata associated with the Producer ID." - case ErrReassignmentInProgress: - return "kafka server: A partition reassignment is in progress." - case ErrDelegationTokenAuthDisabled: - return "kafka server: Delegation Token feature is not enabled." - case ErrDelegationTokenNotFound: - return "kafka server: Delegation Token is not found on server." - case ErrDelegationTokenOwnerMismatch: - return "kafka server: Specified Principal is not valid Owner/Renewer." - case ErrDelegationTokenRequestNotAllowed: - return "kafka server: Delegation Token requests are not allowed on PLAINTEXT/1-way SSL channels and on delegation token authenticated channels." - case ErrDelegationTokenAuthorizationFailed: - return "kafka server: Delegation Token authorization failed." - case ErrDelegationTokenExpired: - return "kafka server: Delegation Token is expired." - case ErrInvalidPrincipalType: - return "kafka server: Supplied principalType is not supported." - case ErrNonEmptyGroup: - return "kafka server: The group is not empty." - case ErrGroupIDNotFound: - return "kafka server: The group id does not exist." - case ErrFetchSessionIDNotFound: - return "kafka server: The fetch session ID was not found." - case ErrInvalidFetchSessionEpoch: - return "kafka server: The fetch session epoch is invalid." - case ErrListenerNotFound: - return "kafka server: There is no listener on the leader broker that matches the listener on which metadata request was processed." - } - - return fmt.Sprintf("Unknown error, how did this happen? Error code = %d", err) -} diff --git a/third/github.com/Shopify/sarama/fetch_request.go b/third/github.com/Shopify/sarama/fetch_request.go deleted file mode 100644 index 462ab8afb..000000000 --- a/third/github.com/Shopify/sarama/fetch_request.go +++ /dev/null @@ -1,170 +0,0 @@ -package sarama - -type fetchRequestBlock struct { - fetchOffset int64 - maxBytes int32 -} - -func (b *fetchRequestBlock) encode(pe packetEncoder) error { - pe.putInt64(b.fetchOffset) - pe.putInt32(b.maxBytes) - return nil -} - -func (b *fetchRequestBlock) decode(pd packetDecoder) (err error) { - if b.fetchOffset, err = pd.getInt64(); err != nil { - return err - } - if b.maxBytes, err = pd.getInt32(); err != nil { - return err - } - return nil -} - -// FetchRequest (API key 1) will fetch Kafka messages. Version 3 introduced the MaxBytes field. See -// https://issues.apache.org/jira/browse/KAFKA-2063 for a discussion of the issues leading up to that. The KIP is at -// https://cwiki.apache.org/confluence/display/KAFKA/KIP-74%3A+Add+Fetch+Response+Size+Limit+in+Bytes -type FetchRequest struct { - MaxWaitTime int32 - MinBytes int32 - MaxBytes int32 - Version int16 - Isolation IsolationLevel - blocks map[string]map[int32]*fetchRequestBlock -} - -type IsolationLevel int8 - -const ( - ReadUncommitted IsolationLevel = 0 - ReadCommitted IsolationLevel = 1 -) - -func (r *FetchRequest) encode(pe packetEncoder) (err error) { - pe.putInt32(-1) // replica ID is always -1 for clients - pe.putInt32(r.MaxWaitTime) - pe.putInt32(r.MinBytes) - if r.Version >= 3 { - pe.putInt32(r.MaxBytes) - } - if r.Version >= 4 { - pe.putInt8(int8(r.Isolation)) - } - err = pe.putArrayLength(len(r.blocks)) - if err != nil { - return err - } - for topic, blocks := range r.blocks { - err = pe.putString(topic) - if err != nil { - return err - } - err = pe.putArrayLength(len(blocks)) - if err != nil { - return err - } - for partition, block := range blocks { - pe.putInt32(partition) - err = block.encode(pe) - if err != nil { - return err - } - } - } - return nil -} - -func (r *FetchRequest) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - if _, err = pd.getInt32(); err != nil { - return err - } - if r.MaxWaitTime, err = pd.getInt32(); err != nil { - return err - } - if r.MinBytes, err = pd.getInt32(); err != nil { - return err - } - if r.Version >= 3 { - if r.MaxBytes, err = pd.getInt32(); err != nil { - return err - } - } - if r.Version >= 4 { - isolation, err := pd.getInt8() - if err != nil { - return err - } - r.Isolation = IsolationLevel(isolation) - } - topicCount, err := pd.getArrayLength() - if err != nil { - return err - } - if topicCount == 0 { - return nil - } - r.blocks = make(map[string]map[int32]*fetchRequestBlock) - for i := 0; i < topicCount; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - partitionCount, err := pd.getArrayLength() - if err != nil { - return err - } - r.blocks[topic] = make(map[int32]*fetchRequestBlock) - for j := 0; j < partitionCount; j++ { - partition, err := pd.getInt32() - if err != nil { - return err - } - fetchBlock := &fetchRequestBlock{} - if err = fetchBlock.decode(pd); err != nil { - return err - } - r.blocks[topic][partition] = fetchBlock - } - } - return nil -} - -func (r *FetchRequest) key() int16 { - return 1 -} - -func (r *FetchRequest) version() int16 { - return r.Version -} - -func (r *FetchRequest) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_9_0_0 - case 2: - return V0_10_0_0 - case 3: - return V0_10_1_0 - case 4: - return V0_11_0_0 - default: - return MinVersion - } -} - -func (r *FetchRequest) AddBlock(topic string, partitionID int32, fetchOffset int64, maxBytes int32) { - if r.blocks == nil { - r.blocks = make(map[string]map[int32]*fetchRequestBlock) - } - - if r.blocks[topic] == nil { - r.blocks[topic] = make(map[int32]*fetchRequestBlock) - } - - tmp := new(fetchRequestBlock) - tmp.maxBytes = maxBytes - tmp.fetchOffset = fetchOffset - - r.blocks[topic][partitionID] = tmp -} diff --git a/third/github.com/Shopify/sarama/fetch_request_test.go b/third/github.com/Shopify/sarama/fetch_request_test.go deleted file mode 100644 index 1a94c2d1f..000000000 --- a/third/github.com/Shopify/sarama/fetch_request_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package sarama - -import "testing" - -var ( - fetchRequestNoBlocks = []byte{ - 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00} - - fetchRequestWithProperties = []byte{ - 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0xEF, - 0x00, 0x00, 0x00, 0x00} - - fetchRequestOneBlock = []byte{ - 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x05, 't', 'o', 'p', 'i', 'c', - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x56} - - fetchRequestOneBlockV4 = []byte{ - 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0xFF, - 0x01, - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x05, 't', 'o', 'p', 'i', 'c', - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x56} -) - -func TestFetchRequest(t *testing.T) { - request := new(FetchRequest) - testRequest(t, "no blocks", request, fetchRequestNoBlocks) - - request.MaxWaitTime = 0x20 - request.MinBytes = 0xEF - testRequest(t, "with properties", request, fetchRequestWithProperties) - - request.MaxWaitTime = 0 - request.MinBytes = 0 - request.AddBlock("topic", 0x12, 0x34, 0x56) - testRequest(t, "one block", request, fetchRequestOneBlock) - - request.Version = 4 - request.MaxBytes = 0xFF - request.Isolation = ReadCommitted - testRequest(t, "one block v4", request, fetchRequestOneBlockV4) -} diff --git a/third/github.com/Shopify/sarama/fetch_response.go b/third/github.com/Shopify/sarama/fetch_response.go deleted file mode 100644 index 90acfc280..000000000 --- a/third/github.com/Shopify/sarama/fetch_response.go +++ /dev/null @@ -1,396 +0,0 @@ -package sarama - -import ( - "time" -) - -type AbortedTransaction struct { - ProducerID int64 - FirstOffset int64 -} - -func (t *AbortedTransaction) decode(pd packetDecoder) (err error) { - if t.ProducerID, err = pd.getInt64(); err != nil { - return err - } - - if t.FirstOffset, err = pd.getInt64(); err != nil { - return err - } - - return nil -} - -func (t *AbortedTransaction) encode(pe packetEncoder) (err error) { - pe.putInt64(t.ProducerID) - pe.putInt64(t.FirstOffset) - - return nil -} - -type FetchResponseBlock struct { - Err KError - HighWaterMarkOffset int64 - LastStableOffset int64 - AbortedTransactions []*AbortedTransaction - Records *Records // deprecated: use FetchResponseBlock.RecordsSet - RecordsSet []*Records - Partial bool -} - -func (b *FetchResponseBlock) decode(pd packetDecoder, version int16) (err error) { - tmp, err := pd.getInt16() - if err != nil { - return err - } - b.Err = KError(tmp) - - b.HighWaterMarkOffset, err = pd.getInt64() - if err != nil { - return err - } - - if version >= 4 { - b.LastStableOffset, err = pd.getInt64() - if err != nil { - return err - } - - numTransact, err := pd.getArrayLength() - if err != nil { - return err - } - - if numTransact >= 0 { - b.AbortedTransactions = make([]*AbortedTransaction, numTransact) - } - - for i := 0; i < numTransact; i++ { - transact := new(AbortedTransaction) - if err = transact.decode(pd); err != nil { - return err - } - b.AbortedTransactions[i] = transact - } - } - - recordsSize, err := pd.getInt32() - if err != nil { - return err - } - - recordsDecoder, err := pd.getSubset(int(recordsSize)) - if err != nil { - return err - } - - b.RecordsSet = []*Records{} - - for recordsDecoder.remaining() > 0 { - records := &Records{} - if err := records.decode(recordsDecoder); err != nil { - // If we have at least one decoded records, this is not an error - if err == ErrInsufficientData { - if len(b.RecordsSet) == 0 { - b.Partial = true - } - break - } - return err - } - - partial, err := records.isPartial() - if err != nil { - return err - } - - n, err := records.numRecords() - if err != nil { - return err - } - - if n > 0 || (partial && len(b.RecordsSet) == 0) { - b.RecordsSet = append(b.RecordsSet, records) - - if b.Records == nil { - b.Records = records - } - } - - overflow, err := records.isOverflow() - if err != nil { - return err - } - - if partial || overflow { - break - } - } - - return nil -} - -func (b *FetchResponseBlock) numRecords() (int, error) { - sum := 0 - - for _, records := range b.RecordsSet { - count, err := records.numRecords() - if err != nil { - return 0, err - } - - sum += count - } - - return sum, nil -} - -func (b *FetchResponseBlock) isPartial() (bool, error) { - if b.Partial { - return true, nil - } - - if len(b.RecordsSet) == 1 { - return b.RecordsSet[0].isPartial() - } - - return false, nil -} - -func (b *FetchResponseBlock) encode(pe packetEncoder, version int16) (err error) { - pe.putInt16(int16(b.Err)) - - pe.putInt64(b.HighWaterMarkOffset) - - if version >= 4 { - pe.putInt64(b.LastStableOffset) - - if err = pe.putArrayLength(len(b.AbortedTransactions)); err != nil { - return err - } - for _, transact := range b.AbortedTransactions { - if err = transact.encode(pe); err != nil { - return err - } - } - } - - pe.push(&lengthField{}) - for _, records := range b.RecordsSet { - err = records.encode(pe) - if err != nil { - return err - } - } - return pe.pop() -} - -type FetchResponse struct { - Blocks map[string]map[int32]*FetchResponseBlock - ThrottleTime time.Duration - Version int16 // v1 requires 0.9+, v2 requires 0.10+ -} - -func (r *FetchResponse) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if r.Version >= 1 { - throttle, err := pd.getInt32() - if err != nil { - return err - } - r.ThrottleTime = time.Duration(throttle) * time.Millisecond - } - - numTopics, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Blocks = make(map[string]map[int32]*FetchResponseBlock, numTopics) - for i := 0; i < numTopics; i++ { - name, err := pd.getString() - if err != nil { - return err - } - - numBlocks, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Blocks[name] = make(map[int32]*FetchResponseBlock, numBlocks) - - for j := 0; j < numBlocks; j++ { - id, err := pd.getInt32() - if err != nil { - return err - } - - block := new(FetchResponseBlock) - err = block.decode(pd, version) - if err != nil { - return err - } - r.Blocks[name][id] = block - } - } - - return nil -} - -func (r *FetchResponse) encode(pe packetEncoder) (err error) { - if r.Version >= 1 { - pe.putInt32(int32(r.ThrottleTime / time.Millisecond)) - } - - err = pe.putArrayLength(len(r.Blocks)) - if err != nil { - return err - } - - for topic, partitions := range r.Blocks { - err = pe.putString(topic) - if err != nil { - return err - } - - err = pe.putArrayLength(len(partitions)) - if err != nil { - return err - } - - for id, block := range partitions { - pe.putInt32(id) - err = block.encode(pe, r.Version) - if err != nil { - return err - } - } - - } - return nil -} - -func (r *FetchResponse) key() int16 { - return 1 -} - -func (r *FetchResponse) version() int16 { - return r.Version -} - -func (r *FetchResponse) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_9_0_0 - case 2: - return V0_10_0_0 - case 3: - return V0_10_1_0 - case 4: - return V0_11_0_0 - default: - return MinVersion - } -} - -func (r *FetchResponse) GetBlock(topic string, partition int32) *FetchResponseBlock { - if r.Blocks == nil { - return nil - } - - if r.Blocks[topic] == nil { - return nil - } - - return r.Blocks[topic][partition] -} - -func (r *FetchResponse) AddError(topic string, partition int32, err KError) { - if r.Blocks == nil { - r.Blocks = make(map[string]map[int32]*FetchResponseBlock) - } - partitions, ok := r.Blocks[topic] - if !ok { - partitions = make(map[int32]*FetchResponseBlock) - r.Blocks[topic] = partitions - } - frb, ok := partitions[partition] - if !ok { - frb = new(FetchResponseBlock) - partitions[partition] = frb - } - frb.Err = err -} - -func (r *FetchResponse) getOrCreateBlock(topic string, partition int32) *FetchResponseBlock { - if r.Blocks == nil { - r.Blocks = make(map[string]map[int32]*FetchResponseBlock) - } - partitions, ok := r.Blocks[topic] - if !ok { - partitions = make(map[int32]*FetchResponseBlock) - r.Blocks[topic] = partitions - } - frb, ok := partitions[partition] - if !ok { - frb = new(FetchResponseBlock) - partitions[partition] = frb - } - - return frb -} - -func encodeKV(key, value Encoder) ([]byte, []byte) { - var kb []byte - var vb []byte - if key != nil { - kb, _ = key.Encode() - } - if value != nil { - vb, _ = value.Encode() - } - - return kb, vb -} - -func (r *FetchResponse) AddMessage(topic string, partition int32, key, value Encoder, offset int64) { - frb := r.getOrCreateBlock(topic, partition) - kb, vb := encodeKV(key, value) - msg := &Message{Key: kb, Value: vb} - msgBlock := &MessageBlock{Msg: msg, Offset: offset} - if len(frb.RecordsSet) == 0 { - records := newLegacyRecords(&MessageSet{}) - frb.RecordsSet = []*Records{&records} - } - set := frb.RecordsSet[0].MsgSet - set.Messages = append(set.Messages, msgBlock) -} - -func (r *FetchResponse) AddRecord(topic string, partition int32, key, value Encoder, offset int64) { - frb := r.getOrCreateBlock(topic, partition) - kb, vb := encodeKV(key, value) - rec := &Record{Key: kb, Value: vb, OffsetDelta: offset} - if len(frb.RecordsSet) == 0 { - records := newDefaultRecords(&RecordBatch{Version: 2}) - frb.RecordsSet = []*Records{&records} - } - batch := frb.RecordsSet[0].RecordBatch - batch.addRecord(rec) -} - -func (r *FetchResponse) SetLastOffsetDelta(topic string, partition int32, offset int32) { - frb := r.getOrCreateBlock(topic, partition) - if len(frb.RecordsSet) == 0 { - records := newDefaultRecords(&RecordBatch{Version: 2}) - frb.RecordsSet = []*Records{&records} - } - batch := frb.RecordsSet[0].RecordBatch - batch.LastOffsetDelta = offset -} - -func (r *FetchResponse) SetLastStableOffset(topic string, partition int32, offset int64) { - frb := r.getOrCreateBlock(topic, partition) - frb.LastStableOffset = offset -} diff --git a/third/github.com/Shopify/sarama/fetch_response_test.go b/third/github.com/Shopify/sarama/fetch_response_test.go deleted file mode 100644 index 917027644..000000000 --- a/third/github.com/Shopify/sarama/fetch_response_test.go +++ /dev/null @@ -1,331 +0,0 @@ -package sarama - -import ( - "bytes" - "testing" -) - -var ( - emptyFetchResponse = []byte{ - 0x00, 0x00, 0x00, 0x00} - - oneMessageFetchResponse = []byte{ - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x05, 't', 'o', 'p', 'i', 'c', - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x05, - 0x00, 0x01, - 0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x10, 0x10, - 0x00, 0x00, 0x00, 0x1C, - // messageSet - 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x10, - // message - 0x23, 0x96, 0x4a, 0xf7, // CRC - 0x00, - 0x00, - 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x00, 0x00, 0x02, 0x00, 0xEE} - - overflowMessageFetchResponse = []byte{ - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x05, 't', 'o', 'p', 'i', 'c', - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x05, - 0x00, 0x01, - 0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x10, 0x10, - 0x00, 0x00, 0x00, 0x30, - // messageSet - 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x10, - // message - 0x23, 0x96, 0x4a, 0xf7, // CRC - 0x00, - 0x00, - 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x00, 0x00, 0x02, 0x00, 0xEE, - // overflow messageSet - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x00, 0x00, 0xFF, - // overflow bytes - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} - - oneRecordFetchResponse = []byte{ - 0x00, 0x00, 0x00, 0x00, // ThrottleTime - 0x00, 0x00, 0x00, 0x01, // Number of Topics - 0x00, 0x05, 't', 'o', 'p', 'i', 'c', // Topic - 0x00, 0x00, 0x00, 0x01, // Number of Partitions - 0x00, 0x00, 0x00, 0x05, // Partition - 0x00, 0x01, // Error - 0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x10, 0x10, // High Watermark Offset - 0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x10, 0x10, // Last Stable Offset - 0x00, 0x00, 0x00, 0x00, // Number of Aborted Transactions - 0x00, 0x00, 0x00, 0x52, // Records length - // recordBatch - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x46, - 0x00, 0x00, 0x00, 0x00, - 0x02, - 0xDB, 0x47, 0x14, 0xC9, - 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0A, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x01, - // record - 0x28, - 0x00, - 0x0A, - 0x00, - 0x08, 0x01, 0x02, 0x03, 0x04, - 0x06, 0x05, 0x06, 0x07, - 0x02, - 0x06, 0x08, 0x09, 0x0A, - 0x04, 0x0B, 0x0C} - - oneMessageFetchResponseV4 = []byte{ - 0x00, 0x00, 0x00, 0x00, // ThrottleTime - 0x00, 0x00, 0x00, 0x01, // Number of Topics - 0x00, 0x05, 't', 'o', 'p', 'i', 'c', // Topic - 0x00, 0x00, 0x00, 0x01, // Number of Partitions - 0x00, 0x00, 0x00, 0x05, // Partition - 0x00, 0x01, // Error - 0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x10, 0x10, // High Watermark Offset - 0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x10, 0x10, // Last Stable Offset - 0x00, 0x00, 0x00, 0x00, // Number of Aborted Transactions - 0x00, 0x00, 0x00, 0x1C, - // messageSet - 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x10, - // message - 0x23, 0x96, 0x4a, 0xf7, // CRC - 0x00, - 0x00, - 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x00, 0x00, 0x02, 0x00, 0xEE} -) - -func TestEmptyFetchResponse(t *testing.T) { - response := FetchResponse{} - testVersionDecodable(t, "empty", &response, emptyFetchResponse, 0) - - if len(response.Blocks) != 0 { - t.Error("Decoding produced topic blocks where there were none.") - } - -} - -func TestOneMessageFetchResponse(t *testing.T) { - response := FetchResponse{} - testVersionDecodable(t, "one message", &response, oneMessageFetchResponse, 0) - - if len(response.Blocks) != 1 { - t.Fatal("Decoding produced incorrect number of topic blocks.") - } - - if len(response.Blocks["topic"]) != 1 { - t.Fatal("Decoding produced incorrect number of partition blocks for topic.") - } - - block := response.GetBlock("topic", 5) - if block == nil { - t.Fatal("GetBlock didn't return block.") - } - if block.Err != ErrOffsetOutOfRange { - t.Error("Decoding didn't produce correct error code.") - } - if block.HighWaterMarkOffset != 0x10101010 { - t.Error("Decoding didn't produce correct high water mark offset.") - } - partial, err := block.isPartial() - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - if partial { - t.Error("Decoding detected a partial trailing message where there wasn't one.") - } - - n, err := block.numRecords() - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - if n != 1 { - t.Fatal("Decoding produced incorrect number of messages.") - } - msgBlock := block.RecordsSet[0].MsgSet.Messages[0] - if msgBlock.Offset != 0x550000 { - t.Error("Decoding produced incorrect message offset.") - } - msg := msgBlock.Msg - if msg.Codec != CompressionNone { - t.Error("Decoding produced incorrect message compression.") - } - if msg.Key != nil { - t.Error("Decoding produced message key where there was none.") - } - if !bytes.Equal(msg.Value, []byte{0x00, 0xEE}) { - t.Error("Decoding produced incorrect message value.") - } -} - -func TestOverflowMessageFetchResponse(t *testing.T) { - response := FetchResponse{} - testVersionDecodable(t, "overflow message", &response, overflowMessageFetchResponse, 0) - - if len(response.Blocks) != 1 { - t.Fatal("Decoding produced incorrect number of topic blocks.") - } - - if len(response.Blocks["topic"]) != 1 { - t.Fatal("Decoding produced incorrect number of partition blocks for topic.") - } - - block := response.GetBlock("topic", 5) - if block == nil { - t.Fatal("GetBlock didn't return block.") - } - if block.Err != ErrOffsetOutOfRange { - t.Error("Decoding didn't produce correct error code.") - } - if block.HighWaterMarkOffset != 0x10101010 { - t.Error("Decoding didn't produce correct high water mark offset.") - } - partial, err := block.Records.isPartial() - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - if partial { - t.Error("Decoding detected a partial trailing message where there wasn't one.") - } - overflow, err := block.Records.isOverflow() - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - if !overflow { - t.Error("Decoding detected a partial trailing message where there wasn't one.") - } - - n, err := block.Records.numRecords() - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - if n != 1 { - t.Fatal("Decoding produced incorrect number of messages.") - } - msgBlock := block.Records.MsgSet.Messages[0] - if msgBlock.Offset != 0x550000 { - t.Error("Decoding produced incorrect message offset.") - } - msg := msgBlock.Msg - if msg.Codec != CompressionNone { - t.Error("Decoding produced incorrect message compression.") - } - if msg.Key != nil { - t.Error("Decoding produced message key where there was none.") - } - if !bytes.Equal(msg.Value, []byte{0x00, 0xEE}) { - t.Error("Decoding produced incorrect message value.") - } -} - -func TestOneRecordFetchResponse(t *testing.T) { - response := FetchResponse{} - testVersionDecodable(t, "one record", &response, oneRecordFetchResponse, 4) - - if len(response.Blocks) != 1 { - t.Fatal("Decoding produced incorrect number of topic blocks.") - } - - if len(response.Blocks["topic"]) != 1 { - t.Fatal("Decoding produced incorrect number of partition blocks for topic.") - } - - block := response.GetBlock("topic", 5) - if block == nil { - t.Fatal("GetBlock didn't return block.") - } - if block.Err != ErrOffsetOutOfRange { - t.Error("Decoding didn't produce correct error code.") - } - if block.HighWaterMarkOffset != 0x10101010 { - t.Error("Decoding didn't produce correct high water mark offset.") - } - partial, err := block.isPartial() - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - if partial { - t.Error("Decoding detected a partial trailing record where there wasn't one.") - } - - n, err := block.numRecords() - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - if n != 1 { - t.Fatal("Decoding produced incorrect number of records.") - } - rec := block.RecordsSet[0].RecordBatch.Records[0] - if !bytes.Equal(rec.Key, []byte{0x01, 0x02, 0x03, 0x04}) { - t.Error("Decoding produced incorrect record key.") - } - if !bytes.Equal(rec.Value, []byte{0x05, 0x06, 0x07}) { - t.Error("Decoding produced incorrect record value.") - } -} - -func TestOneMessageFetchResponseV4(t *testing.T) { - response := FetchResponse{} - testVersionDecodable(t, "one message v4", &response, oneMessageFetchResponseV4, 4) - - if len(response.Blocks) != 1 { - t.Fatal("Decoding produced incorrect number of topic blocks.") - } - - if len(response.Blocks["topic"]) != 1 { - t.Fatal("Decoding produced incorrect number of partition blocks for topic.") - } - - block := response.GetBlock("topic", 5) - if block == nil { - t.Fatal("GetBlock didn't return block.") - } - if block.Err != ErrOffsetOutOfRange { - t.Error("Decoding didn't produce correct error code.") - } - if block.HighWaterMarkOffset != 0x10101010 { - t.Error("Decoding didn't produce correct high water mark offset.") - } - partial, err := block.isPartial() - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - if partial { - t.Error("Decoding detected a partial trailing record where there wasn't one.") - } - - n, err := block.numRecords() - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - if n != 1 { - t.Fatal("Decoding produced incorrect number of records.") - } - msgBlock := block.RecordsSet[0].MsgSet.Messages[0] - if msgBlock.Offset != 0x550000 { - t.Error("Decoding produced incorrect message offset.") - } - msg := msgBlock.Msg - if msg.Codec != CompressionNone { - t.Error("Decoding produced incorrect message compression.") - } - if msg.Key != nil { - t.Error("Decoding produced message key where there was none.") - } - if !bytes.Equal(msg.Value, []byte{0x00, 0xEE}) { - t.Error("Decoding produced incorrect message value.") - } -} diff --git a/third/github.com/Shopify/sarama/find_coordinator_request.go b/third/github.com/Shopify/sarama/find_coordinator_request.go deleted file mode 100644 index 0ab5cb5ff..000000000 --- a/third/github.com/Shopify/sarama/find_coordinator_request.go +++ /dev/null @@ -1,61 +0,0 @@ -package sarama - -type CoordinatorType int8 - -const ( - CoordinatorGroup CoordinatorType = 0 - CoordinatorTransaction CoordinatorType = 1 -) - -type FindCoordinatorRequest struct { - Version int16 - CoordinatorKey string - CoordinatorType CoordinatorType -} - -func (f *FindCoordinatorRequest) encode(pe packetEncoder) error { - if err := pe.putString(f.CoordinatorKey); err != nil { - return err - } - - if f.Version >= 1 { - pe.putInt8(int8(f.CoordinatorType)) - } - - return nil -} - -func (f *FindCoordinatorRequest) decode(pd packetDecoder, version int16) (err error) { - if f.CoordinatorKey, err = pd.getString(); err != nil { - return err - } - - if version >= 1 { - f.Version = version - coordinatorType, err := pd.getInt8() - if err != nil { - return err - } - - f.CoordinatorType = CoordinatorType(coordinatorType) - } - - return nil -} - -func (f *FindCoordinatorRequest) key() int16 { - return 10 -} - -func (f *FindCoordinatorRequest) version() int16 { - return f.Version -} - -func (f *FindCoordinatorRequest) requiredVersion() KafkaVersion { - switch f.Version { - case 1: - return V0_11_0_0 - default: - return V0_8_2_0 - } -} diff --git a/third/github.com/Shopify/sarama/find_coordinator_request_test.go b/third/github.com/Shopify/sarama/find_coordinator_request_test.go deleted file mode 100644 index 7e889b074..000000000 --- a/third/github.com/Shopify/sarama/find_coordinator_request_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package sarama - -import "testing" - -var ( - findCoordinatorRequestConsumerGroup = []byte{ - 0, 5, 'g', 'r', 'o', 'u', 'p', - 0, - } - - findCoordinatorRequestTransaction = []byte{ - 0, 13, 't', 'r', 'a', 'n', 's', 'a', 'c', 't', 'i', 'o', 'n', 'i', 'd', - 1, - } -) - -func TestFindCoordinatorRequest(t *testing.T) { - req := &FindCoordinatorRequest{ - Version: 1, - CoordinatorKey: "group", - CoordinatorType: CoordinatorGroup, - } - - testRequest(t, "version 1 - group", req, findCoordinatorRequestConsumerGroup) - - req = &FindCoordinatorRequest{ - Version: 1, - CoordinatorKey: "transactionid", - CoordinatorType: CoordinatorTransaction, - } - - testRequest(t, "version 1 - transaction", req, findCoordinatorRequestTransaction) -} diff --git a/third/github.com/Shopify/sarama/find_coordinator_response.go b/third/github.com/Shopify/sarama/find_coordinator_response.go deleted file mode 100644 index 9c900e8b7..000000000 --- a/third/github.com/Shopify/sarama/find_coordinator_response.go +++ /dev/null @@ -1,92 +0,0 @@ -package sarama - -import ( - "time" -) - -var NoNode = &Broker{id: -1, addr: ":-1"} - -type FindCoordinatorResponse struct { - Version int16 - ThrottleTime time.Duration - Err KError - ErrMsg *string - Coordinator *Broker -} - -func (f *FindCoordinatorResponse) decode(pd packetDecoder, version int16) (err error) { - if version >= 1 { - f.Version = version - - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - f.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - } - - tmp, err := pd.getInt16() - if err != nil { - return err - } - f.Err = KError(tmp) - - if version >= 1 { - if f.ErrMsg, err = pd.getNullableString(); err != nil { - return err - } - } - - coordinator := new(Broker) - // The version is hardcoded to 0, as version 1 of the Broker-decode - // contains the rack-field which is not present in the FindCoordinatorResponse. - if err := coordinator.decode(pd, 0); err != nil { - return err - } - if coordinator.addr == ":0" { - return nil - } - f.Coordinator = coordinator - - return nil -} - -func (f *FindCoordinatorResponse) encode(pe packetEncoder) error { - if f.Version >= 1 { - pe.putInt32(int32(f.ThrottleTime / time.Millisecond)) - } - - pe.putInt16(int16(f.Err)) - - if f.Version >= 1 { - if err := pe.putNullableString(f.ErrMsg); err != nil { - return err - } - } - - coordinator := f.Coordinator - if coordinator == nil { - coordinator = NoNode - } - if err := coordinator.encode(pe, 0); err != nil { - return err - } - return nil -} - -func (f *FindCoordinatorResponse) key() int16 { - return 10 -} - -func (f *FindCoordinatorResponse) version() int16 { - return f.Version -} - -func (f *FindCoordinatorResponse) requiredVersion() KafkaVersion { - switch f.Version { - case 1: - return V0_11_0_0 - default: - return V0_8_2_0 - } -} diff --git a/third/github.com/Shopify/sarama/find_coordinator_response_test.go b/third/github.com/Shopify/sarama/find_coordinator_response_test.go deleted file mode 100644 index 417a76c6e..000000000 --- a/third/github.com/Shopify/sarama/find_coordinator_response_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -func TestFindCoordinatorResponse(t *testing.T) { - errMsg := "kaboom" - - for _, tc := range []struct { - desc string - response *FindCoordinatorResponse - encoded []byte - }{{ - desc: "version 0 - no error", - response: &FindCoordinatorResponse{ - Version: 0, - Err: ErrNoError, - Coordinator: &Broker{ - id: 7, - addr: "host:9092", - }, - }, - encoded: []byte{ - 0, 0, // Err - 0, 0, 0, 7, // Coordinator.ID - 0, 4, 'h', 'o', 's', 't', // Coordinator.Host - 0, 0, 35, 132, // Coordinator.Port - }, - }, { - desc: "version 1 - no error", - response: &FindCoordinatorResponse{ - Version: 1, - ThrottleTime: 100 * time.Millisecond, - Err: ErrNoError, - Coordinator: &Broker{ - id: 7, - addr: "host:9092", - }, - }, - encoded: []byte{ - 0, 0, 0, 100, // ThrottleTime - 0, 0, // Err - 255, 255, // ErrMsg: empty - 0, 0, 0, 7, // Coordinator.ID - 0, 4, 'h', 'o', 's', 't', // Coordinator.Host - 0, 0, 35, 132, // Coordinator.Port - }, - }, { - desc: "version 0 - error", - response: &FindCoordinatorResponse{ - Version: 0, - Err: ErrConsumerCoordinatorNotAvailable, - Coordinator: NoNode, - }, - encoded: []byte{ - 0, 15, // Err - 255, 255, 255, 255, // Coordinator.ID: -1 - 0, 0, // Coordinator.Host: "" - 255, 255, 255, 255, // Coordinator.Port: -1 - }, - }, { - desc: "version 1 - error", - response: &FindCoordinatorResponse{ - Version: 1, - ThrottleTime: 100 * time.Millisecond, - Err: ErrConsumerCoordinatorNotAvailable, - ErrMsg: &errMsg, - Coordinator: NoNode, - }, - encoded: []byte{ - 0, 0, 0, 100, // ThrottleTime - 0, 15, // Err - 0, 6, 'k', 'a', 'b', 'o', 'o', 'm', // ErrMsg - 255, 255, 255, 255, // Coordinator.ID: -1 - 0, 0, // Coordinator.Host: "" - 255, 255, 255, 255, // Coordinator.Port: -1 - }, - }} { - testResponse(t, tc.desc, tc.response, tc.encoded) - } -} diff --git a/third/github.com/Shopify/sarama/functional_client_test.go b/third/github.com/Shopify/sarama/functional_client_test.go deleted file mode 100644 index 2bf99d252..000000000 --- a/third/github.com/Shopify/sarama/functional_client_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package sarama - -import ( - "fmt" - "testing" - "time" -) - -func TestFuncConnectionFailure(t *testing.T) { - setupFunctionalTest(t) - defer teardownFunctionalTest(t) - - Proxies["kafka1"].Enabled = false - SaveProxy(t, "kafka1") - - config := NewConfig() - config.Metadata.Retry.Max = 1 - - _, err := NewClient([]string{kafkaBrokers[0]}, config) - if err != ErrOutOfBrokers { - t.Fatal("Expected returned error to be ErrOutOfBrokers, but was: ", err) - } -} - -func TestFuncClientMetadata(t *testing.T) { - setupFunctionalTest(t) - defer teardownFunctionalTest(t) - - config := NewConfig() - config.Metadata.Retry.Max = 1 - config.Metadata.Retry.Backoff = 10 * time.Millisecond - client, err := NewClient(kafkaBrokers, config) - if err != nil { - t.Fatal(err) - } - - if err := client.RefreshMetadata("unknown_topic"); err != ErrUnknownTopicOrPartition { - t.Error("Expected ErrUnknownTopicOrPartition, got", err) - } - - if _, err := client.Leader("unknown_topic", 0); err != ErrUnknownTopicOrPartition { - t.Error("Expected ErrUnknownTopicOrPartition, got", err) - } - - if _, err := client.Replicas("invalid/topic", 0); err != ErrUnknownTopicOrPartition { - t.Error("Expected ErrUnknownTopicOrPartition, got", err) - } - - partitions, err := client.Partitions("test.4") - if err != nil { - t.Error(err) - } - if len(partitions) != 4 { - t.Errorf("Expected test.4 topic to have 4 partitions, found %v", partitions) - } - - partitions, err = client.Partitions("test.1") - if err != nil { - t.Error(err) - } - if len(partitions) != 1 { - t.Errorf("Expected test.1 topic to have 1 partitions, found %v", partitions) - } - - safeClose(t, client) -} - -func TestFuncClientCoordinator(t *testing.T) { - checkKafkaVersion(t, "0.8.2") - setupFunctionalTest(t) - defer teardownFunctionalTest(t) - - client, err := NewClient(kafkaBrokers, nil) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < 10; i++ { - broker, err := client.Coordinator(fmt.Sprintf("another_new_consumer_group_%d", i)) - if err != nil { - t.Fatal(err) - } - - if connected, err := broker.Connected(); !connected || err != nil { - t.Errorf("Expected to coordinator %s broker to be properly connected.", broker.Addr()) - } - } - - safeClose(t, client) -} diff --git a/third/github.com/Shopify/sarama/functional_consumer_group_test.go b/third/github.com/Shopify/sarama/functional_consumer_group_test.go deleted file mode 100644 index ae376086d..000000000 --- a/third/github.com/Shopify/sarama/functional_consumer_group_test.go +++ /dev/null @@ -1,418 +0,0 @@ -// +build go1.9 - -package sarama - -import ( - "context" - "fmt" - "log" - "reflect" - "sync" - "sync/atomic" - "testing" - "time" -) - -func TestFuncConsumerGroupPartitioning(t *testing.T) { - checkKafkaVersion(t, "0.10.2") - setupFunctionalTest(t) - defer teardownFunctionalTest(t) - - groupID := testFuncConsumerGroupID(t) - - // start M1 - m1 := runTestFuncConsumerGroupMember(t, groupID, "M1", 0, nil) - defer m1.Stop() - m1.WaitForState(2) - m1.WaitForClaims(map[string]int{"test.4": 4}) - m1.WaitForHandlers(4) - - // start M2 - m2 := runTestFuncConsumerGroupMember(t, groupID, "M2", 0, nil, "test.1", "test.4") - defer m2.Stop() - m2.WaitForState(2) - - // assert that claims are shared among both members - m1.WaitForClaims(map[string]int{"test.4": 2}) - m1.WaitForHandlers(2) - m2.WaitForClaims(map[string]int{"test.1": 1, "test.4": 2}) - m2.WaitForHandlers(3) - - // shutdown M1, wait for M2 to take over - m1.AssertCleanShutdown() - m2.WaitForClaims(map[string]int{"test.1": 1, "test.4": 4}) - m2.WaitForHandlers(5) - - // shutdown M2 - m2.AssertCleanShutdown() -} - -func TestFuncConsumerGroupExcessConsumers(t *testing.T) { - checkKafkaVersion(t, "0.10.2") - setupFunctionalTest(t) - defer teardownFunctionalTest(t) - - groupID := testFuncConsumerGroupID(t) - - // start members - m1 := runTestFuncConsumerGroupMember(t, groupID, "M1", 0, nil) - defer m1.Stop() - m2 := runTestFuncConsumerGroupMember(t, groupID, "M2", 0, nil) - defer m2.Stop() - m3 := runTestFuncConsumerGroupMember(t, groupID, "M3", 0, nil) - defer m3.Stop() - m4 := runTestFuncConsumerGroupMember(t, groupID, "M4", 0, nil) - defer m4.Stop() - - m1.WaitForClaims(map[string]int{"test.4": 1}) - m2.WaitForClaims(map[string]int{"test.4": 1}) - m3.WaitForClaims(map[string]int{"test.4": 1}) - m4.WaitForClaims(map[string]int{"test.4": 1}) - - // start M5 - m5 := runTestFuncConsumerGroupMember(t, groupID, "M5", 0, nil) - defer m5.Stop() - m5.WaitForState(1) - m5.AssertNoErrs() - - // assert that claims are shared among both members - m4.AssertCleanShutdown() - m5.WaitForState(2) - m5.WaitForClaims(map[string]int{"test.4": 1}) - - // shutdown everything - m1.AssertCleanShutdown() - m2.AssertCleanShutdown() - m3.AssertCleanShutdown() - m5.AssertCleanShutdown() -} - -func TestFuncConsumerGroupFuzzy(t *testing.T) { - checkKafkaVersion(t, "0.10.2") - setupFunctionalTest(t) - defer teardownFunctionalTest(t) - - if err := testFuncConsumerGroupFuzzySeed("test.4"); err != nil { - t.Fatal(err) - } - - groupID := testFuncConsumerGroupID(t) - sink := &testFuncConsumerGroupSink{msgs: make(chan testFuncConsumerGroupMessage, 20000)} - waitForMessages := func(t *testing.T, n int) { - t.Helper() - - for i := 0; i < 600; i++ { - if sink.Len() >= n { - break - } - time.Sleep(100 * time.Millisecond) - } - if sz := sink.Len(); sz < n { - log.Fatalf("expected to consume %d messages, but consumed %d", n, sz) - } - } - - defer runTestFuncConsumerGroupMember(t, groupID, "M1", 1500, sink).Stop() - defer runTestFuncConsumerGroupMember(t, groupID, "M2", 3000, sink).Stop() - defer runTestFuncConsumerGroupMember(t, groupID, "M3", 1500, sink).Stop() - defer runTestFuncConsumerGroupMember(t, groupID, "M4", 200, sink).Stop() - defer runTestFuncConsumerGroupMember(t, groupID, "M5", 100, sink).Stop() - waitForMessages(t, 3000) - - defer runTestFuncConsumerGroupMember(t, groupID, "M6", 300, sink).Stop() - defer runTestFuncConsumerGroupMember(t, groupID, "M7", 400, sink).Stop() - defer runTestFuncConsumerGroupMember(t, groupID, "M8", 500, sink).Stop() - defer runTestFuncConsumerGroupMember(t, groupID, "M9", 2000, sink).Stop() - waitForMessages(t, 8000) - - defer runTestFuncConsumerGroupMember(t, groupID, "M10", 1000, sink).Stop() - waitForMessages(t, 10000) - - defer runTestFuncConsumerGroupMember(t, groupID, "M11", 1000, sink).Stop() - defer runTestFuncConsumerGroupMember(t, groupID, "M12", 2500, sink).Stop() - waitForMessages(t, 12000) - - defer runTestFuncConsumerGroupMember(t, groupID, "M13", 1000, sink).Stop() - waitForMessages(t, 15000) - - if umap := sink.Close(); len(umap) != 15000 { - dupes := make(map[string][]string) - for k, v := range umap { - if len(v) > 1 { - dupes[k] = v - } - } - t.Fatalf("expected %d unique messages to be consumed but got %d, including %d duplicates:\n%v", 15000, len(umap), len(dupes), dupes) - } -} - -// -------------------------------------------------------------------- - -func testFuncConsumerGroupID(t *testing.T) string { - return fmt.Sprintf("sarama.%s%d", t.Name(), time.Now().UnixNano()) -} - -func testFuncConsumerGroupFuzzySeed(topic string) error { - client, err := NewClient(kafkaBrokers, nil) - if err != nil { - return err - } - defer func() { _ = client.Close() }() - - total := int64(0) - for pn := int32(0); pn < 4; pn++ { - newest, err := client.GetOffset(topic, pn, OffsetNewest) - if err != nil { - return err - } - oldest, err := client.GetOffset(topic, pn, OffsetOldest) - if err != nil { - return err - } - total = total + newest - oldest - } - if total >= 21000 { - return nil - } - - producer, err := NewAsyncProducerFromClient(client) - if err != nil { - return err - } - for i := total; i < 21000; i++ { - producer.Input() <- &ProducerMessage{Topic: topic, Value: ByteEncoder([]byte("testdata"))} - } - return producer.Close() -} - -type testFuncConsumerGroupMessage struct { - ClientID string - *ConsumerMessage -} - -type testFuncConsumerGroupSink struct { - msgs chan testFuncConsumerGroupMessage - count int32 -} - -func (s *testFuncConsumerGroupSink) Len() int { - if s == nil { - return -1 - } - return int(atomic.LoadInt32(&s.count)) -} - -func (s *testFuncConsumerGroupSink) Push(clientID string, m *ConsumerMessage) { - if s != nil { - s.msgs <- testFuncConsumerGroupMessage{ClientID: clientID, ConsumerMessage: m} - atomic.AddInt32(&s.count, 1) - } -} - -func (s *testFuncConsumerGroupSink) Close() map[string][]string { - close(s.msgs) - - res := make(map[string][]string) - for msg := range s.msgs { - key := fmt.Sprintf("%s-%d:%d", msg.Topic, msg.Partition, msg.Offset) - res[key] = append(res[key], msg.ClientID) - } - return res -} - -type testFuncConsumerGroupMember struct { - ConsumerGroup - clientID string - claims map[string]int - state int32 - handlers int32 - errs []error - maxMessages int32 - isCapped bool - sink *testFuncConsumerGroupSink - - t *testing.T - mu sync.RWMutex -} - -func runTestFuncConsumerGroupMember(t *testing.T, groupID, clientID string, maxMessages int32, sink *testFuncConsumerGroupSink, topics ...string) *testFuncConsumerGroupMember { - t.Helper() - - config := NewConfig() - config.ClientID = clientID - config.Version = V0_10_2_0 - config.Consumer.Return.Errors = true - config.Consumer.Offsets.Initial = OffsetOldest - config.Consumer.Group.Rebalance.Timeout = 10 * time.Second - - group, err := NewConsumerGroup(kafkaBrokers, groupID, config) - if err != nil { - t.Fatal(err) - return nil - } - - if len(topics) == 0 { - topics = []string{"test.4"} - } - - member := &testFuncConsumerGroupMember{ - ConsumerGroup: group, - clientID: clientID, - claims: make(map[string]int), - maxMessages: maxMessages, - isCapped: maxMessages != 0, - sink: sink, - t: t, - } - go member.loop(topics) - return member -} - -func (m *testFuncConsumerGroupMember) AssertCleanShutdown() { - m.t.Helper() - - if err := m.Close(); err != nil { - m.t.Fatalf("unexpected error on Close(): %v", err) - } - m.WaitForState(4) - m.WaitForHandlers(0) - m.AssertNoErrs() -} - -func (m *testFuncConsumerGroupMember) AssertNoErrs() { - m.t.Helper() - - var errs []error - m.mu.RLock() - errs = append(errs, m.errs...) - m.mu.RUnlock() - - if len(errs) != 0 { - m.t.Fatalf("unexpected consumer errors: %v", errs) - } -} - -func (m *testFuncConsumerGroupMember) WaitForState(expected int32) { - m.t.Helper() - - m.waitFor("state", expected, func() (interface{}, error) { - return atomic.LoadInt32(&m.state), nil - }) -} - -func (m *testFuncConsumerGroupMember) WaitForHandlers(expected int) { - m.t.Helper() - - m.waitFor("handlers", expected, func() (interface{}, error) { - return int(atomic.LoadInt32(&m.handlers)), nil - }) -} - -func (m *testFuncConsumerGroupMember) WaitForClaims(expected map[string]int) { - m.t.Helper() - - m.waitFor("claims", expected, func() (interface{}, error) { - m.mu.RLock() - claims := m.claims - m.mu.RUnlock() - return claims, nil - }) -} - -func (m *testFuncConsumerGroupMember) Stop() { _ = m.Close() } - -func (m *testFuncConsumerGroupMember) Setup(s ConsumerGroupSession) error { - // store claims - claims := make(map[string]int) - for topic, partitions := range s.Claims() { - claims[topic] = len(partitions) - } - m.mu.Lock() - m.claims = claims - m.mu.Unlock() - - // enter post-setup state - atomic.StoreInt32(&m.state, 2) - return nil -} -func (m *testFuncConsumerGroupMember) Cleanup(s ConsumerGroupSession) error { - // enter post-cleanup state - atomic.StoreInt32(&m.state, 3) - return nil -} -func (m *testFuncConsumerGroupMember) ConsumeClaim(s ConsumerGroupSession, c ConsumerGroupClaim) error { - atomic.AddInt32(&m.handlers, 1) - defer atomic.AddInt32(&m.handlers, -1) - - for msg := range c.Messages() { - if n := atomic.AddInt32(&m.maxMessages, -1); m.isCapped && n < 0 { - break - } - s.MarkMessage(msg, "") - m.sink.Push(m.clientID, msg) - } - return nil -} - -func (m *testFuncConsumerGroupMember) waitFor(kind string, expected interface{}, factory func() (interface{}, error)) { - m.t.Helper() - - deadline := time.NewTimer(60 * time.Second) - defer deadline.Stop() - - ticker := time.NewTicker(100 * time.Millisecond) - defer ticker.Stop() - - var actual interface{} - for { - var err error - if actual, err = factory(); err != nil { - m.t.Errorf("failed retrieve value, expected %s %#v but received error %v", kind, expected, err) - } - - if reflect.DeepEqual(expected, actual) { - return - } - - select { - case <-deadline.C: - m.t.Fatalf("ttl exceeded, expected %s %#v but got %#v", kind, expected, actual) - return - case <-ticker.C: - } - } -} - -func (m *testFuncConsumerGroupMember) loop(topics []string) { - defer atomic.StoreInt32(&m.state, 4) - - go func() { - for err := range m.Errors() { - _ = m.Close() - - m.mu.Lock() - m.errs = append(m.errs, err) - m.mu.Unlock() - } - }() - - ctx := context.Background() - for { - // set state to pre-consume - atomic.StoreInt32(&m.state, 1) - - if err := m.Consume(ctx, topics, m); err == ErrClosedConsumerGroup { - return - } else if err != nil { - m.mu.Lock() - m.errs = append(m.errs, err) - m.mu.Unlock() - return - } - - // return if capped - if n := atomic.LoadInt32(&m.maxMessages); m.isCapped && n < 0 { - return - } - } -} diff --git a/third/github.com/Shopify/sarama/functional_consumer_test.go b/third/github.com/Shopify/sarama/functional_consumer_test.go deleted file mode 100644 index aa8eccf7c..000000000 --- a/third/github.com/Shopify/sarama/functional_consumer_test.go +++ /dev/null @@ -1,244 +0,0 @@ -package sarama - -import ( - "fmt" - "math" - "os" - "sort" - "sync" - "testing" - "time" -) - -func TestFuncConsumerOffsetOutOfRange(t *testing.T) { - setupFunctionalTest(t) - defer teardownFunctionalTest(t) - - consumer, err := NewConsumer(kafkaBrokers, nil) - if err != nil { - t.Fatal(err) - } - - if _, err := consumer.ConsumePartition("test.1", 0, -10); err != ErrOffsetOutOfRange { - t.Error("Expected ErrOffsetOutOfRange, got:", err) - } - - if _, err := consumer.ConsumePartition("test.1", 0, math.MaxInt64); err != ErrOffsetOutOfRange { - t.Error("Expected ErrOffsetOutOfRange, got:", err) - } - - safeClose(t, consumer) -} - -func TestConsumerHighWaterMarkOffset(t *testing.T) { - setupFunctionalTest(t) - defer teardownFunctionalTest(t) - - p, err := NewSyncProducer(kafkaBrokers, nil) - if err != nil { - t.Fatal(err) - } - defer safeClose(t, p) - - _, offset, err := p.SendMessage(&ProducerMessage{Topic: "test.1", Value: StringEncoder("Test")}) - if err != nil { - t.Fatal(err) - } - - c, err := NewConsumer(kafkaBrokers, nil) - if err != nil { - t.Fatal(err) - } - defer safeClose(t, c) - - pc, err := c.ConsumePartition("test.1", 0, offset) - if err != nil { - t.Fatal(err) - } - - <-pc.Messages() - - if hwmo := pc.HighWaterMarkOffset(); hwmo != offset+1 { - t.Logf("Last produced offset %d; high water mark should be one higher but found %d.", offset, hwmo) - } - - safeClose(t, pc) -} - -// Makes sure that messages produced by all supported client versions/ -// compression codecs (except LZ4) combinations can be consumed by all -// supported consumer versions. It relies on the KAFKA_VERSION environment -// variable to provide the version of the test Kafka cluster. -// -// Note that LZ4 codec was introduced in v0.10.0.0 and therefore is excluded -// from this test case. It has a similar version matrix test case below that -// only checks versions from v0.10.0.0 until KAFKA_VERSION. -func TestVersionMatrix(t *testing.T) { - setupFunctionalTest(t) - defer teardownFunctionalTest(t) - - // Produce lot's of message with all possible combinations of supported - // protocol versions and compressions for the except of LZ4. - testVersions := versionRange(V0_8_2_0) - allCodecsButLZ4 := []CompressionCodec{CompressionNone, CompressionGZIP, CompressionSnappy} - producedMessages := produceMsgs(t, testVersions, allCodecsButLZ4, 17, 100, false) - - // When/Then - consumeMsgs(t, testVersions, producedMessages) -} - -// Support for LZ4 codec was introduced in v0.10.0.0 so a version matrix to -// test LZ4 should start with v0.10.0.0. -func TestVersionMatrixLZ4(t *testing.T) { - setupFunctionalTest(t) - defer teardownFunctionalTest(t) - - // Produce lot's of message with all possible combinations of supported - // protocol versions starting with v0.10 (first where LZ4 was supported) - // and all possible compressions. - testVersions := versionRange(V0_10_0_0) - allCodecs := []CompressionCodec{CompressionNone, CompressionGZIP, CompressionSnappy, CompressionLZ4} - producedMessages := produceMsgs(t, testVersions, allCodecs, 17, 100, false) - - // When/Then - consumeMsgs(t, testVersions, producedMessages) -} - -func TestVersionMatrixIdempotent(t *testing.T) { - setupFunctionalTest(t) - defer teardownFunctionalTest(t) - - // Produce lot's of message with all possible combinations of supported - // protocol versions starting with v0.11 (first where idempotent was supported) - testVersions := versionRange(V0_11_0_0) - producedMessages := produceMsgs(t, testVersions, []CompressionCodec{CompressionNone}, 17, 100, true) - - // When/Then - consumeMsgs(t, testVersions, producedMessages) -} - -func prodMsg2Str(prodMsg *ProducerMessage) string { - return fmt.Sprintf("{offset: %d, value: %s}", prodMsg.Offset, string(prodMsg.Value.(StringEncoder))) -} - -func consMsg2Str(consMsg *ConsumerMessage) string { - return fmt.Sprintf("{offset: %d, value: %s}", consMsg.Offset, string(consMsg.Value)) -} - -func versionRange(lower KafkaVersion) []KafkaVersion { - // Get the test cluster version from the environment. If there is nothing - // there then assume the highest. - upper, err := ParseKafkaVersion(os.Getenv("KAFKA_VERSION")) - if err != nil { - upper = MaxVersion - } - - versions := make([]KafkaVersion, 0, len(SupportedVersions)) - for _, v := range SupportedVersions { - if !v.IsAtLeast(lower) { - continue - } - if !upper.IsAtLeast(v) { - return versions - } - versions = append(versions, v) - } - return versions -} - -func produceMsgs(t *testing.T, clientVersions []KafkaVersion, codecs []CompressionCodec, flush int, countPerVerCodec int, idempotent bool) []*ProducerMessage { - var wg sync.WaitGroup - var producedMessagesMu sync.Mutex - var producedMessages []*ProducerMessage - for _, prodVer := range clientVersions { - for _, codec := range codecs { - prodCfg := NewConfig() - prodCfg.Version = prodVer - prodCfg.Producer.Return.Successes = true - prodCfg.Producer.Return.Errors = true - prodCfg.Producer.Flush.MaxMessages = flush - prodCfg.Producer.Compression = codec - prodCfg.Producer.Idempotent = idempotent - if idempotent { - prodCfg.Producer.RequiredAcks = WaitForAll - prodCfg.Net.MaxOpenRequests = 1 - } - - p, err := NewSyncProducer(kafkaBrokers, prodCfg) - if err != nil { - t.Errorf("Failed to create producer: version=%s, compression=%s, err=%v", prodVer, codec, err) - continue - } - defer safeClose(t, p) - for i := 0; i < countPerVerCodec; i++ { - msg := &ProducerMessage{ - Topic: "test.1", - Value: StringEncoder(fmt.Sprintf("msg:%s:%s:%d", prodVer, codec, i)), - } - wg.Add(1) - go func() { - defer wg.Done() - _, _, err := p.SendMessage(msg) - if err != nil { - t.Errorf("Failed to produce message: %s, err=%v", msg.Value, err) - } - producedMessagesMu.Lock() - producedMessages = append(producedMessages, msg) - producedMessagesMu.Unlock() - }() - } - } - } - wg.Wait() - - // Sort produced message in ascending offset order. - sort.Slice(producedMessages, func(i, j int) bool { - return producedMessages[i].Offset < producedMessages[j].Offset - }) - t.Logf("*** Total produced %d, firstOffset=%d, lastOffset=%d\n", - len(producedMessages), producedMessages[0].Offset, producedMessages[len(producedMessages)-1].Offset) - return producedMessages -} - -func consumeMsgs(t *testing.T, clientVersions []KafkaVersion, producedMessages []*ProducerMessage) { - // Consume all produced messages with all client versions supported by the - // cluster. -consumerVersionLoop: - for _, consVer := range clientVersions { - t.Logf("*** Consuming with client version %s\n", consVer) - // Create a partition consumer that should start from the first produced - // message. - consCfg := NewConfig() - consCfg.Version = consVer - c, err := NewConsumer(kafkaBrokers, consCfg) - if err != nil { - t.Fatal(err) - } - defer safeClose(t, c) - pc, err := c.ConsumePartition("test.1", 0, producedMessages[0].Offset) - if err != nil { - t.Fatal(err) - } - defer safeClose(t, pc) - - // Consume as many messages as there have been produced and make sure that - // order is preserved. - for i, prodMsg := range producedMessages { - select { - case consMsg := <-pc.Messages(): - if consMsg.Offset != prodMsg.Offset { - t.Errorf("Consumed unexpected offset: version=%s, index=%d, want=%s, got=%s", - consVer, i, prodMsg2Str(prodMsg), consMsg2Str(consMsg)) - continue consumerVersionLoop - } - if string(consMsg.Value) != string(prodMsg.Value.(StringEncoder)) { - t.Errorf("Consumed unexpected msg: version=%s, index=%d, want=%s, got=%s", - consVer, i, prodMsg2Str(prodMsg), consMsg2Str(consMsg)) - continue consumerVersionLoop - } - case <-time.After(3 * time.Second): - t.Fatalf("Timeout waiting for: index=%d, offset=%d, msg=%s", i, prodMsg.Offset, prodMsg.Value) - } - } - } -} diff --git a/third/github.com/Shopify/sarama/functional_offset_manager_test.go b/third/github.com/Shopify/sarama/functional_offset_manager_test.go deleted file mode 100644 index 436f35ef4..000000000 --- a/third/github.com/Shopify/sarama/functional_offset_manager_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package sarama - -import ( - "testing" -) - -func TestFuncOffsetManager(t *testing.T) { - checkKafkaVersion(t, "0.8.2") - setupFunctionalTest(t) - defer teardownFunctionalTest(t) - - client, err := NewClient(kafkaBrokers, nil) - if err != nil { - t.Fatal(err) - } - - offsetManager, err := NewOffsetManagerFromClient("sarama.TestFuncOffsetManager", client) - if err != nil { - t.Fatal(err) - } - - pom1, err := offsetManager.ManagePartition("test.1", 0) - if err != nil { - t.Fatal(err) - } - - pom1.MarkOffset(10, "test metadata") - safeClose(t, pom1) - - pom2, err := offsetManager.ManagePartition("test.1", 0) - if err != nil { - t.Fatal(err) - } - - offset, metadata := pom2.NextOffset() - - if offset != 10 { - t.Errorf("Expected the next offset to be 10, found %d.", offset) - } - if metadata != "test metadata" { - t.Errorf("Expected metadata to be 'test metadata', found %s.", metadata) - } - - safeClose(t, pom2) - safeClose(t, offsetManager) - safeClose(t, client) -} diff --git a/third/github.com/Shopify/sarama/functional_producer_test.go b/third/github.com/Shopify/sarama/functional_producer_test.go deleted file mode 100644 index 6843520f7..000000000 --- a/third/github.com/Shopify/sarama/functional_producer_test.go +++ /dev/null @@ -1,323 +0,0 @@ -package sarama - -import ( - "fmt" - "os" - "sync" - "testing" - "time" - - toxiproxy "github.com/gogf/gf/third/github.com/Shopify/toxiproxy/client" - "github.com/gogf/gf/third/github.com/rcrowley/go-metrics" -) - -const TestBatchSize = 1000 - -func TestFuncProducing(t *testing.T) { - config := NewConfig() - testProducingMessages(t, config) -} - -func TestFuncProducingGzip(t *testing.T) { - config := NewConfig() - config.Producer.Compression = CompressionGZIP - testProducingMessages(t, config) -} - -func TestFuncProducingSnappy(t *testing.T) { - config := NewConfig() - config.Producer.Compression = CompressionSnappy - testProducingMessages(t, config) -} - -func TestFuncProducingNoResponse(t *testing.T) { - config := NewConfig() - config.Producer.RequiredAcks = NoResponse - testProducingMessages(t, config) -} - -func TestFuncProducingFlushing(t *testing.T) { - config := NewConfig() - config.Producer.Flush.Messages = TestBatchSize / 8 - config.Producer.Flush.Frequency = 250 * time.Millisecond - testProducingMessages(t, config) -} - -func TestFuncMultiPartitionProduce(t *testing.T) { - setupFunctionalTest(t) - defer teardownFunctionalTest(t) - - config := NewConfig() - config.ChannelBufferSize = 20 - config.Producer.Flush.Frequency = 50 * time.Millisecond - config.Producer.Flush.Messages = 200 - config.Producer.Return.Successes = true - producer, err := NewSyncProducer(kafkaBrokers, config) - if err != nil { - t.Fatal(err) - } - - var wg sync.WaitGroup - wg.Add(TestBatchSize) - - for i := 1; i <= TestBatchSize; i++ { - go func(i int) { - defer wg.Done() - msg := &ProducerMessage{Topic: "test.64", Key: nil, Value: StringEncoder(fmt.Sprintf("hur %d", i))} - if _, _, err := producer.SendMessage(msg); err != nil { - t.Error(i, err) - } - }(i) - } - - wg.Wait() - if err := producer.Close(); err != nil { - t.Error(err) - } -} - -func TestFuncProducingToInvalidTopic(t *testing.T) { - setupFunctionalTest(t) - defer teardownFunctionalTest(t) - - producer, err := NewSyncProducer(kafkaBrokers, nil) - if err != nil { - t.Fatal(err) - } - - if _, _, err := producer.SendMessage(&ProducerMessage{Topic: "in/valid"}); err != ErrUnknownTopicOrPartition { - t.Error("Expected ErrUnknownTopicOrPartition, found", err) - } - - if _, _, err := producer.SendMessage(&ProducerMessage{Topic: "in/valid"}); err != ErrUnknownTopicOrPartition { - t.Error("Expected ErrUnknownTopicOrPartition, found", err) - } - - safeClose(t, producer) -} - -func testProducingMessages(t *testing.T, config *Config) { - setupFunctionalTest(t) - defer teardownFunctionalTest(t) - - // Configure some latency in order to properly validate the request latency metric - for _, proxy := range Proxies { - if _, err := proxy.AddToxic("", "latency", "", 1, toxiproxy.Attributes{"latency": 10}); err != nil { - t.Fatal("Unable to configure latency toxicity", err) - } - } - - config.Producer.Return.Successes = true - config.Consumer.Return.Errors = true - - client, err := NewClient(kafkaBrokers, config) - if err != nil { - t.Fatal(err) - } - - // Keep in mind the current offset - initialOffset, err := client.GetOffset("test.1", 0, OffsetNewest) - if err != nil { - t.Fatal(err) - } - - producer, err := NewAsyncProducerFromClient(client) - if err != nil { - t.Fatal(err) - } - - expectedResponses := TestBatchSize - for i := 1; i <= TestBatchSize; { - msg := &ProducerMessage{Topic: "test.1", Key: nil, Value: StringEncoder(fmt.Sprintf("testing %d", i))} - select { - case producer.Input() <- msg: - i++ - case ret := <-producer.Errors(): - t.Fatal(ret.Err) - case <-producer.Successes(): - expectedResponses-- - } - } - for expectedResponses > 0 { - select { - case ret := <-producer.Errors(): - t.Fatal(ret.Err) - case <-producer.Successes(): - expectedResponses-- - } - } - safeClose(t, producer) - - // Validate producer metrics before using the consumer minus the offset request - validateMetrics(t, client) - - master, err := NewConsumerFromClient(client) - if err != nil { - t.Fatal(err) - } - consumer, err := master.ConsumePartition("test.1", 0, initialOffset) - if err != nil { - t.Fatal(err) - } - - for i := 1; i <= TestBatchSize; i++ { - select { - case <-time.After(10 * time.Second): - t.Fatal("Not received any more events in the last 10 seconds.") - - case err := <-consumer.Errors(): - t.Error(err) - - case message := <-consumer.Messages(): - if string(message.Value) != fmt.Sprintf("testing %d", i) { - t.Fatalf("Unexpected message with index %d: %s", i, message.Value) - } - } - - } - safeClose(t, consumer) - safeClose(t, client) -} - -func validateMetrics(t *testing.T, client Client) { - // Get the broker used by test1 topic - var broker *Broker - if partitions, err := client.Partitions("test.1"); err != nil { - t.Error(err) - } else { - for _, partition := range partitions { - if b, err := client.Leader("test.1", partition); err != nil { - t.Error(err) - } else { - if broker != nil && b != broker { - t.Fatal("Expected only one broker, got at least 2") - } - broker = b - } - } - } - - metricValidators := newMetricValidators() - noResponse := client.Config().Producer.RequiredAcks == NoResponse - compressionEnabled := client.Config().Producer.Compression != CompressionNone - - // We are adding 10ms of latency to all requests with toxiproxy - minRequestLatencyInMs := 10 - if noResponse { - // but when we do not wait for a response it can be less than 1ms - minRequestLatencyInMs = 0 - } - - // We read at least 1 byte from the broker - metricValidators.registerForAllBrokers(broker, minCountMeterValidator("incoming-byte-rate", 1)) - // in at least 3 global requests (1 for metadata request, 1 for offset request and N for produce request) - metricValidators.register(minCountMeterValidator("request-rate", 3)) - metricValidators.register(minCountHistogramValidator("request-size", 3)) - metricValidators.register(minValHistogramValidator("request-size", 1)) - metricValidators.register(minValHistogramValidator("request-latency-in-ms", minRequestLatencyInMs)) - // and at least 2 requests to the registered broker (offset + produces) - metricValidators.registerForBroker(broker, minCountMeterValidator("request-rate", 2)) - metricValidators.registerForBroker(broker, minCountHistogramValidator("request-size", 2)) - metricValidators.registerForBroker(broker, minValHistogramValidator("request-size", 1)) - metricValidators.registerForBroker(broker, minValHistogramValidator("request-latency-in-ms", minRequestLatencyInMs)) - - // We send at least 1 batch - metricValidators.registerForGlobalAndTopic("test_1", minCountHistogramValidator("batch-size", 1)) - metricValidators.registerForGlobalAndTopic("test_1", minValHistogramValidator("batch-size", 1)) - if compressionEnabled { - // We record compression ratios between [0.50,-10.00] (50-1000 with a histogram) for at least one "fake" record - metricValidators.registerForGlobalAndTopic("test_1", minCountHistogramValidator("compression-ratio", 1)) - metricValidators.registerForGlobalAndTopic("test_1", minValHistogramValidator("compression-ratio", 50)) - metricValidators.registerForGlobalAndTopic("test_1", maxValHistogramValidator("compression-ratio", 1000)) - } else { - // We record compression ratios of 1.00 (100 with a histogram) for every TestBatchSize record - metricValidators.registerForGlobalAndTopic("test_1", countHistogramValidator("compression-ratio", TestBatchSize)) - metricValidators.registerForGlobalAndTopic("test_1", minValHistogramValidator("compression-ratio", 100)) - metricValidators.registerForGlobalAndTopic("test_1", maxValHistogramValidator("compression-ratio", 100)) - } - - // We send exactly TestBatchSize messages - metricValidators.registerForGlobalAndTopic("test_1", countMeterValidator("record-send-rate", TestBatchSize)) - // We send at least one record per request - metricValidators.registerForGlobalAndTopic("test_1", minCountHistogramValidator("records-per-request", 1)) - metricValidators.registerForGlobalAndTopic("test_1", minValHistogramValidator("records-per-request", 1)) - - // We receive at least 1 byte from the broker - metricValidators.registerForAllBrokers(broker, minCountMeterValidator("outgoing-byte-rate", 1)) - if noResponse { - // in exactly 2 global responses (metadata + offset) - metricValidators.register(countMeterValidator("response-rate", 2)) - metricValidators.register(minCountHistogramValidator("response-size", 2)) - metricValidators.register(minValHistogramValidator("response-size", 1)) - // and exactly 1 offset response for the registered broker - metricValidators.registerForBroker(broker, countMeterValidator("response-rate", 1)) - metricValidators.registerForBroker(broker, minCountHistogramValidator("response-size", 1)) - metricValidators.registerForBroker(broker, minValHistogramValidator("response-size", 1)) - } else { - // in at least 3 global responses (metadata + offset + produces) - metricValidators.register(minCountMeterValidator("response-rate", 3)) - metricValidators.register(minCountHistogramValidator("response-size", 3)) - metricValidators.register(minValHistogramValidator("response-size", 1)) - // and at least 2 for the registered broker - metricValidators.registerForBroker(broker, minCountMeterValidator("response-rate", 2)) - metricValidators.registerForBroker(broker, minCountHistogramValidator("response-size", 2)) - metricValidators.registerForBroker(broker, minValHistogramValidator("response-size", 1)) - } - - // Run the validators - metricValidators.run(t, client.Config().MetricRegistry) -} - -// Benchmarks - -func BenchmarkProducerSmall(b *testing.B) { - benchmarkProducer(b, nil, "test.64", ByteEncoder(make([]byte, 128))) -} -func BenchmarkProducerMedium(b *testing.B) { - benchmarkProducer(b, nil, "test.64", ByteEncoder(make([]byte, 1024))) -} -func BenchmarkProducerLarge(b *testing.B) { - benchmarkProducer(b, nil, "test.64", ByteEncoder(make([]byte, 8192))) -} -func BenchmarkProducerSmallSinglePartition(b *testing.B) { - benchmarkProducer(b, nil, "test.1", ByteEncoder(make([]byte, 128))) -} -func BenchmarkProducerMediumSnappy(b *testing.B) { - conf := NewConfig() - conf.Producer.Compression = CompressionSnappy - benchmarkProducer(b, conf, "test.1", ByteEncoder(make([]byte, 1024))) -} - -func benchmarkProducer(b *testing.B, conf *Config, topic string, value Encoder) { - setupFunctionalTest(b) - defer teardownFunctionalTest(b) - - metricsDisable := os.Getenv("METRICS_DISABLE") - if metricsDisable != "" { - previousUseNilMetrics := metrics.UseNilMetrics - Logger.Println("Disabling metrics using no-op implementation") - metrics.UseNilMetrics = true - // Restore previous setting - defer func() { - metrics.UseNilMetrics = previousUseNilMetrics - }() - } - - producer, err := NewAsyncProducer(kafkaBrokers, conf) - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - - for i := 1; i <= b.N; { - msg := &ProducerMessage{Topic: topic, Key: StringEncoder(fmt.Sprintf("%d", i)), Value: value} - select { - case producer.Input() <- msg: - i++ - case ret := <-producer.Errors(): - b.Fatal(ret.Err) - } - } - safeClose(b, producer) -} diff --git a/third/github.com/Shopify/sarama/functional_test.go b/third/github.com/Shopify/sarama/functional_test.go deleted file mode 100644 index 116a78c3a..000000000 --- a/third/github.com/Shopify/sarama/functional_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package sarama - -import ( - "log" - "math/rand" - "net" - "os" - "strconv" - "strings" - "testing" - "time" - - toxiproxy "github.com/gogf/gf/third/github.com/Shopify/toxiproxy/client" -) - -const ( - VagrantToxiproxy = "http://192.168.100.67:8474" - VagrantKafkaPeers = "192.168.100.67:9091,192.168.100.67:9092,192.168.100.67:9093,192.168.100.67:9094,192.168.100.67:9095" - VagrantZookeeperPeers = "192.168.100.67:2181,192.168.100.67:2182,192.168.100.67:2183,192.168.100.67:2184,192.168.100.67:2185" -) - -var ( - kafkaAvailable, kafkaRequired bool - kafkaBrokers []string - - proxyClient *toxiproxy.Client - Proxies map[string]*toxiproxy.Proxy - ZKProxies = []string{"zk1", "zk2", "zk3", "zk4", "zk5"} - KafkaProxies = []string{"kafka1", "kafka2", "kafka3", "kafka4", "kafka5"} -) - -func init() { - if os.Getenv("DEBUG") == "true" { - Logger = log.New(os.Stdout, "[sarama] ", log.LstdFlags) - } - - seed := time.Now().UTC().UnixNano() - if tmp := os.Getenv("TEST_SEED"); tmp != "" { - seed, _ = strconv.ParseInt(tmp, 0, 64) - } - Logger.Println("Using random seed:", seed) - rand.Seed(seed) - - proxyAddr := os.Getenv("TOXIPROXY_ADDR") - if proxyAddr == "" { - proxyAddr = VagrantToxiproxy - } - proxyClient = toxiproxy.NewClient(proxyAddr) - - kafkaPeers := os.Getenv("KAFKA_PEERS") - if kafkaPeers == "" { - kafkaPeers = VagrantKafkaPeers - } - kafkaBrokers = strings.Split(kafkaPeers, ",") - - if c, err := net.DialTimeout("tcp", kafkaBrokers[0], 5*time.Second); err == nil { - if err = c.Close(); err == nil { - kafkaAvailable = true - } - } - - kafkaRequired = os.Getenv("CI") != "" -} - -func checkKafkaAvailability(t testing.TB) { - if !kafkaAvailable { - if kafkaRequired { - t.Fatalf("Kafka broker is not available on %s. Set KAFKA_PEERS to connect to Kafka on a different location.", kafkaBrokers[0]) - } else { - t.Skipf("Kafka broker is not available on %s. Set KAFKA_PEERS to connect to Kafka on a different location.", kafkaBrokers[0]) - } - } -} - -func checkKafkaVersion(t testing.TB, requiredVersion string) { - kafkaVersion := os.Getenv("KAFKA_VERSION") - if kafkaVersion == "" { - t.Logf("No KAFKA_VERSION set. This test requires Kafka version %s or higher. Continuing...", requiredVersion) - } else { - available := parseKafkaVersion(kafkaVersion) - required := parseKafkaVersion(requiredVersion) - if !available.satisfies(required) { - t.Skipf("Kafka version %s is required for this test; you have %s. Skipping...", requiredVersion, kafkaVersion) - } - } -} - -func resetProxies(t testing.TB) { - if err := proxyClient.ResetState(); err != nil { - t.Error(err) - } - Proxies = nil -} - -func fetchProxies(t testing.TB) { - var err error - Proxies, err = proxyClient.Proxies() - if err != nil { - t.Fatal(err) - } -} - -func SaveProxy(t *testing.T, px string) { - if err := Proxies[px].Save(); err != nil { - t.Fatal(err) - } -} - -func setupFunctionalTest(t testing.TB) { - checkKafkaAvailability(t) - resetProxies(t) - fetchProxies(t) -} - -func teardownFunctionalTest(t testing.TB) { - resetProxies(t) -} - -type kafkaVersion []int - -func (kv kafkaVersion) satisfies(other kafkaVersion) bool { - var ov int - for index, v := range kv { - if len(other) <= index { - ov = 0 - } else { - ov = other[index] - } - - if v < ov { - return false - } else if v > ov { - return true - } - } - return true -} - -func parseKafkaVersion(version string) kafkaVersion { - numbers := strings.Split(version, ".") - result := make(kafkaVersion, 0, len(numbers)) - for _, number := range numbers { - nr, _ := strconv.Atoi(number) - result = append(result, nr) - } - - return result -} diff --git a/third/github.com/Shopify/sarama/heartbeat_request.go b/third/github.com/Shopify/sarama/heartbeat_request.go deleted file mode 100644 index ce49c4739..000000000 --- a/third/github.com/Shopify/sarama/heartbeat_request.go +++ /dev/null @@ -1,47 +0,0 @@ -package sarama - -type HeartbeatRequest struct { - GroupId string - GenerationId int32 - MemberId string -} - -func (r *HeartbeatRequest) encode(pe packetEncoder) error { - if err := pe.putString(r.GroupId); err != nil { - return err - } - - pe.putInt32(r.GenerationId) - - if err := pe.putString(r.MemberId); err != nil { - return err - } - - return nil -} - -func (r *HeartbeatRequest) decode(pd packetDecoder, version int16) (err error) { - if r.GroupId, err = pd.getString(); err != nil { - return - } - if r.GenerationId, err = pd.getInt32(); err != nil { - return - } - if r.MemberId, err = pd.getString(); err != nil { - return - } - - return nil -} - -func (r *HeartbeatRequest) key() int16 { - return 12 -} - -func (r *HeartbeatRequest) version() int16 { - return 0 -} - -func (r *HeartbeatRequest) requiredVersion() KafkaVersion { - return V0_9_0_0 -} diff --git a/third/github.com/Shopify/sarama/heartbeat_request_test.go b/third/github.com/Shopify/sarama/heartbeat_request_test.go deleted file mode 100644 index da6cd18f5..000000000 --- a/third/github.com/Shopify/sarama/heartbeat_request_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package sarama - -import "testing" - -var ( - basicHeartbeatRequest = []byte{ - 0, 3, 'f', 'o', 'o', // Group ID - 0x00, 0x01, 0x02, 0x03, // Generatiuon ID - 0, 3, 'b', 'a', 'z', // Member ID - } -) - -func TestHeartbeatRequest(t *testing.T) { - var request *HeartbeatRequest - - request = new(HeartbeatRequest) - request.GroupId = "foo" - request.GenerationId = 66051 - request.MemberId = "baz" - testRequest(t, "basic", request, basicHeartbeatRequest) -} diff --git a/third/github.com/Shopify/sarama/heartbeat_response.go b/third/github.com/Shopify/sarama/heartbeat_response.go deleted file mode 100644 index 766f5fdec..000000000 --- a/third/github.com/Shopify/sarama/heartbeat_response.go +++ /dev/null @@ -1,32 +0,0 @@ -package sarama - -type HeartbeatResponse struct { - Err KError -} - -func (r *HeartbeatResponse) encode(pe packetEncoder) error { - pe.putInt16(int16(r.Err)) - return nil -} - -func (r *HeartbeatResponse) decode(pd packetDecoder, version int16) error { - kerr, err := pd.getInt16() - if err != nil { - return err - } - r.Err = KError(kerr) - - return nil -} - -func (r *HeartbeatResponse) key() int16 { - return 12 -} - -func (r *HeartbeatResponse) version() int16 { - return 0 -} - -func (r *HeartbeatResponse) requiredVersion() KafkaVersion { - return V0_9_0_0 -} diff --git a/third/github.com/Shopify/sarama/heartbeat_response_test.go b/third/github.com/Shopify/sarama/heartbeat_response_test.go deleted file mode 100644 index 5bcbec985..000000000 --- a/third/github.com/Shopify/sarama/heartbeat_response_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package sarama - -import "testing" - -var ( - heartbeatResponseNoError = []byte{ - 0x00, 0x00} -) - -func TestHeartbeatResponse(t *testing.T) { - var response *HeartbeatResponse - - response = new(HeartbeatResponse) - testVersionDecodable(t, "no error", response, heartbeatResponseNoError, 0) - if response.Err != ErrNoError { - t.Error("Decoding error failed: no error expected but found", response.Err) - } -} diff --git a/third/github.com/Shopify/sarama/init_producer_id_request.go b/third/github.com/Shopify/sarama/init_producer_id_request.go deleted file mode 100644 index 8ceb6c232..000000000 --- a/third/github.com/Shopify/sarama/init_producer_id_request.go +++ /dev/null @@ -1,43 +0,0 @@ -package sarama - -import "time" - -type InitProducerIDRequest struct { - TransactionalID *string - TransactionTimeout time.Duration -} - -func (i *InitProducerIDRequest) encode(pe packetEncoder) error { - if err := pe.putNullableString(i.TransactionalID); err != nil { - return err - } - pe.putInt32(int32(i.TransactionTimeout / time.Millisecond)) - - return nil -} - -func (i *InitProducerIDRequest) decode(pd packetDecoder, version int16) (err error) { - if i.TransactionalID, err = pd.getNullableString(); err != nil { - return err - } - - timeout, err := pd.getInt32() - if err != nil { - return err - } - i.TransactionTimeout = time.Duration(timeout) * time.Millisecond - - return nil -} - -func (i *InitProducerIDRequest) key() int16 { - return 22 -} - -func (i *InitProducerIDRequest) version() int16 { - return 0 -} - -func (i *InitProducerIDRequest) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/third/github.com/Shopify/sarama/init_producer_id_request_test.go b/third/github.com/Shopify/sarama/init_producer_id_request_test.go deleted file mode 100644 index 5c83d8514..000000000 --- a/third/github.com/Shopify/sarama/init_producer_id_request_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -var ( - initProducerIDRequestNull = []byte{ - 255, 255, - 0, 0, 0, 100, - } - - initProducerIDRequest = []byte{ - 0, 3, 't', 'x', 'n', - 0, 0, 0, 100, - } -) - -func TestInitProducerIDRequest(t *testing.T) { - req := &InitProducerIDRequest{ - TransactionTimeout: 100 * time.Millisecond, - } - - testRequest(t, "null transaction id", req, initProducerIDRequestNull) - - transactionID := "txn" - req.TransactionalID = &transactionID - - testRequest(t, "transaction id", req, initProducerIDRequest) -} diff --git a/third/github.com/Shopify/sarama/init_producer_id_response.go b/third/github.com/Shopify/sarama/init_producer_id_response.go deleted file mode 100644 index 1b32eb085..000000000 --- a/third/github.com/Shopify/sarama/init_producer_id_response.go +++ /dev/null @@ -1,55 +0,0 @@ -package sarama - -import "time" - -type InitProducerIDResponse struct { - ThrottleTime time.Duration - Err KError - ProducerID int64 - ProducerEpoch int16 -} - -func (i *InitProducerIDResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(i.ThrottleTime / time.Millisecond)) - pe.putInt16(int16(i.Err)) - pe.putInt64(i.ProducerID) - pe.putInt16(i.ProducerEpoch) - - return nil -} - -func (i *InitProducerIDResponse) decode(pd packetDecoder, version int16) (err error) { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - i.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - kerr, err := pd.getInt16() - if err != nil { - return err - } - i.Err = KError(kerr) - - if i.ProducerID, err = pd.getInt64(); err != nil { - return err - } - - if i.ProducerEpoch, err = pd.getInt16(); err != nil { - return err - } - - return nil -} - -func (i *InitProducerIDResponse) key() int16 { - return 22 -} - -func (i *InitProducerIDResponse) version() int16 { - return 0 -} - -func (i *InitProducerIDResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/third/github.com/Shopify/sarama/init_producer_id_response_test.go b/third/github.com/Shopify/sarama/init_producer_id_response_test.go deleted file mode 100644 index b0649386a..000000000 --- a/third/github.com/Shopify/sarama/init_producer_id_response_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -var ( - initProducerIDResponse = []byte{ - 0, 0, 0, 100, - 0, 0, - 0, 0, 0, 0, 0, 0, 31, 64, // producerID = 8000 - 0, 0, // epoch - } - - initProducerIDRequestError = []byte{ - 0, 0, 0, 100, - 0, 51, - 255, 255, 255, 255, 255, 255, 255, 255, - 0, 0, - } -) - -func TestInitProducerIDResponse(t *testing.T) { - resp := &InitProducerIDResponse{ - ThrottleTime: 100 * time.Millisecond, - ProducerID: 8000, - ProducerEpoch: 0, - } - - testResponse(t, "", resp, initProducerIDResponse) - - resp.Err = ErrConcurrentTransactions - resp.ProducerID = -1 - - testResponse(t, "with error", resp, initProducerIDRequestError) -} diff --git a/third/github.com/Shopify/sarama/join_group_request.go b/third/github.com/Shopify/sarama/join_group_request.go deleted file mode 100644 index 97e9299ea..000000000 --- a/third/github.com/Shopify/sarama/join_group_request.go +++ /dev/null @@ -1,163 +0,0 @@ -package sarama - -type GroupProtocol struct { - Name string - Metadata []byte -} - -func (p *GroupProtocol) decode(pd packetDecoder) (err error) { - p.Name, err = pd.getString() - if err != nil { - return err - } - p.Metadata, err = pd.getBytes() - return err -} - -func (p *GroupProtocol) encode(pe packetEncoder) (err error) { - if err := pe.putString(p.Name); err != nil { - return err - } - if err := pe.putBytes(p.Metadata); err != nil { - return err - } - return nil -} - -type JoinGroupRequest struct { - Version int16 - GroupId string - SessionTimeout int32 - RebalanceTimeout int32 - MemberId string - ProtocolType string - GroupProtocols map[string][]byte // deprecated; use OrderedGroupProtocols - OrderedGroupProtocols []*GroupProtocol -} - -func (r *JoinGroupRequest) encode(pe packetEncoder) error { - if err := pe.putString(r.GroupId); err != nil { - return err - } - pe.putInt32(r.SessionTimeout) - if r.Version >= 1 { - pe.putInt32(r.RebalanceTimeout) - } - if err := pe.putString(r.MemberId); err != nil { - return err - } - if err := pe.putString(r.ProtocolType); err != nil { - return err - } - - if len(r.GroupProtocols) > 0 { - if len(r.OrderedGroupProtocols) > 0 { - return PacketDecodingError{"cannot specify both GroupProtocols and OrderedGroupProtocols on JoinGroupRequest"} - } - - if err := pe.putArrayLength(len(r.GroupProtocols)); err != nil { - return err - } - for name, metadata := range r.GroupProtocols { - if err := pe.putString(name); err != nil { - return err - } - if err := pe.putBytes(metadata); err != nil { - return err - } - } - } else { - if err := pe.putArrayLength(len(r.OrderedGroupProtocols)); err != nil { - return err - } - for _, protocol := range r.OrderedGroupProtocols { - if err := protocol.encode(pe); err != nil { - return err - } - } - } - - return nil -} - -func (r *JoinGroupRequest) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if r.GroupId, err = pd.getString(); err != nil { - return - } - - if r.SessionTimeout, err = pd.getInt32(); err != nil { - return - } - - if version >= 1 { - if r.RebalanceTimeout, err = pd.getInt32(); err != nil { - return err - } - } - - if r.MemberId, err = pd.getString(); err != nil { - return - } - - if r.ProtocolType, err = pd.getString(); err != nil { - return - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - if n == 0 { - return nil - } - - r.GroupProtocols = make(map[string][]byte) - for i := 0; i < n; i++ { - protocol := &GroupProtocol{} - if err := protocol.decode(pd); err != nil { - return err - } - r.GroupProtocols[protocol.Name] = protocol.Metadata - r.OrderedGroupProtocols = append(r.OrderedGroupProtocols, protocol) - } - - return nil -} - -func (r *JoinGroupRequest) key() int16 { - return 11 -} - -func (r *JoinGroupRequest) version() int16 { - return r.Version -} - -func (r *JoinGroupRequest) requiredVersion() KafkaVersion { - switch r.Version { - case 2: - return V0_11_0_0 - case 1: - return V0_10_1_0 - default: - return V0_9_0_0 - } -} - -func (r *JoinGroupRequest) AddGroupProtocol(name string, metadata []byte) { - r.OrderedGroupProtocols = append(r.OrderedGroupProtocols, &GroupProtocol{ - Name: name, - Metadata: metadata, - }) -} - -func (r *JoinGroupRequest) AddGroupProtocolMetadata(name string, metadata *ConsumerGroupMemberMetadata) error { - bin, err := encode(metadata, nil) - if err != nil { - return err - } - - r.AddGroupProtocol(name, bin) - return nil -} diff --git a/third/github.com/Shopify/sarama/join_group_request_test.go b/third/github.com/Shopify/sarama/join_group_request_test.go deleted file mode 100644 index a2e17f980..000000000 --- a/third/github.com/Shopify/sarama/join_group_request_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package sarama - -import "testing" - -var ( - joinGroupRequestV0_NoProtocols = []byte{ - 0, 9, 'T', 'e', 's', 't', 'G', 'r', 'o', 'u', 'p', // Group ID - 0, 0, 0, 100, // Session timeout - 0, 0, // Member ID - 0, 8, 'c', 'o', 'n', 's', 'u', 'm', 'e', 'r', // Protocol Type - 0, 0, 0, 0, // 0 protocol groups - } - - joinGroupRequestV0_OneProtocol = []byte{ - 0, 9, 'T', 'e', 's', 't', 'G', 'r', 'o', 'u', 'p', // Group ID - 0, 0, 0, 100, // Session timeout - 0, 11, 'O', 'n', 'e', 'P', 'r', 'o', 't', 'o', 'c', 'o', 'l', // Member ID - 0, 8, 'c', 'o', 'n', 's', 'u', 'm', 'e', 'r', // Protocol Type - 0, 0, 0, 1, // 1 group protocol - 0, 3, 'o', 'n', 'e', // Protocol name - 0, 0, 0, 3, 0x01, 0x02, 0x03, // protocol metadata - } - - joinGroupRequestV1 = []byte{ - 0, 9, 'T', 'e', 's', 't', 'G', 'r', 'o', 'u', 'p', // Group ID - 0, 0, 0, 100, // Session timeout - 0, 0, 0, 200, // Rebalance timeout - 0, 11, 'O', 'n', 'e', 'P', 'r', 'o', 't', 'o', 'c', 'o', 'l', // Member ID - 0, 8, 'c', 'o', 'n', 's', 'u', 'm', 'e', 'r', // Protocol Type - 0, 0, 0, 1, // 1 group protocol - 0, 3, 'o', 'n', 'e', // Protocol name - 0, 0, 0, 3, 0x01, 0x02, 0x03, // protocol metadata - } -) - -func TestJoinGroupRequest(t *testing.T) { - request := new(JoinGroupRequest) - request.GroupId = "TestGroup" - request.SessionTimeout = 100 - request.ProtocolType = "consumer" - testRequest(t, "V0: no protocols", request, joinGroupRequestV0_NoProtocols) -} - -func TestJoinGroupRequestV0_OneProtocol(t *testing.T) { - request := new(JoinGroupRequest) - request.GroupId = "TestGroup" - request.SessionTimeout = 100 - request.MemberId = "OneProtocol" - request.ProtocolType = "consumer" - request.AddGroupProtocol("one", []byte{0x01, 0x02, 0x03}) - packet := testRequestEncode(t, "V0: one protocol", request, joinGroupRequestV0_OneProtocol) - request.GroupProtocols = make(map[string][]byte) - request.GroupProtocols["one"] = []byte{0x01, 0x02, 0x03} - testRequestDecode(t, "V0: one protocol", request, packet) -} - -func TestJoinGroupRequestDeprecatedEncode(t *testing.T) { - request := new(JoinGroupRequest) - request.GroupId = "TestGroup" - request.SessionTimeout = 100 - request.MemberId = "OneProtocol" - request.ProtocolType = "consumer" - request.GroupProtocols = make(map[string][]byte) - request.GroupProtocols["one"] = []byte{0x01, 0x02, 0x03} - packet := testRequestEncode(t, "V0: one protocol", request, joinGroupRequestV0_OneProtocol) - request.AddGroupProtocol("one", []byte{0x01, 0x02, 0x03}) - testRequestDecode(t, "V0: one protocol", request, packet) -} - -func TestJoinGroupRequestV1(t *testing.T) { - request := new(JoinGroupRequest) - request.Version = 1 - request.GroupId = "TestGroup" - request.SessionTimeout = 100 - request.RebalanceTimeout = 200 - request.MemberId = "OneProtocol" - request.ProtocolType = "consumer" - request.AddGroupProtocol("one", []byte{0x01, 0x02, 0x03}) - packet := testRequestEncode(t, "V1", request, joinGroupRequestV1) - request.GroupProtocols = make(map[string][]byte) - request.GroupProtocols["one"] = []byte{0x01, 0x02, 0x03} - testRequestDecode(t, "V1", request, packet) -} diff --git a/third/github.com/Shopify/sarama/join_group_response.go b/third/github.com/Shopify/sarama/join_group_response.go deleted file mode 100644 index 5752acc8a..000000000 --- a/third/github.com/Shopify/sarama/join_group_response.go +++ /dev/null @@ -1,135 +0,0 @@ -package sarama - -type JoinGroupResponse struct { - Version int16 - ThrottleTime int32 - Err KError - GenerationId int32 - GroupProtocol string - LeaderId string - MemberId string - Members map[string][]byte -} - -func (r *JoinGroupResponse) GetMembers() (map[string]ConsumerGroupMemberMetadata, error) { - members := make(map[string]ConsumerGroupMemberMetadata, len(r.Members)) - for id, bin := range r.Members { - meta := new(ConsumerGroupMemberMetadata) - if err := decode(bin, meta); err != nil { - return nil, err - } - members[id] = *meta - } - return members, nil -} - -func (r *JoinGroupResponse) encode(pe packetEncoder) error { - if r.Version >= 2 { - pe.putInt32(r.ThrottleTime) - } - pe.putInt16(int16(r.Err)) - pe.putInt32(r.GenerationId) - - if err := pe.putString(r.GroupProtocol); err != nil { - return err - } - if err := pe.putString(r.LeaderId); err != nil { - return err - } - if err := pe.putString(r.MemberId); err != nil { - return err - } - - if err := pe.putArrayLength(len(r.Members)); err != nil { - return err - } - - for memberId, memberMetadata := range r.Members { - if err := pe.putString(memberId); err != nil { - return err - } - - if err := pe.putBytes(memberMetadata); err != nil { - return err - } - } - - return nil -} - -func (r *JoinGroupResponse) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if version >= 2 { - if r.ThrottleTime, err = pd.getInt32(); err != nil { - return - } - } - - kerr, err := pd.getInt16() - if err != nil { - return err - } - - r.Err = KError(kerr) - - if r.GenerationId, err = pd.getInt32(); err != nil { - return - } - - if r.GroupProtocol, err = pd.getString(); err != nil { - return - } - - if r.LeaderId, err = pd.getString(); err != nil { - return - } - - if r.MemberId, err = pd.getString(); err != nil { - return - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - if n == 0 { - return nil - } - - r.Members = make(map[string][]byte) - for i := 0; i < n; i++ { - memberId, err := pd.getString() - if err != nil { - return err - } - - memberMetadata, err := pd.getBytes() - if err != nil { - return err - } - - r.Members[memberId] = memberMetadata - } - - return nil -} - -func (r *JoinGroupResponse) key() int16 { - return 11 -} - -func (r *JoinGroupResponse) version() int16 { - return r.Version -} - -func (r *JoinGroupResponse) requiredVersion() KafkaVersion { - switch r.Version { - case 2: - return V0_11_0_0 - case 1: - return V0_10_1_0 - default: - return V0_9_0_0 - } -} diff --git a/third/github.com/Shopify/sarama/join_group_response_test.go b/third/github.com/Shopify/sarama/join_group_response_test.go deleted file mode 100644 index a43b37a95..000000000 --- a/third/github.com/Shopify/sarama/join_group_response_test.go +++ /dev/null @@ -1,172 +0,0 @@ -package sarama - -import ( - "reflect" - "testing" -) - -var ( - joinGroupResponseV0_NoError = []byte{ - 0x00, 0x00, // No error - 0x00, 0x01, 0x02, 0x03, // Generation ID - 0, 8, 'p', 'r', 'o', 't', 'o', 'c', 'o', 'l', // Protocol name chosen - 0, 3, 'f', 'o', 'o', // Leader ID - 0, 3, 'b', 'a', 'r', // Member ID - 0, 0, 0, 0, // No member info - } - - joinGroupResponseV0_WithError = []byte{ - 0, 23, // Error: inconsistent group protocol - 0x00, 0x00, 0x00, 0x00, // Generation ID - 0, 0, // Protocol name chosen - 0, 0, // Leader ID - 0, 0, // Member ID - 0, 0, 0, 0, // No member info - } - - joinGroupResponseV0_Leader = []byte{ - 0x00, 0x00, // No error - 0x00, 0x01, 0x02, 0x03, // Generation ID - 0, 8, 'p', 'r', 'o', 't', 'o', 'c', 'o', 'l', // Protocol name chosen - 0, 3, 'f', 'o', 'o', // Leader ID - 0, 3, 'f', 'o', 'o', // Member ID == Leader ID - 0, 0, 0, 1, // 1 member - 0, 3, 'f', 'o', 'o', // Member ID - 0, 0, 0, 3, 0x01, 0x02, 0x03, // Member metadata - } - - joinGroupResponseV1 = []byte{ - 0x00, 0x00, // No error - 0x00, 0x01, 0x02, 0x03, // Generation ID - 0, 8, 'p', 'r', 'o', 't', 'o', 'c', 'o', 'l', // Protocol name chosen - 0, 3, 'f', 'o', 'o', // Leader ID - 0, 3, 'b', 'a', 'r', // Member ID - 0, 0, 0, 0, // No member info - } - - joinGroupResponseV2 = []byte{ - 0, 0, 0, 100, - 0x00, 0x00, // No error - 0x00, 0x01, 0x02, 0x03, // Generation ID - 0, 8, 'p', 'r', 'o', 't', 'o', 'c', 'o', 'l', // Protocol name chosen - 0, 3, 'f', 'o', 'o', // Leader ID - 0, 3, 'b', 'a', 'r', // Member ID - 0, 0, 0, 0, // No member info - } -) - -func TestJoinGroupResponseV0(t *testing.T) { - var response *JoinGroupResponse - - response = new(JoinGroupResponse) - testVersionDecodable(t, "no error", response, joinGroupResponseV0_NoError, 0) - if response.Err != ErrNoError { - t.Error("Decoding Err failed: no error expected but found", response.Err) - } - if response.GenerationId != 66051 { - t.Error("Decoding GenerationId failed, found:", response.GenerationId) - } - if response.LeaderId != "foo" { - t.Error("Decoding LeaderId failed, found:", response.LeaderId) - } - if response.MemberId != "bar" { - t.Error("Decoding MemberId failed, found:", response.MemberId) - } - if len(response.Members) != 0 { - t.Error("Decoding Members failed, found:", response.Members) - } - - response = new(JoinGroupResponse) - testVersionDecodable(t, "with error", response, joinGroupResponseV0_WithError, 0) - if response.Err != ErrInconsistentGroupProtocol { - t.Error("Decoding Err failed: ErrInconsistentGroupProtocol expected but found", response.Err) - } - if response.GenerationId != 0 { - t.Error("Decoding GenerationId failed, found:", response.GenerationId) - } - if response.LeaderId != "" { - t.Error("Decoding LeaderId failed, found:", response.LeaderId) - } - if response.MemberId != "" { - t.Error("Decoding MemberId failed, found:", response.MemberId) - } - if len(response.Members) != 0 { - t.Error("Decoding Members failed, found:", response.Members) - } - - response = new(JoinGroupResponse) - testVersionDecodable(t, "with error", response, joinGroupResponseV0_Leader, 0) - if response.Err != ErrNoError { - t.Error("Decoding Err failed: ErrNoError expected but found", response.Err) - } - if response.GenerationId != 66051 { - t.Error("Decoding GenerationId failed, found:", response.GenerationId) - } - if response.LeaderId != "foo" { - t.Error("Decoding LeaderId failed, found:", response.LeaderId) - } - if response.MemberId != "foo" { - t.Error("Decoding MemberId failed, found:", response.MemberId) - } - if len(response.Members) != 1 { - t.Error("Decoding Members failed, found:", response.Members) - } - if !reflect.DeepEqual(response.Members["foo"], []byte{0x01, 0x02, 0x03}) { - t.Error("Decoding foo member failed, found:", response.Members["foo"]) - } -} - -func TestJoinGroupResponseV1(t *testing.T) { - response := new(JoinGroupResponse) - testVersionDecodable(t, "no error", response, joinGroupResponseV1, 1) - if response.Err != ErrNoError { - t.Error("Decoding Err failed: no error expected but found", response.Err) - } - if response.GenerationId != 66051 { - t.Error("Decoding GenerationId failed, found:", response.GenerationId) - } - if response.GroupProtocol != "protocol" { - t.Error("Decoding GroupProtocol failed, found:", response.GroupProtocol) - } - if response.LeaderId != "foo" { - t.Error("Decoding LeaderId failed, found:", response.LeaderId) - } - if response.MemberId != "bar" { - t.Error("Decoding MemberId failed, found:", response.MemberId) - } - if response.Version != 1 { - t.Error("Decoding Version failed, found:", response.Version) - } - if len(response.Members) != 0 { - t.Error("Decoding Members failed, found:", response.Members) - } -} - -func TestJoinGroupResponseV2(t *testing.T) { - response := new(JoinGroupResponse) - testVersionDecodable(t, "no error", response, joinGroupResponseV2, 2) - if response.ThrottleTime != 100 { - t.Error("Decoding ThrottleTime failed, found:", response.ThrottleTime) - } - if response.Err != ErrNoError { - t.Error("Decoding Err failed: no error expected but found", response.Err) - } - if response.GenerationId != 66051 { - t.Error("Decoding GenerationId failed, found:", response.GenerationId) - } - if response.GroupProtocol != "protocol" { - t.Error("Decoding GroupProtocol failed, found:", response.GroupProtocol) - } - if response.LeaderId != "foo" { - t.Error("Decoding LeaderId failed, found:", response.LeaderId) - } - if response.MemberId != "bar" { - t.Error("Decoding MemberId failed, found:", response.MemberId) - } - if response.Version != 2 { - t.Error("Decoding Version failed, found:", response.Version) - } - if len(response.Members) != 0 { - t.Error("Decoding Members failed, found:", response.Members) - } -} diff --git a/third/github.com/Shopify/sarama/leave_group_request.go b/third/github.com/Shopify/sarama/leave_group_request.go deleted file mode 100644 index e17742748..000000000 --- a/third/github.com/Shopify/sarama/leave_group_request.go +++ /dev/null @@ -1,40 +0,0 @@ -package sarama - -type LeaveGroupRequest struct { - GroupId string - MemberId string -} - -func (r *LeaveGroupRequest) encode(pe packetEncoder) error { - if err := pe.putString(r.GroupId); err != nil { - return err - } - if err := pe.putString(r.MemberId); err != nil { - return err - } - - return nil -} - -func (r *LeaveGroupRequest) decode(pd packetDecoder, version int16) (err error) { - if r.GroupId, err = pd.getString(); err != nil { - return - } - if r.MemberId, err = pd.getString(); err != nil { - return - } - - return nil -} - -func (r *LeaveGroupRequest) key() int16 { - return 13 -} - -func (r *LeaveGroupRequest) version() int16 { - return 0 -} - -func (r *LeaveGroupRequest) requiredVersion() KafkaVersion { - return V0_9_0_0 -} diff --git a/third/github.com/Shopify/sarama/leave_group_request_test.go b/third/github.com/Shopify/sarama/leave_group_request_test.go deleted file mode 100644 index c1fed6d25..000000000 --- a/third/github.com/Shopify/sarama/leave_group_request_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package sarama - -import "testing" - -var ( - basicLeaveGroupRequest = []byte{ - 0, 3, 'f', 'o', 'o', - 0, 3, 'b', 'a', 'r', - } -) - -func TestLeaveGroupRequest(t *testing.T) { - var request *LeaveGroupRequest - - request = new(LeaveGroupRequest) - request.GroupId = "foo" - request.MemberId = "bar" - testRequest(t, "basic", request, basicLeaveGroupRequest) -} diff --git a/third/github.com/Shopify/sarama/leave_group_response.go b/third/github.com/Shopify/sarama/leave_group_response.go deleted file mode 100644 index d60c626da..000000000 --- a/third/github.com/Shopify/sarama/leave_group_response.go +++ /dev/null @@ -1,32 +0,0 @@ -package sarama - -type LeaveGroupResponse struct { - Err KError -} - -func (r *LeaveGroupResponse) encode(pe packetEncoder) error { - pe.putInt16(int16(r.Err)) - return nil -} - -func (r *LeaveGroupResponse) decode(pd packetDecoder, version int16) (err error) { - kerr, err := pd.getInt16() - if err != nil { - return err - } - r.Err = KError(kerr) - - return nil -} - -func (r *LeaveGroupResponse) key() int16 { - return 13 -} - -func (r *LeaveGroupResponse) version() int16 { - return 0 -} - -func (r *LeaveGroupResponse) requiredVersion() KafkaVersion { - return V0_9_0_0 -} diff --git a/third/github.com/Shopify/sarama/leave_group_response_test.go b/third/github.com/Shopify/sarama/leave_group_response_test.go deleted file mode 100644 index 9207c6668..000000000 --- a/third/github.com/Shopify/sarama/leave_group_response_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package sarama - -import "testing" - -var ( - leaveGroupResponseNoError = []byte{0x00, 0x00} - leaveGroupResponseWithError = []byte{0, 25} -) - -func TestLeaveGroupResponse(t *testing.T) { - var response *LeaveGroupResponse - - response = new(LeaveGroupResponse) - testVersionDecodable(t, "no error", response, leaveGroupResponseNoError, 0) - if response.Err != ErrNoError { - t.Error("Decoding error failed: no error expected but found", response.Err) - } - - response = new(LeaveGroupResponse) - testVersionDecodable(t, "with error", response, leaveGroupResponseWithError, 0) - if response.Err != ErrUnknownMemberId { - t.Error("Decoding error failed: ErrUnknownMemberId expected but found", response.Err) - } -} diff --git a/third/github.com/Shopify/sarama/length_field.go b/third/github.com/Shopify/sarama/length_field.go deleted file mode 100644 index da199a70a..000000000 --- a/third/github.com/Shopify/sarama/length_field.go +++ /dev/null @@ -1,82 +0,0 @@ -package sarama - -import "encoding/binary" - -// LengthField implements the PushEncoder and PushDecoder interfaces for calculating 4-byte lengths. -type lengthField struct { - startOffset int - length int32 -} - -func (l *lengthField) decode(pd packetDecoder) error { - var err error - l.length, err = pd.getInt32() - if err != nil { - return err - } - if l.length > int32(pd.remaining()) { - return ErrInsufficientData - } - return nil -} - -func (l *lengthField) saveOffset(in int) { - l.startOffset = in -} - -func (l *lengthField) reserveLength() int { - return 4 -} - -func (l *lengthField) run(curOffset int, buf []byte) error { - binary.BigEndian.PutUint32(buf[l.startOffset:], uint32(curOffset-l.startOffset-4)) - return nil -} - -func (l *lengthField) check(curOffset int, buf []byte) error { - if int32(curOffset-l.startOffset-4) != l.length { - return PacketDecodingError{"length field invalid"} - } - - return nil -} - -type varintLengthField struct { - startOffset int - length int64 -} - -func (l *varintLengthField) decode(pd packetDecoder) error { - var err error - l.length, err = pd.getVarint() - return err -} - -func (l *varintLengthField) saveOffset(in int) { - l.startOffset = in -} - -func (l *varintLengthField) adjustLength(currOffset int) int { - oldFieldSize := l.reserveLength() - l.length = int64(currOffset - l.startOffset - oldFieldSize) - - return l.reserveLength() - oldFieldSize -} - -func (l *varintLengthField) reserveLength() int { - var tmp [binary.MaxVarintLen64]byte - return binary.PutVarint(tmp[:], l.length) -} - -func (l *varintLengthField) run(curOffset int, buf []byte) error { - binary.PutVarint(buf[l.startOffset:], l.length) - return nil -} - -func (l *varintLengthField) check(curOffset int, buf []byte) error { - if int64(curOffset-l.startOffset-l.reserveLength()) != l.length { - return PacketDecodingError{"length field invalid"} - } - - return nil -} diff --git a/third/github.com/Shopify/sarama/list_groups_request.go b/third/github.com/Shopify/sarama/list_groups_request.go deleted file mode 100644 index 3b16abf7f..000000000 --- a/third/github.com/Shopify/sarama/list_groups_request.go +++ /dev/null @@ -1,24 +0,0 @@ -package sarama - -type ListGroupsRequest struct { -} - -func (r *ListGroupsRequest) encode(pe packetEncoder) error { - return nil -} - -func (r *ListGroupsRequest) decode(pd packetDecoder, version int16) (err error) { - return nil -} - -func (r *ListGroupsRequest) key() int16 { - return 16 -} - -func (r *ListGroupsRequest) version() int16 { - return 0 -} - -func (r *ListGroupsRequest) requiredVersion() KafkaVersion { - return V0_9_0_0 -} diff --git a/third/github.com/Shopify/sarama/list_groups_request_test.go b/third/github.com/Shopify/sarama/list_groups_request_test.go deleted file mode 100644 index 2e977d9a5..000000000 --- a/third/github.com/Shopify/sarama/list_groups_request_test.go +++ /dev/null @@ -1,7 +0,0 @@ -package sarama - -import "testing" - -func TestListGroupsRequest(t *testing.T) { - testRequest(t, "ListGroupsRequest", &ListGroupsRequest{}, []byte{}) -} diff --git a/third/github.com/Shopify/sarama/list_groups_response.go b/third/github.com/Shopify/sarama/list_groups_response.go deleted file mode 100644 index 56115d4c7..000000000 --- a/third/github.com/Shopify/sarama/list_groups_response.go +++ /dev/null @@ -1,69 +0,0 @@ -package sarama - -type ListGroupsResponse struct { - Err KError - Groups map[string]string -} - -func (r *ListGroupsResponse) encode(pe packetEncoder) error { - pe.putInt16(int16(r.Err)) - - if err := pe.putArrayLength(len(r.Groups)); err != nil { - return err - } - for groupId, protocolType := range r.Groups { - if err := pe.putString(groupId); err != nil { - return err - } - if err := pe.putString(protocolType); err != nil { - return err - } - } - - return nil -} - -func (r *ListGroupsResponse) decode(pd packetDecoder, version int16) error { - kerr, err := pd.getInt16() - if err != nil { - return err - } - - r.Err = KError(kerr) - - n, err := pd.getArrayLength() - if err != nil { - return err - } - if n == 0 { - return nil - } - - r.Groups = make(map[string]string) - for i := 0; i < n; i++ { - groupId, err := pd.getString() - if err != nil { - return err - } - protocolType, err := pd.getString() - if err != nil { - return err - } - - r.Groups[groupId] = protocolType - } - - return nil -} - -func (r *ListGroupsResponse) key() int16 { - return 16 -} - -func (r *ListGroupsResponse) version() int16 { - return 0 -} - -func (r *ListGroupsResponse) requiredVersion() KafkaVersion { - return V0_9_0_0 -} diff --git a/third/github.com/Shopify/sarama/list_groups_response_test.go b/third/github.com/Shopify/sarama/list_groups_response_test.go deleted file mode 100644 index 41ab822f9..000000000 --- a/third/github.com/Shopify/sarama/list_groups_response_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package sarama - -import ( - "testing" -) - -var ( - listGroupsResponseEmpty = []byte{ - 0, 0, // no error - 0, 0, 0, 0, // no groups - } - - listGroupsResponseError = []byte{ - 0, 31, // no error - 0, 0, 0, 0, // ErrClusterAuthorizationFailed - } - - listGroupsResponseWithConsumer = []byte{ - 0, 0, // no error - 0, 0, 0, 1, // 1 group - 0, 3, 'f', 'o', 'o', // group name - 0, 8, 'c', 'o', 'n', 's', 'u', 'm', 'e', 'r', // protocol type - } -) - -func TestListGroupsResponse(t *testing.T) { - var response *ListGroupsResponse - - response = new(ListGroupsResponse) - testVersionDecodable(t, "no error", response, listGroupsResponseEmpty, 0) - if response.Err != ErrNoError { - t.Error("Expected no gerror, found:", response.Err) - } - if len(response.Groups) != 0 { - t.Error("Expected no groups") - } - - response = new(ListGroupsResponse) - testVersionDecodable(t, "no error", response, listGroupsResponseError, 0) - if response.Err != ErrClusterAuthorizationFailed { - t.Error("Expected no gerror, found:", response.Err) - } - if len(response.Groups) != 0 { - t.Error("Expected no groups") - } - - response = new(ListGroupsResponse) - testVersionDecodable(t, "no error", response, listGroupsResponseWithConsumer, 0) - if response.Err != ErrNoError { - t.Error("Expected no gerror, found:", response.Err) - } - if len(response.Groups) != 1 { - t.Error("Expected one group") - } - if response.Groups["foo"] != "consumer" { - t.Error("Expected foo group to use consumer protocol") - } -} diff --git a/third/github.com/Shopify/sarama/message.go b/third/github.com/Shopify/sarama/message.go deleted file mode 100644 index 51d3309c0..000000000 --- a/third/github.com/Shopify/sarama/message.go +++ /dev/null @@ -1,157 +0,0 @@ -package sarama - -import ( - "fmt" - "time" -) - -// CompressionCodec represents the various compression codecs recognized by Kafka in messages. -type CompressionCodec int8 - -// The lowest 3 bits contain the compression codec used for the message -const compressionCodecMask int8 = 0x07 - -const ( - CompressionNone CompressionCodec = 0 - CompressionGZIP CompressionCodec = 1 - CompressionSnappy CompressionCodec = 2 - CompressionLZ4 CompressionCodec = 3 - CompressionZSTD CompressionCodec = 4 -) - -func (cc CompressionCodec) String() string { - return []string{ - "none", - "gzip", - "snappy", - "lz4", - }[int(cc)] -} - -// CompressionLevelDefault is the constant to use in CompressionLevel -// to have the default compression level for any codec. The value is picked -// that we don't use any existing compression levels. -const CompressionLevelDefault = -1000 - -type Message struct { - Codec CompressionCodec // codec used to compress the message contents - CompressionLevel int // compression level - Key []byte // the message key, may be nil - Value []byte // the message contents - Set *MessageSet // the message set a message might wrap - Version int8 // v1 requires Kafka 0.10 - Timestamp time.Time // the timestamp of the message (version 1+ only) - - compressedCache []byte - compressedSize int // used for computing the compression ratio metrics -} - -func (m *Message) encode(pe packetEncoder) error { - pe.push(newCRC32Field(crcIEEE)) - - pe.putInt8(m.Version) - - attributes := int8(m.Codec) & compressionCodecMask - pe.putInt8(attributes) - - if m.Version >= 1 { - if err := (Timestamp{&m.Timestamp}).encode(pe); err != nil { - return err - } - } - - err := pe.putBytes(m.Key) - if err != nil { - return err - } - - var payload []byte - - if m.compressedCache != nil { - payload = m.compressedCache - m.compressedCache = nil - } else if m.Value != nil { - - payload, err = compress(m.Codec, m.CompressionLevel, m.Value) - if err != nil { - return err - } - m.compressedCache = payload - // Keep in mind the compressed payload size for metric gathering - m.compressedSize = len(payload) - } - - if err = pe.putBytes(payload); err != nil { - return err - } - - return pe.pop() -} - -func (m *Message) decode(pd packetDecoder) (err error) { - err = pd.push(newCRC32Field(crcIEEE)) - if err != nil { - return err - } - - m.Version, err = pd.getInt8() - if err != nil { - return err - } - - if m.Version > 1 { - return PacketDecodingError{fmt.Sprintf("unknown magic byte (%v)", m.Version)} - } - - attribute, err := pd.getInt8() - if err != nil { - return err - } - m.Codec = CompressionCodec(attribute & compressionCodecMask) - - if m.Version == 1 { - if err := (Timestamp{&m.Timestamp}).decode(pd); err != nil { - return err - } - } - - m.Key, err = pd.getBytes() - if err != nil { - return err - } - - m.Value, err = pd.getBytes() - if err != nil { - return err - } - - // Required for deep equal assertion during tests but might be useful - // for future metrics about the compression ratio in fetch requests - m.compressedSize = len(m.Value) - - switch m.Codec { - case CompressionNone: - // nothing to do - default: - if m.Value == nil { - break - } - - m.Value, err = decompress(m.Codec, m.Value) - if err != nil { - return err - } - if err := m.decodeSet(); err != nil { - return err - } - } - - return pd.pop() -} - -// decodes a message set from a previousy encoded bulk-message -func (m *Message) decodeSet() (err error) { - pd := realDecoder{raw: m.Value} - m.Set = &MessageSet{} - return m.Set.decode(&pd) -} diff --git a/third/github.com/Shopify/sarama/message_set.go b/third/github.com/Shopify/sarama/message_set.go deleted file mode 100644 index 600c7c4df..000000000 --- a/third/github.com/Shopify/sarama/message_set.go +++ /dev/null @@ -1,108 +0,0 @@ -package sarama - -type MessageBlock struct { - Offset int64 - Msg *Message -} - -// Messages convenience helper which returns either all the -// messages that are wrapped in this block -func (msb *MessageBlock) Messages() []*MessageBlock { - if msb.Msg.Set != nil { - return msb.Msg.Set.Messages - } - return []*MessageBlock{msb} -} - -func (msb *MessageBlock) encode(pe packetEncoder) error { - pe.putInt64(msb.Offset) - pe.push(&lengthField{}) - err := msb.Msg.encode(pe) - if err != nil { - return err - } - return pe.pop() -} - -func (msb *MessageBlock) decode(pd packetDecoder) (err error) { - if msb.Offset, err = pd.getInt64(); err != nil { - return err - } - - if err = pd.push(&lengthField{}); err != nil { - return err - } - - msb.Msg = new(Message) - if err = msb.Msg.decode(pd); err != nil { - return err - } - - if err = pd.pop(); err != nil { - return err - } - - return nil -} - -type MessageSet struct { - PartialTrailingMessage bool // whether the set on the wire contained an incomplete trailing MessageBlock - OverflowMessage bool // whether the set on the wire contained an overflow message - Messages []*MessageBlock -} - -func (ms *MessageSet) encode(pe packetEncoder) error { - for i := range ms.Messages { - err := ms.Messages[i].encode(pe) - if err != nil { - return err - } - } - return nil -} - -func (ms *MessageSet) decode(pd packetDecoder) (err error) { - ms.Messages = nil - - for pd.remaining() > 0 { - magic, err := magicValue(pd) - if err != nil { - if err == ErrInsufficientData { - ms.PartialTrailingMessage = true - return nil - } - return err - } - - if magic > 1 { - return nil - } - - msb := new(MessageBlock) - err = msb.decode(pd) - switch err { - case nil: - ms.Messages = append(ms.Messages, msb) - case ErrInsufficientData: - // As an optimization the server is allowed to return a partial message at the - // end of the message set. Clients should handle this case. So we just ignore such things. - if msb.Offset == -1 { - // This is an overflow message caused by chunked down conversion - ms.OverflowMessage = true - } else { - ms.PartialTrailingMessage = true - } - return nil - default: - return err - } - } - - return nil -} - -func (ms *MessageSet) addMessage(msg *Message) { - block := new(MessageBlock) - block.Msg = msg - ms.Messages = append(ms.Messages, block) -} diff --git a/third/github.com/Shopify/sarama/message_test.go b/third/github.com/Shopify/sarama/message_test.go deleted file mode 100644 index aa103d43f..000000000 --- a/third/github.com/Shopify/sarama/message_test.go +++ /dev/null @@ -1,240 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -var ( - emptyMessage = []byte{ - 167, 236, 104, 3, // CRC - 0x00, // magic version byte - 0x00, // attribute flags - 0xFF, 0xFF, 0xFF, 0xFF, // key - 0xFF, 0xFF, 0xFF, 0xFF} // value - - emptyV1Message = []byte{ - 204, 47, 121, 217, // CRC - 0x01, // magic version byte - 0x00, // attribute flags - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // timestamp - 0xFF, 0xFF, 0xFF, 0xFF, // key - 0xFF, 0xFF, 0xFF, 0xFF} // value - - emptyV2Message = []byte{ - 167, 236, 104, 3, // CRC - 0x02, // magic version byte - 0x00, // attribute flags - 0xFF, 0xFF, 0xFF, 0xFF, // key - 0xFF, 0xFF, 0xFF, 0xFF} // value - - emptyGzipMessage = []byte{ - 132, 99, 80, 148, //CRC - 0x00, // magic version byte - 0x01, // attribute flags - 0xFF, 0xFF, 0xFF, 0xFF, // key - // value - 0x00, 0x00, 0x00, 0x17, - 0x1f, 0x8b, - 0x08, - 0, 0, 0, 0, 0, 0, 255, 1, 0, 0, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0} - - emptyLZ4Message = []byte{ - 132, 219, 238, 101, // CRC - 0x01, // version byte - 0x03, // attribute flags: lz4 - 0, 0, 1, 88, 141, 205, 89, 56, // timestamp - 0xFF, 0xFF, 0xFF, 0xFF, // key - 0x00, 0x00, 0x00, 0x0f, // len - 0x04, 0x22, 0x4D, 0x18, // LZ4 magic number - 100, // LZ4 flags: version 01, block indepedant, content checksum - 112, 185, 0, 0, 0, 0, // LZ4 data - 5, 93, 204, 2, // LZ4 checksum - } - - emptyZSTDMessage = []byte{ - 180, 172, 84, 179, // CRC - 0x01, // version byte - 0x04, // attribute flags: zstd - 0, 0, 1, 88, 141, 205, 89, 56, // timestamp - 0xFF, 0xFF, 0xFF, 0xFF, // key - 0x00, 0x00, 0x00, 0x09, // len - // ZSTD data - 0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x00, 0x01, 0x00, 0x00, - } - - emptyBulkSnappyMessage = []byte{ - 180, 47, 53, 209, //CRC - 0x00, // magic version byte - 0x02, // attribute flags - 0xFF, 0xFF, 0xFF, 0xFF, // key - 0, 0, 0, 42, - 130, 83, 78, 65, 80, 80, 89, 0, // SNAPPY magic - 0, 0, 0, 1, // min version - 0, 0, 0, 1, // default version - 0, 0, 0, 22, 52, 0, 0, 25, 1, 16, 14, 227, 138, 104, 118, 25, 15, 13, 1, 8, 1, 0, 0, 62, 26, 0} - - emptyBulkGzipMessage = []byte{ - 139, 160, 63, 141, //CRC - 0x00, // magic version byte - 0x01, // attribute flags - 0xFF, 0xFF, 0xFF, 0xFF, // key - 0x00, 0x00, 0x00, 0x27, // len - 0x1f, 0x8b, // Gzip Magic - 0x08, // deflate compressed - 0, 0, 0, 0, 0, 0, 0, 99, 96, 128, 3, 190, 202, 112, 143, 7, 12, 12, 255, 129, 0, 33, 200, 192, 136, 41, 3, 0, 199, 226, 155, 70, 52, 0, 0, 0} - - emptyBulkLZ4Message = []byte{ - 246, 12, 188, 129, // CRC - 0x01, // Version - 0x03, // attribute flags (LZ4) - 255, 255, 249, 209, 212, 181, 73, 201, // timestamp - 0xFF, 0xFF, 0xFF, 0xFF, // key - 0x00, 0x00, 0x00, 0x47, // len - 0x04, 0x22, 0x4D, 0x18, // magic number lz4 - 100, // lz4 flags 01100100 - // version: 01, block indep: 1, block checksum: 0, content size: 0, content checksum: 1, reserved: 00 - 112, 185, 52, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 121, 87, 72, 224, 0, 0, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 14, 121, 87, 72, 224, 0, 0, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, - 71, 129, 23, 111, // LZ4 checksum - } - - emptyBulkZSTDMessage = []byte{ - 203, 151, 133, 28, // CRC - 0x01, // Version - 0x04, // attribute flags (ZSTD) - 255, 255, 249, 209, 212, 181, 73, 201, // timestamp - 0xFF, 0xFF, 0xFF, 0xFF, // key - 0x00, 0x00, 0x00, 0x26, // len - // ZSTD data - 0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x34, 0xcd, 0x0, 0x0, 0x78, 0x0, 0x0, 0xe, 0x79, 0x57, 0x48, 0xe0, 0x0, 0x0, 0xff, 0xff, 0xff, 0xff, 0x0, 0x1, 0x3, 0x0, 0x3d, 0xbd, 0x0, 0x3b, 0x15, 0x0, 0xb, 0xd2, 0x34, 0xc1, 0x78, - } -) - -func TestMessageEncoding(t *testing.T) { - message := Message{} - testEncodable(t, "empty", &message, emptyMessage) - - message.Value = []byte{} - message.Codec = CompressionGZIP - testEncodable(t, "empty gzip", &message, emptyGzipMessage) - - message.Value = []byte{} - message.Codec = CompressionLZ4 - message.Timestamp = time.Unix(1479847795, 0) - message.Version = 1 - testEncodable(t, "empty lz4", &message, emptyLZ4Message) - - message.Value = []byte{} - message.Codec = CompressionZSTD - message.Timestamp = time.Unix(1479847795, 0) - message.Version = 1 - testEncodable(t, "empty zstd", &message, emptyZSTDMessage) -} - -func TestMessageDecoding(t *testing.T) { - message := Message{} - testDecodable(t, "empty", &message, emptyMessage) - if message.Codec != CompressionNone { - t.Error("Decoding produced compression codec where there was none.") - } - if message.Key != nil { - t.Error("Decoding produced key where there was none.") - } - if message.Value != nil { - t.Error("Decoding produced value where there was none.") - } - if message.Set != nil { - t.Error("Decoding produced set where there was none.") - } - - testDecodable(t, "empty gzip", &message, emptyGzipMessage) - if message.Codec != CompressionGZIP { - t.Error("Decoding produced incorrect compression codec (was gzip).") - } - if message.Key != nil { - t.Error("Decoding produced key where there was none.") - } - if message.Value == nil || len(message.Value) != 0 { - t.Error("Decoding produced nil or content-ful value where there was an empty array.") - } -} - -func TestMessageDecodingBulkSnappy(t *testing.T) { - message := Message{} - testDecodable(t, "bulk snappy", &message, emptyBulkSnappyMessage) - if message.Codec != CompressionSnappy { - t.Errorf("Decoding produced codec %d, but expected %d.", message.Codec, CompressionSnappy) - } - if message.Key != nil { - t.Errorf("Decoding produced key %+v, but none was expected.", message.Key) - } - if message.Set == nil { - t.Error("Decoding produced no set, but one was expected.") - } else if len(message.Set.Messages) != 2 { - t.Errorf("Decoding produced a set with %d messages, but 2 were expected.", len(message.Set.Messages)) - } -} - -func TestMessageDecodingBulkGzip(t *testing.T) { - message := Message{} - testDecodable(t, "bulk gzip", &message, emptyBulkGzipMessage) - if message.Codec != CompressionGZIP { - t.Errorf("Decoding produced codec %d, but expected %d.", message.Codec, CompressionGZIP) - } - if message.Key != nil { - t.Errorf("Decoding produced key %+v, but none was expected.", message.Key) - } - if message.Set == nil { - t.Error("Decoding produced no set, but one was expected.") - } else if len(message.Set.Messages) != 2 { - t.Errorf("Decoding produced a set with %d messages, but 2 were expected.", len(message.Set.Messages)) - } -} - -func TestMessageDecodingBulkLZ4(t *testing.T) { - message := Message{} - testDecodable(t, "bulk lz4", &message, emptyBulkLZ4Message) - if message.Codec != CompressionLZ4 { - t.Errorf("Decoding produced codec %d, but expected %d.", message.Codec, CompressionLZ4) - } - if message.Key != nil { - t.Errorf("Decoding produced key %+v, but none was expected.", message.Key) - } - if message.Set == nil { - t.Error("Decoding produced no set, but one was expected.") - } else if len(message.Set.Messages) != 2 { - t.Errorf("Decoding produced a set with %d messages, but 2 were expected.", len(message.Set.Messages)) - } -} - -func TestMessageDecodingBulkZSTD(t *testing.T) { - message := Message{} - testDecodable(t, "bulk zstd", &message, emptyBulkZSTDMessage) - if message.Codec != CompressionZSTD { - t.Errorf("Decoding produced codec %d, but expected %d.", message.Codec, CompressionZSTD) - } - if message.Key != nil { - t.Errorf("Decoding produced key %+v, but none was expected.", message.Key) - } - if message.Set == nil { - t.Error("Decoding produced no set, but one was expected.") - } else if len(message.Set.Messages) != 2 { - t.Errorf("Decoding produced a set with %d messages, but 2 were expected.", len(message.Set.Messages)) - } -} - -func TestMessageDecodingVersion1(t *testing.T) { - message := Message{Version: 1} - testDecodable(t, "decoding empty v1 message", &message, emptyV1Message) -} - -func TestMessageDecodingUnknownVersions(t *testing.T) { - message := Message{Version: 2} - err := decode(emptyV2Message, &message) - if err == nil { - t.Error("Decoding did not produce an error for an unknown magic byte") - } - if err.Error() != "kafka: error decoding packet: unknown magic byte (2)" { - t.Error("Decoding an unknown magic byte produced an unknown error ", err) - } -} diff --git a/third/github.com/Shopify/sarama/metadata_request.go b/third/github.com/Shopify/sarama/metadata_request.go deleted file mode 100644 index 17dc4289a..000000000 --- a/third/github.com/Shopify/sarama/metadata_request.go +++ /dev/null @@ -1,88 +0,0 @@ -package sarama - -type MetadataRequest struct { - Version int16 - Topics []string - AllowAutoTopicCreation bool -} - -func (r *MetadataRequest) encode(pe packetEncoder) error { - if r.Version < 0 || r.Version > 5 { - return PacketEncodingError{"invalid or unsupported MetadataRequest version field"} - } - if r.Version == 0 || len(r.Topics) > 0 { - err := pe.putArrayLength(len(r.Topics)) - if err != nil { - return err - } - - for i := range r.Topics { - err = pe.putString(r.Topics[i]) - if err != nil { - return err - } - } - } else { - pe.putInt32(-1) - } - if r.Version > 3 { - pe.putBool(r.AllowAutoTopicCreation) - } - return nil -} - -func (r *MetadataRequest) decode(pd packetDecoder, version int16) error { - r.Version = version - size, err := pd.getInt32() - if err != nil { - return err - } - if size < 0 { - return nil - } else { - topicCount := size - if topicCount == 0 { - return nil - } - - r.Topics = make([]string, topicCount) - for i := range r.Topics { - topic, err := pd.getString() - if err != nil { - return err - } - r.Topics[i] = topic - } - } - if r.Version > 3 { - autoCreation, err := pd.getBool() - if err != nil { - return err - } - r.AllowAutoTopicCreation = autoCreation - } - return nil -} - -func (r *MetadataRequest) key() int16 { - return 3 -} - -func (r *MetadataRequest) version() int16 { - return r.Version -} - -func (r *MetadataRequest) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_10_0_0 - case 2: - return V0_10_1_0 - case 3, 4: - return V0_11_0_0 - case 5: - return V1_0_0_0 - default: - return MinVersion - } -} diff --git a/third/github.com/Shopify/sarama/metadata_request_test.go b/third/github.com/Shopify/sarama/metadata_request_test.go deleted file mode 100644 index 727e48a2c..000000000 --- a/third/github.com/Shopify/sarama/metadata_request_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package sarama - -import "testing" - -var ( - metadataRequestNoTopicsV0 = []byte{ - 0x00, 0x00, 0x00, 0x00} - - metadataRequestOneTopicV0 = []byte{ - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x06, 't', 'o', 'p', 'i', 'c', '1'} - - metadataRequestThreeTopicsV0 = []byte{ - 0x00, 0x00, 0x00, 0x03, - 0x00, 0x03, 'f', 'o', 'o', - 0x00, 0x03, 'b', 'a', 'r', - 0x00, 0x03, 'b', 'a', 'z'} - - metadataRequestNoTopicsV1 = []byte{ - 0xff, 0xff, 0xff, 0xff} - - metadataRequestAutoCreateV4 = append(metadataRequestOneTopicV0, byte(1)) - metadataRequestNoAutoCreateV4 = append(metadataRequestOneTopicV0, byte(0)) -) - -func TestMetadataRequestV0(t *testing.T) { - request := new(MetadataRequest) - testRequest(t, "no topics", request, metadataRequestNoTopicsV0) - - request.Topics = []string{"topic1"} - testRequest(t, "one topic", request, metadataRequestOneTopicV0) - - request.Topics = []string{"foo", "bar", "baz"} - testRequest(t, "three topics", request, metadataRequestThreeTopicsV0) -} - -func TestMetadataRequestV1(t *testing.T) { - request := new(MetadataRequest) - request.Version = 1 - testRequest(t, "no topics", request, metadataRequestNoTopicsV1) - - request.Topics = []string{"topic1"} - testRequest(t, "one topic", request, metadataRequestOneTopicV0) - - request.Topics = []string{"foo", "bar", "baz"} - testRequest(t, "three topics", request, metadataRequestThreeTopicsV0) -} - -func TestMetadataRequestV2(t *testing.T) { - request := new(MetadataRequest) - request.Version = 2 - testRequest(t, "no topics", request, metadataRequestNoTopicsV1) - - request.Topics = []string{"topic1"} - testRequest(t, "one topic", request, metadataRequestOneTopicV0) -} - -func TestMetadataRequestV3(t *testing.T) { - request := new(MetadataRequest) - request.Version = 3 - testRequest(t, "no topics", request, metadataRequestNoTopicsV1) - - request.Topics = []string{"topic1"} - testRequest(t, "one topic", request, metadataRequestOneTopicV0) -} - -func TestMetadataRequestV4(t *testing.T) { - request := new(MetadataRequest) - request.Version = 4 - request.Topics = []string{"topic1"} - request.AllowAutoTopicCreation = true - testRequest(t, "one topic", request, metadataRequestAutoCreateV4) - - request.AllowAutoTopicCreation = false - testRequest(t, "one topic", request, metadataRequestNoAutoCreateV4) -} diff --git a/third/github.com/Shopify/sarama/metadata_response.go b/third/github.com/Shopify/sarama/metadata_response.go deleted file mode 100644 index c402d05fa..000000000 --- a/third/github.com/Shopify/sarama/metadata_response.go +++ /dev/null @@ -1,321 +0,0 @@ -package sarama - -type PartitionMetadata struct { - Err KError - ID int32 - Leader int32 - Replicas []int32 - Isr []int32 - OfflineReplicas []int32 -} - -func (pm *PartitionMetadata) decode(pd packetDecoder, version int16) (err error) { - tmp, err := pd.getInt16() - if err != nil { - return err - } - pm.Err = KError(tmp) - - pm.ID, err = pd.getInt32() - if err != nil { - return err - } - - pm.Leader, err = pd.getInt32() - if err != nil { - return err - } - - pm.Replicas, err = pd.getInt32Array() - if err != nil { - return err - } - - pm.Isr, err = pd.getInt32Array() - if err != nil { - return err - } - - if version >= 5 { - pm.OfflineReplicas, err = pd.getInt32Array() - if err != nil { - return err - } - } - - return nil -} - -func (pm *PartitionMetadata) encode(pe packetEncoder, version int16) (err error) { - pe.putInt16(int16(pm.Err)) - pe.putInt32(pm.ID) - pe.putInt32(pm.Leader) - - err = pe.putInt32Array(pm.Replicas) - if err != nil { - return err - } - - err = pe.putInt32Array(pm.Isr) - if err != nil { - return err - } - - if version >= 5 { - err = pe.putInt32Array(pm.OfflineReplicas) - if err != nil { - return err - } - } - - return nil -} - -type TopicMetadata struct { - Err KError - Name string - IsInternal bool // Only valid for Version >= 1 - Partitions []*PartitionMetadata -} - -func (tm *TopicMetadata) decode(pd packetDecoder, version int16) (err error) { - tmp, err := pd.getInt16() - if err != nil { - return err - } - tm.Err = KError(tmp) - - tm.Name, err = pd.getString() - if err != nil { - return err - } - - if version >= 1 { - tm.IsInternal, err = pd.getBool() - if err != nil { - return err - } - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - tm.Partitions = make([]*PartitionMetadata, n) - for i := 0; i < n; i++ { - tm.Partitions[i] = new(PartitionMetadata) - err = tm.Partitions[i].decode(pd, version) - if err != nil { - return err - } - } - - return nil -} - -func (tm *TopicMetadata) encode(pe packetEncoder, version int16) (err error) { - pe.putInt16(int16(tm.Err)) - - err = pe.putString(tm.Name) - if err != nil { - return err - } - - if version >= 1 { - pe.putBool(tm.IsInternal) - } - - err = pe.putArrayLength(len(tm.Partitions)) - if err != nil { - return err - } - - for _, pm := range tm.Partitions { - err = pm.encode(pe, version) - if err != nil { - return err - } - } - - return nil -} - -type MetadataResponse struct { - Version int16 - ThrottleTimeMs int32 - Brokers []*Broker - ClusterID *string - ControllerID int32 - Topics []*TopicMetadata -} - -func (r *MetadataResponse) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if version >= 3 { - r.ThrottleTimeMs, err = pd.getInt32() - if err != nil { - return err - } - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Brokers = make([]*Broker, n) - for i := 0; i < n; i++ { - r.Brokers[i] = new(Broker) - err = r.Brokers[i].decode(pd, version) - if err != nil { - return err - } - } - - if version >= 2 { - r.ClusterID, err = pd.getNullableString() - if err != nil { - return err - } - } - - if version >= 1 { - r.ControllerID, err = pd.getInt32() - if err != nil { - return err - } - } else { - r.ControllerID = -1 - } - - n, err = pd.getArrayLength() - if err != nil { - return err - } - - r.Topics = make([]*TopicMetadata, n) - for i := 0; i < n; i++ { - r.Topics[i] = new(TopicMetadata) - err = r.Topics[i].decode(pd, version) - if err != nil { - return err - } - } - - return nil -} - -func (r *MetadataResponse) encode(pe packetEncoder) error { - if r.Version >= 3 { - pe.putInt32(r.ThrottleTimeMs) - } - - err := pe.putArrayLength(len(r.Brokers)) - if err != nil { - return err - } - for _, broker := range r.Brokers { - err = broker.encode(pe, r.Version) - if err != nil { - return err - } - } - - if r.Version >= 2 { - err := pe.putNullableString(r.ClusterID) - if err != nil { - return err - } - } - - if r.Version >= 1 { - pe.putInt32(r.ControllerID) - } - - err = pe.putArrayLength(len(r.Topics)) - if err != nil { - return err - } - for _, tm := range r.Topics { - err = tm.encode(pe, r.Version) - if err != nil { - return err - } - } - - return nil -} - -func (r *MetadataResponse) key() int16 { - return 3 -} - -func (r *MetadataResponse) version() int16 { - return r.Version -} - -func (r *MetadataResponse) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_10_0_0 - case 2: - return V0_10_1_0 - case 3, 4: - return V0_11_0_0 - case 5: - return V1_0_0_0 - default: - return MinVersion - } -} - -// testing API - -func (r *MetadataResponse) AddBroker(addr string, id int32) { - r.Brokers = append(r.Brokers, &Broker{id: id, addr: addr}) -} - -func (r *MetadataResponse) AddTopic(topic string, err KError) *TopicMetadata { - var tmatch *TopicMetadata - - for _, tm := range r.Topics { - if tm.Name == topic { - tmatch = tm - goto foundTopic - } - } - - tmatch = new(TopicMetadata) - tmatch.Name = topic - r.Topics = append(r.Topics, tmatch) - -foundTopic: - - tmatch.Err = err - return tmatch -} - -func (r *MetadataResponse) AddTopicPartition(topic string, partition, brokerID int32, replicas, isr []int32, err KError) { - tmatch := r.AddTopic(topic, ErrNoError) - var pmatch *PartitionMetadata - - for _, pm := range tmatch.Partitions { - if pm.ID == partition { - pmatch = pm - goto foundPartition - } - } - - pmatch = new(PartitionMetadata) - pmatch.ID = partition - tmatch.Partitions = append(tmatch.Partitions, pmatch) - -foundPartition: - - pmatch.Leader = brokerID - pmatch.Replicas = replicas - pmatch.Isr = isr - pmatch.Err = err - -} diff --git a/third/github.com/Shopify/sarama/metadata_response_test.go b/third/github.com/Shopify/sarama/metadata_response_test.go deleted file mode 100644 index 04a4ce7fc..000000000 --- a/third/github.com/Shopify/sarama/metadata_response_test.go +++ /dev/null @@ -1,290 +0,0 @@ -package sarama - -import "testing" - -var ( - emptyMetadataResponseV0 = []byte{ - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00} - - brokersNoTopicsMetadataResponseV0 = []byte{ - 0x00, 0x00, 0x00, 0x02, - - 0x00, 0x00, 0xab, 0xff, - 0x00, 0x09, 'l', 'o', 'c', 'a', 'l', 'h', 'o', 's', 't', - 0x00, 0x00, 0x00, 0x33, - - 0x00, 0x01, 0x02, 0x03, - 0x00, 0x0a, 'g', 'o', 'o', 'g', 'l', 'e', '.', 'c', 'o', 'm', - 0x00, 0x00, 0x01, 0x11, - - 0x00, 0x00, 0x00, 0x00} - - topicsNoBrokersMetadataResponseV0 = []byte{ - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x02, - - 0x00, 0x00, - 0x00, 0x03, 'f', 'o', 'o', - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x04, - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x07, - 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, - 0x00, 0x00, 0x00, 0x00, - - 0x00, 0x00, - 0x00, 0x03, 'b', 'a', 'r', - 0x00, 0x00, 0x00, 0x00} - - brokersNoTopicsMetadataResponseV1 = []byte{ - 0x00, 0x00, 0x00, 0x02, - - 0x00, 0x00, 0xab, 0xff, - 0x00, 0x09, 'l', 'o', 'c', 'a', 'l', 'h', 'o', 's', 't', - 0x00, 0x00, 0x00, 0x33, - 0x00, 0x05, 'r', 'a', 'c', 'k', '0', - - 0x00, 0x01, 0x02, 0x03, - 0x00, 0x0a, 'g', 'o', 'o', 'g', 'l', 'e', '.', 'c', 'o', 'm', - 0x00, 0x00, 0x01, 0x11, - 0x00, 0x05, 'r', 'a', 'c', 'k', '1', - - 0x00, 0x00, 0x00, 0x01, - - 0x00, 0x00, 0x00, 0x00} - - topicsNoBrokersMetadataResponseV1 = []byte{ - 0x00, 0x00, 0x00, 0x00, - - 0x00, 0x00, 0x00, 0x04, - - 0x00, 0x00, 0x00, 0x02, - - 0x00, 0x00, - 0x00, 0x03, 'f', 'o', 'o', - 0x00, - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x04, - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x07, - 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, - 0x00, 0x00, 0x00, 0x00, - - 0x00, 0x00, - 0x00, 0x03, 'b', 'a', 'r', - 0x01, - 0x00, 0x00, 0x00, 0x00} - - noBrokersNoTopicsWithThrottleTimeAndClusterIDV3 = []byte{ - 0x00, 0x00, 0x00, 0x10, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x09, 'c', 'l', 'u', 's', 't', 'e', 'r', 'I', 'd', - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x00} - - noBrokersOneTopicWithOfflineReplicasV5 = []byte{ - 0x00, 0x00, 0x00, 0x05, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x09, 'c', 'l', 'u', 's', 't', 'e', 'r', 'I', 'd', - 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, - 0x00, 0x03, 'f', 'o', 'o', - 0x00, - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x04, - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x07, - 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, - 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x03, - } -) - -func TestEmptyMetadataResponseV0(t *testing.T) { - response := MetadataResponse{} - - testVersionDecodable(t, "empty, V0", &response, emptyMetadataResponseV0, 0) - if len(response.Brokers) != 0 { - t.Error("Decoding produced", len(response.Brokers), "brokers where there were none!") - } - if len(response.Topics) != 0 { - t.Error("Decoding produced", len(response.Topics), "topics where there were none!") - } -} - -func TestMetadataResponseWithBrokersV0(t *testing.T) { - response := MetadataResponse{} - - testVersionDecodable(t, "brokers, no topics, V0", &response, brokersNoTopicsMetadataResponseV0, 0) - if len(response.Brokers) != 2 { - t.Fatal("Decoding produced", len(response.Brokers), "brokers where there were two!") - } - - if response.Brokers[0].id != 0xabff { - t.Error("Decoding produced invalid broker 0 id.") - } - if response.Brokers[0].addr != "localhost:51" { - t.Error("Decoding produced invalid broker 0 address.") - } - if response.Brokers[1].id != 0x010203 { - t.Error("Decoding produced invalid broker 1 id.") - } - if response.Brokers[1].addr != "google.com:273" { - t.Error("Decoding produced invalid broker 1 address.") - } - - if len(response.Topics) != 0 { - t.Error("Decoding produced", len(response.Topics), "topics where there were none!") - } -} - -func TestMetadataResponseWithTopicsV0(t *testing.T) { - response := MetadataResponse{} - - testVersionDecodable(t, "topics, no brokers, V0", &response, topicsNoBrokersMetadataResponseV0, 0) - if len(response.Brokers) != 0 { - t.Error("Decoding produced", len(response.Brokers), "brokers where there were none!") - } - - if len(response.Topics) != 2 { - t.Fatal("Decoding produced", len(response.Topics), "topics where there were two!") - } - - if response.Topics[0].Err != ErrNoError { - t.Error("Decoding produced invalid topic 0 error.") - } - - if response.Topics[0].Name != "foo" { - t.Error("Decoding produced invalid topic 0 name.") - } - - if len(response.Topics[0].Partitions) != 1 { - t.Fatal("Decoding produced invalid partition count for topic 0.") - } - - if response.Topics[0].Partitions[0].Err != ErrInvalidMessageSize { - t.Error("Decoding produced invalid topic 0 partition 0 error.") - } - - if response.Topics[0].Partitions[0].ID != 0x01 { - t.Error("Decoding produced invalid topic 0 partition 0 id.") - } - - if response.Topics[0].Partitions[0].Leader != 0x07 { - t.Error("Decoding produced invalid topic 0 partition 0 leader.") - } - - if len(response.Topics[0].Partitions[0].Replicas) != 3 { - t.Fatal("Decoding produced invalid topic 0 partition 0 replicas.") - } - for i := 0; i < 3; i++ { - if response.Topics[0].Partitions[0].Replicas[i] != int32(i+1) { - t.Error("Decoding produced invalid topic 0 partition 0 replica", i) - } - } - - if len(response.Topics[0].Partitions[0].Isr) != 0 { - t.Error("Decoding produced invalid topic 0 partition 0 isr length.") - } - - if response.Topics[1].Err != ErrNoError { - t.Error("Decoding produced invalid topic 1 error.") - } - - if response.Topics[1].Name != "bar" { - t.Error("Decoding produced invalid topic 0 name.") - } - - if len(response.Topics[1].Partitions) != 0 { - t.Error("Decoding produced invalid partition count for topic 1.") - } -} - -func TestMetadataResponseWithBrokersV1(t *testing.T) { - response := MetadataResponse{} - - testVersionDecodable(t, "topics, V1", &response, brokersNoTopicsMetadataResponseV1, 1) - if len(response.Brokers) != 2 { - t.Error("Decoding produced", len(response.Brokers), "brokers where there were 2!") - } - if response.Brokers[0].rack == nil || *response.Brokers[0].rack != "rack0" { - t.Error("Decoding produced invalid broker 0 rack.") - } - if response.Brokers[1].rack == nil || *response.Brokers[1].rack != "rack1" { - t.Error("Decoding produced invalid broker 1 rack.") - } - if response.ControllerID != 1 { - t.Error("Decoding produced", response.ControllerID, "should have been 1!") - } - if len(response.Topics) != 0 { - t.Error("Decoding produced", len(response.Brokers), "brokers where there were none!") - } -} - -func TestMetadataResponseWithTopicsV1(t *testing.T) { - response := MetadataResponse{} - - testVersionDecodable(t, "topics, V1", &response, topicsNoBrokersMetadataResponseV1, 1) - if len(response.Brokers) != 0 { - t.Error("Decoding produced", len(response.Brokers), "brokers where there were none!") - } - if response.ControllerID != 4 { - t.Error("Decoding produced", response.ControllerID, "should have been 4!") - } - if len(response.Topics) != 2 { - t.Error("Decoding produced", len(response.Topics), "topics where there were 2!") - } - if response.Topics[0].IsInternal { - t.Error("Decoding produced", response.Topics[0], "topic0 should have been false!") - } - if !response.Topics[1].IsInternal { - t.Error("Decoding produced", response.Topics[1], "topic1 should have been true!") - } -} - -func TestMetadataResponseWithThrottleTime(t *testing.T) { - response := MetadataResponse{} - - testVersionDecodable(t, "no topics, no brokers, throttle time and cluster Id V3", &response, noBrokersNoTopicsWithThrottleTimeAndClusterIDV3, 3) - if response.ThrottleTimeMs != int32(16) { - t.Error("Decoding produced", response.ThrottleTimeMs, "should have been 16!") - } - if len(response.Brokers) != 0 { - t.Error("Decoding produced", response.Brokers, "should have been 0!") - } - if response.ControllerID != int32(1) { - t.Error("Decoding produced", response.ControllerID, "should have been 1!") - } - if *response.ClusterID != "clusterId" { - t.Error("Decoding produced", response.ClusterID, "should have been clusterId!") - } - if len(response.Topics) != 0 { - t.Error("Decoding produced", len(response.Topics), "should have been 0!") - } -} - -func TestMetadataResponseWithOfflineReplicasV5(t *testing.T) { - response := MetadataResponse{} - - testVersionDecodable(t, "no brokers, 1 topic with offline replica V5", &response, noBrokersOneTopicWithOfflineReplicasV5, 5) - if response.ThrottleTimeMs != int32(5) { - t.Error("Decoding produced", response.ThrottleTimeMs, "should have been 5!") - } - if len(response.Brokers) != 0 { - t.Error("Decoding produced", response.Brokers, "should have been 0!") - } - if response.ControllerID != int32(2) { - t.Error("Decoding produced", response.ControllerID, "should have been 21!") - } - if *response.ClusterID != "clusterId" { - t.Error("Decoding produced", response.ClusterID, "should have been clusterId!") - } - if len(response.Topics) != 1 { - t.Error("Decoding produced", len(response.Topics), "should have been 1!") - } - if len(response.Topics[0].Partitions[0].OfflineReplicas) != 1 { - t.Error("Decoding produced", len(response.Topics[0].Partitions[0].OfflineReplicas), "should have been 1!") - } -} diff --git a/third/github.com/Shopify/sarama/metrics.go b/third/github.com/Shopify/sarama/metrics.go deleted file mode 100644 index 9935a3e2b..000000000 --- a/third/github.com/Shopify/sarama/metrics.go +++ /dev/null @@ -1,51 +0,0 @@ -package sarama - -import ( - "fmt" - "strings" - - "github.com/gogf/gf/third/github.com/rcrowley/go-metrics" -) - -// Use exponentially decaying reservoir for sampling histograms with the same defaults as the Java library: -// 1028 elements, which offers a 99.9% confidence level with a 5% margin of error assuming a normal distribution, -// and an alpha factor of 0.015, which heavily biases the reservoir to the past 5 minutes of measurements. -// See https://github.com/dropwizard/metrics/blob/v3.1.0/metrics-core/src/main/java/com/codahale/metrics/ExponentiallyDecayingReservoir.java#L38 -const ( - metricsReservoirSize = 1028 - metricsAlphaFactor = 0.015 -) - -func getOrRegisterHistogram(name string, r metrics.Registry) metrics.Histogram { - return r.GetOrRegister(name, func() metrics.Histogram { - return metrics.NewHistogram(metrics.NewExpDecaySample(metricsReservoirSize, metricsAlphaFactor)) - }).(metrics.Histogram) -} - -func getMetricNameForBroker(name string, broker *Broker) string { - // Use broker id like the Java client as it does not contain '.' or ':' characters that - // can be interpreted as special character by monitoring tool (e.g. Graphite) - return fmt.Sprintf(name+"-for-broker-%d", broker.ID()) -} - -func getOrRegisterBrokerMeter(name string, broker *Broker, r metrics.Registry) metrics.Meter { - return metrics.GetOrRegisterMeter(getMetricNameForBroker(name, broker), r) -} - -func getOrRegisterBrokerHistogram(name string, broker *Broker, r metrics.Registry) metrics.Histogram { - return getOrRegisterHistogram(getMetricNameForBroker(name, broker), r) -} - -func getMetricNameForTopic(name string, topic string) string { - // Convert dot to _ since reporters like Graphite typically use dot to represent hierarchy - // cf. KAFKA-1902 and KAFKA-2337 - return fmt.Sprintf(name+"-for-topic-%s", strings.Replace(topic, ".", "_", -1)) -} - -func getOrRegisterTopicMeter(name string, topic string, r metrics.Registry) metrics.Meter { - return metrics.GetOrRegisterMeter(getMetricNameForTopic(name, topic), r) -} - -func getOrRegisterTopicHistogram(name string, topic string, r metrics.Registry) metrics.Histogram { - return getOrRegisterHistogram(getMetricNameForTopic(name, topic), r) -} diff --git a/third/github.com/Shopify/sarama/metrics_test.go b/third/github.com/Shopify/sarama/metrics_test.go deleted file mode 100644 index 1f7fae21c..000000000 --- a/third/github.com/Shopify/sarama/metrics_test.go +++ /dev/null @@ -1,172 +0,0 @@ -package sarama - -import ( - "testing" - - "github.com/gogf/gf/third/github.com/rcrowley/go-metrics" -) - -func TestGetOrRegisterHistogram(t *testing.T) { - metricRegistry := metrics.NewRegistry() - histogram := getOrRegisterHistogram("name", metricRegistry) - - if histogram == nil { - t.Error("Unexpected nil histogram") - } - - // Fetch the metric - foundHistogram := metricRegistry.Get("name") - - if foundHistogram != histogram { - t.Error("Unexpected different histogram", foundHistogram, histogram) - } - - // Try to register the metric again - sameHistogram := getOrRegisterHistogram("name", metricRegistry) - - if sameHistogram != histogram { - t.Error("Unexpected different histogram", sameHistogram, histogram) - } -} - -func TestGetMetricNameForBroker(t *testing.T) { - metricName := getMetricNameForBroker("name", &Broker{id: 1}) - - if metricName != "name-for-broker-1" { - t.Error("Unexpected metric name", metricName) - } -} - -// Common type and functions for metric validation -type metricValidator struct { - name string - validator func(*testing.T, interface{}) -} - -type metricValidators []*metricValidator - -func newMetricValidators() metricValidators { - return make([]*metricValidator, 0, 32) -} - -func (m *metricValidators) register(validator *metricValidator) { - *m = append(*m, validator) -} - -func (m *metricValidators) registerForBroker(broker *Broker, validator *metricValidator) { - m.register(&metricValidator{getMetricNameForBroker(validator.name, broker), validator.validator}) -} - -func (m *metricValidators) registerForGlobalAndTopic(topic string, validator *metricValidator) { - m.register(&metricValidator{validator.name, validator.validator}) - m.register(&metricValidator{getMetricNameForTopic(validator.name, topic), validator.validator}) -} - -func (m *metricValidators) registerForAllBrokers(broker *Broker, validator *metricValidator) { - m.register(validator) - m.registerForBroker(broker, validator) -} - -func (m metricValidators) run(t *testing.T, r metrics.Registry) { - for _, metricValidator := range m { - metric := r.Get(metricValidator.name) - if metric == nil { - t.Error("No metric named", metricValidator.name) - } else { - metricValidator.validator(t, metric) - } - } -} - -func meterValidator(name string, extraValidator func(*testing.T, metrics.Meter)) *metricValidator { - return &metricValidator{ - name: name, - validator: func(t *testing.T, metric interface{}) { - if meter, ok := metric.(metrics.Meter); !ok { - t.Errorf("Expected meter metric for '%s', got %T", name, metric) - } else { - extraValidator(t, meter) - } - }, - } -} - -func countMeterValidator(name string, expectedCount int) *metricValidator { - return meterValidator(name, func(t *testing.T, meter metrics.Meter) { - count := meter.Count() - if count != int64(expectedCount) { - t.Errorf("Expected meter metric '%s' count = %d, got %d", name, expectedCount, count) - } - }) -} - -func minCountMeterValidator(name string, minCount int) *metricValidator { - return meterValidator(name, func(t *testing.T, meter metrics.Meter) { - count := meter.Count() - if count < int64(minCount) { - t.Errorf("Expected meter metric '%s' count >= %d, got %d", name, minCount, count) - } - }) -} - -func histogramValidator(name string, extraValidator func(*testing.T, metrics.Histogram)) *metricValidator { - return &metricValidator{ - name: name, - validator: func(t *testing.T, metric interface{}) { - if histogram, ok := metric.(metrics.Histogram); !ok { - t.Errorf("Expected histogram metric for '%s', got %T", name, metric) - } else { - extraValidator(t, histogram) - } - }, - } -} - -func countHistogramValidator(name string, expectedCount int) *metricValidator { - return histogramValidator(name, func(t *testing.T, histogram metrics.Histogram) { - count := histogram.Count() - if count != int64(expectedCount) { - t.Errorf("Expected histogram metric '%s' count = %d, got %d", name, expectedCount, count) - } - }) -} - -func minCountHistogramValidator(name string, minCount int) *metricValidator { - return histogramValidator(name, func(t *testing.T, histogram metrics.Histogram) { - count := histogram.Count() - if count < int64(minCount) { - t.Errorf("Expected histogram metric '%s' count >= %d, got %d", name, minCount, count) - } - }) -} - -func minMaxHistogramValidator(name string, expectedMin int, expectedMax int) *metricValidator { - return histogramValidator(name, func(t *testing.T, histogram metrics.Histogram) { - min := int(histogram.Min()) - if min != expectedMin { - t.Errorf("Expected histogram metric '%s' min = %d, got %d", name, expectedMin, min) - } - max := int(histogram.Max()) - if max != expectedMax { - t.Errorf("Expected histogram metric '%s' max = %d, got %d", name, expectedMax, max) - } - }) -} - -func minValHistogramValidator(name string, minMin int) *metricValidator { - return histogramValidator(name, func(t *testing.T, histogram metrics.Histogram) { - min := int(histogram.Min()) - if min < minMin { - t.Errorf("Expected histogram metric '%s' min >= %d, got %d", name, minMin, min) - } - }) -} - -func maxValHistogramValidator(name string, maxMax int) *metricValidator { - return histogramValidator(name, func(t *testing.T, histogram metrics.Histogram) { - max := int(histogram.Max()) - if max > maxMax { - t.Errorf("Expected histogram metric '%s' max <= %d, got %d", name, maxMax, max) - } - }) -} diff --git a/third/github.com/Shopify/sarama/mockbroker.go b/third/github.com/Shopify/sarama/mockbroker.go deleted file mode 100644 index 23a958092..000000000 --- a/third/github.com/Shopify/sarama/mockbroker.go +++ /dev/null @@ -1,330 +0,0 @@ -package sarama - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "net" - "reflect" - "strconv" - "sync" - "time" - - "github.com/gogf/gf/third/github.com/davecgh/go-spew/spew" -) - -const ( - expectationTimeout = 500 * time.Millisecond -) - -type requestHandlerFunc func(req *request) (res encoder) - -// RequestNotifierFunc is invoked when a mock broker processes a request successfully -// and will provides the number of bytes read and written. -type RequestNotifierFunc func(bytesRead, bytesWritten int) - -// MockBroker is a mock Kafka broker that is used in unit tests. It is exposed -// to facilitate testing of higher level or specialized consumers and producers -// built on top of Sarama. Note that it does not 'mimic' the Kafka API protocol, -// but rather provides a facility to do that. It takes care of the TCP -// transport, request unmarshaling, response marshaling, and makes it the test -// writer responsibility to program correct according to the Kafka API protocol -// MockBroker behaviour. -// -// MockBroker is implemented as a TCP server listening on a kernel-selected -// localhost port that can accept many connections. It reads Kafka requests -// from that connection and returns responses programmed by the SetHandlerByMap -// function. If a MockBroker receives a request that it has no programmed -// response for, then it returns nothing and the request times out. -// -// A set of MockRequest builders to define mappings used by MockBroker is -// provided by Sarama. But users can develop MockRequests of their own and use -// them along with or instead of the standard ones. -// -// When running tests with MockBroker it is strongly recommended to specify -// a timeout to `go test` so that if the broker hangs waiting for a response, -// the test panics. -// -// It is not necessary to prefix message length or correlation ID to your -// response bytes, the server does that automatically as a convenience. -type MockBroker struct { - brokerID int32 - port int32 - closing chan none - stopper chan none - expectations chan encoder - listener net.Listener - t TestReporter - latency time.Duration - handler requestHandlerFunc - notifier RequestNotifierFunc - history []RequestResponse - lock sync.Mutex -} - -// RequestResponse represents a Request/Response pair processed by MockBroker. -type RequestResponse struct { - Request protocolBody - Response encoder -} - -// SetLatency makes broker pause for the specified period every time before -// replying. -func (b *MockBroker) SetLatency(latency time.Duration) { - b.latency = latency -} - -// SetHandlerByMap defines mapping of Request types to MockResponses. When a -// request is received by the broker, it looks up the request type in the map -// and uses the found MockResponse instance to generate an appropriate reply. -// If the request type is not found in the map then nothing is sent. -func (b *MockBroker) SetHandlerByMap(handlerMap map[string]MockResponse) { - b.setHandler(func(req *request) (res encoder) { - reqTypeName := reflect.TypeOf(req.body).Elem().Name() - mockResponse := handlerMap[reqTypeName] - if mockResponse == nil { - return nil - } - return mockResponse.For(req.body) - }) -} - -// SetNotifier set a function that will get invoked whenever a request has been -// processed successfully and will provide the number of bytes read and written -func (b *MockBroker) SetNotifier(notifier RequestNotifierFunc) { - b.lock.Lock() - b.notifier = notifier - b.lock.Unlock() -} - -// BrokerID returns broker ID assigned to the broker. -func (b *MockBroker) BrokerID() int32 { - return b.brokerID -} - -// History returns a slice of RequestResponse pairs in the order they were -// processed by the broker. Note that in case of multiple connections to the -// broker the order expected by a test can be different from the order recorded -// in the history, unless some synchronization is implemented in the test. -func (b *MockBroker) History() []RequestResponse { - b.lock.Lock() - history := make([]RequestResponse, len(b.history)) - copy(history, b.history) - b.lock.Unlock() - return history -} - -// Port returns the TCP port number the broker is listening for requests on. -func (b *MockBroker) Port() int32 { - return b.port -} - -// Addr returns the broker connection string in the form "
:". -func (b *MockBroker) Addr() string { - return b.listener.Addr().String() -} - -// Close terminates the broker blocking until it stops internal goroutines and -// releases all resources. -func (b *MockBroker) Close() { - close(b.expectations) - if len(b.expectations) > 0 { - buf := bytes.NewBufferString(fmt.Sprintf("mockbroker/%d: not all expectations were satisfied! Still waiting on:\n", b.BrokerID())) - for e := range b.expectations { - _, _ = buf.WriteString(spew.Sdump(e)) - } - b.t.Error(buf.String()) - } - close(b.closing) - <-b.stopper -} - -// setHandler sets the specified function as the request handler. Whenever -// a mock broker reads a request from the wire it passes the request to the -// function and sends back whatever the handler function returns. -func (b *MockBroker) setHandler(handler requestHandlerFunc) { - b.lock.Lock() - b.handler = handler - b.lock.Unlock() -} - -func (b *MockBroker) serverLoop() { - defer close(b.stopper) - var err error - var conn net.Conn - - go func() { - <-b.closing - err := b.listener.Close() - if err != nil { - b.t.Error(err) - } - }() - - wg := &sync.WaitGroup{} - i := 0 - for conn, err = b.listener.Accept(); err == nil; conn, err = b.listener.Accept() { - wg.Add(1) - go b.handleRequests(conn, i, wg) - i++ - } - wg.Wait() - Logger.Printf("*** mockbroker/%d: listener closed, err=%v", b.BrokerID(), err) -} - -func (b *MockBroker) handleRequests(conn net.Conn, idx int, wg *sync.WaitGroup) { - defer wg.Done() - defer func() { - _ = conn.Close() - }() - Logger.Printf("*** mockbroker/%d/%d: connection opened", b.BrokerID(), idx) - var err error - - abort := make(chan none) - defer close(abort) - go func() { - select { - case <-b.closing: - _ = conn.Close() - case <-abort: - } - }() - - resHeader := make([]byte, 8) - for { - req, bytesRead, err := decodeRequest(conn) - if err != nil { - Logger.Printf("*** mockbroker/%d/%d: invalid request: err=%+v, %+v", b.brokerID, idx, err, spew.Sdump(req)) - b.serverError(err) - break - } - - if b.latency > 0 { - time.Sleep(b.latency) - } - - b.lock.Lock() - res := b.handler(req) - b.history = append(b.history, RequestResponse{req.body, res}) - b.lock.Unlock() - - if res == nil { - Logger.Printf("*** mockbroker/%d/%d: ignored %v", b.brokerID, idx, spew.Sdump(req)) - continue - } - Logger.Printf("*** mockbroker/%d/%d: served %v -> %v", b.brokerID, idx, req, res) - - encodedRes, err := encode(res, nil) - if err != nil { - b.serverError(err) - break - } - if len(encodedRes) == 0 { - b.lock.Lock() - if b.notifier != nil { - b.notifier(bytesRead, 0) - } - b.lock.Unlock() - continue - } - - binary.BigEndian.PutUint32(resHeader, uint32(len(encodedRes)+4)) - binary.BigEndian.PutUint32(resHeader[4:], uint32(req.correlationID)) - if _, err = conn.Write(resHeader); err != nil { - b.serverError(err) - break - } - if _, err = conn.Write(encodedRes); err != nil { - b.serverError(err) - break - } - - b.lock.Lock() - if b.notifier != nil { - b.notifier(bytesRead, len(resHeader)+len(encodedRes)) - } - b.lock.Unlock() - } - Logger.Printf("*** mockbroker/%d/%d: connection closed, err=%v", b.BrokerID(), idx, err) -} - -func (b *MockBroker) defaultRequestHandler(req *request) (res encoder) { - select { - case res, ok := <-b.expectations: - if !ok { - return nil - } - return res - case <-time.After(expectationTimeout): - return nil - } -} - -func (b *MockBroker) serverError(err error) { - isConnectionClosedError := false - if _, ok := err.(*net.OpError); ok { - isConnectionClosedError = true - } else if err == io.EOF { - isConnectionClosedError = true - } else if err.Error() == "use of closed network connection" { - isConnectionClosedError = true - } - - if isConnectionClosedError { - return - } - - b.t.Errorf(err.Error()) -} - -// NewMockBroker launches a fake Kafka broker. It takes a TestReporter as provided by the -// test framework and a channel of responses to use. If an error occurs it is -// simply logged to the TestReporter and the broker exits. -func NewMockBroker(t TestReporter, brokerID int32) *MockBroker { - return NewMockBrokerAddr(t, brokerID, "localhost:0") -} - -// NewMockBrokerAddr behaves like newMockBroker but listens on the address you give -// it rather than just some ephemeral port. -func NewMockBrokerAddr(t TestReporter, brokerID int32, addr string) *MockBroker { - listener, err := net.Listen("tcp", addr) - if err != nil { - t.Fatal(err) - } - return NewMockBrokerListener(t, brokerID, listener) -} - -// NewMockBrokerListener behaves like newMockBrokerAddr but accepts connections on the listener specified. -func NewMockBrokerListener(t TestReporter, brokerID int32, listener net.Listener) *MockBroker { - var err error - - broker := &MockBroker{ - closing: make(chan none), - stopper: make(chan none), - t: t, - brokerID: brokerID, - expectations: make(chan encoder, 512), - listener: listener, - } - broker.handler = broker.defaultRequestHandler - - Logger.Printf("*** mockbroker/%d listening on %s\n", brokerID, broker.listener.Addr().String()) - _, portStr, err := net.SplitHostPort(broker.listener.Addr().String()) - if err != nil { - t.Fatal(err) - } - tmp, err := strconv.ParseInt(portStr, 10, 32) - if err != nil { - t.Fatal(err) - } - broker.port = int32(tmp) - - go broker.serverLoop() - - return broker -} - -func (b *MockBroker) Returns(e encoder) { - b.expectations <- e -} diff --git a/third/github.com/Shopify/sarama/mockresponses.go b/third/github.com/Shopify/sarama/mockresponses.go deleted file mode 100644 index fe55200c6..000000000 --- a/third/github.com/Shopify/sarama/mockresponses.go +++ /dev/null @@ -1,727 +0,0 @@ -package sarama - -import ( - "fmt" -) - -// TestReporter has methods matching go's testing.T to avoid importing -// `testing` in the main part of the library. -type TestReporter interface { - Error(...interface{}) - Errorf(string, ...interface{}) - Fatal(...interface{}) - Fatalf(string, ...interface{}) -} - -// MockResponse is a response builder interface it defines one method that -// allows generating a response based on a request body. MockResponses are used -// to program behavior of MockBroker in tests. -type MockResponse interface { - For(reqBody versionedDecoder) (res encoder) -} - -// MockWrapper is a mock response builder that returns a particular concrete -// response regardless of the actual request passed to the `For` method. -type MockWrapper struct { - res encoder -} - -func (mw *MockWrapper) For(reqBody versionedDecoder) (res encoder) { - return mw.res -} - -func NewMockWrapper(res encoder) *MockWrapper { - return &MockWrapper{res: res} -} - -// MockSequence is a mock response builder that is created from a sequence of -// concrete responses. Every time when a `MockBroker` calls its `For` method -// the next response from the sequence is returned. When the end of the -// sequence is reached the last element from the sequence is returned. -type MockSequence struct { - responses []MockResponse -} - -func NewMockSequence(responses ...interface{}) *MockSequence { - ms := &MockSequence{} - ms.responses = make([]MockResponse, len(responses)) - for i, res := range responses { - switch res := res.(type) { - case MockResponse: - ms.responses[i] = res - case encoder: - ms.responses[i] = NewMockWrapper(res) - default: - panic(fmt.Sprintf("Unexpected response type: %T", res)) - } - } - return ms -} - -func (mc *MockSequence) For(reqBody versionedDecoder) (res encoder) { - res = mc.responses[0].For(reqBody) - if len(mc.responses) > 1 { - mc.responses = mc.responses[1:] - } - return res -} - -// MockMetadataResponse is a `MetadataResponse` builder. -type MockMetadataResponse struct { - controllerID int32 - leaders map[string]map[int32]int32 - brokers map[string]int32 - t TestReporter -} - -func NewMockMetadataResponse(t TestReporter) *MockMetadataResponse { - return &MockMetadataResponse{ - leaders: make(map[string]map[int32]int32), - brokers: make(map[string]int32), - t: t, - } -} - -func (mmr *MockMetadataResponse) SetLeader(topic string, partition, brokerID int32) *MockMetadataResponse { - partitions := mmr.leaders[topic] - if partitions == nil { - partitions = make(map[int32]int32) - mmr.leaders[topic] = partitions - } - partitions[partition] = brokerID - return mmr -} - -func (mmr *MockMetadataResponse) SetBroker(addr string, brokerID int32) *MockMetadataResponse { - mmr.brokers[addr] = brokerID - return mmr -} - -func (mmr *MockMetadataResponse) SetController(brokerID int32) *MockMetadataResponse { - mmr.controllerID = brokerID - return mmr -} - -func (mmr *MockMetadataResponse) For(reqBody versionedDecoder) encoder { - metadataRequest := reqBody.(*MetadataRequest) - metadataResponse := &MetadataResponse{ - Version: metadataRequest.version(), - ControllerID: mmr.controllerID, - } - for addr, brokerID := range mmr.brokers { - metadataResponse.AddBroker(addr, brokerID) - } - if len(metadataRequest.Topics) == 0 { - for topic, partitions := range mmr.leaders { - for partition, brokerID := range partitions { - metadataResponse.AddTopicPartition(topic, partition, brokerID, nil, nil, ErrNoError) - } - } - return metadataResponse - } - for _, topic := range metadataRequest.Topics { - for partition, brokerID := range mmr.leaders[topic] { - metadataResponse.AddTopicPartition(topic, partition, brokerID, nil, nil, ErrNoError) - } - } - return metadataResponse -} - -// MockOffsetResponse is an `OffsetResponse` builder. -type MockOffsetResponse struct { - offsets map[string]map[int32]map[int64]int64 - t TestReporter - version int16 -} - -func NewMockOffsetResponse(t TestReporter) *MockOffsetResponse { - return &MockOffsetResponse{ - offsets: make(map[string]map[int32]map[int64]int64), - t: t, - } -} - -func (mor *MockOffsetResponse) SetVersion(version int16) *MockOffsetResponse { - mor.version = version - return mor -} - -func (mor *MockOffsetResponse) SetOffset(topic string, partition int32, time, offset int64) *MockOffsetResponse { - partitions := mor.offsets[topic] - if partitions == nil { - partitions = make(map[int32]map[int64]int64) - mor.offsets[topic] = partitions - } - times := partitions[partition] - if times == nil { - times = make(map[int64]int64) - partitions[partition] = times - } - times[time] = offset - return mor -} - -func (mor *MockOffsetResponse) For(reqBody versionedDecoder) encoder { - offsetRequest := reqBody.(*OffsetRequest) - offsetResponse := &OffsetResponse{Version: mor.version} - for topic, partitions := range offsetRequest.blocks { - for partition, block := range partitions { - offset := mor.getOffset(topic, partition, block.time) - offsetResponse.AddTopicPartition(topic, partition, offset) - } - } - return offsetResponse -} - -func (mor *MockOffsetResponse) getOffset(topic string, partition int32, time int64) int64 { - partitions := mor.offsets[topic] - if partitions == nil { - mor.t.Errorf("missing topic: %s", topic) - } - times := partitions[partition] - if times == nil { - mor.t.Errorf("missing partition: %d", partition) - } - offset, ok := times[time] - if !ok { - mor.t.Errorf("missing time: %d", time) - } - return offset -} - -// MockFetchResponse is a `FetchResponse` builder. -type MockFetchResponse struct { - messages map[string]map[int32]map[int64]Encoder - highWaterMarks map[string]map[int32]int64 - t TestReporter - batchSize int - version int16 -} - -func NewMockFetchResponse(t TestReporter, batchSize int) *MockFetchResponse { - return &MockFetchResponse{ - messages: make(map[string]map[int32]map[int64]Encoder), - highWaterMarks: make(map[string]map[int32]int64), - t: t, - batchSize: batchSize, - } -} - -func (mfr *MockFetchResponse) SetVersion(version int16) *MockFetchResponse { - mfr.version = version - return mfr -} - -func (mfr *MockFetchResponse) SetMessage(topic string, partition int32, offset int64, msg Encoder) *MockFetchResponse { - partitions := mfr.messages[topic] - if partitions == nil { - partitions = make(map[int32]map[int64]Encoder) - mfr.messages[topic] = partitions - } - messages := partitions[partition] - if messages == nil { - messages = make(map[int64]Encoder) - partitions[partition] = messages - } - messages[offset] = msg - return mfr -} - -func (mfr *MockFetchResponse) SetHighWaterMark(topic string, partition int32, offset int64) *MockFetchResponse { - partitions := mfr.highWaterMarks[topic] - if partitions == nil { - partitions = make(map[int32]int64) - mfr.highWaterMarks[topic] = partitions - } - partitions[partition] = offset - return mfr -} - -func (mfr *MockFetchResponse) For(reqBody versionedDecoder) encoder { - fetchRequest := reqBody.(*FetchRequest) - res := &FetchResponse{ - Version: mfr.version, - } - for topic, partitions := range fetchRequest.blocks { - for partition, block := range partitions { - initialOffset := block.fetchOffset - offset := initialOffset - maxOffset := initialOffset + int64(mfr.getMessageCount(topic, partition)) - for i := 0; i < mfr.batchSize && offset < maxOffset; { - msg := mfr.getMessage(topic, partition, offset) - if msg != nil { - res.AddMessage(topic, partition, nil, msg, offset) - i++ - } - offset++ - } - fb := res.GetBlock(topic, partition) - if fb == nil { - res.AddError(topic, partition, ErrNoError) - fb = res.GetBlock(topic, partition) - } - fb.HighWaterMarkOffset = mfr.getHighWaterMark(topic, partition) - } - } - return res -} - -func (mfr *MockFetchResponse) getMessage(topic string, partition int32, offset int64) Encoder { - partitions := mfr.messages[topic] - if partitions == nil { - return nil - } - messages := partitions[partition] - if messages == nil { - return nil - } - return messages[offset] -} - -func (mfr *MockFetchResponse) getMessageCount(topic string, partition int32) int { - partitions := mfr.messages[topic] - if partitions == nil { - return 0 - } - messages := partitions[partition] - if messages == nil { - return 0 - } - return len(messages) -} - -func (mfr *MockFetchResponse) getHighWaterMark(topic string, partition int32) int64 { - partitions := mfr.highWaterMarks[topic] - if partitions == nil { - return 0 - } - return partitions[partition] -} - -// MockConsumerMetadataResponse is a `ConsumerMetadataResponse` builder. -type MockConsumerMetadataResponse struct { - coordinators map[string]interface{} - t TestReporter -} - -func NewMockConsumerMetadataResponse(t TestReporter) *MockConsumerMetadataResponse { - return &MockConsumerMetadataResponse{ - coordinators: make(map[string]interface{}), - t: t, - } -} - -func (mr *MockConsumerMetadataResponse) SetCoordinator(group string, broker *MockBroker) *MockConsumerMetadataResponse { - mr.coordinators[group] = broker - return mr -} - -func (mr *MockConsumerMetadataResponse) SetError(group string, kerror KError) *MockConsumerMetadataResponse { - mr.coordinators[group] = kerror - return mr -} - -func (mr *MockConsumerMetadataResponse) For(reqBody versionedDecoder) encoder { - req := reqBody.(*ConsumerMetadataRequest) - group := req.ConsumerGroup - res := &ConsumerMetadataResponse{} - v := mr.coordinators[group] - switch v := v.(type) { - case *MockBroker: - res.Coordinator = &Broker{id: v.BrokerID(), addr: v.Addr()} - case KError: - res.Err = v - } - return res -} - -// MockFindCoordinatorResponse is a `FindCoordinatorResponse` builder. -type MockFindCoordinatorResponse struct { - groupCoordinators map[string]interface{} - transCoordinators map[string]interface{} - t TestReporter -} - -func NewMockFindCoordinatorResponse(t TestReporter) *MockFindCoordinatorResponse { - return &MockFindCoordinatorResponse{ - groupCoordinators: make(map[string]interface{}), - transCoordinators: make(map[string]interface{}), - t: t, - } -} - -func (mr *MockFindCoordinatorResponse) SetCoordinator(coordinatorType CoordinatorType, group string, broker *MockBroker) *MockFindCoordinatorResponse { - switch coordinatorType { - case CoordinatorGroup: - mr.groupCoordinators[group] = broker - case CoordinatorTransaction: - mr.transCoordinators[group] = broker - } - return mr -} - -func (mr *MockFindCoordinatorResponse) SetError(coordinatorType CoordinatorType, group string, kerror KError) *MockFindCoordinatorResponse { - switch coordinatorType { - case CoordinatorGroup: - mr.groupCoordinators[group] = kerror - case CoordinatorTransaction: - mr.transCoordinators[group] = kerror - } - return mr -} - -func (mr *MockFindCoordinatorResponse) For(reqBody versionedDecoder) encoder { - req := reqBody.(*FindCoordinatorRequest) - res := &FindCoordinatorResponse{} - var v interface{} - switch req.CoordinatorType { - case CoordinatorGroup: - v = mr.groupCoordinators[req.CoordinatorKey] - case CoordinatorTransaction: - v = mr.transCoordinators[req.CoordinatorKey] - } - switch v := v.(type) { - case *MockBroker: - res.Coordinator = &Broker{id: v.BrokerID(), addr: v.Addr()} - case KError: - res.Err = v - } - return res -} - -// MockOffsetCommitResponse is a `OffsetCommitResponse` builder. -type MockOffsetCommitResponse struct { - errors map[string]map[string]map[int32]KError - t TestReporter -} - -func NewMockOffsetCommitResponse(t TestReporter) *MockOffsetCommitResponse { - return &MockOffsetCommitResponse{t: t} -} - -func (mr *MockOffsetCommitResponse) SetError(group, topic string, partition int32, kerror KError) *MockOffsetCommitResponse { - if mr.errors == nil { - mr.errors = make(map[string]map[string]map[int32]KError) - } - topics := mr.errors[group] - if topics == nil { - topics = make(map[string]map[int32]KError) - mr.errors[group] = topics - } - partitions := topics[topic] - if partitions == nil { - partitions = make(map[int32]KError) - topics[topic] = partitions - } - partitions[partition] = kerror - return mr -} - -func (mr *MockOffsetCommitResponse) For(reqBody versionedDecoder) encoder { - req := reqBody.(*OffsetCommitRequest) - group := req.ConsumerGroup - res := &OffsetCommitResponse{} - for topic, partitions := range req.blocks { - for partition := range partitions { - res.AddError(topic, partition, mr.getError(group, topic, partition)) - } - } - return res -} - -func (mr *MockOffsetCommitResponse) getError(group, topic string, partition int32) KError { - topics := mr.errors[group] - if topics == nil { - return ErrNoError - } - partitions := topics[topic] - if partitions == nil { - return ErrNoError - } - kerror, ok := partitions[partition] - if !ok { - return ErrNoError - } - return kerror -} - -// MockProduceResponse is a `ProduceResponse` builder. -type MockProduceResponse struct { - version int16 - errors map[string]map[int32]KError - t TestReporter -} - -func NewMockProduceResponse(t TestReporter) *MockProduceResponse { - return &MockProduceResponse{t: t} -} - -func (mr *MockProduceResponse) SetVersion(version int16) *MockProduceResponse { - mr.version = version - return mr -} - -func (mr *MockProduceResponse) SetError(topic string, partition int32, kerror KError) *MockProduceResponse { - if mr.errors == nil { - mr.errors = make(map[string]map[int32]KError) - } - partitions := mr.errors[topic] - if partitions == nil { - partitions = make(map[int32]KError) - mr.errors[topic] = partitions - } - partitions[partition] = kerror - return mr -} - -func (mr *MockProduceResponse) For(reqBody versionedDecoder) encoder { - req := reqBody.(*ProduceRequest) - res := &ProduceResponse{ - Version: mr.version, - } - for topic, partitions := range req.records { - for partition := range partitions { - res.AddTopicPartition(topic, partition, mr.getError(topic, partition)) - } - } - return res -} - -func (mr *MockProduceResponse) getError(topic string, partition int32) KError { - partitions := mr.errors[topic] - if partitions == nil { - return ErrNoError - } - kerror, ok := partitions[partition] - if !ok { - return ErrNoError - } - return kerror -} - -// MockOffsetFetchResponse is a `OffsetFetchResponse` builder. -type MockOffsetFetchResponse struct { - offsets map[string]map[string]map[int32]*OffsetFetchResponseBlock - t TestReporter -} - -func NewMockOffsetFetchResponse(t TestReporter) *MockOffsetFetchResponse { - return &MockOffsetFetchResponse{t: t} -} - -func (mr *MockOffsetFetchResponse) SetOffset(group, topic string, partition int32, offset int64, metadata string, kerror KError) *MockOffsetFetchResponse { - if mr.offsets == nil { - mr.offsets = make(map[string]map[string]map[int32]*OffsetFetchResponseBlock) - } - topics := mr.offsets[group] - if topics == nil { - topics = make(map[string]map[int32]*OffsetFetchResponseBlock) - mr.offsets[group] = topics - } - partitions := topics[topic] - if partitions == nil { - partitions = make(map[int32]*OffsetFetchResponseBlock) - topics[topic] = partitions - } - partitions[partition] = &OffsetFetchResponseBlock{offset, 0, metadata, kerror} - return mr -} - -func (mr *MockOffsetFetchResponse) For(reqBody versionedDecoder) encoder { - req := reqBody.(*OffsetFetchRequest) - group := req.ConsumerGroup - res := &OffsetFetchResponse{} - for topic, partitions := range mr.offsets[group] { - for partition, block := range partitions { - res.AddBlock(topic, partition, block) - } - } - return res -} - -type MockCreateTopicsResponse struct { - t TestReporter -} - -func NewMockCreateTopicsResponse(t TestReporter) *MockCreateTopicsResponse { - return &MockCreateTopicsResponse{t: t} -} - -func (mr *MockCreateTopicsResponse) For(reqBody versionedDecoder) encoder { - req := reqBody.(*CreateTopicsRequest) - res := &CreateTopicsResponse{} - res.TopicErrors = make(map[string]*TopicError) - - for topic, _ := range req.TopicDetails { - res.TopicErrors[topic] = &TopicError{Err: ErrNoError} - } - return res -} - -type MockDeleteTopicsResponse struct { - t TestReporter -} - -func NewMockDeleteTopicsResponse(t TestReporter) *MockDeleteTopicsResponse { - return &MockDeleteTopicsResponse{t: t} -} - -func (mr *MockDeleteTopicsResponse) For(reqBody versionedDecoder) encoder { - req := reqBody.(*DeleteTopicsRequest) - res := &DeleteTopicsResponse{} - res.TopicErrorCodes = make(map[string]KError) - - for _, topic := range req.Topics { - res.TopicErrorCodes[topic] = ErrNoError - } - return res -} - -type MockCreatePartitionsResponse struct { - t TestReporter -} - -func NewMockCreatePartitionsResponse(t TestReporter) *MockCreatePartitionsResponse { - return &MockCreatePartitionsResponse{t: t} -} - -func (mr *MockCreatePartitionsResponse) For(reqBody versionedDecoder) encoder { - req := reqBody.(*CreatePartitionsRequest) - res := &CreatePartitionsResponse{} - res.TopicPartitionErrors = make(map[string]*TopicPartitionError) - - for topic, _ := range req.TopicPartitions { - res.TopicPartitionErrors[topic] = &TopicPartitionError{Err: ErrNoError} - } - return res -} - -type MockDeleteRecordsResponse struct { - t TestReporter -} - -func NewMockDeleteRecordsResponse(t TestReporter) *MockDeleteRecordsResponse { - return &MockDeleteRecordsResponse{t: t} -} - -func (mr *MockDeleteRecordsResponse) For(reqBody versionedDecoder) encoder { - req := reqBody.(*DeleteRecordsRequest) - res := &DeleteRecordsResponse{} - res.Topics = make(map[string]*DeleteRecordsResponseTopic) - - for topic, deleteRecordRequestTopic := range req.Topics { - partitions := make(map[int32]*DeleteRecordsResponsePartition) - for partition, _ := range deleteRecordRequestTopic.PartitionOffsets { - partitions[partition] = &DeleteRecordsResponsePartition{Err: ErrNoError} - } - res.Topics[topic] = &DeleteRecordsResponseTopic{Partitions: partitions} - } - return res -} - -type MockDescribeConfigsResponse struct { - t TestReporter -} - -func NewMockDescribeConfigsResponse(t TestReporter) *MockDescribeConfigsResponse { - return &MockDescribeConfigsResponse{t: t} -} - -func (mr *MockDescribeConfigsResponse) For(reqBody versionedDecoder) encoder { - req := reqBody.(*DescribeConfigsRequest) - res := &DescribeConfigsResponse{} - - var configEntries []*ConfigEntry - configEntries = append(configEntries, &ConfigEntry{Name: "my_topic", - Value: "my_topic", - ReadOnly: true, - Default: true, - Sensitive: false, - }) - - for _, r := range req.Resources { - res.Resources = append(res.Resources, &ResourceResponse{Name: r.Name, Configs: configEntries}) - } - return res -} - -type MockAlterConfigsResponse struct { - t TestReporter -} - -func NewMockAlterConfigsResponse(t TestReporter) *MockAlterConfigsResponse { - return &MockAlterConfigsResponse{t: t} -} - -func (mr *MockAlterConfigsResponse) For(reqBody versionedDecoder) encoder { - req := reqBody.(*AlterConfigsRequest) - res := &AlterConfigsResponse{} - - for _, r := range req.Resources { - res.Resources = append(res.Resources, &AlterConfigsResourceResponse{Name: r.Name, - Type: TopicResource, - ErrorMsg: "", - }) - } - return res -} - -type MockCreateAclsResponse struct { - t TestReporter -} - -func NewMockCreateAclsResponse(t TestReporter) *MockCreateAclsResponse { - return &MockCreateAclsResponse{t: t} -} - -func (mr *MockCreateAclsResponse) For(reqBody versionedDecoder) encoder { - req := reqBody.(*CreateAclsRequest) - res := &CreateAclsResponse{} - - for range req.AclCreations { - res.AclCreationResponses = append(res.AclCreationResponses, &AclCreationResponse{Err: ErrNoError}) - } - return res -} - -type MockListAclsResponse struct { - t TestReporter -} - -func NewMockListAclsResponse(t TestReporter) *MockListAclsResponse { - return &MockListAclsResponse{t: t} -} - -func (mr *MockListAclsResponse) For(reqBody versionedDecoder) encoder { - req := reqBody.(*DescribeAclsRequest) - res := &DescribeAclsResponse{} - - res.Err = ErrNoError - acl := &ResourceAcls{} - acl.Resource.ResourceName = *req.ResourceName - acl.Resource.ResourceType = req.ResourceType - acl.Acls = append(acl.Acls, &Acl{}) - res.ResourceAcls = append(res.ResourceAcls, acl) - - return res -} - -type MockDeleteAclsResponse struct { - t TestReporter -} - -func NewMockDeleteAclsResponse(t TestReporter) *MockDeleteAclsResponse { - return &MockDeleteAclsResponse{t: t} -} - -func (mr *MockDeleteAclsResponse) For(reqBody versionedDecoder) encoder { - req := reqBody.(*DeleteAclsRequest) - res := &DeleteAclsResponse{} - - for range req.Filters { - response := &FilterResponse{Err: ErrNoError} - response.MatchingAcls = append(response.MatchingAcls, &MatchingAcl{Err: ErrNoError}) - res.FilterResponses = append(res.FilterResponses, response) - } - return res -} diff --git a/third/github.com/Shopify/sarama/mocks/README.md b/third/github.com/Shopify/sarama/mocks/README.md deleted file mode 100644 index 55a6c2e61..000000000 --- a/third/github.com/Shopify/sarama/mocks/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# sarama/mocks - -The `mocks` subpackage includes mock implementations that implement the interfaces of the major sarama types. -You can use them to test your sarama applications using dependency injection. - -The following mock objects are available: - -- [Consumer](https://godoc.org/github.com/Shopify/sarama/mocks#Consumer), which will create [PartitionConsumer](https://godoc.org/github.com/Shopify/sarama/mocks#PartitionConsumer) mocks. -- [AsyncProducer](https://godoc.org/github.com/Shopify/sarama/mocks#AsyncProducer) -- [SyncProducer](https://godoc.org/github.com/Shopify/sarama/mocks#SyncProducer) - -The mocks allow you to set expectations on them. When you close the mocks, the expectations will be verified, -and the results will be reported to the `*testing.T` object you provided when creating the mock. diff --git a/third/github.com/Shopify/sarama/mocks/async_producer.go b/third/github.com/Shopify/sarama/mocks/async_producer.go deleted file mode 100644 index 53bc10c67..000000000 --- a/third/github.com/Shopify/sarama/mocks/async_producer.go +++ /dev/null @@ -1,173 +0,0 @@ -package mocks - -import ( - "sync" - - "github.com/gogf/gf/third/github.com/Shopify/sarama" -) - -// AsyncProducer implements sarama's Producer interface for testing purposes. -// Before you can send messages to it's Input channel, you have to set expectations -// so it knows how to handle the input; it returns an error if the number of messages -// received is bigger then the number of expectations set. You can also set a -// function in each expectation so that the message value is checked by this function -// and an error is returned if the match fails. -type AsyncProducer struct { - l sync.Mutex - t ErrorReporter - expectations []*producerExpectation - closed chan struct{} - input chan *sarama.ProducerMessage - successes chan *sarama.ProducerMessage - errors chan *sarama.ProducerError - lastOffset int64 -} - -// NewAsyncProducer instantiates a new Producer mock. The t argument should -// be the *testing.T instance of your test method. An error will be written to it if -// an expectation is violated. The config argument is used to determine whether it -// should ack successes on the Successes channel. -func NewAsyncProducer(t ErrorReporter, config *sarama.Config) *AsyncProducer { - if config == nil { - config = sarama.NewConfig() - } - mp := &AsyncProducer{ - t: t, - closed: make(chan struct{}, 0), - expectations: make([]*producerExpectation, 0), - input: make(chan *sarama.ProducerMessage, config.ChannelBufferSize), - successes: make(chan *sarama.ProducerMessage, config.ChannelBufferSize), - errors: make(chan *sarama.ProducerError, config.ChannelBufferSize), - } - - go func() { - defer func() { - close(mp.successes) - close(mp.errors) - close(mp.closed) - }() - - for msg := range mp.input { - mp.l.Lock() - if mp.expectations == nil || len(mp.expectations) == 0 { - mp.expectations = nil - mp.t.Errorf("No more expectation set on this mock producer to handle the input message.") - } else { - expectation := mp.expectations[0] - mp.expectations = mp.expectations[1:] - if expectation.CheckFunction != nil { - if val, err := msg.Value.Encode(); err != nil { - mp.t.Errorf("Input message encoding failed: %s", err.Error()) - mp.errors <- &sarama.ProducerError{Err: err, Msg: msg} - } else { - err = expectation.CheckFunction(val) - if err != nil { - mp.t.Errorf("Check function returned an error: %s", err.Error()) - mp.errors <- &sarama.ProducerError{Err: err, Msg: msg} - } - } - } - if expectation.Result == errProduceSuccess { - mp.lastOffset++ - if config.Producer.Return.Successes { - msg.Offset = mp.lastOffset - mp.successes <- msg - } - } else { - if config.Producer.Return.Errors { - mp.errors <- &sarama.ProducerError{Err: expectation.Result, Msg: msg} - } - } - } - mp.l.Unlock() - } - - mp.l.Lock() - if len(mp.expectations) > 0 { - mp.t.Errorf("Expected to exhaust all expectations, but %d are left.", len(mp.expectations)) - } - mp.l.Unlock() - }() - - return mp -} - -//////////////////////////////////////////////// -// Implement Producer interface -//////////////////////////////////////////////// - -// AsyncClose corresponds with the AsyncClose method of sarama's Producer implementation. -// By closing a mock producer, you also tell it that no more input will be provided, so it will -// write an error to the test state if there's any remaining expectations. -func (mp *AsyncProducer) AsyncClose() { - close(mp.input) -} - -// Close corresponds with the Close method of sarama's Producer implementation. -// By closing a mock producer, you also tell it that no more input will be provided, so it will -// write an error to the test state if there's any remaining expectations. -func (mp *AsyncProducer) Close() error { - mp.AsyncClose() - <-mp.closed - return nil -} - -// Input corresponds with the Input method of sarama's Producer implementation. -// You have to set expectations on the mock producer before writing messages to the Input -// channel, so it knows how to handle them. If there is no more remaining expectations and -// a messages is written to the Input channel, the mock producer will write an error to the test -// state object. -func (mp *AsyncProducer) Input() chan<- *sarama.ProducerMessage { - return mp.input -} - -// Successes corresponds with the Successes method of sarama's Producer implementation. -func (mp *AsyncProducer) Successes() <-chan *sarama.ProducerMessage { - return mp.successes -} - -// Errors corresponds with the Errors method of sarama's Producer implementation. -func (mp *AsyncProducer) Errors() <-chan *sarama.ProducerError { - return mp.errors -} - -//////////////////////////////////////////////// -// Setting expectations -//////////////////////////////////////////////// - -// ExpectInputWithCheckerFunctionAndSucceed sets an expectation on the mock producer that a message -// will be provided on the input channel. The mock producer will call the given function to check -// the message value. If an error is returned it will be made available on the Errors channel -// otherwise the mock will handle the message as if it produced successfully, i.e. it will make -// it available on the Successes channel if the Producer.Return.Successes setting is set to true. -func (mp *AsyncProducer) ExpectInputWithCheckerFunctionAndSucceed(cf ValueChecker) { - mp.l.Lock() - defer mp.l.Unlock() - mp.expectations = append(mp.expectations, &producerExpectation{Result: errProduceSuccess, CheckFunction: cf}) -} - -// ExpectInputWithCheckerFunctionAndFail sets an expectation on the mock producer that a message -// will be provided on the input channel. The mock producer will first call the given function to -// check the message value. If an error is returned it will be made available on the Errors channel -// otherwise the mock will handle the message as if it failed to produce successfully. This means -// it will make a ProducerError available on the Errors channel. -func (mp *AsyncProducer) ExpectInputWithCheckerFunctionAndFail(cf ValueChecker, err error) { - mp.l.Lock() - defer mp.l.Unlock() - mp.expectations = append(mp.expectations, &producerExpectation{Result: err, CheckFunction: cf}) -} - -// ExpectInputAndSucceed sets an expectation on the mock producer that a message will be provided -// on the input channel. The mock producer will handle the message as if it is produced successfully, -// i.e. it will make it available on the Successes channel if the Producer.Return.Successes setting -// is set to true. -func (mp *AsyncProducer) ExpectInputAndSucceed() { - mp.ExpectInputWithCheckerFunctionAndSucceed(nil) -} - -// ExpectInputAndFail sets an expectation on the mock producer that a message will be provided -// on the input channel. The mock producer will handle the message as if it failed to produce -// successfully. This means it will make a ProducerError available on the Errors channel. -func (mp *AsyncProducer) ExpectInputAndFail(err error) { - mp.ExpectInputWithCheckerFunctionAndFail(nil, err) -} diff --git a/third/github.com/Shopify/sarama/mocks/async_producer_test.go b/third/github.com/Shopify/sarama/mocks/async_producer_test.go deleted file mode 100644 index 827b14111..000000000 --- a/third/github.com/Shopify/sarama/mocks/async_producer_test.go +++ /dev/null @@ -1,132 +0,0 @@ -package mocks - -import ( - "errors" - "fmt" - "regexp" - "strings" - "testing" - - "github.com/gogf/gf/third/github.com/Shopify/sarama" -) - -func generateRegexpChecker(re string) func([]byte) error { - return func(val []byte) error { - matched, err := regexp.MatchString(re, string(val)) - if err != nil { - return errors.New("Error while trying to match the input message with the expected pattern: " + err.Error()) - } - if !matched { - return fmt.Errorf("No match between input value \"%s\" and expected pattern \"%s\"", val, re) - } - return nil - } -} - -type testReporterMock struct { - errors []string -} - -func newTestReporterMock() *testReporterMock { - return &testReporterMock{errors: make([]string, 0)} -} - -func (trm *testReporterMock) Errorf(format string, args ...interface{}) { - trm.errors = append(trm.errors, fmt.Sprintf(format, args...)) -} - -func TestMockAsyncProducerImplementsAsyncProducerInterface(t *testing.T) { - var mp interface{} = &AsyncProducer{} - if _, ok := mp.(sarama.AsyncProducer); !ok { - t.Error("The mock producer should implement the sarama.Producer interface.") - } -} - -func TestProducerReturnsExpectationsToChannels(t *testing.T) { - config := sarama.NewConfig() - config.Producer.Return.Successes = true - mp := NewAsyncProducer(t, config) - - mp.ExpectInputAndSucceed() - mp.ExpectInputAndSucceed() - mp.ExpectInputAndFail(sarama.ErrOutOfBrokers) - - mp.Input() <- &sarama.ProducerMessage{Topic: "test 1"} - mp.Input() <- &sarama.ProducerMessage{Topic: "test 2"} - mp.Input() <- &sarama.ProducerMessage{Topic: "test 3"} - - msg1 := <-mp.Successes() - msg2 := <-mp.Successes() - err1 := <-mp.Errors() - - if msg1.Topic != "test 1" { - t.Error("Expected message 1 to be returned first") - } - - if msg2.Topic != "test 2" { - t.Error("Expected message 2 to be returned second") - } - - if err1.Msg.Topic != "test 3" || err1.Err != sarama.ErrOutOfBrokers { - t.Error("Expected message 3 to be returned as error") - } - - if err := mp.Close(); err != nil { - t.Error(err) - } -} - -func TestProducerWithTooFewExpectations(t *testing.T) { - trm := newTestReporterMock() - mp := NewAsyncProducer(trm, nil) - mp.ExpectInputAndSucceed() - - mp.Input() <- &sarama.ProducerMessage{Topic: "test"} - mp.Input() <- &sarama.ProducerMessage{Topic: "test"} - - if err := mp.Close(); err != nil { - t.Error(err) - } - - if len(trm.errors) != 1 { - t.Error("Expected to report an error") - } -} - -func TestProducerWithTooManyExpectations(t *testing.T) { - trm := newTestReporterMock() - mp := NewAsyncProducer(trm, nil) - mp.ExpectInputAndSucceed() - mp.ExpectInputAndFail(sarama.ErrOutOfBrokers) - - mp.Input() <- &sarama.ProducerMessage{Topic: "test"} - if err := mp.Close(); err != nil { - t.Error(err) - } - - if len(trm.errors) != 1 { - t.Error("Expected to report an error") - } -} - -func TestProducerWithCheckerFunction(t *testing.T) { - trm := newTestReporterMock() - mp := NewAsyncProducer(trm, nil) - mp.ExpectInputWithCheckerFunctionAndSucceed(generateRegexpChecker("^tes")) - mp.ExpectInputWithCheckerFunctionAndSucceed(generateRegexpChecker("^tes$")) - - mp.Input() <- &sarama.ProducerMessage{Topic: "test", Value: sarama.StringEncoder("test")} - mp.Input() <- &sarama.ProducerMessage{Topic: "test", Value: sarama.StringEncoder("test")} - if err := mp.Close(); err != nil { - t.Error(err) - } - - if len(mp.Errors()) != 1 { - t.Error("Expected to report an error") - } - - err1 := <-mp.Errors() - if !strings.HasPrefix(err1.Err.Error(), "No match") { - t.Error("Expected to report a value check error, found: ", err1.Err) - } -} diff --git a/third/github.com/Shopify/sarama/mocks/consumer.go b/third/github.com/Shopify/sarama/mocks/consumer.go deleted file mode 100644 index f6679f7d7..000000000 --- a/third/github.com/Shopify/sarama/mocks/consumer.go +++ /dev/null @@ -1,315 +0,0 @@ -package mocks - -import ( - "sync" - "sync/atomic" - - "github.com/gogf/gf/third/github.com/Shopify/sarama" -) - -// Consumer implements sarama's Consumer interface for testing purposes. -// Before you can start consuming from this consumer, you have to register -// topic/partitions using ExpectConsumePartition, and set expectations on them. -type Consumer struct { - l sync.Mutex - t ErrorReporter - config *sarama.Config - partitionConsumers map[string]map[int32]*PartitionConsumer - metadata map[string][]int32 -} - -// NewConsumer returns a new mock Consumer instance. The t argument should -// be the *testing.T instance of your test method. An error will be written to it if -// an expectation is violated. The config argument can be set to nil. -func NewConsumer(t ErrorReporter, config *sarama.Config) *Consumer { - if config == nil { - config = sarama.NewConfig() - } - - c := &Consumer{ - t: t, - config: config, - partitionConsumers: make(map[string]map[int32]*PartitionConsumer), - } - return c -} - -/////////////////////////////////////////////////// -// Consumer interface implementation -/////////////////////////////////////////////////// - -// ConsumePartition implements the ConsumePartition method from the sarama.Consumer interface. -// Before you can start consuming a partition, you have to set expectations on it using -// ExpectConsumePartition. You can only consume a partition once per consumer. -func (c *Consumer) ConsumePartition(topic string, partition int32, offset int64) (sarama.PartitionConsumer, error) { - c.l.Lock() - defer c.l.Unlock() - - if c.partitionConsumers[topic] == nil || c.partitionConsumers[topic][partition] == nil { - c.t.Errorf("No expectations set for %s/%d", topic, partition) - return nil, errOutOfExpectations - } - - pc := c.partitionConsumers[topic][partition] - if pc.consumed { - return nil, sarama.ConfigurationError("The topic/partition is already being consumed") - } - - if pc.offset != AnyOffset && pc.offset != offset { - c.t.Errorf("Unexpected offset when calling ConsumePartition for %s/%d. Expected %d, got %d.", topic, partition, pc.offset, offset) - } - - pc.consumed = true - return pc, nil -} - -// Topics returns a list of topics, as registered with SetMetadata -func (c *Consumer) Topics() ([]string, error) { - c.l.Lock() - defer c.l.Unlock() - - if c.metadata == nil { - c.t.Errorf("Unexpected call to Topics. Initialize the mock's topic metadata with SetMetadata.") - return nil, sarama.ErrOutOfBrokers - } - - var result []string - for topic := range c.metadata { - result = append(result, topic) - } - return result, nil -} - -// Partitions returns the list of parititons for the given topic, as registered with SetMetadata -func (c *Consumer) Partitions(topic string) ([]int32, error) { - c.l.Lock() - defer c.l.Unlock() - - if c.metadata == nil { - c.t.Errorf("Unexpected call to Partitions. Initialize the mock's topic metadata with SetMetadata.") - return nil, sarama.ErrOutOfBrokers - } - if c.metadata[topic] == nil { - return nil, sarama.ErrUnknownTopicOrPartition - } - - return c.metadata[topic], nil -} - -func (c *Consumer) HighWaterMarks() map[string]map[int32]int64 { - c.l.Lock() - defer c.l.Unlock() - - hwms := make(map[string]map[int32]int64, len(c.partitionConsumers)) - for topic, partitionConsumers := range c.partitionConsumers { - hwm := make(map[int32]int64, len(partitionConsumers)) - for partition, pc := range partitionConsumers { - hwm[partition] = pc.HighWaterMarkOffset() - } - hwms[topic] = hwm - } - - return hwms -} - -// Close implements the Close method from the sarama.Consumer interface. It will close -// all registered PartitionConsumer instances. -func (c *Consumer) Close() error { - c.l.Lock() - defer c.l.Unlock() - - for _, partitions := range c.partitionConsumers { - for _, partitionConsumer := range partitions { - _ = partitionConsumer.Close() - } - } - - return nil -} - -/////////////////////////////////////////////////// -// Expectation API -/////////////////////////////////////////////////// - -// SetTopicMetadata sets the clusters topic/partition metadata, -// which will be returned by Topics() and Partitions(). -func (c *Consumer) SetTopicMetadata(metadata map[string][]int32) { - c.l.Lock() - defer c.l.Unlock() - - c.metadata = metadata -} - -// ExpectConsumePartition will register a topic/partition, so you can set expectations on it. -// The registered PartitionConsumer will be returned, so you can set expectations -// on it using method chaining. Once a topic/partition is registered, you are -// expected to start consuming it using ConsumePartition. If that doesn't happen, -// an error will be written to the error reporter once the mock consumer is closed. It will -// also expect that the -func (c *Consumer) ExpectConsumePartition(topic string, partition int32, offset int64) *PartitionConsumer { - c.l.Lock() - defer c.l.Unlock() - - if c.partitionConsumers[topic] == nil { - c.partitionConsumers[topic] = make(map[int32]*PartitionConsumer) - } - - if c.partitionConsumers[topic][partition] == nil { - c.partitionConsumers[topic][partition] = &PartitionConsumer{ - t: c.t, - topic: topic, - partition: partition, - offset: offset, - messages: make(chan *sarama.ConsumerMessage, c.config.ChannelBufferSize), - errors: make(chan *sarama.ConsumerError, c.config.ChannelBufferSize), - } - } - - return c.partitionConsumers[topic][partition] -} - -/////////////////////////////////////////////////// -// PartitionConsumer mock type -/////////////////////////////////////////////////// - -// PartitionConsumer implements sarama's PartitionConsumer interface for testing purposes. -// It is returned by the mock Consumers ConsumePartitionMethod, but only if it is -// registered first using the Consumer's ExpectConsumePartition method. Before consuming the -// Errors and Messages channel, you should specify what values will be provided on these -// channels using YieldMessage and YieldError. -type PartitionConsumer struct { - highWaterMarkOffset int64 // must be at the top of the struct because https://golang.org/pkg/sync/atomic/#pkg-note-BUG - l sync.Mutex - t ErrorReporter - topic string - partition int32 - offset int64 - messages chan *sarama.ConsumerMessage - errors chan *sarama.ConsumerError - singleClose sync.Once - consumed bool - errorsShouldBeDrained bool - messagesShouldBeDrained bool -} - -/////////////////////////////////////////////////// -// PartitionConsumer interface implementation -/////////////////////////////////////////////////// - -// AsyncClose implements the AsyncClose method from the sarama.PartitionConsumer interface. -func (pc *PartitionConsumer) AsyncClose() { - pc.singleClose.Do(func() { - close(pc.messages) - close(pc.errors) - }) -} - -// Close implements the Close method from the sarama.PartitionConsumer interface. It will -// verify whether the partition consumer was actually started. -func (pc *PartitionConsumer) Close() error { - if !pc.consumed { - pc.t.Errorf("Expectations set on %s/%d, but no partition consumer was started.", pc.topic, pc.partition) - return errPartitionConsumerNotStarted - } - - if pc.errorsShouldBeDrained && len(pc.errors) > 0 { - pc.t.Errorf("Expected the errors channel for %s/%d to be drained on close, but found %d errors.", pc.topic, pc.partition, len(pc.errors)) - } - - if pc.messagesShouldBeDrained && len(pc.messages) > 0 { - pc.t.Errorf("Expected the messages channel for %s/%d to be drained on close, but found %d messages.", pc.topic, pc.partition, len(pc.messages)) - } - - pc.AsyncClose() - - var ( - closeErr error - wg sync.WaitGroup - ) - - wg.Add(1) - go func() { - defer wg.Done() - - var errs = make(sarama.ConsumerErrors, 0) - for err := range pc.errors { - errs = append(errs, err) - } - - if len(errs) > 0 { - closeErr = errs - } - }() - - wg.Add(1) - go func() { - defer wg.Done() - for range pc.messages { - // drain - } - }() - - wg.Wait() - return closeErr -} - -// Errors implements the Errors method from the sarama.PartitionConsumer interface. -func (pc *PartitionConsumer) Errors() <-chan *sarama.ConsumerError { - return pc.errors -} - -// Messages implements the Messages method from the sarama.PartitionConsumer interface. -func (pc *PartitionConsumer) Messages() <-chan *sarama.ConsumerMessage { - return pc.messages -} - -func (pc *PartitionConsumer) HighWaterMarkOffset() int64 { - return atomic.LoadInt64(&pc.highWaterMarkOffset) + 1 -} - -/////////////////////////////////////////////////// -// Expectation API -/////////////////////////////////////////////////// - -// YieldMessage will yield a messages Messages channel of this partition consumer -// when it is consumed. By default, the mock consumer will not verify whether this -// message was consumed from the Messages channel, because there are legitimate -// reasons forthis not to happen. ou can call ExpectMessagesDrainedOnClose so it will -// verify that the channel is empty on close. -func (pc *PartitionConsumer) YieldMessage(msg *sarama.ConsumerMessage) { - pc.l.Lock() - defer pc.l.Unlock() - - msg.Topic = pc.topic - msg.Partition = pc.partition - msg.Offset = atomic.AddInt64(&pc.highWaterMarkOffset, 1) - - pc.messages <- msg -} - -// YieldError will yield an error on the Errors channel of this partition consumer -// when it is consumed. By default, the mock consumer will not verify whether this error was -// consumed from the Errors channel, because there are legitimate reasons for this -// not to happen. You can call ExpectErrorsDrainedOnClose so it will verify that -// the channel is empty on close. -func (pc *PartitionConsumer) YieldError(err error) { - pc.errors <- &sarama.ConsumerError{ - Topic: pc.topic, - Partition: pc.partition, - Err: err, - } -} - -// ExpectMessagesDrainedOnClose sets an expectation on the partition consumer -// that the messages channel will be fully drained when Close is called. If this -// expectation is not met, an error is reported to the error reporter. -func (pc *PartitionConsumer) ExpectMessagesDrainedOnClose() { - pc.messagesShouldBeDrained = true -} - -// ExpectErrorsDrainedOnClose sets an expectation on the partition consumer -// that the errors channel will be fully drained when Close is called. If this -// expectation is not met, an error is reported to the error reporter. -func (pc *PartitionConsumer) ExpectErrorsDrainedOnClose() { - pc.errorsShouldBeDrained = true -} diff --git a/third/github.com/Shopify/sarama/mocks/consumer_test.go b/third/github.com/Shopify/sarama/mocks/consumer_test.go deleted file mode 100644 index 6f08d386c..000000000 --- a/third/github.com/Shopify/sarama/mocks/consumer_test.go +++ /dev/null @@ -1,249 +0,0 @@ -package mocks - -import ( - "sort" - "testing" - - "github.com/gogf/gf/third/github.com/Shopify/sarama" -) - -func TestMockConsumerImplementsConsumerInterface(t *testing.T) { - var c interface{} = &Consumer{} - if _, ok := c.(sarama.Consumer); !ok { - t.Error("The mock consumer should implement the sarama.Consumer interface.") - } - - var pc interface{} = &PartitionConsumer{} - if _, ok := pc.(sarama.PartitionConsumer); !ok { - t.Error("The mock partitionconsumer should implement the sarama.PartitionConsumer interface.") - } -} - -func TestConsumerHandlesExpectations(t *testing.T) { - consumer := NewConsumer(t, nil) - defer func() { - if err := consumer.Close(); err != nil { - t.Error(err) - } - }() - - consumer.ExpectConsumePartition("test", 0, sarama.OffsetOldest).YieldMessage(&sarama.ConsumerMessage{Value: []byte("hello world")}) - consumer.ExpectConsumePartition("test", 0, sarama.OffsetOldest).YieldError(sarama.ErrOutOfBrokers) - consumer.ExpectConsumePartition("test", 1, sarama.OffsetOldest).YieldMessage(&sarama.ConsumerMessage{Value: []byte("hello world again")}) - consumer.ExpectConsumePartition("other", 0, AnyOffset).YieldMessage(&sarama.ConsumerMessage{Value: []byte("hello other")}) - - pc_test0, err := consumer.ConsumePartition("test", 0, sarama.OffsetOldest) - if err != nil { - t.Fatal(err) - } - test0_msg := <-pc_test0.Messages() - if test0_msg.Topic != "test" || test0_msg.Partition != 0 || string(test0_msg.Value) != "hello world" { - t.Error("Message was not as expected:", test0_msg) - } - test0_err := <-pc_test0.Errors() - if test0_err.Err != sarama.ErrOutOfBrokers { - t.Error("Expected sarama.ErrOutOfBrokers, found:", test0_err.Err) - } - - pc_test1, err := consumer.ConsumePartition("test", 1, sarama.OffsetOldest) - if err != nil { - t.Fatal(err) - } - test1_msg := <-pc_test1.Messages() - if test1_msg.Topic != "test" || test1_msg.Partition != 1 || string(test1_msg.Value) != "hello world again" { - t.Error("Message was not as expected:", test1_msg) - } - - pc_other0, err := consumer.ConsumePartition("other", 0, sarama.OffsetNewest) - if err != nil { - t.Fatal(err) - } - other0_msg := <-pc_other0.Messages() - if other0_msg.Topic != "other" || other0_msg.Partition != 0 || string(other0_msg.Value) != "hello other" { - t.Error("Message was not as expected:", other0_msg) - } -} - -func TestConsumerReturnsNonconsumedErrorsOnClose(t *testing.T) { - consumer := NewConsumer(t, nil) - consumer.ExpectConsumePartition("test", 0, sarama.OffsetOldest).YieldError(sarama.ErrOutOfBrokers) - consumer.ExpectConsumePartition("test", 0, sarama.OffsetOldest).YieldError(sarama.ErrOutOfBrokers) - - pc, err := consumer.ConsumePartition("test", 0, sarama.OffsetOldest) - if err != nil { - t.Fatal(err) - } - - select { - case <-pc.Messages(): - t.Error("Did not epxect a message on the messages channel.") - case err := <-pc.Errors(): - if err.Err != sarama.ErrOutOfBrokers { - t.Error("Expected sarama.ErrOutOfBrokers, found", err) - } - } - - errs := pc.Close().(sarama.ConsumerErrors) - if len(errs) != 1 && errs[0].Err != sarama.ErrOutOfBrokers { - t.Error("Expected Close to return the remaining sarama.ErrOutOfBrokers") - } -} - -func TestConsumerWithoutExpectationsOnPartition(t *testing.T) { - trm := newTestReporterMock() - consumer := NewConsumer(trm, nil) - - _, err := consumer.ConsumePartition("test", 1, sarama.OffsetOldest) - if err != errOutOfExpectations { - t.Error("Expected ConsumePartition to return errOutOfExpectations") - } - - if err := consumer.Close(); err != nil { - t.Error("No error expected on close, but found:", err) - } - - if len(trm.errors) != 1 { - t.Errorf("Expected an expectation failure to be set on the error reporter.") - } -} - -func TestConsumerWithExpectationsOnUnconsumedPartition(t *testing.T) { - trm := newTestReporterMock() - consumer := NewConsumer(trm, nil) - consumer.ExpectConsumePartition("test", 0, sarama.OffsetOldest).YieldMessage(&sarama.ConsumerMessage{Value: []byte("hello world")}) - - if err := consumer.Close(); err != nil { - t.Error("No error expected on close, but found:", err) - } - - if len(trm.errors) != 1 { - t.Errorf("Expected an expectation failure to be set on the error reporter.") - } -} - -func TestConsumerWithWrongOffsetExpectation(t *testing.T) { - trm := newTestReporterMock() - consumer := NewConsumer(trm, nil) - consumer.ExpectConsumePartition("test", 0, sarama.OffsetOldest) - - _, err := consumer.ConsumePartition("test", 0, sarama.OffsetNewest) - if err != nil { - t.Error("Did not expect error, found:", err) - } - - if len(trm.errors) != 1 { - t.Errorf("Expected an expectation failure to be set on the error reporter.") - } - - if err := consumer.Close(); err != nil { - t.Error(err) - } -} - -func TestConsumerViolatesMessagesDrainedExpectation(t *testing.T) { - trm := newTestReporterMock() - consumer := NewConsumer(trm, nil) - pcmock := consumer.ExpectConsumePartition("test", 0, sarama.OffsetOldest) - pcmock.YieldMessage(&sarama.ConsumerMessage{Value: []byte("hello")}) - pcmock.YieldMessage(&sarama.ConsumerMessage{Value: []byte("hello")}) - pcmock.ExpectMessagesDrainedOnClose() - - pc, err := consumer.ConsumePartition("test", 0, sarama.OffsetOldest) - if err != nil { - t.Error(err) - } - - // consume first message, not second one - <-pc.Messages() - - if err := consumer.Close(); err != nil { - t.Error(err) - } - - if len(trm.errors) != 1 { - t.Errorf("Expected an expectation failure to be set on the error reporter.") - } -} - -func TestConsumerMeetsErrorsDrainedExpectation(t *testing.T) { - trm := newTestReporterMock() - consumer := NewConsumer(trm, nil) - - pcmock := consumer.ExpectConsumePartition("test", 0, sarama.OffsetOldest) - pcmock.YieldError(sarama.ErrInvalidMessage) - pcmock.YieldError(sarama.ErrInvalidMessage) - pcmock.ExpectErrorsDrainedOnClose() - - pc, err := consumer.ConsumePartition("test", 0, sarama.OffsetOldest) - if err != nil { - t.Error(err) - } - - // consume first and second error, - <-pc.Errors() - <-pc.Errors() - - if err := consumer.Close(); err != nil { - t.Error(err) - } - - if len(trm.errors) != 0 { - t.Errorf("Expected no expectation failures to be set on the error reporter.") - } -} - -func TestConsumerTopicMetadata(t *testing.T) { - trm := newTestReporterMock() - consumer := NewConsumer(trm, nil) - - consumer.SetTopicMetadata(map[string][]int32{ - "test1": {0, 1, 2, 3}, - "test2": {0, 1, 2, 3, 4, 5, 6, 7}, - }) - - topics, err := consumer.Topics() - if err != nil { - t.Error(t) - } - - sortedTopics := sort.StringSlice(topics) - sortedTopics.Sort() - if len(sortedTopics) != 2 || sortedTopics[0] != "test1" || sortedTopics[1] != "test2" { - t.Error("Unexpected topics returned:", sortedTopics) - } - - partitions1, err := consumer.Partitions("test1") - if err != nil { - t.Error(t) - } - - if len(partitions1) != 4 { - t.Error("Unexpected partitions returned:", len(partitions1)) - } - - partitions2, err := consumer.Partitions("test2") - if err != nil { - t.Error(t) - } - - if len(partitions2) != 8 { - t.Error("Unexpected partitions returned:", len(partitions2)) - } - - if len(trm.errors) != 0 { - t.Errorf("Expected no expectation failures to be set on the error reporter.") - } -} - -func TestConsumerUnexpectedTopicMetadata(t *testing.T) { - trm := newTestReporterMock() - consumer := NewConsumer(trm, nil) - - if _, err := consumer.Topics(); err != sarama.ErrOutOfBrokers { - t.Error("Expected sarama.ErrOutOfBrokers, found", err) - } - - if len(trm.errors) != 1 { - t.Errorf("Expected an expectation failure to be set on the error reporter.") - } -} diff --git a/third/github.com/Shopify/sarama/mocks/mocks.go b/third/github.com/Shopify/sarama/mocks/mocks.go deleted file mode 100644 index b0368b1a8..000000000 --- a/third/github.com/Shopify/sarama/mocks/mocks.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Package mocks provides mocks that can be used for testing applications -that use Sarama. The mock types provided by this package implement the -interfaces Sarama exports, so you can use them for dependency injection -in your tests. - -All mock instances require you to set expectations on them before you -can use them. It will determine how the mock will behave. If an -expectation is not met, it will make your test fail. - -NOTE: this package currently does not fall under the API stability -guarantee of Sarama as it is still considered experimental. -*/ -package mocks - -import ( - "errors" - - "github.com/gogf/gf/third/github.com/Shopify/sarama" -) - -// ErrorReporter is a simple interface that includes the testing.T methods we use to report -// expectation violations when using the mock objects. -type ErrorReporter interface { - Errorf(string, ...interface{}) -} - -// ValueChecker is a function type to be set in each expectation of the producer mocks -// to check the value passed. -type ValueChecker func(val []byte) error - -var ( - errProduceSuccess error = nil - errOutOfExpectations = errors.New("No more expectations set on mock") - errPartitionConsumerNotStarted = errors.New("The partition consumer was never started") -) - -const AnyOffset int64 = -1000 - -type producerExpectation struct { - Result error - CheckFunction ValueChecker -} - -type consumerExpectation struct { - Err error - Msg *sarama.ConsumerMessage -} diff --git a/third/github.com/Shopify/sarama/mocks/sync_producer.go b/third/github.com/Shopify/sarama/mocks/sync_producer.go deleted file mode 100644 index c88fac6eb..000000000 --- a/third/github.com/Shopify/sarama/mocks/sync_producer.go +++ /dev/null @@ -1,157 +0,0 @@ -package mocks - -import ( - "sync" - - "github.com/gogf/gf/third/github.com/Shopify/sarama" -) - -// SyncProducer implements sarama's SyncProducer interface for testing purposes. -// Before you can use it, you have to set expectations on the mock SyncProducer -// to tell it how to handle calls to SendMessage, so you can easily test success -// and failure scenarios. -type SyncProducer struct { - l sync.Mutex - t ErrorReporter - expectations []*producerExpectation - lastOffset int64 -} - -// NewSyncProducer instantiates a new SyncProducer mock. The t argument should -// be the *testing.T instance of your test method. An error will be written to it if -// an expectation is violated. The config argument is currently unused, but is -// maintained to be compatible with the async Producer. -func NewSyncProducer(t ErrorReporter, config *sarama.Config) *SyncProducer { - return &SyncProducer{ - t: t, - expectations: make([]*producerExpectation, 0), - } -} - -//////////////////////////////////////////////// -// Implement SyncProducer interface -//////////////////////////////////////////////// - -// SendMessage corresponds with the SendMessage method of sarama's SyncProducer implementation. -// You have to set expectations on the mock producer before calling SendMessage, so it knows -// how to handle them. You can set a function in each expectation so that the message value -// checked by this function and an error is returned if the match fails. -// If there is no more remaining expectation when SendMessage is called, -// the mock producer will write an error to the test state object. -func (sp *SyncProducer) SendMessage(msg *sarama.ProducerMessage) (partition int32, offset int64, err error) { - sp.l.Lock() - defer sp.l.Unlock() - - if len(sp.expectations) > 0 { - expectation := sp.expectations[0] - sp.expectations = sp.expectations[1:] - if expectation.CheckFunction != nil { - val, err := msg.Value.Encode() - if err != nil { - sp.t.Errorf("Input message encoding failed: %s", err.Error()) - return -1, -1, err - } - - errCheck := expectation.CheckFunction(val) - if errCheck != nil { - sp.t.Errorf("Check function returned an error: %s", errCheck.Error()) - return -1, -1, errCheck - } - } - if expectation.Result == errProduceSuccess { - sp.lastOffset++ - msg.Offset = sp.lastOffset - return 0, msg.Offset, nil - } - return -1, -1, expectation.Result - } - sp.t.Errorf("No more expectation set on this mock producer to handle the input message.") - return -1, -1, errOutOfExpectations -} - -// SendMessages corresponds with the SendMessages method of sarama's SyncProducer implementation. -// You have to set expectations on the mock producer before calling SendMessages, so it knows -// how to handle them. If there is no more remaining expectations when SendMessages is called, -// the mock producer will write an error to the test state object. -func (sp *SyncProducer) SendMessages(msgs []*sarama.ProducerMessage) error { - sp.l.Lock() - defer sp.l.Unlock() - - if len(sp.expectations) >= len(msgs) { - expectations := sp.expectations[0:len(msgs)] - sp.expectations = sp.expectations[len(msgs):] - - for i, expectation := range expectations { - if expectation.CheckFunction != nil { - val, err := msgs[i].Value.Encode() - if err != nil { - sp.t.Errorf("Input message encoding failed: %s", err.Error()) - return err - } - errCheck := expectation.CheckFunction(val) - if errCheck != nil { - sp.t.Errorf("Check function returned an error: %s", errCheck.Error()) - return errCheck - } - } - if expectation.Result != errProduceSuccess { - return expectation.Result - } - } - return nil - } - sp.t.Errorf("Insufficient expectations set on this mock producer to handle the input messages.") - return errOutOfExpectations -} - -// Close corresponds with the Close method of sarama's SyncProducer implementation. -// By closing a mock syncproducer, you also tell it that no more SendMessage calls will follow, -// so it will write an error to the test state if there's any remaining expectations. -func (sp *SyncProducer) Close() error { - sp.l.Lock() - defer sp.l.Unlock() - - if len(sp.expectations) > 0 { - sp.t.Errorf("Expected to exhaust all expectations, but %d are left.", len(sp.expectations)) - } - - return nil -} - -//////////////////////////////////////////////// -// Setting expectations -//////////////////////////////////////////////// - -// ExpectSendMessageWithCheckerFunctionAndSucceed sets an expectation on the mock producer that SendMessage -// will be called. The mock producer will first call the given function to check the message value. -// It will cascade the error of the function, if any, or handle the message as if it produced -// successfully, i.e. by returning a valid partition, and offset, and a nil error. -func (sp *SyncProducer) ExpectSendMessageWithCheckerFunctionAndSucceed(cf ValueChecker) { - sp.l.Lock() - defer sp.l.Unlock() - sp.expectations = append(sp.expectations, &producerExpectation{Result: errProduceSuccess, CheckFunction: cf}) -} - -// ExpectSendMessageWithCheckerFunctionAndFail sets an expectation on the mock producer that SendMessage will be -// called. The mock producer will first call the given function to check the message value. -// It will cascade the error of the function, if any, or handle the message as if it failed -// to produce successfully, i.e. by returning the provided error. -func (sp *SyncProducer) ExpectSendMessageWithCheckerFunctionAndFail(cf ValueChecker, err error) { - sp.l.Lock() - defer sp.l.Unlock() - sp.expectations = append(sp.expectations, &producerExpectation{Result: err, CheckFunction: cf}) -} - -// ExpectSendMessageAndSucceed sets an expectation on the mock producer that SendMessage will be -// called. The mock producer will handle the message as if it produced successfully, i.e. by -// returning a valid partition, and offset, and a nil error. -func (sp *SyncProducer) ExpectSendMessageAndSucceed() { - sp.ExpectSendMessageWithCheckerFunctionAndSucceed(nil) -} - -// ExpectSendMessageAndFail sets an expectation on the mock producer that SendMessage will be -// called. The mock producer will handle the message as if it failed to produce -// successfully, i.e. by returning the provided error. -func (sp *SyncProducer) ExpectSendMessageAndFail(err error) { - sp.ExpectSendMessageWithCheckerFunctionAndFail(nil, err) -} diff --git a/third/github.com/Shopify/sarama/mocks/sync_producer_test.go b/third/github.com/Shopify/sarama/mocks/sync_producer_test.go deleted file mode 100644 index b967666fa..000000000 --- a/third/github.com/Shopify/sarama/mocks/sync_producer_test.go +++ /dev/null @@ -1,250 +0,0 @@ -package mocks - -import ( - "errors" - "strings" - "testing" - - "github.com/gogf/gf/third/github.com/Shopify/sarama" -) - -func TestMockSyncProducerImplementsSyncProducerInterface(t *testing.T) { - var mp interface{} = &SyncProducer{} - if _, ok := mp.(sarama.SyncProducer); !ok { - t.Error("The mock async producer should implement the sarama.SyncProducer interface.") - } -} - -func TestSyncProducerReturnsExpectationsToSendMessage(t *testing.T) { - sp := NewSyncProducer(t, nil) - defer func() { - if err := sp.Close(); err != nil { - t.Error(err) - } - }() - - sp.ExpectSendMessageAndSucceed() - sp.ExpectSendMessageAndSucceed() - sp.ExpectSendMessageAndFail(sarama.ErrOutOfBrokers) - - msg := &sarama.ProducerMessage{Topic: "test", Value: sarama.StringEncoder("test")} - - _, offset, err := sp.SendMessage(msg) - if err != nil { - t.Errorf("The first message should have been produced successfully, but got %s", err) - } - if offset != 1 || offset != msg.Offset { - t.Errorf("The first message should have been assigned offset 1, but got %d", msg.Offset) - } - - _, offset, err = sp.SendMessage(msg) - if err != nil { - t.Errorf("The second message should have been produced successfully, but got %s", err) - } - if offset != 2 || offset != msg.Offset { - t.Errorf("The second message should have been assigned offset 2, but got %d", offset) - } - - _, _, err = sp.SendMessage(msg) - if err != sarama.ErrOutOfBrokers { - t.Errorf("The third message should not have been produced successfully") - } - - if err := sp.Close(); err != nil { - t.Error(err) - } -} - -func TestSyncProducerWithTooManyExpectations(t *testing.T) { - trm := newTestReporterMock() - - sp := NewSyncProducer(trm, nil) - sp.ExpectSendMessageAndSucceed() - sp.ExpectSendMessageAndFail(sarama.ErrOutOfBrokers) - - msg := &sarama.ProducerMessage{Topic: "test", Value: sarama.StringEncoder("test")} - if _, _, err := sp.SendMessage(msg); err != nil { - t.Error("No error expected on first SendMessage call", err) - } - - if err := sp.Close(); err != nil { - t.Error(err) - } - - if len(trm.errors) != 1 { - t.Error("Expected to report an error") - } -} - -func TestSyncProducerWithTooFewExpectations(t *testing.T) { - trm := newTestReporterMock() - - sp := NewSyncProducer(trm, nil) - sp.ExpectSendMessageAndSucceed() - - msg := &sarama.ProducerMessage{Topic: "test", Value: sarama.StringEncoder("test")} - if _, _, err := sp.SendMessage(msg); err != nil { - t.Error("No error expected on first SendMessage call", err) - } - if _, _, err := sp.SendMessage(msg); err != errOutOfExpectations { - t.Error("errOutOfExpectations expected on second SendMessage call, found:", err) - } - - if err := sp.Close(); err != nil { - t.Error(err) - } - - if len(trm.errors) != 1 { - t.Error("Expected to report an error") - } -} - -func TestSyncProducerWithCheckerFunction(t *testing.T) { - trm := newTestReporterMock() - - sp := NewSyncProducer(trm, nil) - sp.ExpectSendMessageWithCheckerFunctionAndSucceed(generateRegexpChecker("^tes")) - sp.ExpectSendMessageWithCheckerFunctionAndSucceed(generateRegexpChecker("^tes$")) - - msg := &sarama.ProducerMessage{Topic: "test", Value: sarama.StringEncoder("test")} - if _, _, err := sp.SendMessage(msg); err != nil { - t.Error("No error expected on first SendMessage call, found: ", err) - } - msg = &sarama.ProducerMessage{Topic: "test", Value: sarama.StringEncoder("test")} - if _, _, err := sp.SendMessage(msg); err == nil || !strings.HasPrefix(err.Error(), "No match") { - t.Error("Error during value check expected on second SendMessage call, found:", err) - } - - if err := sp.Close(); err != nil { - t.Error(err) - } - - if len(trm.errors) != 1 { - t.Error("Expected to report an error") - } -} - -func TestSyncProducerWithCheckerFunctionForSendMessagesWithError(t *testing.T) { - trm := newTestReporterMock() - - sp := NewSyncProducer(trm, nil) - sp.ExpectSendMessageWithCheckerFunctionAndSucceed(generateRegexpChecker("^tes")) - sp.ExpectSendMessageWithCheckerFunctionAndSucceed(generateRegexpChecker("^tes$")) - - msg1 := &sarama.ProducerMessage{Topic: "test", Value: sarama.StringEncoder("test")} - msg2 := &sarama.ProducerMessage{Topic: "test", Value: sarama.StringEncoder("test")} - msgs := []*sarama.ProducerMessage{msg1, msg2} - - if err := sp.SendMessages(msgs); err == nil || !strings.HasPrefix(err.Error(), "No match") { - t.Error("Error during value check expected on second message, found: ", err) - } - - if err := sp.Close(); err != nil { - t.Error(err) - } - - if len(trm.errors) != 1 { - t.Error("Expected to report an error") - } -} - -func TestSyncProducerWithCheckerFunctionForSendMessagesWithoutError(t *testing.T) { - trm := newTestReporterMock() - - sp := NewSyncProducer(trm, nil) - sp.ExpectSendMessageWithCheckerFunctionAndSucceed(generateRegexpChecker("^tes")) - - msg1 := &sarama.ProducerMessage{Topic: "test", Value: sarama.StringEncoder("test")} - msgs := []*sarama.ProducerMessage{msg1} - - if err := sp.SendMessages(msgs); err != nil { - t.Error("No error expected on SendMessages call, found: ", err) - } - - if err := sp.Close(); err != nil { - t.Error(err) - } - - if len(trm.errors) != 0 { - t.Errorf("Expected to not report any errors, found: %v", trm.errors) - } -} - -func TestSyncProducerSendMessagesExpectationsMismatchTooFew(t *testing.T) { - trm := newTestReporterMock() - - sp := NewSyncProducer(trm, nil) - sp.ExpectSendMessageWithCheckerFunctionAndSucceed(generateRegexpChecker("^tes")) - - msg1 := &sarama.ProducerMessage{Topic: "test", Value: sarama.StringEncoder("test")} - msg2 := &sarama.ProducerMessage{Topic: "test", Value: sarama.StringEncoder("test")} - - msgs := []*sarama.ProducerMessage{msg1, msg2} - - if err := sp.SendMessages(msgs); err == nil { - t.Error("Error during value check expected on second message, found: ", err) - } - - if err := sp.Close(); err != nil { - t.Error(err) - } - - if len(trm.errors) != 2 { - t.Error("Expected to report 2 errors") - } -} - -func TestSyncProducerSendMessagesExpectationsMismatchTooMany(t *testing.T) { - trm := newTestReporterMock() - - sp := NewSyncProducer(trm, nil) - sp.ExpectSendMessageWithCheckerFunctionAndSucceed(generateRegexpChecker("^tes")) - sp.ExpectSendMessageWithCheckerFunctionAndSucceed(generateRegexpChecker("^tes")) - - msg1 := &sarama.ProducerMessage{Topic: "test", Value: sarama.StringEncoder("test")} - msgs := []*sarama.ProducerMessage{msg1} - - if err := sp.SendMessages(msgs); err != nil { - t.Error("No error expected on SendMessages call, found: ", err) - } - - if err := sp.Close(); err != nil { - t.Error(err) - } - - if len(trm.errors) != 1 { - t.Error("Expected to report 1 errors") - } -} - -func TestSyncProducerSendMessagesFaultyEncoder(t *testing.T) { - trm := newTestReporterMock() - - sp := NewSyncProducer(trm, nil) - sp.ExpectSendMessageWithCheckerFunctionAndSucceed(generateRegexpChecker("^tes")) - - msg1 := &sarama.ProducerMessage{Topic: "test", Value: faultyEncoder("123")} - msgs := []*sarama.ProducerMessage{msg1} - - if err := sp.SendMessages(msgs); err == nil || !strings.HasPrefix(err.Error(), "encode error") { - t.Error("Encoding error expected, found: ", err) - } - - if err := sp.Close(); err != nil { - t.Error(err) - } - - if len(trm.errors) != 1 { - t.Error("Expected to report 1 errors") - } -} - -type faultyEncoder []byte - -func (f faultyEncoder) Encode() ([]byte, error) { - return nil, errors.New("encode error") -} - -func (f faultyEncoder) Length() int { - return len(f) -} diff --git a/third/github.com/Shopify/sarama/offset_commit_request.go b/third/github.com/Shopify/sarama/offset_commit_request.go deleted file mode 100644 index 1ec583e6d..000000000 --- a/third/github.com/Shopify/sarama/offset_commit_request.go +++ /dev/null @@ -1,210 +0,0 @@ -package sarama - -import "errors" - -// ReceiveTime is a special value for the timestamp field of Offset Commit Requests which -// tells the broker to set the timestamp to the time at which the request was received. -// The timestamp is only used if message version 1 is used, which requires kafka 0.8.2. -const ReceiveTime int64 = -1 - -// GroupGenerationUndefined is a special value for the group generation field of -// Offset Commit Requests that should be used when a consumer group does not rely -// on Kafka for partition management. -const GroupGenerationUndefined = -1 - -type offsetCommitRequestBlock struct { - offset int64 - timestamp int64 - metadata string -} - -func (b *offsetCommitRequestBlock) encode(pe packetEncoder, version int16) error { - pe.putInt64(b.offset) - if version == 1 { - pe.putInt64(b.timestamp) - } else if b.timestamp != 0 { - Logger.Println("Non-zero timestamp specified for OffsetCommitRequest not v1, it will be ignored") - } - - return pe.putString(b.metadata) -} - -func (b *offsetCommitRequestBlock) decode(pd packetDecoder, version int16) (err error) { - if b.offset, err = pd.getInt64(); err != nil { - return err - } - if version == 1 { - if b.timestamp, err = pd.getInt64(); err != nil { - return err - } - } - b.metadata, err = pd.getString() - return err -} - -type OffsetCommitRequest struct { - ConsumerGroup string - ConsumerGroupGeneration int32 // v1 or later - ConsumerID string // v1 or later - RetentionTime int64 // v2 or later - - // Version can be: - // - 0 (kafka 0.8.1 and later) - // - 1 (kafka 0.8.2 and later) - // - 2 (kafka 0.9.0 and later) - // - 3 (kafka 0.11.0 and later) - // - 4 (kafka 2.0.0 and later) - Version int16 - blocks map[string]map[int32]*offsetCommitRequestBlock -} - -func (r *OffsetCommitRequest) encode(pe packetEncoder) error { - if r.Version < 0 || r.Version > 4 { - return PacketEncodingError{"invalid or unsupported OffsetCommitRequest version field"} - } - - if err := pe.putString(r.ConsumerGroup); err != nil { - return err - } - - if r.Version >= 1 { - pe.putInt32(r.ConsumerGroupGeneration) - if err := pe.putString(r.ConsumerID); err != nil { - return err - } - } else { - if r.ConsumerGroupGeneration != 0 { - Logger.Println("Non-zero ConsumerGroupGeneration specified for OffsetCommitRequest v0, it will be ignored") - } - if r.ConsumerID != "" { - Logger.Println("Non-empty ConsumerID specified for OffsetCommitRequest v0, it will be ignored") - } - } - - if r.Version >= 2 { - pe.putInt64(r.RetentionTime) - } else if r.RetentionTime != 0 { - Logger.Println("Non-zero RetentionTime specified for OffsetCommitRequest version <2, it will be ignored") - } - - if err := pe.putArrayLength(len(r.blocks)); err != nil { - return err - } - for topic, partitions := range r.blocks { - if err := pe.putString(topic); err != nil { - return err - } - if err := pe.putArrayLength(len(partitions)); err != nil { - return err - } - for partition, block := range partitions { - pe.putInt32(partition) - if err := block.encode(pe, r.Version); err != nil { - return err - } - } - } - return nil -} - -func (r *OffsetCommitRequest) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if r.ConsumerGroup, err = pd.getString(); err != nil { - return err - } - - if r.Version >= 1 { - if r.ConsumerGroupGeneration, err = pd.getInt32(); err != nil { - return err - } - if r.ConsumerID, err = pd.getString(); err != nil { - return err - } - } - - if r.Version >= 2 { - if r.RetentionTime, err = pd.getInt64(); err != nil { - return err - } - } - - topicCount, err := pd.getArrayLength() - if err != nil { - return err - } - if topicCount == 0 { - return nil - } - r.blocks = make(map[string]map[int32]*offsetCommitRequestBlock) - for i := 0; i < topicCount; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - partitionCount, err := pd.getArrayLength() - if err != nil { - return err - } - r.blocks[topic] = make(map[int32]*offsetCommitRequestBlock) - for j := 0; j < partitionCount; j++ { - partition, err := pd.getInt32() - if err != nil { - return err - } - block := &offsetCommitRequestBlock{} - if err := block.decode(pd, r.Version); err != nil { - return err - } - r.blocks[topic][partition] = block - } - } - return nil -} - -func (r *OffsetCommitRequest) key() int16 { - return 8 -} - -func (r *OffsetCommitRequest) version() int16 { - return r.Version -} - -func (r *OffsetCommitRequest) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_8_2_0 - case 2: - return V0_9_0_0 - case 3: - return V0_11_0_0 - case 4: - return V2_0_0_0 - default: - return MinVersion - } -} - -func (r *OffsetCommitRequest) AddBlock(topic string, partitionID int32, offset int64, timestamp int64, metadata string) { - if r.blocks == nil { - r.blocks = make(map[string]map[int32]*offsetCommitRequestBlock) - } - - if r.blocks[topic] == nil { - r.blocks[topic] = make(map[int32]*offsetCommitRequestBlock) - } - - r.blocks[topic][partitionID] = &offsetCommitRequestBlock{offset, timestamp, metadata} -} - -func (r *OffsetCommitRequest) Offset(topic string, partitionID int32) (int64, string, error) { - partitions := r.blocks[topic] - if partitions == nil { - return 0, "", errors.New("No such offset") - } - block := partitions[partitionID] - if block == nil { - return 0, "", errors.New("No such offset") - } - return block.offset, block.metadata, nil -} diff --git a/third/github.com/Shopify/sarama/offset_commit_request_test.go b/third/github.com/Shopify/sarama/offset_commit_request_test.go deleted file mode 100644 index efb3d33f1..000000000 --- a/third/github.com/Shopify/sarama/offset_commit_request_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package sarama - -import ( - "fmt" - "testing" -) - -var ( - offsetCommitRequestNoBlocksV0 = []byte{ - 0x00, 0x06, 'f', 'o', 'o', 'b', 'a', 'r', - 0x00, 0x00, 0x00, 0x00} - - offsetCommitRequestNoBlocksV1 = []byte{ - 0x00, 0x06, 'f', 'o', 'o', 'b', 'a', 'r', - 0x00, 0x00, 0x11, 0x22, - 0x00, 0x04, 'c', 'o', 'n', 's', - 0x00, 0x00, 0x00, 0x00} - - offsetCommitRequestNoBlocksV2 = []byte{ - 0x00, 0x06, 'f', 'o', 'o', 'b', 'a', 'r', - 0x00, 0x00, 0x11, 0x22, - 0x00, 0x04, 'c', 'o', 'n', 's', - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x33, - 0x00, 0x00, 0x00, 0x00} - - offsetCommitRequestOneBlockV0 = []byte{ - 0x00, 0x06, 'f', 'o', 'o', 'b', 'a', 'r', - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x05, 't', 'o', 'p', 'i', 'c', - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x52, 0x21, - 0x00, 0x00, 0x00, 0x00, 0xDE, 0xAD, 0xBE, 0xEF, - 0x00, 0x08, 'm', 'e', 't', 'a', 'd', 'a', 't', 'a'} - - offsetCommitRequestOneBlockV1 = []byte{ - 0x00, 0x06, 'f', 'o', 'o', 'b', 'a', 'r', - 0x00, 0x00, 0x11, 0x22, - 0x00, 0x04, 'c', 'o', 'n', 's', - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x05, 't', 'o', 'p', 'i', 'c', - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x52, 0x21, - 0x00, 0x00, 0x00, 0x00, 0xDE, 0xAD, 0xBE, 0xEF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x08, 'm', 'e', 't', 'a', 'd', 'a', 't', 'a'} - - offsetCommitRequestOneBlockV2 = []byte{ - 0x00, 0x06, 'f', 'o', 'o', 'b', 'a', 'r', - 0x00, 0x00, 0x11, 0x22, - 0x00, 0x04, 'c', 'o', 'n', 's', - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x33, - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x05, 't', 'o', 'p', 'i', 'c', - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x52, 0x21, - 0x00, 0x00, 0x00, 0x00, 0xDE, 0xAD, 0xBE, 0xEF, - 0x00, 0x08, 'm', 'e', 't', 'a', 'd', 'a', 't', 'a'} -) - -func TestOffsetCommitRequestV0(t *testing.T) { - request := new(OffsetCommitRequest) - request.Version = 0 - request.ConsumerGroup = "foobar" - testRequest(t, "no blocks v0", request, offsetCommitRequestNoBlocksV0) - - request.AddBlock("topic", 0x5221, 0xDEADBEEF, 0, "metadata") - testRequest(t, "one block v0", request, offsetCommitRequestOneBlockV0) -} - -func TestOffsetCommitRequestV1(t *testing.T) { - request := new(OffsetCommitRequest) - request.ConsumerGroup = "foobar" - request.ConsumerID = "cons" - request.ConsumerGroupGeneration = 0x1122 - request.Version = 1 - testRequest(t, "no blocks v1", request, offsetCommitRequestNoBlocksV1) - - request.AddBlock("topic", 0x5221, 0xDEADBEEF, ReceiveTime, "metadata") - testRequest(t, "one block v1", request, offsetCommitRequestOneBlockV1) -} - -func TestOffsetCommitRequestV2ToV4(t *testing.T) { - for version := 2; version <= 4; version++ { - request := new(OffsetCommitRequest) - request.ConsumerGroup = "foobar" - request.ConsumerID = "cons" - request.ConsumerGroupGeneration = 0x1122 - request.RetentionTime = 0x4433 - request.Version = int16(version) - testRequest(t, fmt.Sprintf("no blocks v%d", version), request, offsetCommitRequestNoBlocksV2) - - request.AddBlock("topic", 0x5221, 0xDEADBEEF, 0, "metadata") - testRequest(t, fmt.Sprintf("one block v%d", version), request, offsetCommitRequestOneBlockV2) - } -} diff --git a/third/github.com/Shopify/sarama/offset_commit_response.go b/third/github.com/Shopify/sarama/offset_commit_response.go deleted file mode 100644 index e842298db..000000000 --- a/third/github.com/Shopify/sarama/offset_commit_response.go +++ /dev/null @@ -1,110 +0,0 @@ -package sarama - -type OffsetCommitResponse struct { - Version int16 - ThrottleTimeMs int32 - Errors map[string]map[int32]KError -} - -func (r *OffsetCommitResponse) AddError(topic string, partition int32, kerror KError) { - if r.Errors == nil { - r.Errors = make(map[string]map[int32]KError) - } - partitions := r.Errors[topic] - if partitions == nil { - partitions = make(map[int32]KError) - r.Errors[topic] = partitions - } - partitions[partition] = kerror -} - -func (r *OffsetCommitResponse) encode(pe packetEncoder) error { - if r.Version >= 3 { - pe.putInt32(r.ThrottleTimeMs) - } - if err := pe.putArrayLength(len(r.Errors)); err != nil { - return err - } - for topic, partitions := range r.Errors { - if err := pe.putString(topic); err != nil { - return err - } - if err := pe.putArrayLength(len(partitions)); err != nil { - return err - } - for partition, kerror := range partitions { - pe.putInt32(partition) - pe.putInt16(int16(kerror)) - } - } - return nil -} - -func (r *OffsetCommitResponse) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if version >= 3 { - r.ThrottleTimeMs, err = pd.getInt32() - if err != nil { - return err - } - } - - numTopics, err := pd.getArrayLength() - if err != nil || numTopics == 0 { - return err - } - - r.Errors = make(map[string]map[int32]KError, numTopics) - for i := 0; i < numTopics; i++ { - name, err := pd.getString() - if err != nil { - return err - } - - numErrors, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Errors[name] = make(map[int32]KError, numErrors) - - for j := 0; j < numErrors; j++ { - id, err := pd.getInt32() - if err != nil { - return err - } - - tmp, err := pd.getInt16() - if err != nil { - return err - } - r.Errors[name][id] = KError(tmp) - } - } - - return nil -} - -func (r *OffsetCommitResponse) key() int16 { - return 8 -} - -func (r *OffsetCommitResponse) version() int16 { - return r.Version -} - -func (r *OffsetCommitResponse) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_8_2_0 - case 2: - return V0_9_0_0 - case 3: - return V0_11_0_0 - case 4: - return V2_0_0_0 - default: - return MinVersion - } -} diff --git a/third/github.com/Shopify/sarama/offset_commit_response_test.go b/third/github.com/Shopify/sarama/offset_commit_response_test.go deleted file mode 100644 index 3c85713c7..000000000 --- a/third/github.com/Shopify/sarama/offset_commit_response_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package sarama - -import ( - "fmt" - "testing" -) - -var ( - emptyOffsetCommitResponse = []byte{ - 0x00, 0x00, 0x00, 0x00} -) - -func TestEmptyOffsetCommitResponse(t *testing.T) { - response := OffsetCommitResponse{} - testResponse(t, "empty", &response, emptyOffsetCommitResponse) -} - -func TestNormalOffsetCommitResponse(t *testing.T) { - response := OffsetCommitResponse{} - response.AddError("t", 0, ErrNotLeaderForPartition) - response.Errors["m"] = make(map[int32]KError) - // The response encoded form cannot be checked for it varies due to - // unpredictable map traversal order. - testResponse(t, "normal", &response, nil) -} - -func TestOffsetCommitResponseWithThrottleTime(t *testing.T) { - for version := 3; version <= 4; version++ { - response := OffsetCommitResponse{ - Version: int16(version), - ThrottleTimeMs: 123, - } - response.AddError("t", 0, ErrNotLeaderForPartition) - response.Errors["m"] = make(map[int32]KError) - // The response encoded form cannot be checked for it varies due to - // unpredictable map traversal order. - testResponse(t, fmt.Sprintf("v%d with throttle time", version), &response, nil) - } -} diff --git a/third/github.com/Shopify/sarama/offset_fetch_request.go b/third/github.com/Shopify/sarama/offset_fetch_request.go deleted file mode 100644 index 68608241f..000000000 --- a/third/github.com/Shopify/sarama/offset_fetch_request.go +++ /dev/null @@ -1,100 +0,0 @@ -package sarama - -type OffsetFetchRequest struct { - Version int16 - ConsumerGroup string - partitions map[string][]int32 -} - -func (r *OffsetFetchRequest) encode(pe packetEncoder) (err error) { - if r.Version < 0 || r.Version > 5 { - return PacketEncodingError{"invalid or unsupported OffsetFetchRequest version field"} - } - - if err = pe.putString(r.ConsumerGroup); err != nil { - return err - } - - if r.Version >= 2 && r.partitions == nil { - pe.putInt32(-1) - } else { - if err = pe.putArrayLength(len(r.partitions)); err != nil { - return err - } - for topic, partitions := range r.partitions { - if err = pe.putString(topic); err != nil { - return err - } - if err = pe.putInt32Array(partitions); err != nil { - return err - } - } - } - return nil -} - -func (r *OffsetFetchRequest) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - if r.ConsumerGroup, err = pd.getString(); err != nil { - return err - } - partitionCount, err := pd.getArrayLength() - if err != nil { - return err - } - if (partitionCount == 0 && version < 2) || partitionCount < 0 { - return nil - } - r.partitions = make(map[string][]int32) - for i := 0; i < partitionCount; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - partitions, err := pd.getInt32Array() - if err != nil { - return err - } - r.partitions[topic] = partitions - } - return nil -} - -func (r *OffsetFetchRequest) key() int16 { - return 9 -} - -func (r *OffsetFetchRequest) version() int16 { - return r.Version -} - -func (r *OffsetFetchRequest) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_8_2_0 - case 2: - return V0_10_2_0 - case 3: - return V0_11_0_0 - case 4: - return V2_0_0_0 - case 5: - return V2_1_0_0 - default: - return MinVersion - } -} - -func (r *OffsetFetchRequest) ZeroPartitions() { - if r.partitions == nil && r.Version >= 2 { - r.partitions = make(map[string][]int32) - } -} - -func (r *OffsetFetchRequest) AddPartition(topic string, partitionID int32) { - if r.partitions == nil { - r.partitions = make(map[string][]int32) - } - - r.partitions[topic] = append(r.partitions[topic], partitionID) -} diff --git a/third/github.com/Shopify/sarama/offset_fetch_request_test.go b/third/github.com/Shopify/sarama/offset_fetch_request_test.go deleted file mode 100644 index 55b46eea7..000000000 --- a/third/github.com/Shopify/sarama/offset_fetch_request_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package sarama - -import ( - "fmt" - "testing" -) - -var ( - offsetFetchRequestNoGroupNoPartitions = []byte{ - 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00} - - offsetFetchRequestNoPartitions = []byte{ - 0x00, 0x04, 'b', 'l', 'a', 'h', - 0x00, 0x00, 0x00, 0x00} - - offsetFetchRequestOnePartition = []byte{ - 0x00, 0x04, 'b', 'l', 'a', 'h', - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x0D, 't', 'o', 'p', 'i', 'c', 'T', 'h', 'e', 'F', 'i', 'r', 's', 't', - 0x00, 0x00, 0x00, 0x01, - 0x4F, 0x4F, 0x4F, 0x4F} - - offsetFetchRequestAllPartitions = []byte{ - 0x00, 0x04, 'b', 'l', 'a', 'h', - 0xff, 0xff, 0xff, 0xff} -) - -func TestOffsetFetchRequestNoPartitions(t *testing.T) { - for version := 0; version <= 5; version++ { - request := new(OffsetFetchRequest) - request.Version = int16(version) - request.ZeroPartitions() - testRequest(t, fmt.Sprintf("no group, no partitions %d", version), request, offsetFetchRequestNoGroupNoPartitions) - - request.ConsumerGroup = "blah" - testRequest(t, fmt.Sprintf("no partitions %d", version), request, offsetFetchRequestNoPartitions) - } -} -func TestOffsetFetchRequest(t *testing.T) { - for version := 0; version <= 5; version++ { - request := new(OffsetFetchRequest) - request.Version = int16(version) - request.ConsumerGroup = "blah" - request.AddPartition("topicTheFirst", 0x4F4F4F4F) - testRequest(t, fmt.Sprintf("one partition %d", version), request, offsetFetchRequestOnePartition) - } -} - -func TestOffsetFetchRequestAllPartitions(t *testing.T) { - for version := 2; version <= 5; version++ { - request := &OffsetFetchRequest{Version: int16(version), ConsumerGroup: "blah"} - testRequest(t, fmt.Sprintf("all partitions %d", version), request, offsetFetchRequestAllPartitions) - } -} diff --git a/third/github.com/Shopify/sarama/offset_fetch_response.go b/third/github.com/Shopify/sarama/offset_fetch_response.go deleted file mode 100644 index 9e2570280..000000000 --- a/third/github.com/Shopify/sarama/offset_fetch_response.go +++ /dev/null @@ -1,197 +0,0 @@ -package sarama - -type OffsetFetchResponseBlock struct { - Offset int64 - LeaderEpoch int32 - Metadata string - Err KError -} - -func (b *OffsetFetchResponseBlock) decode(pd packetDecoder, version int16) (err error) { - b.Offset, err = pd.getInt64() - if err != nil { - return err - } - - if version >= 5 { - b.LeaderEpoch, err = pd.getInt32() - if err != nil { - return err - } - } - - b.Metadata, err = pd.getString() - if err != nil { - return err - } - - tmp, err := pd.getInt16() - if err != nil { - return err - } - b.Err = KError(tmp) - - return nil -} - -func (b *OffsetFetchResponseBlock) encode(pe packetEncoder, version int16) (err error) { - pe.putInt64(b.Offset) - - if version >= 5 { - pe.putInt32(b.LeaderEpoch) - } - - err = pe.putString(b.Metadata) - if err != nil { - return err - } - - pe.putInt16(int16(b.Err)) - - return nil -} - -type OffsetFetchResponse struct { - Version int16 - ThrottleTimeMs int32 - Blocks map[string]map[int32]*OffsetFetchResponseBlock - Err KError -} - -func (r *OffsetFetchResponse) encode(pe packetEncoder) error { - if r.Version >= 3 { - pe.putInt32(r.ThrottleTimeMs) - } - - if err := pe.putArrayLength(len(r.Blocks)); err != nil { - return err - } - for topic, partitions := range r.Blocks { - if err := pe.putString(topic); err != nil { - return err - } - if err := pe.putArrayLength(len(partitions)); err != nil { - return err - } - for partition, block := range partitions { - pe.putInt32(partition) - if err := block.encode(pe, r.Version); err != nil { - return err - } - } - } - if r.Version >= 2 { - pe.putInt16(int16(r.Err)) - } - return nil -} - -func (r *OffsetFetchResponse) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if version >= 3 { - r.ThrottleTimeMs, err = pd.getInt32() - if err != nil { - return err - } - } - - numTopics, err := pd.getArrayLength() - if err != nil { - return err - } - - if numTopics > 0 { - r.Blocks = make(map[string]map[int32]*OffsetFetchResponseBlock, numTopics) - for i := 0; i < numTopics; i++ { - name, err := pd.getString() - if err != nil { - return err - } - - numBlocks, err := pd.getArrayLength() - if err != nil { - return err - } - - if numBlocks == 0 { - r.Blocks[name] = nil - continue - } - r.Blocks[name] = make(map[int32]*OffsetFetchResponseBlock, numBlocks) - - for j := 0; j < numBlocks; j++ { - id, err := pd.getInt32() - if err != nil { - return err - } - - block := new(OffsetFetchResponseBlock) - err = block.decode(pd, version) - if err != nil { - return err - } - r.Blocks[name][id] = block - } - } - } - - if version >= 2 { - kerr, err := pd.getInt16() - if err != nil { - return err - } - r.Err = KError(kerr) - } - - return nil -} - -func (r *OffsetFetchResponse) key() int16 { - return 9 -} - -func (r *OffsetFetchResponse) version() int16 { - return r.Version -} - -func (r *OffsetFetchResponse) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_8_2_0 - case 2: - return V0_10_2_0 - case 3: - return V0_11_0_0 - case 4: - return V2_0_0_0 - case 5: - return V2_1_0_0 - default: - return MinVersion - } -} - -func (r *OffsetFetchResponse) GetBlock(topic string, partition int32) *OffsetFetchResponseBlock { - if r.Blocks == nil { - return nil - } - - if r.Blocks[topic] == nil { - return nil - } - - return r.Blocks[topic][partition] -} - -func (r *OffsetFetchResponse) AddBlock(topic string, partition int32, block *OffsetFetchResponseBlock) { - if r.Blocks == nil { - r.Blocks = make(map[string]map[int32]*OffsetFetchResponseBlock) - } - partitions := r.Blocks[topic] - if partitions == nil { - partitions = make(map[int32]*OffsetFetchResponseBlock) - r.Blocks[topic] = partitions - } - partitions[partition] = block -} diff --git a/third/github.com/Shopify/sarama/offset_fetch_response_test.go b/third/github.com/Shopify/sarama/offset_fetch_response_test.go deleted file mode 100644 index b564f70f9..000000000 --- a/third/github.com/Shopify/sarama/offset_fetch_response_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package sarama - -import ( - "fmt" - "testing" -) - -var ( - emptyOffsetFetchResponse = []byte{ - 0x00, 0x00, 0x00, 0x00} - - emptyOffsetFetchResponseV2 = []byte{ - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x2A} - - emptyOffsetFetchResponseV3 = []byte{ - 0x00, 0x00, 0x00, 0x09, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x2A} -) - -func TestEmptyOffsetFetchResponse(t *testing.T) { - for version := 0; version <= 1; version++ { - response := OffsetFetchResponse{Version: int16(version)} - testResponse(t, fmt.Sprintf("empty v%d", version), &response, emptyOffsetFetchResponse) - } - - responseV2 := OffsetFetchResponse{Version: 2, Err: ErrInvalidRequest} - testResponse(t, "empty V2", &responseV2, emptyOffsetFetchResponseV2) - - for version := 3; version <= 5; version++ { - responseV3 := OffsetFetchResponse{Version: int16(version), Err: ErrInvalidRequest, ThrottleTimeMs: 9} - testResponse(t, fmt.Sprintf("empty v%d", version), &responseV3, emptyOffsetFetchResponseV3) - } -} - -func TestNormalOffsetFetchResponse(t *testing.T) { - // The response encoded form cannot be checked for it varies due to - // unpredictable map traversal order. - // Hence the 'nil' as byte[] parameter in the 'testResponse(..)' calls - - for version := 0; version <= 1; version++ { - response := OffsetFetchResponse{Version: int16(version)} - response.AddBlock("t", 0, &OffsetFetchResponseBlock{0, 0, "md", ErrRequestTimedOut}) - response.Blocks["m"] = nil - testResponse(t, fmt.Sprintf("Normal v%d", version), &response, nil) - } - - responseV2 := OffsetFetchResponse{Version: 2, Err: ErrInvalidRequest} - responseV2.AddBlock("t", 0, &OffsetFetchResponseBlock{0, 0, "md", ErrRequestTimedOut}) - responseV2.Blocks["m"] = nil - testResponse(t, "normal V2", &responseV2, nil) - - for version := 3; version <= 4; version++ { - responseV3 := OffsetFetchResponse{Version: int16(version), Err: ErrInvalidRequest, ThrottleTimeMs: 9} - responseV3.AddBlock("t", 0, &OffsetFetchResponseBlock{0, 0, "md", ErrRequestTimedOut}) - responseV3.Blocks["m"] = nil - testResponse(t, fmt.Sprintf("Normal v%d", version), &responseV3, nil) - } - - responseV5 := OffsetFetchResponse{Version: 5, Err: ErrInvalidRequest, ThrottleTimeMs: 9} - responseV5.AddBlock("t", 0, &OffsetFetchResponseBlock{Offset: 10, LeaderEpoch: 100, Metadata: "md", Err: ErrRequestTimedOut}) - responseV5.Blocks["m"] = nil - testResponse(t, "normal V5", &responseV5, nil) -} diff --git a/third/github.com/Shopify/sarama/offset_manager.go b/third/github.com/Shopify/sarama/offset_manager.go deleted file mode 100644 index 8ea857f83..000000000 --- a/third/github.com/Shopify/sarama/offset_manager.go +++ /dev/null @@ -1,572 +0,0 @@ -package sarama - -import ( - "sync" - "time" -) - -// Offset Manager - -// OffsetManager uses Kafka to store and fetch consumed partition offsets. -type OffsetManager interface { - // ManagePartition creates a PartitionOffsetManager on the given topic/partition. - // It will return an error if this OffsetManager is already managing the given - // topic/partition. - ManagePartition(topic string, partition int32) (PartitionOffsetManager, error) - - // Close stops the OffsetManager from managing offsets. It is required to call - // this function before an OffsetManager object passes out of scope, as it - // will otherwise leak memory. You must call this after all the - // PartitionOffsetManagers are closed. - Close() error -} - -type offsetManager struct { - client Client - conf *Config - group string - ticker *time.Ticker - - memberID string - generation int32 - - broker *Broker - brokerLock sync.RWMutex - - poms map[string]map[int32]*partitionOffsetManager - pomsLock sync.RWMutex - - closeOnce sync.Once - closing chan none - closed chan none -} - -// NewOffsetManagerFromClient creates a new OffsetManager from the given client. -// It is still necessary to call Close() on the underlying client when finished with the partition manager. -func NewOffsetManagerFromClient(group string, client Client) (OffsetManager, error) { - return newOffsetManagerFromClient(group, "", GroupGenerationUndefined, client) -} - -func newOffsetManagerFromClient(group, memberID string, generation int32, client Client) (*offsetManager, error) { - // Check that we are not dealing with a closed Client before processing any other arguments - if client.Closed() { - return nil, ErrClosedClient - } - - conf := client.Config() - om := &offsetManager{ - client: client, - conf: conf, - group: group, - ticker: time.NewTicker(conf.Consumer.Offsets.CommitInterval), - poms: make(map[string]map[int32]*partitionOffsetManager), - - memberID: memberID, - generation: generation, - - closing: make(chan none), - closed: make(chan none), - } - go withRecover(om.mainLoop) - - return om, nil -} - -func (om *offsetManager) ManagePartition(topic string, partition int32) (PartitionOffsetManager, error) { - pom, err := om.newPartitionOffsetManager(topic, partition) - if err != nil { - return nil, err - } - - om.pomsLock.Lock() - defer om.pomsLock.Unlock() - - topicManagers := om.poms[topic] - if topicManagers == nil { - topicManagers = make(map[int32]*partitionOffsetManager) - om.poms[topic] = topicManagers - } - - if topicManagers[partition] != nil { - return nil, ConfigurationError("That topic/partition is already being managed") - } - - topicManagers[partition] = pom - return pom, nil -} - -func (om *offsetManager) Close() error { - om.closeOnce.Do(func() { - // exit the mainLoop - close(om.closing) - <-om.closed - - // mark all POMs as closed - om.asyncClosePOMs() - - // flush one last time - for attempt := 0; attempt <= om.conf.Consumer.Offsets.Retry.Max; attempt++ { - om.flushToBroker() - if om.releasePOMs(false) == 0 { - break - } - } - - om.releasePOMs(true) - om.brokerLock.Lock() - om.broker = nil - om.brokerLock.Unlock() - }) - return nil -} - -func (om *offsetManager) fetchInitialOffset(topic string, partition int32, retries int) (int64, string, error) { - broker, err := om.coordinator() - if err != nil { - if retries <= 0 { - return 0, "", err - } - return om.fetchInitialOffset(topic, partition, retries-1) - } - - req := new(OffsetFetchRequest) - req.Version = 1 - req.ConsumerGroup = om.group - req.AddPartition(topic, partition) - - resp, err := broker.FetchOffset(req) - if err != nil { - if retries <= 0 { - return 0, "", err - } - om.releaseCoordinator(broker) - return om.fetchInitialOffset(topic, partition, retries-1) - } - - block := resp.GetBlock(topic, partition) - if block == nil { - return 0, "", ErrIncompleteResponse - } - - switch block.Err { - case ErrNoError: - return block.Offset, block.Metadata, nil - case ErrNotCoordinatorForConsumer: - if retries <= 0 { - return 0, "", block.Err - } - om.releaseCoordinator(broker) - return om.fetchInitialOffset(topic, partition, retries-1) - case ErrOffsetsLoadInProgress: - if retries <= 0 { - return 0, "", block.Err - } - select { - case <-om.closing: - return 0, "", block.Err - case <-time.After(om.conf.Metadata.Retry.Backoff): - } - return om.fetchInitialOffset(topic, partition, retries-1) - default: - return 0, "", block.Err - } -} - -func (om *offsetManager) coordinator() (*Broker, error) { - om.brokerLock.RLock() - broker := om.broker - om.brokerLock.RUnlock() - - if broker != nil { - return broker, nil - } - - om.brokerLock.Lock() - defer om.brokerLock.Unlock() - - if broker := om.broker; broker != nil { - return broker, nil - } - - if err := om.client.RefreshCoordinator(om.group); err != nil { - return nil, err - } - - broker, err := om.client.Coordinator(om.group) - if err != nil { - return nil, err - } - - om.broker = broker - return broker, nil -} - -func (om *offsetManager) releaseCoordinator(b *Broker) { - om.brokerLock.Lock() - if om.broker == b { - om.broker = nil - } - om.brokerLock.Unlock() -} - -func (om *offsetManager) mainLoop() { - defer om.ticker.Stop() - defer close(om.closed) - - for { - select { - case <-om.ticker.C: - om.flushToBroker() - om.releasePOMs(false) - case <-om.closing: - return - } - } -} - -func (om *offsetManager) flushToBroker() { - req := om.constructRequest() - if req == nil { - return - } - - broker, err := om.coordinator() - if err != nil { - om.handleError(err) - return - } - - resp, err := broker.CommitOffset(req) - if err != nil { - om.handleError(err) - om.releaseCoordinator(broker) - _ = broker.Close() - return - } - - om.handleResponse(broker, req, resp) -} - -func (om *offsetManager) constructRequest() *OffsetCommitRequest { - var r *OffsetCommitRequest - var perPartitionTimestamp int64 - if om.conf.Consumer.Offsets.Retention == 0 { - perPartitionTimestamp = ReceiveTime - r = &OffsetCommitRequest{ - Version: 1, - ConsumerGroup: om.group, - ConsumerID: om.memberID, - ConsumerGroupGeneration: om.generation, - } - } else { - r = &OffsetCommitRequest{ - Version: 2, - RetentionTime: int64(om.conf.Consumer.Offsets.Retention / time.Millisecond), - ConsumerGroup: om.group, - ConsumerID: om.memberID, - ConsumerGroupGeneration: om.generation, - } - - } - - om.pomsLock.RLock() - defer om.pomsLock.RUnlock() - - for _, topicManagers := range om.poms { - for _, pom := range topicManagers { - pom.lock.Lock() - if pom.dirty { - r.AddBlock(pom.topic, pom.partition, pom.offset, perPartitionTimestamp, pom.metadata) - } - pom.lock.Unlock() - } - } - - if len(r.blocks) > 0 { - return r - } - - return nil -} - -func (om *offsetManager) handleResponse(broker *Broker, req *OffsetCommitRequest, resp *OffsetCommitResponse) { - om.pomsLock.RLock() - defer om.pomsLock.RUnlock() - - for _, topicManagers := range om.poms { - for _, pom := range topicManagers { - if req.blocks[pom.topic] == nil || req.blocks[pom.topic][pom.partition] == nil { - continue - } - - var err KError - var ok bool - - if resp.Errors[pom.topic] == nil { - pom.handleError(ErrIncompleteResponse) - continue - } - if err, ok = resp.Errors[pom.topic][pom.partition]; !ok { - pom.handleError(ErrIncompleteResponse) - continue - } - - switch err { - case ErrNoError: - block := req.blocks[pom.topic][pom.partition] - pom.updateCommitted(block.offset, block.metadata) - case ErrNotLeaderForPartition, ErrLeaderNotAvailable, - ErrConsumerCoordinatorNotAvailable, ErrNotCoordinatorForConsumer: - // not a critical error, we just need to redispatch - om.releaseCoordinator(broker) - case ErrOffsetMetadataTooLarge, ErrInvalidCommitOffsetSize: - // nothing we can do about this, just tell the user and carry on - pom.handleError(err) - case ErrOffsetsLoadInProgress: - // nothing wrong but we didn't commit, we'll get it next time round - break - case ErrUnknownTopicOrPartition: - // let the user know *and* try redispatching - if topic-auto-create is - // enabled, redispatching should trigger a metadata req and create the - // topic; if not then re-dispatching won't help, but we've let the user - // know and it shouldn't hurt either (see https://github.com/Shopify/sarama/issues/706) - fallthrough - default: - // dunno, tell the user and try redispatching - pom.handleError(err) - om.releaseCoordinator(broker) - } - } - } -} - -func (om *offsetManager) handleError(err error) { - om.pomsLock.RLock() - defer om.pomsLock.RUnlock() - - for _, topicManagers := range om.poms { - for _, pom := range topicManagers { - pom.handleError(err) - } - } -} - -func (om *offsetManager) asyncClosePOMs() { - om.pomsLock.RLock() - defer om.pomsLock.RUnlock() - - for _, topicManagers := range om.poms { - for _, pom := range topicManagers { - pom.AsyncClose() - } - } -} - -// Releases/removes closed POMs once they are clean (or when forced) -func (om *offsetManager) releasePOMs(force bool) (remaining int) { - om.pomsLock.Lock() - defer om.pomsLock.Unlock() - - for topic, topicManagers := range om.poms { - for partition, pom := range topicManagers { - pom.lock.Lock() - releaseDue := pom.done && (force || !pom.dirty) - pom.lock.Unlock() - - if releaseDue { - pom.release() - - delete(om.poms[topic], partition) - if len(om.poms[topic]) == 0 { - delete(om.poms, topic) - } - } - } - remaining += len(om.poms[topic]) - } - return -} - -func (om *offsetManager) findPOM(topic string, partition int32) *partitionOffsetManager { - om.pomsLock.RLock() - defer om.pomsLock.RUnlock() - - if partitions, ok := om.poms[topic]; ok { - if pom, ok := partitions[partition]; ok { - return pom - } - } - return nil -} - -// Partition Offset Manager - -// PartitionOffsetManager uses Kafka to store and fetch consumed partition offsets. You MUST call Close() -// on a partition offset manager to avoid leaks, it will not be garbage-collected automatically when it passes -// out of scope. -type PartitionOffsetManager interface { - // NextOffset returns the next offset that should be consumed for the managed - // partition, accompanied by metadata which can be used to reconstruct the state - // of the partition consumer when it resumes. NextOffset() will return - // `config.Consumer.Offsets.Initial` and an empty metadata string if no offset - // was committed for this partition yet. - NextOffset() (int64, string) - - // MarkOffset marks the provided offset, alongside a metadata string - // that represents the state of the partition consumer at that point in time. The - // metadata string can be used by another consumer to restore that state, so it - // can resume consumption. - // - // To follow upstream conventions, you are expected to mark the offset of the - // next message to read, not the last message read. Thus, when calling `MarkOffset` - // you should typically add one to the offset of the last consumed message. - // - // Note: calling MarkOffset does not necessarily commit the offset to the backend - // store immediately for efficiency reasons, and it may never be committed if - // your application crashes. This means that you may end up processing the same - // message twice, and your processing should ideally be idempotent. - MarkOffset(offset int64, metadata string) - - // ResetOffset resets to the provided offset, alongside a metadata string that - // represents the state of the partition consumer at that point in time. Reset - // acts as a counterpart to MarkOffset, the difference being that it allows to - // reset an offset to an earlier or smaller value, where MarkOffset only - // allows incrementing the offset. cf MarkOffset for more details. - ResetOffset(offset int64, metadata string) - - // Errors returns a read channel of errors that occur during offset management, if - // enabled. By default, errors are logged and not returned over this channel. If - // you want to implement any custom error handling, set your config's - // Consumer.Return.Errors setting to true, and read from this channel. - Errors() <-chan *ConsumerError - - // AsyncClose initiates a shutdown of the PartitionOffsetManager. This method will - // return immediately, after which you should wait until the 'errors' channel has - // been drained and closed. It is required to call this function, or Close before - // a consumer object passes out of scope, as it will otherwise leak memory. You - // must call this before calling Close on the underlying client. - AsyncClose() - - // Close stops the PartitionOffsetManager from managing offsets. It is required to - // call this function (or AsyncClose) before a PartitionOffsetManager object - // passes out of scope, as it will otherwise leak memory. You must call this - // before calling Close on the underlying client. - Close() error -} - -type partitionOffsetManager struct { - parent *offsetManager - topic string - partition int32 - - lock sync.Mutex - offset int64 - metadata string - dirty bool - done bool - - releaseOnce sync.Once - errors chan *ConsumerError -} - -func (om *offsetManager) newPartitionOffsetManager(topic string, partition int32) (*partitionOffsetManager, error) { - offset, metadata, err := om.fetchInitialOffset(topic, partition, om.conf.Metadata.Retry.Max) - if err != nil { - return nil, err - } - - return &partitionOffsetManager{ - parent: om, - topic: topic, - partition: partition, - errors: make(chan *ConsumerError, om.conf.ChannelBufferSize), - offset: offset, - metadata: metadata, - }, nil -} - -func (pom *partitionOffsetManager) Errors() <-chan *ConsumerError { - return pom.errors -} - -func (pom *partitionOffsetManager) MarkOffset(offset int64, metadata string) { - pom.lock.Lock() - defer pom.lock.Unlock() - - if offset > pom.offset { - pom.offset = offset - pom.metadata = metadata - pom.dirty = true - } -} - -func (pom *partitionOffsetManager) ResetOffset(offset int64, metadata string) { - pom.lock.Lock() - defer pom.lock.Unlock() - - if offset <= pom.offset { - pom.offset = offset - pom.metadata = metadata - pom.dirty = true - } -} - -func (pom *partitionOffsetManager) updateCommitted(offset int64, metadata string) { - pom.lock.Lock() - defer pom.lock.Unlock() - - if pom.offset == offset && pom.metadata == metadata { - pom.dirty = false - } -} - -func (pom *partitionOffsetManager) NextOffset() (int64, string) { - pom.lock.Lock() - defer pom.lock.Unlock() - - if pom.offset >= 0 { - return pom.offset, pom.metadata - } - - return pom.parent.conf.Consumer.Offsets.Initial, "" -} - -func (pom *partitionOffsetManager) AsyncClose() { - pom.lock.Lock() - pom.done = true - pom.lock.Unlock() -} - -func (pom *partitionOffsetManager) Close() error { - pom.AsyncClose() - - var errors ConsumerErrors - for err := range pom.errors { - errors = append(errors, err) - } - - if len(errors) > 0 { - return errors - } - return nil -} - -func (pom *partitionOffsetManager) handleError(err error) { - cErr := &ConsumerError{ - Topic: pom.topic, - Partition: pom.partition, - Err: err, - } - - if pom.parent.conf.Consumer.Return.Errors { - pom.errors <- cErr - } else { - Logger.Println(cErr) - } -} - -func (pom *partitionOffsetManager) release() { - pom.releaseOnce.Do(func() { - go close(pom.errors) - }) -} diff --git a/third/github.com/Shopify/sarama/offset_manager_test.go b/third/github.com/Shopify/sarama/offset_manager_test.go deleted file mode 100644 index 86d6f4eb4..000000000 --- a/third/github.com/Shopify/sarama/offset_manager_test.go +++ /dev/null @@ -1,421 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -func initOffsetManager(t *testing.T, retention time.Duration) (om OffsetManager, - testClient Client, broker, coordinator *MockBroker) { - - config := NewConfig() - config.Metadata.Retry.Max = 1 - config.Consumer.Offsets.CommitInterval = 1 * time.Millisecond - config.Version = V0_9_0_0 - if retention > 0 { - config.Consumer.Offsets.Retention = retention - } - - broker = NewMockBroker(t, 1) - coordinator = NewMockBroker(t, 2) - - seedMeta := new(MetadataResponse) - seedMeta.AddBroker(coordinator.Addr(), coordinator.BrokerID()) - seedMeta.AddTopicPartition("my_topic", 0, 1, []int32{}, []int32{}, ErrNoError) - seedMeta.AddTopicPartition("my_topic", 1, 1, []int32{}, []int32{}, ErrNoError) - broker.Returns(seedMeta) - - var err error - testClient, err = NewClient([]string{broker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - broker.Returns(&ConsumerMetadataResponse{ - CoordinatorID: coordinator.BrokerID(), - CoordinatorHost: "127.0.0.1", - CoordinatorPort: coordinator.Port(), - }) - - om, err = NewOffsetManagerFromClient("group", testClient) - if err != nil { - t.Fatal(err) - } - - return om, testClient, broker, coordinator -} - -func initPartitionOffsetManager(t *testing.T, om OffsetManager, - coordinator *MockBroker, initialOffset int64, metadata string) PartitionOffsetManager { - - fetchResponse := new(OffsetFetchResponse) - fetchResponse.AddBlock("my_topic", 0, &OffsetFetchResponseBlock{ - Err: ErrNoError, - Offset: initialOffset, - Metadata: metadata, - }) - coordinator.Returns(fetchResponse) - - pom, err := om.ManagePartition("my_topic", 0) - if err != nil { - t.Fatal(err) - } - - return pom -} - -func TestNewOffsetManager(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - seedBroker.Returns(new(MetadataResponse)) - defer seedBroker.Close() - - testClient, err := NewClient([]string{seedBroker.Addr()}, nil) - if err != nil { - t.Fatal(err) - } - - om, err := NewOffsetManagerFromClient("group", testClient) - if err != nil { - t.Error(err) - } - safeClose(t, om) - safeClose(t, testClient) - - _, err = NewOffsetManagerFromClient("group", testClient) - if err != ErrClosedClient { - t.Errorf("Error expected for closed client; actual value: %v", err) - } -} - -// Test recovery from ErrNotCoordinatorForConsumer -// on first fetchInitialOffset call -func TestOffsetManagerFetchInitialFail(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t, 0) - - // Error on first fetchInitialOffset call - responseBlock := OffsetFetchResponseBlock{ - Err: ErrNotCoordinatorForConsumer, - Offset: 5, - Metadata: "test_meta", - } - - fetchResponse := new(OffsetFetchResponse) - fetchResponse.AddBlock("my_topic", 0, &responseBlock) - coordinator.Returns(fetchResponse) - - // Refresh coordinator - newCoordinator := NewMockBroker(t, 3) - broker.Returns(&ConsumerMetadataResponse{ - CoordinatorID: newCoordinator.BrokerID(), - CoordinatorHost: "127.0.0.1", - CoordinatorPort: newCoordinator.Port(), - }) - - // Second fetchInitialOffset call is fine - fetchResponse2 := new(OffsetFetchResponse) - responseBlock2 := responseBlock - responseBlock2.Err = ErrNoError - fetchResponse2.AddBlock("my_topic", 0, &responseBlock2) - newCoordinator.Returns(fetchResponse2) - - pom, err := om.ManagePartition("my_topic", 0) - if err != nil { - t.Error(err) - } - - broker.Close() - coordinator.Close() - newCoordinator.Close() - safeClose(t, pom) - safeClose(t, om) - safeClose(t, testClient) -} - -// Test fetchInitialOffset retry on ErrOffsetsLoadInProgress -func TestOffsetManagerFetchInitialLoadInProgress(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t, 0) - - // Error on first fetchInitialOffset call - responseBlock := OffsetFetchResponseBlock{ - Err: ErrOffsetsLoadInProgress, - Offset: 5, - Metadata: "test_meta", - } - - fetchResponse := new(OffsetFetchResponse) - fetchResponse.AddBlock("my_topic", 0, &responseBlock) - coordinator.Returns(fetchResponse) - - // Second fetchInitialOffset call is fine - fetchResponse2 := new(OffsetFetchResponse) - responseBlock2 := responseBlock - responseBlock2.Err = ErrNoError - fetchResponse2.AddBlock("my_topic", 0, &responseBlock2) - coordinator.Returns(fetchResponse2) - - pom, err := om.ManagePartition("my_topic", 0) - if err != nil { - t.Error(err) - } - - broker.Close() - coordinator.Close() - safeClose(t, pom) - safeClose(t, om) - safeClose(t, testClient) -} - -func TestPartitionOffsetManagerInitialOffset(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t, 0) - testClient.Config().Consumer.Offsets.Initial = OffsetOldest - - // Kafka returns -1 if no offset has been stored for this partition yet. - pom := initPartitionOffsetManager(t, om, coordinator, -1, "") - - offset, meta := pom.NextOffset() - if offset != OffsetOldest { - t.Errorf("Expected offset 5. Actual: %v", offset) - } - if meta != "" { - t.Errorf("Expected metadata to be empty. Actual: %q", meta) - } - - safeClose(t, pom) - safeClose(t, om) - broker.Close() - coordinator.Close() - safeClose(t, testClient) -} - -func TestPartitionOffsetManagerNextOffset(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t, 0) - pom := initPartitionOffsetManager(t, om, coordinator, 5, "test_meta") - - offset, meta := pom.NextOffset() - if offset != 5 { - t.Errorf("Expected offset 5. Actual: %v", offset) - } - if meta != "test_meta" { - t.Errorf("Expected metadata \"test_meta\". Actual: %q", meta) - } - - safeClose(t, pom) - safeClose(t, om) - broker.Close() - coordinator.Close() - safeClose(t, testClient) -} - -func TestPartitionOffsetManagerResetOffset(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t, 0) - pom := initPartitionOffsetManager(t, om, coordinator, 5, "original_meta") - - ocResponse := new(OffsetCommitResponse) - ocResponse.AddError("my_topic", 0, ErrNoError) - coordinator.Returns(ocResponse) - - expected := int64(1) - pom.ResetOffset(expected, "modified_meta") - actual, meta := pom.NextOffset() - - if actual != expected { - t.Errorf("Expected offset %v. Actual: %v", expected, actual) - } - if meta != "modified_meta" { - t.Errorf("Expected metadata \"modified_meta\". Actual: %q", meta) - } - - safeClose(t, pom) - safeClose(t, om) - safeClose(t, testClient) - broker.Close() - coordinator.Close() -} - -func TestPartitionOffsetManagerResetOffsetWithRetention(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t, time.Hour) - pom := initPartitionOffsetManager(t, om, coordinator, 5, "original_meta") - - ocResponse := new(OffsetCommitResponse) - ocResponse.AddError("my_topic", 0, ErrNoError) - handler := func(req *request) (res encoder) { - if req.body.version() != 2 { - t.Errorf("Expected to be using version 2. Actual: %v", req.body.version()) - } - offsetCommitRequest := req.body.(*OffsetCommitRequest) - if offsetCommitRequest.RetentionTime != (60 * 60 * 1000) { - t.Errorf("Expected an hour retention time. Actual: %v", offsetCommitRequest.RetentionTime) - } - return ocResponse - } - coordinator.setHandler(handler) - - expected := int64(1) - pom.ResetOffset(expected, "modified_meta") - actual, meta := pom.NextOffset() - - if actual != expected { - t.Errorf("Expected offset %v. Actual: %v", expected, actual) - } - if meta != "modified_meta" { - t.Errorf("Expected metadata \"modified_meta\". Actual: %q", meta) - } - - safeClose(t, pom) - safeClose(t, om) - safeClose(t, testClient) - broker.Close() - coordinator.Close() -} - -func TestPartitionOffsetManagerMarkOffset(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t, 0) - pom := initPartitionOffsetManager(t, om, coordinator, 5, "original_meta") - - ocResponse := new(OffsetCommitResponse) - ocResponse.AddError("my_topic", 0, ErrNoError) - coordinator.Returns(ocResponse) - - pom.MarkOffset(100, "modified_meta") - offset, meta := pom.NextOffset() - - if offset != 100 { - t.Errorf("Expected offset 100. Actual: %v", offset) - } - if meta != "modified_meta" { - t.Errorf("Expected metadata \"modified_meta\". Actual: %q", meta) - } - - safeClose(t, pom) - safeClose(t, om) - safeClose(t, testClient) - broker.Close() - coordinator.Close() -} - -func TestPartitionOffsetManagerMarkOffsetWithRetention(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t, time.Hour) - pom := initPartitionOffsetManager(t, om, coordinator, 5, "original_meta") - - ocResponse := new(OffsetCommitResponse) - ocResponse.AddError("my_topic", 0, ErrNoError) - handler := func(req *request) (res encoder) { - if req.body.version() != 2 { - t.Errorf("Expected to be using version 2. Actual: %v", req.body.version()) - } - offsetCommitRequest := req.body.(*OffsetCommitRequest) - if offsetCommitRequest.RetentionTime != (60 * 60 * 1000) { - t.Errorf("Expected an hour retention time. Actual: %v", offsetCommitRequest.RetentionTime) - } - return ocResponse - } - coordinator.setHandler(handler) - - pom.MarkOffset(100, "modified_meta") - offset, meta := pom.NextOffset() - - if offset != 100 { - t.Errorf("Expected offset 100. Actual: %v", offset) - } - if meta != "modified_meta" { - t.Errorf("Expected metadata \"modified_meta\". Actual: %q", meta) - } - - safeClose(t, pom) - safeClose(t, om) - safeClose(t, testClient) - broker.Close() - coordinator.Close() -} - -func TestPartitionOffsetManagerCommitErr(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t, 0) - pom := initPartitionOffsetManager(t, om, coordinator, 5, "meta") - - // Error on one partition - ocResponse := new(OffsetCommitResponse) - ocResponse.AddError("my_topic", 0, ErrOffsetOutOfRange) - ocResponse.AddError("my_topic", 1, ErrNoError) - coordinator.Returns(ocResponse) - - newCoordinator := NewMockBroker(t, 3) - - // For RefreshCoordinator() - broker.Returns(&ConsumerMetadataResponse{ - CoordinatorID: newCoordinator.BrokerID(), - CoordinatorHost: "127.0.0.1", - CoordinatorPort: newCoordinator.Port(), - }) - - // Nothing in response.Errors at all - ocResponse2 := new(OffsetCommitResponse) - newCoordinator.Returns(ocResponse2) - - // No error, no need to refresh coordinator - - // Error on the wrong partition for this pom - ocResponse3 := new(OffsetCommitResponse) - ocResponse3.AddError("my_topic", 1, ErrNoError) - newCoordinator.Returns(ocResponse3) - - // No error, no need to refresh coordinator - - // ErrUnknownTopicOrPartition/ErrNotLeaderForPartition/ErrLeaderNotAvailable block - ocResponse4 := new(OffsetCommitResponse) - ocResponse4.AddError("my_topic", 0, ErrUnknownTopicOrPartition) - newCoordinator.Returns(ocResponse4) - - // For RefreshCoordinator() - broker.Returns(&ConsumerMetadataResponse{ - CoordinatorID: newCoordinator.BrokerID(), - CoordinatorHost: "127.0.0.1", - CoordinatorPort: newCoordinator.Port(), - }) - - // Normal error response - ocResponse5 := new(OffsetCommitResponse) - ocResponse5.AddError("my_topic", 0, ErrNoError) - newCoordinator.Returns(ocResponse5) - - pom.MarkOffset(100, "modified_meta") - - err := pom.Close() - if err != nil { - t.Error(err) - } - - broker.Close() - coordinator.Close() - newCoordinator.Close() - safeClose(t, om) - safeClose(t, testClient) -} - -// Test of recovery from abort -func TestAbortPartitionOffsetManager(t *testing.T) { - om, testClient, broker, coordinator := initOffsetManager(t, 0) - pom := initPartitionOffsetManager(t, om, coordinator, 5, "meta") - - // this triggers an error in the CommitOffset request, - // which leads to the abort call - coordinator.Close() - - // Response to refresh coordinator request - newCoordinator := NewMockBroker(t, 3) - broker.Returns(&ConsumerMetadataResponse{ - CoordinatorID: newCoordinator.BrokerID(), - CoordinatorHost: "127.0.0.1", - CoordinatorPort: newCoordinator.Port(), - }) - - ocResponse := new(OffsetCommitResponse) - ocResponse.AddError("my_topic", 0, ErrNoError) - newCoordinator.Returns(ocResponse) - - pom.MarkOffset(100, "modified_meta") - - safeClose(t, pom) - safeClose(t, om) - broker.Close() - safeClose(t, testClient) -} diff --git a/third/github.com/Shopify/sarama/offset_request.go b/third/github.com/Shopify/sarama/offset_request.go deleted file mode 100644 index 326c3720c..000000000 --- a/third/github.com/Shopify/sarama/offset_request.go +++ /dev/null @@ -1,156 +0,0 @@ -package sarama - -type offsetRequestBlock struct { - time int64 - maxOffsets int32 // Only used in version 0 -} - -func (b *offsetRequestBlock) encode(pe packetEncoder, version int16) error { - pe.putInt64(int64(b.time)) - if version == 0 { - pe.putInt32(b.maxOffsets) - } - - return nil -} - -func (b *offsetRequestBlock) decode(pd packetDecoder, version int16) (err error) { - if b.time, err = pd.getInt64(); err != nil { - return err - } - if version == 0 { - if b.maxOffsets, err = pd.getInt32(); err != nil { - return err - } - } - return nil -} - -type OffsetRequest struct { - Version int16 - replicaID int32 - isReplicaIDSet bool - blocks map[string]map[int32]*offsetRequestBlock -} - -func (r *OffsetRequest) encode(pe packetEncoder) error { - if r.isReplicaIDSet { - pe.putInt32(r.replicaID) - } else { - // default replica ID is always -1 for clients - pe.putInt32(-1) - } - - err := pe.putArrayLength(len(r.blocks)) - if err != nil { - return err - } - for topic, partitions := range r.blocks { - err = pe.putString(topic) - if err != nil { - return err - } - err = pe.putArrayLength(len(partitions)) - if err != nil { - return err - } - for partition, block := range partitions { - pe.putInt32(partition) - if err = block.encode(pe, r.Version); err != nil { - return err - } - } - } - return nil -} - -func (r *OffsetRequest) decode(pd packetDecoder, version int16) error { - r.Version = version - - replicaID, err := pd.getInt32() - if err != nil { - return err - } - if replicaID >= 0 { - r.SetReplicaID(replicaID) - } - - blockCount, err := pd.getArrayLength() - if err != nil { - return err - } - if blockCount == 0 { - return nil - } - r.blocks = make(map[string]map[int32]*offsetRequestBlock) - for i := 0; i < blockCount; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - partitionCount, err := pd.getArrayLength() - if err != nil { - return err - } - r.blocks[topic] = make(map[int32]*offsetRequestBlock) - for j := 0; j < partitionCount; j++ { - partition, err := pd.getInt32() - if err != nil { - return err - } - block := &offsetRequestBlock{} - if err := block.decode(pd, version); err != nil { - return err - } - r.blocks[topic][partition] = block - } - } - return nil -} - -func (r *OffsetRequest) key() int16 { - return 2 -} - -func (r *OffsetRequest) version() int16 { - return r.Version -} - -func (r *OffsetRequest) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_10_1_0 - default: - return MinVersion - } -} - -func (r *OffsetRequest) SetReplicaID(id int32) { - r.replicaID = id - r.isReplicaIDSet = true -} - -func (r *OffsetRequest) ReplicaID() int32 { - if r.isReplicaIDSet { - return r.replicaID - } - return -1 -} - -func (r *OffsetRequest) AddBlock(topic string, partitionID int32, time int64, maxOffsets int32) { - if r.blocks == nil { - r.blocks = make(map[string]map[int32]*offsetRequestBlock) - } - - if r.blocks[topic] == nil { - r.blocks[topic] = make(map[int32]*offsetRequestBlock) - } - - tmp := new(offsetRequestBlock) - tmp.time = time - if r.Version == 0 { - tmp.maxOffsets = maxOffsets - } - - r.blocks[topic][partitionID] = tmp -} diff --git a/third/github.com/Shopify/sarama/offset_request_test.go b/third/github.com/Shopify/sarama/offset_request_test.go deleted file mode 100644 index 8ca818e49..000000000 --- a/third/github.com/Shopify/sarama/offset_request_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package sarama - -import "testing" - -var ( - offsetRequestNoBlocks = []byte{ - 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x00, 0x00, 0x00} - - offsetRequestOneBlock = []byte{ - 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x03, 'f', 'o', 'o', - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x04, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x02} - - offsetRequestOneBlockV1 = []byte{ - 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x03, 'b', 'a', 'r', - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x04, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01} - - offsetRequestReplicaID = []byte{ - 0x00, 0x00, 0x00, 0x2a, - 0x00, 0x00, 0x00, 0x00} -) - -func TestOffsetRequest(t *testing.T) { - request := new(OffsetRequest) - testRequest(t, "no blocks", request, offsetRequestNoBlocks) - - request.AddBlock("foo", 4, 1, 2) - testRequest(t, "one block", request, offsetRequestOneBlock) -} - -func TestOffsetRequestV1(t *testing.T) { - request := new(OffsetRequest) - request.Version = 1 - testRequest(t, "no blocks", request, offsetRequestNoBlocks) - - request.AddBlock("bar", 4, 1, 2) // Last argument is ignored for V1 - testRequest(t, "one block", request, offsetRequestOneBlockV1) -} - -func TestOffsetRequestReplicaID(t *testing.T) { - request := new(OffsetRequest) - replicaID := int32(42) - request.SetReplicaID(replicaID) - - if found := request.ReplicaID(); found != replicaID { - t.Errorf("replicaID: expected %v, found %v", replicaID, found) - } - - testRequest(t, "with replica ID", request, offsetRequestReplicaID) -} diff --git a/third/github.com/Shopify/sarama/offset_response.go b/third/github.com/Shopify/sarama/offset_response.go deleted file mode 100644 index 8b2193f9a..000000000 --- a/third/github.com/Shopify/sarama/offset_response.go +++ /dev/null @@ -1,174 +0,0 @@ -package sarama - -type OffsetResponseBlock struct { - Err KError - Offsets []int64 // Version 0 - Offset int64 // Version 1 - Timestamp int64 // Version 1 -} - -func (b *OffsetResponseBlock) decode(pd packetDecoder, version int16) (err error) { - tmp, err := pd.getInt16() - if err != nil { - return err - } - b.Err = KError(tmp) - - if version == 0 { - b.Offsets, err = pd.getInt64Array() - - return err - } - - b.Timestamp, err = pd.getInt64() - if err != nil { - return err - } - - b.Offset, err = pd.getInt64() - if err != nil { - return err - } - - // For backwards compatibility put the offset in the offsets array too - b.Offsets = []int64{b.Offset} - - return nil -} - -func (b *OffsetResponseBlock) encode(pe packetEncoder, version int16) (err error) { - pe.putInt16(int16(b.Err)) - - if version == 0 { - return pe.putInt64Array(b.Offsets) - } - - pe.putInt64(b.Timestamp) - pe.putInt64(b.Offset) - - return nil -} - -type OffsetResponse struct { - Version int16 - Blocks map[string]map[int32]*OffsetResponseBlock -} - -func (r *OffsetResponse) decode(pd packetDecoder, version int16) (err error) { - numTopics, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Blocks = make(map[string]map[int32]*OffsetResponseBlock, numTopics) - for i := 0; i < numTopics; i++ { - name, err := pd.getString() - if err != nil { - return err - } - - numBlocks, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Blocks[name] = make(map[int32]*OffsetResponseBlock, numBlocks) - - for j := 0; j < numBlocks; j++ { - id, err := pd.getInt32() - if err != nil { - return err - } - - block := new(OffsetResponseBlock) - err = block.decode(pd, version) - if err != nil { - return err - } - r.Blocks[name][id] = block - } - } - - return nil -} - -func (r *OffsetResponse) GetBlock(topic string, partition int32) *OffsetResponseBlock { - if r.Blocks == nil { - return nil - } - - if r.Blocks[topic] == nil { - return nil - } - - return r.Blocks[topic][partition] -} - -/* -// [0 0 0 1 ntopics -0 8 109 121 95 116 111 112 105 99 topic -0 0 0 1 npartitions -0 0 0 0 id -0 0 - -0 0 0 1 0 0 0 0 -0 1 1 1 0 0 0 1 -0 8 109 121 95 116 111 112 -105 99 0 0 0 1 0 0 -0 0 0 0 0 0 0 1 -0 0 0 0 0 1 1 1] - -*/ -func (r *OffsetResponse) encode(pe packetEncoder) (err error) { - if err = pe.putArrayLength(len(r.Blocks)); err != nil { - return err - } - - for topic, partitions := range r.Blocks { - if err = pe.putString(topic); err != nil { - return err - } - if err = pe.putArrayLength(len(partitions)); err != nil { - return err - } - for partition, block := range partitions { - pe.putInt32(partition) - if err = block.encode(pe, r.version()); err != nil { - return err - } - } - } - - return nil -} - -func (r *OffsetResponse) key() int16 { - return 2 -} - -func (r *OffsetResponse) version() int16 { - return r.Version -} - -func (r *OffsetResponse) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_10_1_0 - default: - return MinVersion - } -} - -// testing API - -func (r *OffsetResponse) AddTopicPartition(topic string, partition int32, offset int64) { - if r.Blocks == nil { - r.Blocks = make(map[string]map[int32]*OffsetResponseBlock) - } - byTopic, ok := r.Blocks[topic] - if !ok { - byTopic = make(map[int32]*OffsetResponseBlock) - r.Blocks[topic] = byTopic - } - byTopic[partition] = &OffsetResponseBlock{Offsets: []int64{offset}, Offset: offset} -} diff --git a/third/github.com/Shopify/sarama/offset_response_test.go b/third/github.com/Shopify/sarama/offset_response_test.go deleted file mode 100644 index 0df6c9f3e..000000000 --- a/third/github.com/Shopify/sarama/offset_response_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package sarama - -import "testing" - -var ( - emptyOffsetResponse = []byte{ - 0x00, 0x00, 0x00, 0x00} - - normalOffsetResponse = []byte{ - 0x00, 0x00, 0x00, 0x02, - - 0x00, 0x01, 'a', - 0x00, 0x00, 0x00, 0x00, - - 0x00, 0x01, 'z', - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, - 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06} - - normalOffsetResponseV1 = []byte{ - 0x00, 0x00, 0x00, 0x02, - - 0x00, 0x01, 'a', - 0x00, 0x00, 0x00, 0x00, - - 0x00, 0x01, 'z', - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, - 0x00, 0x00, 0x01, 0x58, 0x1A, 0xE6, 0x48, 0x86, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06} -) - -func TestEmptyOffsetResponse(t *testing.T) { - response := OffsetResponse{} - - testVersionDecodable(t, "empty", &response, emptyOffsetResponse, 0) - if len(response.Blocks) != 0 { - t.Error("Decoding produced", len(response.Blocks), "topics where there were none.") - } - - response = OffsetResponse{} - - testVersionDecodable(t, "empty", &response, emptyOffsetResponse, 1) - if len(response.Blocks) != 0 { - t.Error("Decoding produced", len(response.Blocks), "topics where there were none.") - } -} - -func TestNormalOffsetResponse(t *testing.T) { - response := OffsetResponse{} - - testVersionDecodable(t, "normal", &response, normalOffsetResponse, 0) - - if len(response.Blocks) != 2 { - t.Fatal("Decoding produced", len(response.Blocks), "topics where there were two.") - } - - if len(response.Blocks["a"]) != 0 { - t.Fatal("Decoding produced", len(response.Blocks["a"]), "partitions for topic 'a' where there were none.") - } - - if len(response.Blocks["z"]) != 1 { - t.Fatal("Decoding produced", len(response.Blocks["z"]), "partitions for topic 'z' where there was one.") - } - - if response.Blocks["z"][2].Err != ErrNoError { - t.Fatal("Decoding produced invalid error for topic z partition 2.") - } - - if len(response.Blocks["z"][2].Offsets) != 2 { - t.Fatal("Decoding produced invalid number of offsets for topic z partition 2.") - } - - if response.Blocks["z"][2].Offsets[0] != 5 || response.Blocks["z"][2].Offsets[1] != 6 { - t.Fatal("Decoding produced invalid offsets for topic z partition 2.") - } -} - -func TestNormalOffsetResponseV1(t *testing.T) { - response := OffsetResponse{} - - testVersionDecodable(t, "normal", &response, normalOffsetResponseV1, 1) - - if len(response.Blocks) != 2 { - t.Fatal("Decoding produced", len(response.Blocks), "topics where there were two.") - } - - if len(response.Blocks["a"]) != 0 { - t.Fatal("Decoding produced", len(response.Blocks["a"]), "partitions for topic 'a' where there were none.") - } - - if len(response.Blocks["z"]) != 1 { - t.Fatal("Decoding produced", len(response.Blocks["z"]), "partitions for topic 'z' where there was one.") - } - - if response.Blocks["z"][2].Err != ErrNoError { - t.Fatal("Decoding produced invalid error for topic z partition 2.") - } - - if response.Blocks["z"][2].Timestamp != 1477920049286 { - t.Fatal("Decoding produced invalid timestamp for topic z partition 2.", response.Blocks["z"][2].Timestamp) - } - - if response.Blocks["z"][2].Offset != 6 { - t.Fatal("Decoding produced invalid offsets for topic z partition 2.") - } -} diff --git a/third/github.com/Shopify/sarama/packet_decoder.go b/third/github.com/Shopify/sarama/packet_decoder.go deleted file mode 100644 index 74805ccbf..000000000 --- a/third/github.com/Shopify/sarama/packet_decoder.go +++ /dev/null @@ -1,60 +0,0 @@ -package sarama - -// PacketDecoder is the interface providing helpers for reading with Kafka's encoding rules. -// Types implementing Decoder only need to worry about calling methods like GetString, -// not about how a string is represented in Kafka. -type packetDecoder interface { - // Primitives - getInt8() (int8, error) - getInt16() (int16, error) - getInt32() (int32, error) - getInt64() (int64, error) - getVarint() (int64, error) - getArrayLength() (int, error) - getBool() (bool, error) - - // Collections - getBytes() ([]byte, error) - getVarintBytes() ([]byte, error) - getRawBytes(length int) ([]byte, error) - getString() (string, error) - getNullableString() (*string, error) - getInt32Array() ([]int32, error) - getInt64Array() ([]int64, error) - getStringArray() ([]string, error) - - // Subsets - remaining() int - getSubset(length int) (packetDecoder, error) - peek(offset, length int) (packetDecoder, error) // similar to getSubset, but it doesn't advance the offset - - // Stacks, see PushDecoder - push(in pushDecoder) error - pop() error -} - -// PushDecoder is the interface for decoding fields like CRCs and lengths where the validity -// of the field depends on what is after it in the packet. Start them with PacketDecoder.Push() where -// the actual value is located in the packet, then PacketDecoder.Pop() them when all the bytes they -// depend upon have been decoded. -type pushDecoder interface { - // Saves the offset into the input buffer as the location to actually read the calculated value when able. - saveOffset(in int) - - // Returns the length of data to reserve for the input of this encoder (eg 4 bytes for a CRC32). - reserveLength() int - - // Indicates that all required data is now available to calculate and check the field. - // SaveOffset is guaranteed to have been called first. The implementation should read ReserveLength() bytes - // of data from the saved offset, and verify it based on the data between the saved offset and curOffset. - check(curOffset int, buf []byte) error -} - -// dynamicPushDecoder extends the interface of pushDecoder for uses cases where the length of the -// fields itself is unknown until its value was decoded (for instance varint encoded length -// fields). -// During push, dynamicPushDecoder.decode() method will be called instead of reserveLength() -type dynamicPushDecoder interface { - pushDecoder - decoder -} diff --git a/third/github.com/Shopify/sarama/packet_encoder.go b/third/github.com/Shopify/sarama/packet_encoder.go deleted file mode 100644 index fb771daab..000000000 --- a/third/github.com/Shopify/sarama/packet_encoder.go +++ /dev/null @@ -1,65 +0,0 @@ -package sarama - -import "github.com/gogf/gf/third/github.com/rcrowley/go-metrics" - -// PacketEncoder is the interface providing helpers for writing with Kafka's encoding rules. -// Types implementing Encoder only need to worry about calling methods like PutString, -// not about how a string is represented in Kafka. -type packetEncoder interface { - // Primitives - putInt8(in int8) - putInt16(in int16) - putInt32(in int32) - putInt64(in int64) - putVarint(in int64) - putArrayLength(in int) error - putBool(in bool) - - // Collections - putBytes(in []byte) error - putVarintBytes(in []byte) error - putRawBytes(in []byte) error - putString(in string) error - putNullableString(in *string) error - putStringArray(in []string) error - putInt32Array(in []int32) error - putInt64Array(in []int64) error - - // Provide the current offset to record the batch size metric - offset() int - - // Stacks, see PushEncoder - push(in pushEncoder) - pop() error - - // To record metrics when provided - metricRegistry() metrics.Registry -} - -// PushEncoder is the interface for encoding fields like CRCs and lengths where the value -// of the field depends on what is encoded after it in the packet. Start them with PacketEncoder.Push() where -// the actual value is located in the packet, then PacketEncoder.Pop() them when all the bytes they -// depend upon have been written. -type pushEncoder interface { - // Saves the offset into the input buffer as the location to actually write the calculated value when able. - saveOffset(in int) - - // Returns the length of data to reserve for the output of this encoder (eg 4 bytes for a CRC32). - reserveLength() int - - // Indicates that all required data is now available to calculate and write the field. - // SaveOffset is guaranteed to have been called first. The implementation should write ReserveLength() bytes - // of data to the saved offset, based on the data between the saved offset and curOffset. - run(curOffset int, buf []byte) error -} - -// dynamicPushEncoder extends the interface of pushEncoder for uses cases where the length of the -// fields itself is unknown until its value was computed (for instance varint encoded length -// fields). -type dynamicPushEncoder interface { - pushEncoder - - // Called during pop() to adjust the length of the field. - // It should return the difference in bytes between the last computed length and current length. - adjustLength(currOffset int) int -} diff --git a/third/github.com/Shopify/sarama/partitioner.go b/third/github.com/Shopify/sarama/partitioner.go deleted file mode 100644 index 6a708e729..000000000 --- a/third/github.com/Shopify/sarama/partitioner.go +++ /dev/null @@ -1,217 +0,0 @@ -package sarama - -import ( - "hash" - "hash/fnv" - "math/rand" - "time" -) - -// Partitioner is anything that, given a Kafka message and a number of partitions indexed [0...numPartitions-1], -// decides to which partition to send the message. RandomPartitioner, RoundRobinPartitioner and HashPartitioner are provided -// as simple default implementations. -type Partitioner interface { - // Partition takes a message and partition count and chooses a partition - Partition(message *ProducerMessage, numPartitions int32) (int32, error) - - // RequiresConsistency indicates to the user of the partitioner whether the - // mapping of key->partition is consistent or not. Specifically, if a - // partitioner requires consistency then it must be allowed to choose from all - // partitions (even ones known to be unavailable), and its choice must be - // respected by the caller. The obvious example is the HashPartitioner. - RequiresConsistency() bool -} - -// DynamicConsistencyPartitioner can optionally be implemented by Partitioners -// in order to allow more flexibility than is originally allowed by the -// RequiresConsistency method in the Partitioner interface. This allows -// partitioners to require consistency sometimes, but not all times. It's useful -// for, e.g., the HashPartitioner, which does not require consistency if the -// message key is nil. -type DynamicConsistencyPartitioner interface { - Partitioner - - // MessageRequiresConsistency is similar to Partitioner.RequiresConsistency, - // but takes in the message being partitioned so that the partitioner can - // make a per-message determination. - MessageRequiresConsistency(message *ProducerMessage) bool -} - -// PartitionerConstructor is the type for a function capable of constructing new Partitioners. -type PartitionerConstructor func(topic string) Partitioner - -type manualPartitioner struct{} - -// HashPartitionOption lets you modify default values of the partitioner -type HashPartitionerOption func(*hashPartitioner) - -// WithAbsFirst means that the partitioner handles absolute values -// in the same way as the reference Java implementation -func WithAbsFirst() HashPartitionerOption { - return func(hp *hashPartitioner) { - hp.referenceAbs = true - } -} - -// WithCustomHashFunction lets you specify what hash function to use for the partitioning -func WithCustomHashFunction(hasher func() hash.Hash32) HashPartitionerOption { - return func(hp *hashPartitioner) { - hp.hasher = hasher() - } -} - -// WithCustomFallbackPartitioner lets you specify what HashPartitioner should be used in case a Distribution Key is empty -func WithCustomFallbackPartitioner(randomHP *hashPartitioner) HashPartitionerOption { - return func(hp *hashPartitioner) { - hp.random = hp - } -} - -// NewManualPartitioner returns a Partitioner which uses the partition manually set in the provided -// ProducerMessage's Partition field as the partition to produce to. -func NewManualPartitioner(topic string) Partitioner { - return new(manualPartitioner) -} - -func (p *manualPartitioner) Partition(message *ProducerMessage, numPartitions int32) (int32, error) { - return message.Partition, nil -} - -func (p *manualPartitioner) RequiresConsistency() bool { - return true -} - -type randomPartitioner struct { - generator *rand.Rand -} - -// NewRandomPartitioner returns a Partitioner which chooses a random partition each time. -func NewRandomPartitioner(topic string) Partitioner { - p := new(randomPartitioner) - p.generator = rand.New(rand.NewSource(time.Now().UTC().UnixNano())) - return p -} - -func (p *randomPartitioner) Partition(message *ProducerMessage, numPartitions int32) (int32, error) { - return int32(p.generator.Intn(int(numPartitions))), nil -} - -func (p *randomPartitioner) RequiresConsistency() bool { - return false -} - -type roundRobinPartitioner struct { - partition int32 -} - -// NewRoundRobinPartitioner returns a Partitioner which walks through the available partitions one at a time. -func NewRoundRobinPartitioner(topic string) Partitioner { - return &roundRobinPartitioner{} -} - -func (p *roundRobinPartitioner) Partition(message *ProducerMessage, numPartitions int32) (int32, error) { - if p.partition >= numPartitions { - p.partition = 0 - } - ret := p.partition - p.partition++ - return ret, nil -} - -func (p *roundRobinPartitioner) RequiresConsistency() bool { - return false -} - -type hashPartitioner struct { - random Partitioner - hasher hash.Hash32 - referenceAbs bool -} - -// NewCustomHashPartitioner is a wrapper around NewHashPartitioner, allowing the use of custom hasher. -// The argument is a function providing the instance, implementing the hash.Hash32 interface. This is to ensure that -// each partition dispatcher gets its own hasher, to avoid concurrency issues by sharing an instance. -func NewCustomHashPartitioner(hasher func() hash.Hash32) PartitionerConstructor { - return func(topic string) Partitioner { - p := new(hashPartitioner) - p.random = NewRandomPartitioner(topic) - p.hasher = hasher() - p.referenceAbs = false - return p - } -} - -// NewCustomPartitioner creates a default Partitioner but lets you specify the behavior of each component via options -func NewCustomPartitioner(options ...HashPartitionerOption) PartitionerConstructor { - return func(topic string) Partitioner { - p := new(hashPartitioner) - p.random = NewRandomPartitioner(topic) - p.hasher = fnv.New32a() - p.referenceAbs = false - for _, option := range options { - option(p) - } - return p - } -} - -// NewHashPartitioner returns a Partitioner which behaves as follows. If the message's key is nil then a -// random partition is chosen. Otherwise the FNV-1a hash of the encoded bytes of the message key is used, -// modulus the number of partitions. This ensures that messages with the same key always end up on the -// same partition. -func NewHashPartitioner(topic string) Partitioner { - p := new(hashPartitioner) - p.random = NewRandomPartitioner(topic) - p.hasher = fnv.New32a() - p.referenceAbs = false - return p -} - -// NewReferenceHashPartitioner is like NewHashPartitioner except that it handles absolute values -// in the same way as the reference Java implementation. NewHashPartitioner was supposed to do -// that but it had a mistake and now there are people depending on both behaviours. This will -// all go away on the next major version bump. -func NewReferenceHashPartitioner(topic string) Partitioner { - p := new(hashPartitioner) - p.random = NewRandomPartitioner(topic) - p.hasher = fnv.New32a() - p.referenceAbs = true - return p -} - -func (p *hashPartitioner) Partition(message *ProducerMessage, numPartitions int32) (int32, error) { - if message.Key == nil { - return p.random.Partition(message, numPartitions) - } - bytes, err := message.Key.Encode() - if err != nil { - return -1, err - } - p.hasher.Reset() - _, err = p.hasher.Write(bytes) - if err != nil { - return -1, err - } - var partition int32 - // Turns out we were doing our absolute value in a subtly different way from the upstream - // implementation, but now we need to maintain backwards compat for people who started using - // the old version; if referenceAbs is set we are compatible with the reference java client - // but not past Sarama versions - if p.referenceAbs { - partition = (int32(p.hasher.Sum32()) & 0x7fffffff) % numPartitions - } else { - partition = int32(p.hasher.Sum32()) % numPartitions - if partition < 0 { - partition = -partition - } - } - return partition, nil -} - -func (p *hashPartitioner) RequiresConsistency() bool { - return true -} - -func (p *hashPartitioner) MessageRequiresConsistency(message *ProducerMessage) bool { - return message.Key != nil -} diff --git a/third/github.com/Shopify/sarama/partitioner_test.go b/third/github.com/Shopify/sarama/partitioner_test.go deleted file mode 100644 index f6dde0204..000000000 --- a/third/github.com/Shopify/sarama/partitioner_test.go +++ /dev/null @@ -1,283 +0,0 @@ -package sarama - -import ( - "crypto/rand" - "hash/fnv" - "log" - "testing" -) - -func assertPartitioningConsistent(t *testing.T, partitioner Partitioner, message *ProducerMessage, numPartitions int32) { - choice, err := partitioner.Partition(message, numPartitions) - if err != nil { - t.Error(partitioner, err) - } - if choice < 0 || choice >= numPartitions { - t.Error(partitioner, "returned partition", choice, "outside of range for", message) - } - for i := 1; i < 50; i++ { - newChoice, err := partitioner.Partition(message, numPartitions) - if err != nil { - t.Error(partitioner, err) - } - if newChoice != choice { - t.Error(partitioner, "returned partition", newChoice, "inconsistent with", choice, ".") - } - } -} - -func TestRandomPartitioner(t *testing.T) { - partitioner := NewRandomPartitioner("mytopic") - - choice, err := partitioner.Partition(nil, 1) - if err != nil { - t.Error(partitioner, err) - } - if choice != 0 { - t.Error("Returned non-zero partition when only one available.") - } - - for i := 1; i < 50; i++ { - choice, err := partitioner.Partition(nil, 50) - if err != nil { - t.Error(partitioner, err) - } - if choice < 0 || choice >= 50 { - t.Error("Returned partition", choice, "outside of range.") - } - } -} - -func TestRoundRobinPartitioner(t *testing.T) { - partitioner := NewRoundRobinPartitioner("mytopic") - - choice, err := partitioner.Partition(nil, 1) - if err != nil { - t.Error(partitioner, err) - } - if choice != 0 { - t.Error("Returned non-zero partition when only one available.") - } - - var i int32 - for i = 1; i < 50; i++ { - choice, err := partitioner.Partition(nil, 7) - if err != nil { - t.Error(partitioner, err) - } - if choice != i%7 { - t.Error("Returned partition", choice, "expecting", i%7) - } - } -} - -func TestNewHashPartitionerWithHasher(t *testing.T) { - // use the current default hasher fnv.New32a() - partitioner := NewCustomHashPartitioner(fnv.New32a)("mytopic") - - choice, err := partitioner.Partition(&ProducerMessage{}, 1) - if err != nil { - t.Error(partitioner, err) - } - if choice != 0 { - t.Error("Returned non-zero partition when only one available.") - } - - for i := 1; i < 50; i++ { - choice, err := partitioner.Partition(&ProducerMessage{}, 50) - if err != nil { - t.Error(partitioner, err) - } - if choice < 0 || choice >= 50 { - t.Error("Returned partition", choice, "outside of range for nil key.") - } - } - - buf := make([]byte, 256) - for i := 1; i < 50; i++ { - if _, err := rand.Read(buf); err != nil { - t.Error(err) - } - assertPartitioningConsistent(t, partitioner, &ProducerMessage{Key: ByteEncoder(buf)}, 50) - } -} - -func TestHashPartitionerWithHasherMinInt32(t *testing.T) { - // use the current default hasher fnv.New32a() - partitioner := NewCustomHashPartitioner(fnv.New32a)("mytopic") - - msg := ProducerMessage{} - // "1468509572224" generates 2147483648 (uint32) result from Sum32 function - // which is -2147483648 or int32's min value - msg.Key = StringEncoder("1468509572224") - - choice, err := partitioner.Partition(&msg, 50) - if err != nil { - t.Error(partitioner, err) - } - if choice < 0 || choice >= 50 { - t.Error("Returned partition", choice, "outside of range for nil key.") - } -} - -func TestHashPartitioner(t *testing.T) { - partitioner := NewHashPartitioner("mytopic") - - choice, err := partitioner.Partition(&ProducerMessage{}, 1) - if err != nil { - t.Error(partitioner, err) - } - if choice != 0 { - t.Error("Returned non-zero partition when only one available.") - } - - for i := 1; i < 50; i++ { - choice, err := partitioner.Partition(&ProducerMessage{}, 50) - if err != nil { - t.Error(partitioner, err) - } - if choice < 0 || choice >= 50 { - t.Error("Returned partition", choice, "outside of range for nil key.") - } - } - - buf := make([]byte, 256) - for i := 1; i < 50; i++ { - if _, err := rand.Read(buf); err != nil { - t.Error(err) - } - assertPartitioningConsistent(t, partitioner, &ProducerMessage{Key: ByteEncoder(buf)}, 50) - } -} - -func TestHashPartitionerConsistency(t *testing.T) { - partitioner := NewHashPartitioner("mytopic") - ep, ok := partitioner.(DynamicConsistencyPartitioner) - - if !ok { - t.Error("Hash partitioner does not implement DynamicConsistencyPartitioner") - } - - consistency := ep.MessageRequiresConsistency(&ProducerMessage{Key: StringEncoder("hi")}) - if !consistency { - t.Error("Messages with keys should require consistency") - } - consistency = ep.MessageRequiresConsistency(&ProducerMessage{}) - if consistency { - t.Error("Messages without keys should require consistency") - } -} - -func TestHashPartitionerMinInt32(t *testing.T) { - partitioner := NewHashPartitioner("mytopic") - - msg := ProducerMessage{} - // "1468509572224" generates 2147483648 (uint32) result from Sum32 function - // which is -2147483648 or int32's min value - msg.Key = StringEncoder("1468509572224") - - choice, err := partitioner.Partition(&msg, 50) - if err != nil { - t.Error(partitioner, err) - } - if choice < 0 || choice >= 50 { - t.Error("Returned partition", choice, "outside of range for nil key.") - } -} - -func TestManualPartitioner(t *testing.T) { - partitioner := NewManualPartitioner("mytopic") - - choice, err := partitioner.Partition(&ProducerMessage{}, 1) - if err != nil { - t.Error(partitioner, err) - } - if choice != 0 { - t.Error("Returned non-zero partition when only one available.") - } - - for i := int32(1); i < 50; i++ { - choice, err := partitioner.Partition(&ProducerMessage{Partition: i}, 50) - if err != nil { - t.Error(partitioner, err) - } - if choice != i { - t.Error("Returned partition not the same as the input partition") - } - } -} - -// By default, Sarama uses the message's key to consistently assign a partition to -// a message using hashing. If no key is set, a random partition will be chosen. -// This example shows how you can partition messages randomly, even when a key is set, -// by overriding Config.Producer.Partitioner. -func ExamplePartitioner_random() { - config := NewConfig() - config.Producer.Partitioner = NewRandomPartitioner - - producer, err := NewSyncProducer([]string{"localhost:9092"}, config) - if err != nil { - log.Fatal(err) - } - defer func() { - if err := producer.Close(); err != nil { - log.Println("Failed to close producer:", err) - } - }() - - msg := &ProducerMessage{Topic: "test", Key: StringEncoder("key is set"), Value: StringEncoder("test")} - partition, offset, err := producer.SendMessage(msg) - if err != nil { - log.Fatalln("Failed to produce message to kafka cluster.") - } - - log.Printf("Produced message to partition %d with offset %d", partition, offset) -} - -// This example shows how to assign partitions to your messages manually. -func ExamplePartitioner_manual() { - config := NewConfig() - - // First, we tell the producer that we are going to partition ourselves. - config.Producer.Partitioner = NewManualPartitioner - - producer, err := NewSyncProducer([]string{"localhost:9092"}, config) - if err != nil { - log.Fatal(err) - } - defer func() { - if err := producer.Close(); err != nil { - log.Println("Failed to close producer:", err) - } - }() - - // Now, we set the Partition field of the ProducerMessage struct. - msg := &ProducerMessage{Topic: "test", Partition: 6, Value: StringEncoder("test")} - - partition, offset, err := producer.SendMessage(msg) - if err != nil { - log.Fatalln("Failed to produce message to kafka cluster.") - } - - if partition != 6 { - log.Fatal("Message should have been produced to partition 6!") - } - - log.Printf("Produced message to partition %d with offset %d", partition, offset) -} - -// This example shows how to set a different partitioner depending on the topic. -func ExamplePartitioner_per_topic() { - config := NewConfig() - config.Producer.Partitioner = func(topic string) Partitioner { - switch topic { - case "access_log", "error_log": - return NewRandomPartitioner(topic) - - default: - return NewHashPartitioner(topic) - } - } - - // ... -} diff --git a/third/github.com/Shopify/sarama/prep_encoder.go b/third/github.com/Shopify/sarama/prep_encoder.go deleted file mode 100644 index 85ee89cc2..000000000 --- a/third/github.com/Shopify/sarama/prep_encoder.go +++ /dev/null @@ -1,153 +0,0 @@ -package sarama - -import ( - "encoding/binary" - "fmt" - "math" - - "github.com/gogf/gf/third/github.com/rcrowley/go-metrics" -) - -type prepEncoder struct { - stack []pushEncoder - length int -} - -// primitives - -func (pe *prepEncoder) putInt8(in int8) { - pe.length++ -} - -func (pe *prepEncoder) putInt16(in int16) { - pe.length += 2 -} - -func (pe *prepEncoder) putInt32(in int32) { - pe.length += 4 -} - -func (pe *prepEncoder) putInt64(in int64) { - pe.length += 8 -} - -func (pe *prepEncoder) putVarint(in int64) { - var buf [binary.MaxVarintLen64]byte - pe.length += binary.PutVarint(buf[:], in) -} - -func (pe *prepEncoder) putArrayLength(in int) error { - if in > math.MaxInt32 { - return PacketEncodingError{fmt.Sprintf("array too long (%d)", in)} - } - pe.length += 4 - return nil -} - -func (pe *prepEncoder) putBool(in bool) { - pe.length++ -} - -// arrays - -func (pe *prepEncoder) putBytes(in []byte) error { - pe.length += 4 - if in == nil { - return nil - } - return pe.putRawBytes(in) -} - -func (pe *prepEncoder) putVarintBytes(in []byte) error { - if in == nil { - pe.putVarint(-1) - return nil - } - pe.putVarint(int64(len(in))) - return pe.putRawBytes(in) -} - -func (pe *prepEncoder) putRawBytes(in []byte) error { - if len(in) > math.MaxInt32 { - return PacketEncodingError{fmt.Sprintf("byteslice too long (%d)", len(in))} - } - pe.length += len(in) - return nil -} - -func (pe *prepEncoder) putNullableString(in *string) error { - if in == nil { - pe.length += 2 - return nil - } - return pe.putString(*in) -} - -func (pe *prepEncoder) putString(in string) error { - pe.length += 2 - if len(in) > math.MaxInt16 { - return PacketEncodingError{fmt.Sprintf("string too long (%d)", len(in))} - } - pe.length += len(in) - return nil -} - -func (pe *prepEncoder) putStringArray(in []string) error { - err := pe.putArrayLength(len(in)) - if err != nil { - return err - } - - for _, str := range in { - if err := pe.putString(str); err != nil { - return err - } - } - - return nil -} - -func (pe *prepEncoder) putInt32Array(in []int32) error { - err := pe.putArrayLength(len(in)) - if err != nil { - return err - } - pe.length += 4 * len(in) - return nil -} - -func (pe *prepEncoder) putInt64Array(in []int64) error { - err := pe.putArrayLength(len(in)) - if err != nil { - return err - } - pe.length += 8 * len(in) - return nil -} - -func (pe *prepEncoder) offset() int { - return pe.length -} - -// stackable - -func (pe *prepEncoder) push(in pushEncoder) { - in.saveOffset(pe.length) - pe.length += in.reserveLength() - pe.stack = append(pe.stack, in) -} - -func (pe *prepEncoder) pop() error { - in := pe.stack[len(pe.stack)-1] - pe.stack = pe.stack[:len(pe.stack)-1] - if dpe, ok := in.(dynamicPushEncoder); ok { - pe.length += dpe.adjustLength(pe.length) - } - - return nil -} - -// we do not record metrics during the prep encoder pass -func (pe *prepEncoder) metricRegistry() metrics.Registry { - return nil -} diff --git a/third/github.com/Shopify/sarama/produce_request.go b/third/github.com/Shopify/sarama/produce_request.go deleted file mode 100644 index bbb4c8500..000000000 --- a/third/github.com/Shopify/sarama/produce_request.go +++ /dev/null @@ -1,252 +0,0 @@ -package sarama - -import "github.com/gogf/gf/third/github.com/rcrowley/go-metrics" - -// RequiredAcks is used in Produce Requests to tell the broker how many replica acknowledgements -// it must see before responding. Any of the constants defined here are valid. On broker versions -// prior to 0.8.2.0 any other positive int16 is also valid (the broker will wait for that many -// acknowledgements) but in 0.8.2.0 and later this will raise an exception (it has been replaced -// by setting the `min.isr` value in the brokers configuration). -type RequiredAcks int16 - -const ( - // NoResponse doesn't send any response, the TCP ACK is all you get. - NoResponse RequiredAcks = 0 - // WaitForLocal waits for only the local commit to succeed before responding. - WaitForLocal RequiredAcks = 1 - // WaitForAll waits for all in-sync replicas to commit before responding. - // The minimum number of in-sync replicas is configured on the broker via - // the `min.insync.replicas` configuration key. - WaitForAll RequiredAcks = -1 -) - -type ProduceRequest struct { - TransactionalID *string - RequiredAcks RequiredAcks - Timeout int32 - Version int16 // v1 requires Kafka 0.9, v2 requires Kafka 0.10, v3 requires Kafka 0.11 - records map[string]map[int32]Records -} - -func updateMsgSetMetrics(msgSet *MessageSet, compressionRatioMetric metrics.Histogram, - topicCompressionRatioMetric metrics.Histogram) int64 { - var topicRecordCount int64 - for _, messageBlock := range msgSet.Messages { - // Is this a fake "message" wrapping real messages? - if messageBlock.Msg.Set != nil { - topicRecordCount += int64(len(messageBlock.Msg.Set.Messages)) - } else { - // A single uncompressed message - topicRecordCount++ - } - // Better be safe than sorry when computing the compression ratio - if messageBlock.Msg.compressedSize != 0 { - compressionRatio := float64(len(messageBlock.Msg.Value)) / - float64(messageBlock.Msg.compressedSize) - // Histogram do not support decimal values, let's multiple it by 100 for better precision - intCompressionRatio := int64(100 * compressionRatio) - compressionRatioMetric.Update(intCompressionRatio) - topicCompressionRatioMetric.Update(intCompressionRatio) - } - } - return topicRecordCount -} - -func updateBatchMetrics(recordBatch *RecordBatch, compressionRatioMetric metrics.Histogram, - topicCompressionRatioMetric metrics.Histogram) int64 { - if recordBatch.compressedRecords != nil { - compressionRatio := int64(float64(recordBatch.recordsLen) / float64(len(recordBatch.compressedRecords)) * 100) - compressionRatioMetric.Update(compressionRatio) - topicCompressionRatioMetric.Update(compressionRatio) - } - - return int64(len(recordBatch.Records)) -} - -func (r *ProduceRequest) encode(pe packetEncoder) error { - if r.Version >= 3 { - if err := pe.putNullableString(r.TransactionalID); err != nil { - return err - } - } - pe.putInt16(int16(r.RequiredAcks)) - pe.putInt32(r.Timeout) - metricRegistry := pe.metricRegistry() - var batchSizeMetric metrics.Histogram - var compressionRatioMetric metrics.Histogram - if metricRegistry != nil { - batchSizeMetric = getOrRegisterHistogram("batch-size", metricRegistry) - compressionRatioMetric = getOrRegisterHistogram("compression-ratio", metricRegistry) - } - totalRecordCount := int64(0) - - err := pe.putArrayLength(len(r.records)) - if err != nil { - return err - } - - for topic, partitions := range r.records { - err = pe.putString(topic) - if err != nil { - return err - } - err = pe.putArrayLength(len(partitions)) - if err != nil { - return err - } - topicRecordCount := int64(0) - var topicCompressionRatioMetric metrics.Histogram - if metricRegistry != nil { - topicCompressionRatioMetric = getOrRegisterTopicHistogram("compression-ratio", topic, metricRegistry) - } - for id, records := range partitions { - startOffset := pe.offset() - pe.putInt32(id) - pe.push(&lengthField{}) - err = records.encode(pe) - if err != nil { - return err - } - err = pe.pop() - if err != nil { - return err - } - if metricRegistry != nil { - if r.Version >= 3 { - topicRecordCount += updateBatchMetrics(records.RecordBatch, compressionRatioMetric, topicCompressionRatioMetric) - } else { - topicRecordCount += updateMsgSetMetrics(records.MsgSet, compressionRatioMetric, topicCompressionRatioMetric) - } - batchSize := int64(pe.offset() - startOffset) - batchSizeMetric.Update(batchSize) - getOrRegisterTopicHistogram("batch-size", topic, metricRegistry).Update(batchSize) - } - } - if topicRecordCount > 0 { - getOrRegisterTopicMeter("record-send-rate", topic, metricRegistry).Mark(topicRecordCount) - getOrRegisterTopicHistogram("records-per-request", topic, metricRegistry).Update(topicRecordCount) - totalRecordCount += topicRecordCount - } - } - if totalRecordCount > 0 { - metrics.GetOrRegisterMeter("record-send-rate", metricRegistry).Mark(totalRecordCount) - getOrRegisterHistogram("records-per-request", metricRegistry).Update(totalRecordCount) - } - - return nil -} - -func (r *ProduceRequest) decode(pd packetDecoder, version int16) error { - r.Version = version - - if version >= 3 { - id, err := pd.getNullableString() - if err != nil { - return err - } - r.TransactionalID = id - } - requiredAcks, err := pd.getInt16() - if err != nil { - return err - } - r.RequiredAcks = RequiredAcks(requiredAcks) - if r.Timeout, err = pd.getInt32(); err != nil { - return err - } - topicCount, err := pd.getArrayLength() - if err != nil { - return err - } - if topicCount == 0 { - return nil - } - - r.records = make(map[string]map[int32]Records) - for i := 0; i < topicCount; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - partitionCount, err := pd.getArrayLength() - if err != nil { - return err - } - r.records[topic] = make(map[int32]Records) - - for j := 0; j < partitionCount; j++ { - partition, err := pd.getInt32() - if err != nil { - return err - } - size, err := pd.getInt32() - if err != nil { - return err - } - recordsDecoder, err := pd.getSubset(int(size)) - if err != nil { - return err - } - var records Records - if err := records.decode(recordsDecoder); err != nil { - return err - } - r.records[topic][partition] = records - } - } - - return nil -} - -func (r *ProduceRequest) key() int16 { - return 0 -} - -func (r *ProduceRequest) version() int16 { - return r.Version -} - -func (r *ProduceRequest) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_9_0_0 - case 2: - return V0_10_0_0 - case 3: - return V0_11_0_0 - default: - return MinVersion - } -} - -func (r *ProduceRequest) ensureRecords(topic string, partition int32) { - if r.records == nil { - r.records = make(map[string]map[int32]Records) - } - - if r.records[topic] == nil { - r.records[topic] = make(map[int32]Records) - } -} - -func (r *ProduceRequest) AddMessage(topic string, partition int32, msg *Message) { - r.ensureRecords(topic, partition) - set := r.records[topic][partition].MsgSet - - if set == nil { - set = new(MessageSet) - r.records[topic][partition] = newLegacyRecords(set) - } - - set.addMessage(msg) -} - -func (r *ProduceRequest) AddSet(topic string, partition int32, set *MessageSet) { - r.ensureRecords(topic, partition) - r.records[topic][partition] = newLegacyRecords(set) -} - -func (r *ProduceRequest) AddBatch(topic string, partition int32, batch *RecordBatch) { - r.ensureRecords(topic, partition) - r.records[topic][partition] = newDefaultRecords(batch) -} diff --git a/third/github.com/Shopify/sarama/produce_request_test.go b/third/github.com/Shopify/sarama/produce_request_test.go deleted file mode 100644 index b9896eb61..000000000 --- a/third/github.com/Shopify/sarama/produce_request_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -var ( - produceRequestEmpty = []byte{ - 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00} - - produceRequestHeader = []byte{ - 0x01, 0x23, - 0x00, 0x00, 0x04, 0x44, - 0x00, 0x00, 0x00, 0x00} - - produceRequestOneMessage = []byte{ - 0x01, 0x23, - 0x00, 0x00, 0x04, 0x44, - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x05, 't', 'o', 'p', 'i', 'c', - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0xAD, - 0x00, 0x00, 0x00, 0x1C, - // messageSet - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x10, - // message - 0x23, 0x96, 0x4a, 0xf7, // CRC - 0x00, - 0x00, - 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x00, 0x00, 0x02, 0x00, 0xEE} - - produceRequestOneRecord = []byte{ - 0xFF, 0xFF, // Transaction ID - 0x01, 0x23, // Required Acks - 0x00, 0x00, 0x04, 0x44, // Timeout - 0x00, 0x00, 0x00, 0x01, // Number of Topics - 0x00, 0x05, 't', 'o', 'p', 'i', 'c', // Topic - 0x00, 0x00, 0x00, 0x01, // Number of Partitions - 0x00, 0x00, 0x00, 0xAD, // Partition - 0x00, 0x00, 0x00, 0x52, // Records length - // recordBatch - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x46, - 0x00, 0x00, 0x00, 0x00, - 0x02, - 0xCA, 0x33, 0xBC, 0x05, - 0x00, 0x00, - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x01, 0x58, 0x8D, 0xCD, 0x59, 0x38, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x01, - // record - 0x28, - 0x00, - 0x0A, - 0x00, - 0x08, 0x01, 0x02, 0x03, 0x04, - 0x06, 0x05, 0x06, 0x07, - 0x02, - 0x06, 0x08, 0x09, 0x0A, - 0x04, 0x0B, 0x0C, - } -) - -func TestProduceRequest(t *testing.T) { - request := new(ProduceRequest) - testRequest(t, "empty", request, produceRequestEmpty) - - request.RequiredAcks = 0x123 - request.Timeout = 0x444 - testRequest(t, "header", request, produceRequestHeader) - - request.AddMessage("topic", 0xAD, &Message{Codec: CompressionNone, Key: nil, Value: []byte{0x00, 0xEE}}) - testRequest(t, "one message", request, produceRequestOneMessage) - - request.Version = 3 - batch := &RecordBatch{ - LastOffsetDelta: 1, - Version: 2, - FirstTimestamp: time.Unix(1479847795, 0), - MaxTimestamp: time.Unix(0, 0), - Records: []*Record{{ - TimestampDelta: 5 * time.Millisecond, - Key: []byte{0x01, 0x02, 0x03, 0x04}, - Value: []byte{0x05, 0x06, 0x07}, - Headers: []*RecordHeader{{ - Key: []byte{0x08, 0x09, 0x0A}, - Value: []byte{0x0B, 0x0C}, - }}, - }}, - } - request.AddBatch("topic", 0xAD, batch) - packet := testRequestEncode(t, "one record", request, produceRequestOneRecord) - // compressRecords field is not populated on decoding because consumers - // are only interested in decoded records. - batch.compressedRecords = nil - testRequestDecode(t, "one record", request, packet) -} diff --git a/third/github.com/Shopify/sarama/produce_response.go b/third/github.com/Shopify/sarama/produce_response.go deleted file mode 100644 index 4c5cd3569..000000000 --- a/third/github.com/Shopify/sarama/produce_response.go +++ /dev/null @@ -1,189 +0,0 @@ -package sarama - -import ( - "fmt" - "time" -) - -type ProduceResponseBlock struct { - Err KError - Offset int64 - // only provided if Version >= 2 and the broker is configured with `LogAppendTime` - Timestamp time.Time -} - -func (b *ProduceResponseBlock) decode(pd packetDecoder, version int16) (err error) { - tmp, err := pd.getInt16() - if err != nil { - return err - } - b.Err = KError(tmp) - - b.Offset, err = pd.getInt64() - if err != nil { - return err - } - - if version >= 2 { - if millis, err := pd.getInt64(); err != nil { - return err - } else if millis != -1 { - b.Timestamp = time.Unix(millis/1000, (millis%1000)*int64(time.Millisecond)) - } - } - - return nil -} - -func (b *ProduceResponseBlock) encode(pe packetEncoder, version int16) (err error) { - pe.putInt16(int16(b.Err)) - pe.putInt64(b.Offset) - - if version >= 2 { - timestamp := int64(-1) - if !b.Timestamp.Before(time.Unix(0, 0)) { - timestamp = b.Timestamp.UnixNano() / int64(time.Millisecond) - } else if !b.Timestamp.IsZero() { - return PacketEncodingError{fmt.Sprintf("invalid timestamp (%v)", b.Timestamp)} - } - pe.putInt64(timestamp) - } - - return nil -} - -type ProduceResponse struct { - Blocks map[string]map[int32]*ProduceResponseBlock - Version int16 - ThrottleTime time.Duration // only provided if Version >= 1 -} - -func (r *ProduceResponse) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - numTopics, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Blocks = make(map[string]map[int32]*ProduceResponseBlock, numTopics) - for i := 0; i < numTopics; i++ { - name, err := pd.getString() - if err != nil { - return err - } - - numBlocks, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Blocks[name] = make(map[int32]*ProduceResponseBlock, numBlocks) - - for j := 0; j < numBlocks; j++ { - id, err := pd.getInt32() - if err != nil { - return err - } - - block := new(ProduceResponseBlock) - err = block.decode(pd, version) - if err != nil { - return err - } - r.Blocks[name][id] = block - } - } - - if r.Version >= 1 { - millis, err := pd.getInt32() - if err != nil { - return err - } - - r.ThrottleTime = time.Duration(millis) * time.Millisecond - } - - return nil -} - -func (r *ProduceResponse) encode(pe packetEncoder) error { - err := pe.putArrayLength(len(r.Blocks)) - if err != nil { - return err - } - for topic, partitions := range r.Blocks { - err = pe.putString(topic) - if err != nil { - return err - } - err = pe.putArrayLength(len(partitions)) - if err != nil { - return err - } - for id, prb := range partitions { - pe.putInt32(id) - err = prb.encode(pe, r.Version) - if err != nil { - return err - } - } - } - if r.Version >= 1 { - pe.putInt32(int32(r.ThrottleTime / time.Millisecond)) - } - return nil -} - -func (r *ProduceResponse) key() int16 { - return 0 -} - -func (r *ProduceResponse) version() int16 { - return r.Version -} - -func (r *ProduceResponse) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_9_0_0 - case 2: - return V0_10_0_0 - case 3: - return V0_11_0_0 - default: - return MinVersion - } -} - -func (r *ProduceResponse) GetBlock(topic string, partition int32) *ProduceResponseBlock { - if r.Blocks == nil { - return nil - } - - if r.Blocks[topic] == nil { - return nil - } - - return r.Blocks[topic][partition] -} - -// Testing API - -func (r *ProduceResponse) AddTopicPartition(topic string, partition int32, err KError) { - if r.Blocks == nil { - r.Blocks = make(map[string]map[int32]*ProduceResponseBlock) - } - byTopic, ok := r.Blocks[topic] - if !ok { - byTopic = make(map[int32]*ProduceResponseBlock) - r.Blocks[topic] = byTopic - } - block := &ProduceResponseBlock{ - Err: err, - } - if r.Version >= 2 { - block.Timestamp = time.Now() - } - byTopic[partition] = block -} diff --git a/third/github.com/Shopify/sarama/produce_response_test.go b/third/github.com/Shopify/sarama/produce_response_test.go deleted file mode 100644 index 197c7fb50..000000000 --- a/third/github.com/Shopify/sarama/produce_response_test.go +++ /dev/null @@ -1,128 +0,0 @@ -package sarama - -import ( - "fmt" - "testing" - "time" -) - -var ( - produceResponseNoBlocksV0 = []byte{ - 0x00, 0x00, 0x00, 0x00} - - produceResponseManyBlocksVersions = [][]byte{ - { - 0x00, 0x00, 0x00, 0x01, - - 0x00, 0x03, 'f', 'o', 'o', - 0x00, 0x00, 0x00, 0x01, - - 0x00, 0x00, 0x00, 0x01, // Partition 1 - 0x00, 0x02, // ErrInvalidMessage - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, // Offset 255 - }, { - 0x00, 0x00, 0x00, 0x01, - - 0x00, 0x03, 'f', 'o', 'o', - 0x00, 0x00, 0x00, 0x01, - - 0x00, 0x00, 0x00, 0x01, // Partition 1 - 0x00, 0x02, // ErrInvalidMessage - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, // Offset 255 - - 0x00, 0x00, 0x00, 0x64, // 100 ms throttle time - }, { - 0x00, 0x00, 0x00, 0x01, - - 0x00, 0x03, 'f', 'o', 'o', - 0x00, 0x00, 0x00, 0x01, - - 0x00, 0x00, 0x00, 0x01, // Partition 1 - 0x00, 0x02, // ErrInvalidMessage - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, // Offset 255 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xE8, // Timestamp January 1st 0001 at 00:00:01,000 UTC (LogAppendTime was used) - - 0x00, 0x00, 0x00, 0x64, // 100 ms throttle time - }, - } -) - -func TestProduceResponseDecode(t *testing.T) { - response := ProduceResponse{} - - testVersionDecodable(t, "no blocks", &response, produceResponseNoBlocksV0, 0) - if len(response.Blocks) != 0 { - t.Error("Decoding produced", len(response.Blocks), "topics where there were none") - } - - for v, produceResponseManyBlocks := range produceResponseManyBlocksVersions { - t.Logf("Decoding produceResponseManyBlocks version %d", v) - testVersionDecodable(t, "many blocks", &response, produceResponseManyBlocks, int16(v)) - if len(response.Blocks) != 1 { - t.Error("Decoding produced", len(response.Blocks), "topics where there was 1") - } - if len(response.Blocks["foo"]) != 1 { - t.Error("Decoding produced", len(response.Blocks["foo"]), "partitions for 'foo' where there was one") - } - block := response.GetBlock("foo", 1) - if block == nil { - t.Error("Decoding did not produce a block for foo/1") - } else { - if block.Err != ErrInvalidMessage { - t.Error("Decoding failed for foo/2/Err, got:", int16(block.Err)) - } - if block.Offset != 255 { - t.Error("Decoding failed for foo/1/Offset, got:", block.Offset) - } - if v >= 2 { - if block.Timestamp != time.Unix(1, 0) { - t.Error("Decoding failed for foo/2/Timestamp, got:", block.Timestamp) - } - } - } - if v >= 1 { - if expected := 100 * time.Millisecond; response.ThrottleTime != expected { - t.Error("Failed decoding produced throttle time, expected:", expected, ", got:", response.ThrottleTime) - } - } - } -} - -func TestProduceResponseEncode(t *testing.T) { - response := ProduceResponse{} - response.Blocks = make(map[string]map[int32]*ProduceResponseBlock) - testEncodable(t, "empty", &response, produceResponseNoBlocksV0) - - response.Blocks["foo"] = make(map[int32]*ProduceResponseBlock) - response.Blocks["foo"][1] = &ProduceResponseBlock{ - Err: ErrInvalidMessage, - Offset: 255, - Timestamp: time.Unix(1, 0), - } - response.ThrottleTime = 100 * time.Millisecond - for v, produceResponseManyBlocks := range produceResponseManyBlocksVersions { - response.Version = int16(v) - testEncodable(t, fmt.Sprintf("many blocks version %d", v), &response, produceResponseManyBlocks) - } -} - -func TestProduceResponseEncodeInvalidTimestamp(t *testing.T) { - response := ProduceResponse{} - response.Version = 2 - response.Blocks = make(map[string]map[int32]*ProduceResponseBlock) - response.Blocks["t"] = make(map[int32]*ProduceResponseBlock) - response.Blocks["t"][0] = &ProduceResponseBlock{ - Err: ErrNoError, - Offset: 0, - // Use a timestamp before Unix time - Timestamp: time.Unix(0, 0).Add(-1 * time.Millisecond), - } - response.ThrottleTime = 100 * time.Millisecond - _, err := encode(&response, nil) - if err == nil { - t.Error("Expecting error, got nil") - } - if _, ok := err.(PacketEncodingError); !ok { - t.Error("Expecting PacketEncodingError, got:", err) - } -} diff --git a/third/github.com/Shopify/sarama/produce_set.go b/third/github.com/Shopify/sarama/produce_set.go deleted file mode 100644 index 219ec5f26..000000000 --- a/third/github.com/Shopify/sarama/produce_set.go +++ /dev/null @@ -1,259 +0,0 @@ -package sarama - -import ( - "encoding/binary" - "errors" - "time" -) - -type partitionSet struct { - msgs []*ProducerMessage - recordsToSend Records - bufferBytes int -} - -type produceSet struct { - parent *asyncProducer - msgs map[string]map[int32]*partitionSet - - bufferBytes int - bufferCount int -} - -func newProduceSet(parent *asyncProducer) *produceSet { - return &produceSet{ - msgs: make(map[string]map[int32]*partitionSet), - parent: parent, - } -} - -func (ps *produceSet) add(msg *ProducerMessage) error { - var err error - var key, val []byte - - if msg.Key != nil { - if key, err = msg.Key.Encode(); err != nil { - return err - } - } - - if msg.Value != nil { - if val, err = msg.Value.Encode(); err != nil { - return err - } - } - - timestamp := msg.Timestamp - if msg.Timestamp.IsZero() { - timestamp = time.Now() - } - - partitions := ps.msgs[msg.Topic] - if partitions == nil { - partitions = make(map[int32]*partitionSet) - ps.msgs[msg.Topic] = partitions - } - - var size int - - set := partitions[msg.Partition] - if set == nil { - if ps.parent.conf.Version.IsAtLeast(V0_11_0_0) { - batch := &RecordBatch{ - FirstTimestamp: timestamp, - Version: 2, - Codec: ps.parent.conf.Producer.Compression, - CompressionLevel: ps.parent.conf.Producer.CompressionLevel, - ProducerID: ps.parent.txnmgr.producerID, - ProducerEpoch: ps.parent.txnmgr.producerEpoch, - } - if ps.parent.conf.Producer.Idempotent { - batch.FirstSequence = msg.sequenceNumber - } - set = &partitionSet{recordsToSend: newDefaultRecords(batch)} - size = recordBatchOverhead - } else { - set = &partitionSet{recordsToSend: newLegacyRecords(new(MessageSet))} - } - partitions[msg.Partition] = set - } - set.msgs = append(set.msgs, msg) - - if ps.parent.conf.Version.IsAtLeast(V0_11_0_0) { - if ps.parent.conf.Producer.Idempotent && msg.sequenceNumber < set.recordsToSend.RecordBatch.FirstSequence { - return errors.New("Assertion failed: Message out of sequence added to a batch") - } - // We are being conservative here to avoid having to prep encode the record - size += maximumRecordOverhead - rec := &Record{ - Key: key, - Value: val, - TimestampDelta: timestamp.Sub(set.recordsToSend.RecordBatch.FirstTimestamp), - } - size += len(key) + len(val) - if len(msg.Headers) > 0 { - rec.Headers = make([]*RecordHeader, len(msg.Headers)) - for i := range msg.Headers { - rec.Headers[i] = &msg.Headers[i] - size += len(rec.Headers[i].Key) + len(rec.Headers[i].Value) + 2*binary.MaxVarintLen32 - } - } - set.recordsToSend.RecordBatch.addRecord(rec) - } else { - msgToSend := &Message{Codec: CompressionNone, Key: key, Value: val} - if ps.parent.conf.Version.IsAtLeast(V0_10_0_0) { - msgToSend.Timestamp = timestamp - msgToSend.Version = 1 - } - set.recordsToSend.MsgSet.addMessage(msgToSend) - size = producerMessageOverhead + len(key) + len(val) - } - - set.bufferBytes += size - ps.bufferBytes += size - ps.bufferCount++ - - return nil -} - -func (ps *produceSet) buildRequest() *ProduceRequest { - req := &ProduceRequest{ - RequiredAcks: ps.parent.conf.Producer.RequiredAcks, - Timeout: int32(ps.parent.conf.Producer.Timeout / time.Millisecond), - } - if ps.parent.conf.Version.IsAtLeast(V0_10_0_0) { - req.Version = 2 - } - if ps.parent.conf.Version.IsAtLeast(V0_11_0_0) { - req.Version = 3 - } - - for topic, partitionSets := range ps.msgs { - for partition, set := range partitionSets { - if req.Version >= 3 { - // If the API version we're hitting is 3 or greater, we need to calculate - // offsets for each record in the batch relative to FirstOffset. - // Additionally, we must set LastOffsetDelta to the value of the last offset - // in the batch. Since the OffsetDelta of the first record is 0, we know that the - // final record of any batch will have an offset of (# of records in batch) - 1. - // (See https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets - // under the RecordBatch section for details.) - rb := set.recordsToSend.RecordBatch - if len(rb.Records) > 0 { - rb.LastOffsetDelta = int32(len(rb.Records) - 1) - for i, record := range rb.Records { - record.OffsetDelta = int64(i) - } - } - req.AddBatch(topic, partition, rb) - continue - } - if ps.parent.conf.Producer.Compression == CompressionNone { - req.AddSet(topic, partition, set.recordsToSend.MsgSet) - } else { - // When compression is enabled, the entire set for each partition is compressed - // and sent as the payload of a single fake "message" with the appropriate codec - // set and no key. When the server sees a message with a compression codec, it - // decompresses the payload and treats the result as its message set. - - if ps.parent.conf.Version.IsAtLeast(V0_10_0_0) { - // If our version is 0.10 or later, assign relative offsets - // to the inner messages. This lets the broker avoid - // recompressing the message set. - // (See https://cwiki.apache.org/confluence/display/KAFKA/KIP-31+-+Move+to+relative+offsets+in+compressed+message+sets - // for details on relative offsets.) - for i, msg := range set.recordsToSend.MsgSet.Messages { - msg.Offset = int64(i) - } - } - payload, err := encode(set.recordsToSend.MsgSet, ps.parent.conf.MetricRegistry) - if err != nil { - Logger.Println(err) // if this happens, it's basically our fault. - panic(err) - } - compMsg := &Message{ - Codec: ps.parent.conf.Producer.Compression, - CompressionLevel: ps.parent.conf.Producer.CompressionLevel, - Key: nil, - Value: payload, - Set: set.recordsToSend.MsgSet, // Provide the underlying message set for accurate metrics - } - if ps.parent.conf.Version.IsAtLeast(V0_10_0_0) { - compMsg.Version = 1 - compMsg.Timestamp = set.recordsToSend.MsgSet.Messages[0].Msg.Timestamp - } - req.AddMessage(topic, partition, compMsg) - } - } - } - - return req -} - -func (ps *produceSet) eachPartition(cb func(topic string, partition int32, pSet *partitionSet)) { - for topic, partitionSet := range ps.msgs { - for partition, set := range partitionSet { - cb(topic, partition, set) - } - } -} - -func (ps *produceSet) dropPartition(topic string, partition int32) []*ProducerMessage { - if ps.msgs[topic] == nil { - return nil - } - set := ps.msgs[topic][partition] - if set == nil { - return nil - } - ps.bufferBytes -= set.bufferBytes - ps.bufferCount -= len(set.msgs) - delete(ps.msgs[topic], partition) - return set.msgs -} - -func (ps *produceSet) wouldOverflow(msg *ProducerMessage) bool { - version := 1 - if ps.parent.conf.Version.IsAtLeast(V0_11_0_0) { - version = 2 - } - - switch { - // Would we overflow our maximum possible size-on-the-wire? 10KiB is arbitrary overhead for safety. - case ps.bufferBytes+msg.byteSize(version) >= int(MaxRequestSize-(10*1024)): - return true - // Would we overflow the size-limit of a compressed message-batch for this partition? - case ps.parent.conf.Producer.Compression != CompressionNone && - ps.msgs[msg.Topic] != nil && ps.msgs[msg.Topic][msg.Partition] != nil && - ps.msgs[msg.Topic][msg.Partition].bufferBytes+msg.byteSize(version) >= ps.parent.conf.Producer.MaxMessageBytes: - return true - // Would we overflow simply in number of messages? - case ps.parent.conf.Producer.Flush.MaxMessages > 0 && ps.bufferCount >= ps.parent.conf.Producer.Flush.MaxMessages: - return true - default: - return false - } -} - -func (ps *produceSet) readyToFlush() bool { - switch { - // If we don't have any messages, nothing else matters - case ps.empty(): - return false - // If all three config values are 0, we always flush as-fast-as-possible - case ps.parent.conf.Producer.Flush.Frequency == 0 && ps.parent.conf.Producer.Flush.Bytes == 0 && ps.parent.conf.Producer.Flush.Messages == 0: - return true - // If we've passed the message trigger-point - case ps.parent.conf.Producer.Flush.Messages > 0 && ps.bufferCount >= ps.parent.conf.Producer.Flush.Messages: - return true - // If we've passed the byte trigger-point - case ps.parent.conf.Producer.Flush.Bytes > 0 && ps.bufferBytes >= ps.parent.conf.Producer.Flush.Bytes: - return true - default: - return false - } -} - -func (ps *produceSet) empty() bool { - return ps.bufferCount == 0 -} diff --git a/third/github.com/Shopify/sarama/produce_set_test.go b/third/github.com/Shopify/sarama/produce_set_test.go deleted file mode 100644 index 51d4cef30..000000000 --- a/third/github.com/Shopify/sarama/produce_set_test.go +++ /dev/null @@ -1,346 +0,0 @@ -package sarama - -import ( - "fmt" - "testing" - "time" -) - -func makeProduceSet() (*asyncProducer, *produceSet) { - conf := NewConfig() - txnmgr, _ := newTransactionManager(conf, nil) - parent := &asyncProducer{ - conf: conf, - txnmgr: txnmgr, - } - return parent, newProduceSet(parent) -} - -func safeAddMessage(t *testing.T, ps *produceSet, msg *ProducerMessage) { - if err := ps.add(msg); err != nil { - t.Error(err) - } -} - -func TestProduceSetInitial(t *testing.T) { - _, ps := makeProduceSet() - - if !ps.empty() { - t.Error("New produceSet should be empty") - } - - if ps.readyToFlush() { - t.Error("Empty produceSet must never be ready to flush") - } -} - -func TestProduceSetAddingMessages(t *testing.T) { - parent, ps := makeProduceSet() - parent.conf.Producer.Flush.MaxMessages = 1000 - - msg := &ProducerMessage{Key: StringEncoder(TestMessage), Value: StringEncoder(TestMessage)} - safeAddMessage(t, ps, msg) - - if ps.empty() { - t.Error("set shouldn't be empty when a message is added") - } - - if !ps.readyToFlush() { - t.Error("by default set should be ready to flush when any message is in place") - } - - for i := 0; i < 999; i++ { - if ps.wouldOverflow(msg) { - t.Error("set shouldn't fill up after only", i+1, "messages") - } - safeAddMessage(t, ps, msg) - } - - if !ps.wouldOverflow(msg) { - t.Error("set should be full after 1000 messages") - } -} - -func TestProduceSetPartitionTracking(t *testing.T) { - _, ps := makeProduceSet() - - m1 := &ProducerMessage{Topic: "t1", Partition: 0} - m2 := &ProducerMessage{Topic: "t1", Partition: 1} - m3 := &ProducerMessage{Topic: "t2", Partition: 0} - safeAddMessage(t, ps, m1) - safeAddMessage(t, ps, m2) - safeAddMessage(t, ps, m3) - - seenT1P0 := false - seenT1P1 := false - seenT2P0 := false - - ps.eachPartition(func(topic string, partition int32, pSet *partitionSet) { - if len(pSet.msgs) != 1 { - t.Error("Wrong message count") - } - - if topic == "t1" && partition == 0 { - seenT1P0 = true - } else if topic == "t1" && partition == 1 { - seenT1P1 = true - } else if topic == "t2" && partition == 0 { - seenT2P0 = true - } - }) - - if !seenT1P0 { - t.Error("Didn't see t1p0") - } - if !seenT1P1 { - t.Error("Didn't see t1p1") - } - if !seenT2P0 { - t.Error("Didn't see t2p0") - } - - if len(ps.dropPartition("t1", 1)) != 1 { - t.Error("Got wrong messages back from dropping partition") - } - - if ps.bufferCount != 2 { - t.Error("Incorrect buffer count after dropping partition") - } -} - -func TestProduceSetRequestBuilding(t *testing.T) { - parent, ps := makeProduceSet() - parent.conf.Producer.RequiredAcks = WaitForAll - parent.conf.Producer.Timeout = 10 * time.Second - - msg := &ProducerMessage{ - Topic: "t1", - Partition: 0, - Key: StringEncoder(TestMessage), - Value: StringEncoder(TestMessage), - } - for i := 0; i < 10; i++ { - safeAddMessage(t, ps, msg) - } - msg.Partition = 1 - for i := 0; i < 10; i++ { - safeAddMessage(t, ps, msg) - } - msg.Topic = "t2" - for i := 0; i < 10; i++ { - safeAddMessage(t, ps, msg) - } - - req := ps.buildRequest() - - if req.RequiredAcks != WaitForAll { - t.Error("RequiredAcks not set properly") - } - - if req.Timeout != 10000 { - t.Error("Timeout not set properly") - } - - if len(req.records) != 2 { - t.Error("Wrong number of topics in request") - } -} - -func TestProduceSetCompressedRequestBuilding(t *testing.T) { - parent, ps := makeProduceSet() - parent.conf.Producer.RequiredAcks = WaitForAll - parent.conf.Producer.Timeout = 10 * time.Second - parent.conf.Producer.Compression = CompressionGZIP - parent.conf.Version = V0_10_0_0 - - msg := &ProducerMessage{ - Topic: "t1", - Partition: 0, - Key: StringEncoder(TestMessage), - Value: StringEncoder(TestMessage), - Timestamp: time.Now(), - } - for i := 0; i < 10; i++ { - safeAddMessage(t, ps, msg) - } - - req := ps.buildRequest() - - if req.Version != 2 { - t.Error("Wrong request version") - } - - for _, msgBlock := range req.records["t1"][0].MsgSet.Messages { - msg := msgBlock.Msg - err := msg.decodeSet() - if err != nil { - t.Error("Failed to decode set from payload") - } - for i, compMsgBlock := range msg.Set.Messages { - compMsg := compMsgBlock.Msg - if compMsg.Version != 1 { - t.Error("Wrong compressed message version") - } - if compMsgBlock.Offset != int64(i) { - t.Errorf("Wrong relative inner offset, expected %d, got %d", i, compMsgBlock.Offset) - } - } - if msg.Version != 1 { - t.Error("Wrong compressed parent message version") - } - } -} - -func TestProduceSetV3RequestBuilding(t *testing.T) { - parent, ps := makeProduceSet() - parent.conf.Producer.RequiredAcks = WaitForAll - parent.conf.Producer.Timeout = 10 * time.Second - parent.conf.Version = V0_11_0_0 - - now := time.Now() - msg := &ProducerMessage{ - Topic: "t1", - Partition: 0, - Key: StringEncoder(TestMessage), - Value: StringEncoder(TestMessage), - Headers: []RecordHeader{ - RecordHeader{ - Key: []byte("header-1"), - Value: []byte("value-1"), - }, - RecordHeader{ - Key: []byte("header-2"), - Value: []byte("value-2"), - }, - RecordHeader{ - Key: []byte("header-3"), - Value: []byte("value-3"), - }, - }, - Timestamp: now, - } - for i := 0; i < 10; i++ { - safeAddMessage(t, ps, msg) - msg.Timestamp = msg.Timestamp.Add(time.Second) - } - - req := ps.buildRequest() - - if req.Version != 3 { - t.Error("Wrong request version") - } - - batch := req.records["t1"][0].RecordBatch - if batch.FirstTimestamp != now { - t.Errorf("Wrong first timestamp: %v", batch.FirstTimestamp) - } - for i := 0; i < 10; i++ { - rec := batch.Records[i] - if rec.TimestampDelta != time.Duration(i)*time.Second { - t.Errorf("Wrong timestamp delta: %v", rec.TimestampDelta) - } - - if rec.OffsetDelta != int64(i) { - t.Errorf("Wrong relative inner offset, expected %d, got %d", i, rec.OffsetDelta) - } - - for j, h := range batch.Records[i].Headers { - exp := fmt.Sprintf("header-%d", j+1) - if string(h.Key) != exp { - t.Errorf("Wrong header key, expected %v, got %v", exp, h.Key) - } - exp = fmt.Sprintf("value-%d", j+1) - if string(h.Value) != exp { - t.Errorf("Wrong header value, expected %v, got %v", exp, h.Value) - } - } - } -} - -func TestProduceSetIdempotentRequestBuilding(t *testing.T) { - const pID = 1000 - const pEpoch = 1234 - - config := NewConfig() - config.Producer.RequiredAcks = WaitForAll - config.Producer.Idempotent = true - config.Version = V0_11_0_0 - - parent := &asyncProducer{ - conf: config, - txnmgr: &transactionManager{ - producerID: pID, - producerEpoch: pEpoch, - }, - } - ps := newProduceSet(parent) - - now := time.Now() - msg := &ProducerMessage{ - Topic: "t1", - Partition: 0, - Key: StringEncoder(TestMessage), - Value: StringEncoder(TestMessage), - Headers: []RecordHeader{ - RecordHeader{ - Key: []byte("header-1"), - Value: []byte("value-1"), - }, - RecordHeader{ - Key: []byte("header-2"), - Value: []byte("value-2"), - }, - RecordHeader{ - Key: []byte("header-3"), - Value: []byte("value-3"), - }, - }, - Timestamp: now, - sequenceNumber: 123, - } - for i := 0; i < 10; i++ { - safeAddMessage(t, ps, msg) - msg.Timestamp = msg.Timestamp.Add(time.Second) - } - - req := ps.buildRequest() - - if req.Version != 3 { - t.Error("Wrong request version") - } - - batch := req.records["t1"][0].RecordBatch - if batch.FirstTimestamp != now { - t.Errorf("Wrong first timestamp: %v", batch.FirstTimestamp) - } - if batch.ProducerID != pID { - t.Errorf("Wrong producerID: %v", batch.ProducerID) - } - if batch.ProducerEpoch != pEpoch { - t.Errorf("Wrong producerEpoch: %v", batch.ProducerEpoch) - } - if batch.FirstSequence != 123 { - t.Errorf("Wrong first sequence: %v", batch.FirstSequence) - } - for i := 0; i < 10; i++ { - rec := batch.Records[i] - if rec.TimestampDelta != time.Duration(i)*time.Second { - t.Errorf("Wrong timestamp delta: %v", rec.TimestampDelta) - } - - if rec.OffsetDelta != int64(i) { - t.Errorf("Wrong relative inner offset, expected %d, got %d", i, rec.OffsetDelta) - } - - for j, h := range batch.Records[i].Headers { - exp := fmt.Sprintf("header-%d", j+1) - if string(h.Key) != exp { - t.Errorf("Wrong header key, expected %v, got %v", exp, h.Key) - } - exp = fmt.Sprintf("value-%d", j+1) - if string(h.Value) != exp { - t.Errorf("Wrong header value, expected %v, got %v", exp, h.Value) - } - } - } -} diff --git a/third/github.com/Shopify/sarama/real_decoder.go b/third/github.com/Shopify/sarama/real_decoder.go deleted file mode 100644 index 23045e7d3..000000000 --- a/third/github.com/Shopify/sarama/real_decoder.go +++ /dev/null @@ -1,324 +0,0 @@ -package sarama - -import ( - "encoding/binary" - "math" -) - -var errInvalidArrayLength = PacketDecodingError{"invalid array length"} -var errInvalidByteSliceLength = PacketDecodingError{"invalid byteslice length"} -var errInvalidByteSliceLengthType = PacketDecodingError{"invalid byteslice length type"} -var errInvalidStringLength = PacketDecodingError{"invalid string length"} -var errInvalidSubsetSize = PacketDecodingError{"invalid subset size"} -var errVarintOverflow = PacketDecodingError{"varint overflow"} -var errInvalidBool = PacketDecodingError{"invalid bool"} - -type realDecoder struct { - raw []byte - off int - stack []pushDecoder -} - -// primitives - -func (rd *realDecoder) getInt8() (int8, error) { - if rd.remaining() < 1 { - rd.off = len(rd.raw) - return -1, ErrInsufficientData - } - tmp := int8(rd.raw[rd.off]) - rd.off++ - return tmp, nil -} - -func (rd *realDecoder) getInt16() (int16, error) { - if rd.remaining() < 2 { - rd.off = len(rd.raw) - return -1, ErrInsufficientData - } - tmp := int16(binary.BigEndian.Uint16(rd.raw[rd.off:])) - rd.off += 2 - return tmp, nil -} - -func (rd *realDecoder) getInt32() (int32, error) { - if rd.remaining() < 4 { - rd.off = len(rd.raw) - return -1, ErrInsufficientData - } - tmp := int32(binary.BigEndian.Uint32(rd.raw[rd.off:])) - rd.off += 4 - return tmp, nil -} - -func (rd *realDecoder) getInt64() (int64, error) { - if rd.remaining() < 8 { - rd.off = len(rd.raw) - return -1, ErrInsufficientData - } - tmp := int64(binary.BigEndian.Uint64(rd.raw[rd.off:])) - rd.off += 8 - return tmp, nil -} - -func (rd *realDecoder) getVarint() (int64, error) { - tmp, n := binary.Varint(rd.raw[rd.off:]) - if n == 0 { - rd.off = len(rd.raw) - return -1, ErrInsufficientData - } - if n < 0 { - rd.off -= n - return -1, errVarintOverflow - } - rd.off += n - return tmp, nil -} - -func (rd *realDecoder) getArrayLength() (int, error) { - if rd.remaining() < 4 { - rd.off = len(rd.raw) - return -1, ErrInsufficientData - } - tmp := int(int32(binary.BigEndian.Uint32(rd.raw[rd.off:]))) - rd.off += 4 - if tmp > rd.remaining() { - rd.off = len(rd.raw) - return -1, ErrInsufficientData - } else if tmp > 2*math.MaxUint16 { - return -1, errInvalidArrayLength - } - return tmp, nil -} - -func (rd *realDecoder) getBool() (bool, error) { - b, err := rd.getInt8() - if err != nil || b == 0 { - return false, err - } - if b != 1 { - return false, errInvalidBool - } - return true, nil -} - -// collections - -func (rd *realDecoder) getBytes() ([]byte, error) { - tmp, err := rd.getInt32() - if err != nil { - return nil, err - } - if tmp == -1 { - return nil, nil - } - - return rd.getRawBytes(int(tmp)) -} - -func (rd *realDecoder) getVarintBytes() ([]byte, error) { - tmp, err := rd.getVarint() - if err != nil { - return nil, err - } - if tmp == -1 { - return nil, nil - } - - return rd.getRawBytes(int(tmp)) -} - -func (rd *realDecoder) getStringLength() (int, error) { - length, err := rd.getInt16() - if err != nil { - return 0, err - } - - n := int(length) - - switch { - case n < -1: - return 0, errInvalidStringLength - case n > rd.remaining(): - rd.off = len(rd.raw) - return 0, ErrInsufficientData - } - - return n, nil -} - -func (rd *realDecoder) getString() (string, error) { - n, err := rd.getStringLength() - if err != nil || n == -1 { - return "", err - } - - tmpStr := string(rd.raw[rd.off : rd.off+n]) - rd.off += n - return tmpStr, nil -} - -func (rd *realDecoder) getNullableString() (*string, error) { - n, err := rd.getStringLength() - if err != nil || n == -1 { - return nil, err - } - - tmpStr := string(rd.raw[rd.off : rd.off+n]) - rd.off += n - return &tmpStr, err -} - -func (rd *realDecoder) getInt32Array() ([]int32, error) { - if rd.remaining() < 4 { - rd.off = len(rd.raw) - return nil, ErrInsufficientData - } - n := int(binary.BigEndian.Uint32(rd.raw[rd.off:])) - rd.off += 4 - - if rd.remaining() < 4*n { - rd.off = len(rd.raw) - return nil, ErrInsufficientData - } - - if n == 0 { - return nil, nil - } - - if n < 0 { - return nil, errInvalidArrayLength - } - - ret := make([]int32, n) - for i := range ret { - ret[i] = int32(binary.BigEndian.Uint32(rd.raw[rd.off:])) - rd.off += 4 - } - return ret, nil -} - -func (rd *realDecoder) getInt64Array() ([]int64, error) { - if rd.remaining() < 4 { - rd.off = len(rd.raw) - return nil, ErrInsufficientData - } - n := int(binary.BigEndian.Uint32(rd.raw[rd.off:])) - rd.off += 4 - - if rd.remaining() < 8*n { - rd.off = len(rd.raw) - return nil, ErrInsufficientData - } - - if n == 0 { - return nil, nil - } - - if n < 0 { - return nil, errInvalidArrayLength - } - - ret := make([]int64, n) - for i := range ret { - ret[i] = int64(binary.BigEndian.Uint64(rd.raw[rd.off:])) - rd.off += 8 - } - return ret, nil -} - -func (rd *realDecoder) getStringArray() ([]string, error) { - if rd.remaining() < 4 { - rd.off = len(rd.raw) - return nil, ErrInsufficientData - } - n := int(binary.BigEndian.Uint32(rd.raw[rd.off:])) - rd.off += 4 - - if n == 0 { - return nil, nil - } - - if n < 0 { - return nil, errInvalidArrayLength - } - - ret := make([]string, n) - for i := range ret { - str, err := rd.getString() - if err != nil { - return nil, err - } - - ret[i] = str - } - return ret, nil -} - -// subsets - -func (rd *realDecoder) remaining() int { - return len(rd.raw) - rd.off -} - -func (rd *realDecoder) getSubset(length int) (packetDecoder, error) { - buf, err := rd.getRawBytes(length) - if err != nil { - return nil, err - } - return &realDecoder{raw: buf}, nil -} - -func (rd *realDecoder) getRawBytes(length int) ([]byte, error) { - if length < 0 { - return nil, errInvalidByteSliceLength - } else if length > rd.remaining() { - rd.off = len(rd.raw) - return nil, ErrInsufficientData - } - - start := rd.off - rd.off += length - return rd.raw[start:rd.off], nil -} - -func (rd *realDecoder) peek(offset, length int) (packetDecoder, error) { - if rd.remaining() < offset+length { - return nil, ErrInsufficientData - } - off := rd.off + offset - return &realDecoder{raw: rd.raw[off : off+length]}, nil -} - -// stacks - -func (rd *realDecoder) push(in pushDecoder) error { - in.saveOffset(rd.off) - - var reserve int - if dpd, ok := in.(dynamicPushDecoder); ok { - if err := dpd.decode(rd); err != nil { - return err - } - } else { - reserve = in.reserveLength() - if rd.remaining() < reserve { - rd.off = len(rd.raw) - return ErrInsufficientData - } - } - - rd.stack = append(rd.stack, in) - - rd.off += reserve - - return nil -} - -func (rd *realDecoder) pop() error { - // this is go's ugly pop pattern (the inverse of append) - in := rd.stack[len(rd.stack)-1] - rd.stack = rd.stack[:len(rd.stack)-1] - - return in.check(rd.off, rd.raw) -} diff --git a/third/github.com/Shopify/sarama/real_encoder.go b/third/github.com/Shopify/sarama/real_encoder.go deleted file mode 100644 index f3fbe2795..000000000 --- a/third/github.com/Shopify/sarama/real_encoder.go +++ /dev/null @@ -1,156 +0,0 @@ -package sarama - -import ( - "encoding/binary" - - "github.com/gogf/gf/third/github.com/rcrowley/go-metrics" -) - -type realEncoder struct { - raw []byte - off int - stack []pushEncoder - registry metrics.Registry -} - -// primitives - -func (re *realEncoder) putInt8(in int8) { - re.raw[re.off] = byte(in) - re.off++ -} - -func (re *realEncoder) putInt16(in int16) { - binary.BigEndian.PutUint16(re.raw[re.off:], uint16(in)) - re.off += 2 -} - -func (re *realEncoder) putInt32(in int32) { - binary.BigEndian.PutUint32(re.raw[re.off:], uint32(in)) - re.off += 4 -} - -func (re *realEncoder) putInt64(in int64) { - binary.BigEndian.PutUint64(re.raw[re.off:], uint64(in)) - re.off += 8 -} - -func (re *realEncoder) putVarint(in int64) { - re.off += binary.PutVarint(re.raw[re.off:], in) -} - -func (re *realEncoder) putArrayLength(in int) error { - re.putInt32(int32(in)) - return nil -} - -func (re *realEncoder) putBool(in bool) { - if in { - re.putInt8(1) - return - } - re.putInt8(0) -} - -// collection - -func (re *realEncoder) putRawBytes(in []byte) error { - copy(re.raw[re.off:], in) - re.off += len(in) - return nil -} - -func (re *realEncoder) putBytes(in []byte) error { - if in == nil { - re.putInt32(-1) - return nil - } - re.putInt32(int32(len(in))) - return re.putRawBytes(in) -} - -func (re *realEncoder) putVarintBytes(in []byte) error { - if in == nil { - re.putVarint(-1) - return nil - } - re.putVarint(int64(len(in))) - return re.putRawBytes(in) -} - -func (re *realEncoder) putString(in string) error { - re.putInt16(int16(len(in))) - copy(re.raw[re.off:], in) - re.off += len(in) - return nil -} - -func (re *realEncoder) putNullableString(in *string) error { - if in == nil { - re.putInt16(-1) - return nil - } - return re.putString(*in) -} - -func (re *realEncoder) putStringArray(in []string) error { - err := re.putArrayLength(len(in)) - if err != nil { - return err - } - - for _, val := range in { - if err := re.putString(val); err != nil { - return err - } - } - - return nil -} - -func (re *realEncoder) putInt32Array(in []int32) error { - err := re.putArrayLength(len(in)) - if err != nil { - return err - } - for _, val := range in { - re.putInt32(val) - } - return nil -} - -func (re *realEncoder) putInt64Array(in []int64) error { - err := re.putArrayLength(len(in)) - if err != nil { - return err - } - for _, val := range in { - re.putInt64(val) - } - return nil -} - -func (re *realEncoder) offset() int { - return re.off -} - -// stacks - -func (re *realEncoder) push(in pushEncoder) { - in.saveOffset(re.off) - re.off += in.reserveLength() - re.stack = append(re.stack, in) -} - -func (re *realEncoder) pop() error { - // this is go's ugly pop pattern (the inverse of append) - in := re.stack[len(re.stack)-1] - re.stack = re.stack[:len(re.stack)-1] - - return in.run(re.off, re.raw) -} - -// we do record metrics during the real encoder pass -func (re *realEncoder) metricRegistry() metrics.Registry { - return re.registry -} diff --git a/third/github.com/Shopify/sarama/record.go b/third/github.com/Shopify/sarama/record.go deleted file mode 100644 index cded308cf..000000000 --- a/third/github.com/Shopify/sarama/record.go +++ /dev/null @@ -1,113 +0,0 @@ -package sarama - -import ( - "encoding/binary" - "time" -) - -const ( - controlMask = 0x20 - maximumRecordOverhead = 5*binary.MaxVarintLen32 + binary.MaxVarintLen64 + 1 -) - -type RecordHeader struct { - Key []byte - Value []byte -} - -func (h *RecordHeader) encode(pe packetEncoder) error { - if err := pe.putVarintBytes(h.Key); err != nil { - return err - } - return pe.putVarintBytes(h.Value) -} - -func (h *RecordHeader) decode(pd packetDecoder) (err error) { - if h.Key, err = pd.getVarintBytes(); err != nil { - return err - } - - if h.Value, err = pd.getVarintBytes(); err != nil { - return err - } - return nil -} - -type Record struct { - Attributes int8 - TimestampDelta time.Duration - OffsetDelta int64 - Key []byte - Value []byte - Headers []*RecordHeader - - length varintLengthField -} - -func (r *Record) encode(pe packetEncoder) error { - pe.push(&r.length) - pe.putInt8(r.Attributes) - pe.putVarint(int64(r.TimestampDelta / time.Millisecond)) - pe.putVarint(r.OffsetDelta) - if err := pe.putVarintBytes(r.Key); err != nil { - return err - } - if err := pe.putVarintBytes(r.Value); err != nil { - return err - } - pe.putVarint(int64(len(r.Headers))) - - for _, h := range r.Headers { - if err := h.encode(pe); err != nil { - return err - } - } - - return pe.pop() -} - -func (r *Record) decode(pd packetDecoder) (err error) { - if err = pd.push(&r.length); err != nil { - return err - } - - if r.Attributes, err = pd.getInt8(); err != nil { - return err - } - - timestamp, err := pd.getVarint() - if err != nil { - return err - } - r.TimestampDelta = time.Duration(timestamp) * time.Millisecond - - if r.OffsetDelta, err = pd.getVarint(); err != nil { - return err - } - - if r.Key, err = pd.getVarintBytes(); err != nil { - return err - } - - if r.Value, err = pd.getVarintBytes(); err != nil { - return err - } - - numHeaders, err := pd.getVarint() - if err != nil { - return err - } - - if numHeaders >= 0 { - r.Headers = make([]*RecordHeader, numHeaders) - } - for i := int64(0); i < numHeaders; i++ { - hdr := new(RecordHeader) - if err := hdr.decode(pd); err != nil { - return err - } - r.Headers[i] = hdr - } - - return pd.pop() -} diff --git a/third/github.com/Shopify/sarama/record_batch.go b/third/github.com/Shopify/sarama/record_batch.go deleted file mode 100644 index e0f183f7a..000000000 --- a/third/github.com/Shopify/sarama/record_batch.go +++ /dev/null @@ -1,208 +0,0 @@ -package sarama - -import ( - "fmt" - "time" -) - -const recordBatchOverhead = 49 - -type recordsArray []*Record - -func (e recordsArray) encode(pe packetEncoder) error { - for _, r := range e { - if err := r.encode(pe); err != nil { - return err - } - } - return nil -} - -func (e recordsArray) decode(pd packetDecoder) error { - for i := range e { - rec := &Record{} - if err := rec.decode(pd); err != nil { - return err - } - e[i] = rec - } - return nil -} - -type RecordBatch struct { - FirstOffset int64 - PartitionLeaderEpoch int32 - Version int8 - Codec CompressionCodec - CompressionLevel int - Control bool - LastOffsetDelta int32 - FirstTimestamp time.Time - MaxTimestamp time.Time - ProducerID int64 - ProducerEpoch int16 - FirstSequence int32 - Records []*Record - PartialTrailingRecord bool - - compressedRecords []byte - recordsLen int // uncompressed records size -} - -func (b *RecordBatch) encode(pe packetEncoder) error { - if b.Version != 2 { - return PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", b.Codec)} - } - pe.putInt64(b.FirstOffset) - pe.push(&lengthField{}) - pe.putInt32(b.PartitionLeaderEpoch) - pe.putInt8(b.Version) - pe.push(newCRC32Field(crcCastagnoli)) - pe.putInt16(b.computeAttributes()) - pe.putInt32(b.LastOffsetDelta) - - if err := (Timestamp{&b.FirstTimestamp}).encode(pe); err != nil { - return err - } - - if err := (Timestamp{&b.MaxTimestamp}).encode(pe); err != nil { - return err - } - - pe.putInt64(b.ProducerID) - pe.putInt16(b.ProducerEpoch) - pe.putInt32(b.FirstSequence) - - if err := pe.putArrayLength(len(b.Records)); err != nil { - return err - } - - if b.compressedRecords == nil { - if err := b.encodeRecords(pe); err != nil { - return err - } - } - if err := pe.putRawBytes(b.compressedRecords); err != nil { - return err - } - - if err := pe.pop(); err != nil { - return err - } - return pe.pop() -} - -func (b *RecordBatch) decode(pd packetDecoder) (err error) { - if b.FirstOffset, err = pd.getInt64(); err != nil { - return err - } - - batchLen, err := pd.getInt32() - if err != nil { - return err - } - - if b.PartitionLeaderEpoch, err = pd.getInt32(); err != nil { - return err - } - - if b.Version, err = pd.getInt8(); err != nil { - return err - } - - if err = pd.push(&crc32Field{polynomial: crcCastagnoli}); err != nil { - return err - } - - attributes, err := pd.getInt16() - if err != nil { - return err - } - b.Codec = CompressionCodec(int8(attributes) & compressionCodecMask) - b.Control = attributes&controlMask == controlMask - - if b.LastOffsetDelta, err = pd.getInt32(); err != nil { - return err - } - - if err = (Timestamp{&b.FirstTimestamp}).decode(pd); err != nil { - return err - } - - if err = (Timestamp{&b.MaxTimestamp}).decode(pd); err != nil { - return err - } - - if b.ProducerID, err = pd.getInt64(); err != nil { - return err - } - - if b.ProducerEpoch, err = pd.getInt16(); err != nil { - return err - } - - if b.FirstSequence, err = pd.getInt32(); err != nil { - return err - } - - numRecs, err := pd.getArrayLength() - if err != nil { - return err - } - if numRecs >= 0 { - b.Records = make([]*Record, numRecs) - } - - bufSize := int(batchLen) - recordBatchOverhead - recBuffer, err := pd.getRawBytes(bufSize) - if err != nil { - if err == ErrInsufficientData { - b.PartialTrailingRecord = true - b.Records = nil - return nil - } - return err - } - - if err = pd.pop(); err != nil { - return err - } - - recBuffer, err = decompress(b.Codec, recBuffer) - if err != nil { - return err - } - - b.recordsLen = len(recBuffer) - err = decode(recBuffer, recordsArray(b.Records)) - if err == ErrInsufficientData { - b.PartialTrailingRecord = true - b.Records = nil - return nil - } - return err -} - -func (b *RecordBatch) encodeRecords(pe packetEncoder) error { - var raw []byte - var err error - if raw, err = encode(recordsArray(b.Records), pe.metricRegistry()); err != nil { - return err - } - b.recordsLen = len(raw) - - b.compressedRecords, err = compress(b.Codec, b.CompressionLevel, raw) - return err -} - -func (b *RecordBatch) computeAttributes() int16 { - attr := int16(b.Codec) & int16(compressionCodecMask) - if b.Control { - attr |= controlMask - } - return attr -} - -func (b *RecordBatch) addRecord(r *Record) { - b.Records = append(b.Records, r) -} diff --git a/third/github.com/Shopify/sarama/record_test.go b/third/github.com/Shopify/sarama/record_test.go deleted file mode 100644 index bece24dcf..000000000 --- a/third/github.com/Shopify/sarama/record_test.go +++ /dev/null @@ -1,292 +0,0 @@ -package sarama - -import ( - "reflect" - "runtime" - "strconv" - "strings" - "testing" - "time" - - "github.com/gogf/gf/third/github.com/davecgh/go-spew/spew" -) - -var recordBatchTestCases = []struct { - name string - batch RecordBatch - encoded []byte - oldGoEncoded []byte // used in case of gzipped content for go versions prior to 1.8 -}{ - { - name: "empty record", - batch: RecordBatch{ - Version: 2, - FirstTimestamp: time.Unix(0, 0), - MaxTimestamp: time.Unix(0, 0), - Records: []*Record{}, - }, - encoded: []byte{ - 0, 0, 0, 0, 0, 0, 0, 0, // First Offset - 0, 0, 0, 49, // Length - 0, 0, 0, 0, // Partition Leader Epoch - 2, // Version - 89, 95, 183, 221, // CRC - 0, 0, // Attributes - 0, 0, 0, 0, // Last Offset Delta - 0, 0, 0, 0, 0, 0, 0, 0, // First Timestamp - 0, 0, 0, 0, 0, 0, 0, 0, // Max Timestamp - 0, 0, 0, 0, 0, 0, 0, 0, // Producer ID - 0, 0, // Producer Epoch - 0, 0, 0, 0, // First Sequence - 0, 0, 0, 0, // Number of Records - }, - }, - { - name: "control batch", - batch: RecordBatch{ - Version: 2, - Control: true, - FirstTimestamp: time.Unix(0, 0), - MaxTimestamp: time.Unix(0, 0), - Records: []*Record{}, - }, - encoded: []byte{ - 0, 0, 0, 0, 0, 0, 0, 0, // First Offset - 0, 0, 0, 49, // Length - 0, 0, 0, 0, // Partition Leader Epoch - 2, // Version - 81, 46, 67, 217, // CRC - 0, 32, // Attributes - 0, 0, 0, 0, // Last Offset Delta - 0, 0, 0, 0, 0, 0, 0, 0, // First Timestamp - 0, 0, 0, 0, 0, 0, 0, 0, // Max Timestamp - 0, 0, 0, 0, 0, 0, 0, 0, // Producer ID - 0, 0, // Producer Epoch - 0, 0, 0, 0, // First Sequence - 0, 0, 0, 0, // Number of Records - }, - }, - { - name: "uncompressed record", - batch: RecordBatch{ - Version: 2, - FirstTimestamp: time.Unix(1479847795, 0), - MaxTimestamp: time.Unix(0, 0), - LastOffsetDelta: 0, - Records: []*Record{{ - TimestampDelta: 5 * time.Millisecond, - Key: []byte{1, 2, 3, 4}, - Value: []byte{5, 6, 7}, - Headers: []*RecordHeader{{ - Key: []byte{8, 9, 10}, - Value: []byte{11, 12}, - }}, - }}, - recordsLen: 21, - }, - encoded: []byte{ - 0, 0, 0, 0, 0, 0, 0, 0, // First Offset - 0, 0, 0, 70, // Length - 0, 0, 0, 0, // Partition Leader Epoch - 2, // Version - 84, 121, 97, 253, // CRC - 0, 0, // Attributes - 0, 0, 0, 0, // Last Offset Delta - 0, 0, 1, 88, 141, 205, 89, 56, // First Timestamp - 0, 0, 0, 0, 0, 0, 0, 0, // Max Timestamp - 0, 0, 0, 0, 0, 0, 0, 0, // Producer ID - 0, 0, // Producer Epoch - 0, 0, 0, 0, // First Sequence - 0, 0, 0, 1, // Number of Records - 40, // Record Length - 0, // Attributes - 10, // Timestamp Delta - 0, // Offset Delta - 8, // Key Length - 1, 2, 3, 4, - 6, // Value Length - 5, 6, 7, - 2, // Number of Headers - 6, // Header Key Length - 8, 9, 10, // Header Key - 4, // Header Value Length - 11, 12, // Header Value - }, - }, - { - name: "gzipped record", - batch: RecordBatch{ - Version: 2, - Codec: CompressionGZIP, - CompressionLevel: CompressionLevelDefault, - FirstTimestamp: time.Unix(1479847795, 0), - MaxTimestamp: time.Unix(0, 0), - LastOffsetDelta: 0, - Records: []*Record{{ - TimestampDelta: 5 * time.Millisecond, - Key: []byte{1, 2, 3, 4}, - Value: []byte{5, 6, 7}, - Headers: []*RecordHeader{{ - Key: []byte{8, 9, 10}, - Value: []byte{11, 12}, - }}, - }}, - recordsLen: 21, - }, - encoded: []byte{ - 0, 0, 0, 0, 0, 0, 0, 0, // First Offset - 0, 0, 0, 94, // Length - 0, 0, 0, 0, // Partition Leader Epoch - 2, // Version - 159, 236, 182, 189, // CRC - 0, 1, // Attributes - 0, 0, 0, 0, // Last Offset Delta - 0, 0, 1, 88, 141, 205, 89, 56, // First Timestamp - 0, 0, 0, 0, 0, 0, 0, 0, // Max Timestamp - 0, 0, 0, 0, 0, 0, 0, 0, // Producer ID - 0, 0, // Producer Epoch - 0, 0, 0, 0, // First Sequence - 0, 0, 0, 1, // Number of Records - 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 210, 96, 224, 98, 224, 96, 100, 98, 102, 97, 99, 101, - 99, 103, 98, 227, 224, 228, 98, 225, 230, 1, 4, 0, 0, 255, 255, 173, 201, 88, 103, 21, 0, 0, 0, - }, - oldGoEncoded: []byte{ - 0, 0, 0, 0, 0, 0, 0, 0, // First Offset - 0, 0, 0, 94, // Length - 0, 0, 0, 0, // Partition Leader Epoch - 2, // Version - 0, 216, 14, 210, // CRC - 0, 1, // Attributes - 0, 0, 0, 0, // Last Offset Delta - 0, 0, 1, 88, 141, 205, 89, 56, // First Timestamp - 0, 0, 0, 0, 0, 0, 0, 0, // Max Timestamp - 0, 0, 0, 0, 0, 0, 0, 0, // Producer ID - 0, 0, // Producer Epoch - 0, 0, 0, 0, // First Sequence - 0, 0, 0, 1, // Number of Records - 31, 139, 8, 0, 0, 9, 110, 136, 0, 255, 210, 96, 224, 98, 224, 96, 100, 98, 102, 97, 99, 101, - 99, 103, 98, 227, 224, 228, 98, 225, 230, 1, 4, 0, 0, 255, 255, 173, 201, 88, 103, 21, 0, 0, 0, - }, - }, - { - name: "snappy compressed record", - batch: RecordBatch{ - Version: 2, - Codec: CompressionSnappy, - FirstTimestamp: time.Unix(1479847795, 0), - MaxTimestamp: time.Unix(0, 0), - LastOffsetDelta: 0, - Records: []*Record{{ - TimestampDelta: 5 * time.Millisecond, - Key: []byte{1, 2, 3, 4}, - Value: []byte{5, 6, 7}, - Headers: []*RecordHeader{{ - Key: []byte{8, 9, 10}, - Value: []byte{11, 12}, - }}, - }}, - recordsLen: 21, - }, - encoded: []byte{ - 0, 0, 0, 0, 0, 0, 0, 0, // First Offset - 0, 0, 0, 72, // Length - 0, 0, 0, 0, // Partition Leader Epoch - 2, // Version - 21, 0, 159, 97, // CRC - 0, 2, // Attributes - 0, 0, 0, 0, // Last Offset Delta - 0, 0, 1, 88, 141, 205, 89, 56, // First Timestamp - 0, 0, 0, 0, 0, 0, 0, 0, // Max Timestamp - 0, 0, 0, 0, 0, 0, 0, 0, // Producer ID - 0, 0, // Producer Epoch - 0, 0, 0, 0, // First Sequence - 0, 0, 0, 1, // Number of Records - 21, 80, 40, 0, 10, 0, 8, 1, 2, 3, 4, 6, 5, 6, 7, 2, 6, 8, 9, 10, 4, 11, 12, - }, - }, - { - name: "lz4 compressed record", - batch: RecordBatch{ - Version: 2, - Codec: CompressionLZ4, - FirstTimestamp: time.Unix(1479847795, 0), - MaxTimestamp: time.Unix(0, 0), - LastOffsetDelta: 0, - Records: []*Record{{ - TimestampDelta: 5 * time.Millisecond, - Key: []byte{1, 2, 3, 4}, - Value: []byte{5, 6, 7}, - Headers: []*RecordHeader{{ - Key: []byte{8, 9, 10}, - Value: []byte{11, 12}, - }}, - }}, - recordsLen: 21, - }, - encoded: []byte{ - 0, 0, 0, 0, 0, 0, 0, 0, // First Offset - 0, 0, 0, 89, // Length - 0, 0, 0, 0, // Partition Leader Epoch - 2, // Version - 169, 74, 119, 197, // CRC - 0, 3, // Attributes - 0, 0, 0, 0, // Last Offset Delta - 0, 0, 1, 88, 141, 205, 89, 56, // First Timestamp - 0, 0, 0, 0, 0, 0, 0, 0, // Max Timestamp - 0, 0, 0, 0, 0, 0, 0, 0, // Producer ID - 0, 0, // Producer Epoch - 0, 0, 0, 0, // First Sequence - 0, 0, 0, 1, // Number of Records - 4, 34, 77, 24, 100, 112, 185, 21, 0, 0, 128, 40, 0, 10, 0, 8, 1, 2, 3, 4, 6, 5, 6, 7, 2, - 6, 8, 9, 10, 4, 11, 12, 0, 0, 0, 0, 12, 59, 239, 146, - }, - }, -} - -func isOldGo(t *testing.T) bool { - v := strings.Split(runtime.Version()[2:], ".") - if len(v) < 2 { - t.Logf("Can't parse version: %s", runtime.Version()) - return false - } - maj, err := strconv.Atoi(v[0]) - if err != nil { - t.Logf("Can't parse version: %s", runtime.Version()) - return false - } - min, err := strconv.Atoi(v[1]) - if err != nil { - t.Logf("Can't parse version: %s", runtime.Version()) - return false - } - return maj < 1 || (maj == 1 && min < 8) -} - -func TestRecordBatchEncoding(t *testing.T) { - for _, tc := range recordBatchTestCases { - if tc.oldGoEncoded != nil && isOldGo(t) { - testEncodable(t, tc.name, &tc.batch, tc.oldGoEncoded) - } else { - testEncodable(t, tc.name, &tc.batch, tc.encoded) - } - } -} - -func TestRecordBatchDecoding(t *testing.T) { - for _, tc := range recordBatchTestCases { - batch := RecordBatch{} - testDecodable(t, tc.name, &batch, tc.encoded) - for _, r := range batch.Records { - r.length = varintLengthField{} - } - for _, r := range tc.batch.Records { - r.length = varintLengthField{} - } - // The compression level is not restored on decoding. It is not needed - // anyway. We only set it here to ensure that comparision succeeds. - batch.CompressionLevel = tc.batch.CompressionLevel - if !reflect.DeepEqual(batch, tc.batch) { - t.Errorf(spew.Sprintf("invalid decode of %s\ngot %+v\nwanted %+v", tc.name, batch, tc.batch)) - } - } -} diff --git a/third/github.com/Shopify/sarama/records.go b/third/github.com/Shopify/sarama/records.go deleted file mode 100644 index 192f5927b..000000000 --- a/third/github.com/Shopify/sarama/records.go +++ /dev/null @@ -1,194 +0,0 @@ -package sarama - -import "fmt" - -const ( - unknownRecords = iota - legacyRecords - defaultRecords - - magicOffset = 16 - magicLength = 1 -) - -// Records implements a union type containing either a RecordBatch or a legacy MessageSet. -type Records struct { - recordsType int - MsgSet *MessageSet - RecordBatch *RecordBatch -} - -func newLegacyRecords(msgSet *MessageSet) Records { - return Records{recordsType: legacyRecords, MsgSet: msgSet} -} - -func newDefaultRecords(batch *RecordBatch) Records { - return Records{recordsType: defaultRecords, RecordBatch: batch} -} - -// setTypeFromFields sets type of Records depending on which of MsgSet or RecordBatch is not nil. -// The first return value indicates whether both fields are nil (and the type is not set). -// If both fields are not nil, it returns an error. -func (r *Records) setTypeFromFields() (bool, error) { - if r.MsgSet == nil && r.RecordBatch == nil { - return true, nil - } - if r.MsgSet != nil && r.RecordBatch != nil { - return false, fmt.Errorf("both MsgSet and RecordBatch are set, but record type is unknown") - } - r.recordsType = defaultRecords - if r.MsgSet != nil { - r.recordsType = legacyRecords - } - return false, nil -} - -func (r *Records) encode(pe packetEncoder) error { - if r.recordsType == unknownRecords { - if empty, err := r.setTypeFromFields(); err != nil || empty { - return err - } - } - - switch r.recordsType { - case legacyRecords: - if r.MsgSet == nil { - return nil - } - return r.MsgSet.encode(pe) - case defaultRecords: - if r.RecordBatch == nil { - return nil - } - return r.RecordBatch.encode(pe) - } - - return fmt.Errorf("unknown records type: %v", r.recordsType) -} - -func (r *Records) setTypeFromMagic(pd packetDecoder) error { - magic, err := magicValue(pd) - if err != nil { - return err - } - - r.recordsType = defaultRecords - if magic < 2 { - r.recordsType = legacyRecords - } - - return nil -} - -func (r *Records) decode(pd packetDecoder) error { - if r.recordsType == unknownRecords { - if err := r.setTypeFromMagic(pd); err != nil { - return err - } - } - - switch r.recordsType { - case legacyRecords: - r.MsgSet = &MessageSet{} - return r.MsgSet.decode(pd) - case defaultRecords: - r.RecordBatch = &RecordBatch{} - return r.RecordBatch.decode(pd) - } - return fmt.Errorf("unknown records type: %v", r.recordsType) -} - -func (r *Records) numRecords() (int, error) { - if r.recordsType == unknownRecords { - if empty, err := r.setTypeFromFields(); err != nil || empty { - return 0, err - } - } - - switch r.recordsType { - case legacyRecords: - if r.MsgSet == nil { - return 0, nil - } - return len(r.MsgSet.Messages), nil - case defaultRecords: - if r.RecordBatch == nil { - return 0, nil - } - return len(r.RecordBatch.Records), nil - } - return 0, fmt.Errorf("unknown records type: %v", r.recordsType) -} - -func (r *Records) isPartial() (bool, error) { - if r.recordsType == unknownRecords { - if empty, err := r.setTypeFromFields(); err != nil || empty { - return false, err - } - } - - switch r.recordsType { - case unknownRecords: - return false, nil - case legacyRecords: - if r.MsgSet == nil { - return false, nil - } - return r.MsgSet.PartialTrailingMessage, nil - case defaultRecords: - if r.RecordBatch == nil { - return false, nil - } - return r.RecordBatch.PartialTrailingRecord, nil - } - return false, fmt.Errorf("unknown records type: %v", r.recordsType) -} - -func (r *Records) isControl() (bool, error) { - if r.recordsType == unknownRecords { - if empty, err := r.setTypeFromFields(); err != nil || empty { - return false, err - } - } - - switch r.recordsType { - case legacyRecords: - return false, nil - case defaultRecords: - if r.RecordBatch == nil { - return false, nil - } - return r.RecordBatch.Control, nil - } - return false, fmt.Errorf("unknown records type: %v", r.recordsType) -} - -func (r *Records) isOverflow() (bool, error) { - if r.recordsType == unknownRecords { - if empty, err := r.setTypeFromFields(); err != nil || empty { - return false, err - } - } - - switch r.recordsType { - case unknownRecords: - return false, nil - case legacyRecords: - if r.MsgSet == nil { - return false, nil - } - return r.MsgSet.OverflowMessage, nil - case defaultRecords: - return false, nil - } - return false, fmt.Errorf("unknown records type: %v", r.recordsType) -} - -func magicValue(pd packetDecoder) (int8, error) { - dec, err := pd.peek(magicOffset, magicLength) - if err != nil { - return 0, err - } - - return dec.getInt8() -} diff --git a/third/github.com/Shopify/sarama/records_test.go b/third/github.com/Shopify/sarama/records_test.go deleted file mode 100644 index cd6aa2784..000000000 --- a/third/github.com/Shopify/sarama/records_test.go +++ /dev/null @@ -1,143 +0,0 @@ -package sarama - -import ( - "bytes" - "reflect" - "testing" -) - -func TestLegacyRecords(t *testing.T) { - set := &MessageSet{ - Messages: []*MessageBlock{ - { - Msg: &Message{ - Version: 1, - }, - }, - }, - } - r := newLegacyRecords(set) - - exp, err := encode(set, nil) - if err != nil { - t.Fatal(err) - } - buf, err := encode(&r, nil) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(buf, exp) { - t.Errorf("Wrong encoding for legacy records, wanted %v, got %v", exp, buf) - } - - set = &MessageSet{} - r = Records{} - - err = decode(exp, set) - if err != nil { - t.Fatal(err) - } - err = decode(buf, &r) - if err != nil { - t.Fatal(err) - } - - if r.recordsType != legacyRecords { - t.Fatalf("Wrong records type %v, expected %v", r.recordsType, legacyRecords) - } - if !reflect.DeepEqual(set, r.MsgSet) { - t.Errorf("Wrong decoding for legacy records, wanted %#+v, got %#+v", set, r.MsgSet) - } - - n, err := r.numRecords() - if err != nil { - t.Fatal(err) - } - if n != 1 { - t.Errorf("Wrong number of records, wanted 1, got %d", n) - } - - p, err := r.isPartial() - if err != nil { - t.Fatal(err) - } - if p { - t.Errorf("MessageSet shouldn't have a partial trailing message") - } - - c, err := r.isControl() - if err != nil { - t.Fatal(err) - } - if c { - t.Errorf("MessageSet can't be a control batch") - } -} - -func TestDefaultRecords(t *testing.T) { - batch := &RecordBatch{ - Version: 2, - Records: []*Record{ - { - Value: []byte{1}, - }, - }, - } - - r := newDefaultRecords(batch) - - exp, err := encode(batch, nil) - if err != nil { - t.Fatal(err) - } - buf, err := encode(&r, nil) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(buf, exp) { - t.Errorf("Wrong encoding for default records, wanted %v, got %v", exp, buf) - } - - batch = &RecordBatch{} - r = Records{} - - err = decode(exp, batch) - if err != nil { - t.Fatal(err) - } - err = decode(buf, &r) - if err != nil { - t.Fatal(err) - } - - if r.recordsType != defaultRecords { - t.Fatalf("Wrong records type %v, expected %v", r.recordsType, defaultRecords) - } - if !reflect.DeepEqual(batch, r.RecordBatch) { - t.Errorf("Wrong decoding for default records, wanted %#+v, got %#+v", batch, r.RecordBatch) - } - - n, err := r.numRecords() - if err != nil { - t.Fatal(err) - } - if n != 1 { - t.Errorf("Wrong number of records, wanted 1, got %d", n) - } - - p, err := r.isPartial() - if err != nil { - t.Fatal(err) - } - if p { - t.Errorf("RecordBatch shouldn't have a partial trailing record") - } - - c, err := r.isControl() - if err != nil { - t.Fatal(err) - } - if c { - t.Errorf("RecordBatch shouldn't be a control batch") - } -} diff --git a/third/github.com/Shopify/sarama/request.go b/third/github.com/Shopify/sarama/request.go deleted file mode 100644 index 4d211a14f..000000000 --- a/third/github.com/Shopify/sarama/request.go +++ /dev/null @@ -1,149 +0,0 @@ -package sarama - -import ( - "encoding/binary" - "fmt" - "io" -) - -type protocolBody interface { - encoder - versionedDecoder - key() int16 - version() int16 - requiredVersion() KafkaVersion -} - -type request struct { - correlationID int32 - clientID string - body protocolBody -} - -func (r *request) encode(pe packetEncoder) (err error) { - pe.push(&lengthField{}) - pe.putInt16(r.body.key()) - pe.putInt16(r.body.version()) - pe.putInt32(r.correlationID) - err = pe.putString(r.clientID) - if err != nil { - return err - } - err = r.body.encode(pe) - if err != nil { - return err - } - return pe.pop() -} - -func (r *request) decode(pd packetDecoder) (err error) { - var key int16 - if key, err = pd.getInt16(); err != nil { - return err - } - var version int16 - if version, err = pd.getInt16(); err != nil { - return err - } - if r.correlationID, err = pd.getInt32(); err != nil { - return err - } - r.clientID, err = pd.getString() - - r.body = allocateBody(key, version) - if r.body == nil { - return PacketDecodingError{fmt.Sprintf("unknown request key (%d)", key)} - } - return r.body.decode(pd, version) -} - -func decodeRequest(r io.Reader) (req *request, bytesRead int, err error) { - lengthBytes := make([]byte, 4) - if _, err := io.ReadFull(r, lengthBytes); err != nil { - return nil, bytesRead, err - } - bytesRead += len(lengthBytes) - - length := int32(binary.BigEndian.Uint32(lengthBytes)) - if length <= 4 || length > MaxRequestSize { - return nil, bytesRead, PacketDecodingError{fmt.Sprintf("message of length %d too large or too small", length)} - } - - encodedReq := make([]byte, length) - if _, err := io.ReadFull(r, encodedReq); err != nil { - return nil, bytesRead, err - } - bytesRead += len(encodedReq) - - req = &request{} - if err := decode(encodedReq, req); err != nil { - return nil, bytesRead, err - } - return req, bytesRead, nil -} - -func allocateBody(key, version int16) protocolBody { - switch key { - case 0: - return &ProduceRequest{} - case 1: - return &FetchRequest{} - case 2: - return &OffsetRequest{Version: version} - case 3: - return &MetadataRequest{} - case 8: - return &OffsetCommitRequest{Version: version} - case 9: - return &OffsetFetchRequest{} - case 10: - return &FindCoordinatorRequest{} - case 11: - return &JoinGroupRequest{} - case 12: - return &HeartbeatRequest{} - case 13: - return &LeaveGroupRequest{} - case 14: - return &SyncGroupRequest{} - case 15: - return &DescribeGroupsRequest{} - case 16: - return &ListGroupsRequest{} - case 17: - return &SaslHandshakeRequest{} - case 18: - return &ApiVersionsRequest{} - case 19: - return &CreateTopicsRequest{} - case 20: - return &DeleteTopicsRequest{} - case 21: - return &DeleteRecordsRequest{} - case 22: - return &InitProducerIDRequest{} - case 24: - return &AddPartitionsToTxnRequest{} - case 25: - return &AddOffsetsToTxnRequest{} - case 26: - return &EndTxnRequest{} - case 28: - return &TxnOffsetCommitRequest{} - case 29: - return &DescribeAclsRequest{} - case 30: - return &CreateAclsRequest{} - case 31: - return &DeleteAclsRequest{} - case 32: - return &DescribeConfigsRequest{} - case 33: - return &AlterConfigsRequest{} - case 37: - return &CreatePartitionsRequest{} - case 42: - return &DeleteGroupsRequest{} - } - return nil -} diff --git a/third/github.com/Shopify/sarama/request_test.go b/third/github.com/Shopify/sarama/request_test.go deleted file mode 100644 index 4633dd2df..000000000 --- a/third/github.com/Shopify/sarama/request_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package sarama - -import ( - "bytes" - "reflect" - "testing" - - "github.com/gogf/gf/third/github.com/davecgh/go-spew/spew" -) - -type testRequestBody struct { -} - -func (s *testRequestBody) key() int16 { - return 0x666 -} - -func (s *testRequestBody) version() int16 { - return 0xD2 -} - -func (s *testRequestBody) encode(pe packetEncoder) error { - return pe.putString("abc") -} - -// not specific to request tests, just helper functions for testing structures that -// implement the encoder or decoder interfaces that needed somewhere to live - -func testEncodable(t *testing.T, name string, in encoder, expect []byte) { - packet, err := encode(in, nil) - if err != nil { - t.Error(err) - } else if !bytes.Equal(packet, expect) { - t.Error("Encoding", name, "failed\ngot ", packet, "\nwant", expect) - } -} - -func testDecodable(t *testing.T, name string, out decoder, in []byte) { - err := decode(in, out) - if err != nil { - t.Error("Decoding", name, "failed:", err) - } -} - -func testVersionDecodable(t *testing.T, name string, out versionedDecoder, in []byte, version int16) { - err := versionedDecode(in, out, version) - if err != nil { - t.Error("Decoding", name, "version", version, "failed:", err) - } -} - -func testRequest(t *testing.T, name string, rb protocolBody, expected []byte) { - if !rb.requiredVersion().IsAtLeast(MinVersion) { - t.Errorf("Request %s has invalid required version", name) - } - packet := testRequestEncode(t, name, rb, expected) - testRequestDecode(t, name, rb, packet) -} - -func testRequestEncode(t *testing.T, name string, rb protocolBody, expected []byte) []byte { - req := &request{correlationID: 123, clientID: "foo", body: rb} - packet, err := encode(req, nil) - headerSize := 14 + len("foo") - if err != nil { - t.Error(err) - } else if !bytes.Equal(packet[headerSize:], expected) { - t.Error("Encoding", name, "failed\ngot ", packet[headerSize:], "\nwant", expected) - } - return packet -} - -func testRequestDecode(t *testing.T, name string, rb protocolBody, packet []byte) { - decoded, n, err := decodeRequest(bytes.NewReader(packet)) - if err != nil { - t.Error("Failed to decode request", err) - } else if decoded.correlationID != 123 || decoded.clientID != "foo" { - t.Errorf("Decoded header %q is not valid: %+v", name, decoded) - } else if !reflect.DeepEqual(rb, decoded.body) { - t.Error(spew.Sprintf("Decoded request %q does not match the encoded one\nencoded: %+v\ndecoded: %+v", name, rb, decoded.body)) - } else if n != len(packet) { - t.Errorf("Decoded request %q bytes: %d does not match the encoded one: %d\n", name, n, len(packet)) - } else if rb.version() != decoded.body.version() { - t.Errorf("Decoded request %q version: %d does not match the encoded one: %d\n", name, decoded.body.version(), rb.version()) - } -} - -func testResponse(t *testing.T, name string, res protocolBody, expected []byte) { - encoded, err := encode(res, nil) - if err != nil { - t.Error(err) - } else if expected != nil && !bytes.Equal(encoded, expected) { - t.Error("Encoding", name, "failed\ngot ", encoded, "\nwant", expected) - } - - decoded := reflect.New(reflect.TypeOf(res).Elem()).Interface().(versionedDecoder) - if err := versionedDecode(encoded, decoded, res.version()); err != nil { - t.Error("Decoding", name, "failed:", err) - } - - if !reflect.DeepEqual(decoded, res) { - t.Errorf("Decoded response does not match the encoded one\nencoded: %#v\ndecoded: %#v", res, decoded) - } -} - -func nullString(s string) *string { return &s } diff --git a/third/github.com/Shopify/sarama/response_header.go b/third/github.com/Shopify/sarama/response_header.go deleted file mode 100644 index f3f4d27d6..000000000 --- a/third/github.com/Shopify/sarama/response_header.go +++ /dev/null @@ -1,21 +0,0 @@ -package sarama - -import "fmt" - -type responseHeader struct { - length int32 - correlationID int32 -} - -func (r *responseHeader) decode(pd packetDecoder) (err error) { - r.length, err = pd.getInt32() - if err != nil { - return err - } - if r.length <= 4 || r.length > MaxResponseSize { - return PacketDecodingError{fmt.Sprintf("message of length %d too large or too small", r.length)} - } - - r.correlationID, err = pd.getInt32() - return err -} diff --git a/third/github.com/Shopify/sarama/response_header_test.go b/third/github.com/Shopify/sarama/response_header_test.go deleted file mode 100644 index 8f9fdb80c..000000000 --- a/third/github.com/Shopify/sarama/response_header_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package sarama - -import "testing" - -var ( - responseHeaderBytes = []byte{ - 0x00, 0x00, 0x0f, 0x00, - 0x0a, 0xbb, 0xcc, 0xff} -) - -func TestResponseHeader(t *testing.T) { - header := responseHeader{} - - testDecodable(t, "response header", &header, responseHeaderBytes) - if header.length != 0xf00 { - t.Error("Decoding header length failed, got", header.length) - } - if header.correlationID != 0x0abbccff { - t.Error("Decoding header correlation id failed, got", header.correlationID) - } -} diff --git a/third/github.com/Shopify/sarama/sarama.go b/third/github.com/Shopify/sarama/sarama.go deleted file mode 100644 index 7d5dc60d3..000000000 --- a/third/github.com/Shopify/sarama/sarama.go +++ /dev/null @@ -1,99 +0,0 @@ -/* -Package sarama is a pure Go client library for dealing with Apache Kafka (versions 0.8 and later). It includes a high-level -API for easily producing and consuming messages, and a low-level API for controlling bytes on the wire when the high-level -API is insufficient. Usage examples for the high-level APIs are provided inline with their full documentation. - -To produce messages, use either the AsyncProducer or the SyncProducer. The AsyncProducer accepts messages on a channel -and produces them asynchronously in the background as efficiently as possible; it is preferred in most cases. -The SyncProducer provides a method which will block until Kafka acknowledges the message as produced. This can be -useful but comes with two caveats: it will generally be less efficient, and the actual durability guarantees -depend on the configured value of `Producer.RequiredAcks`. There are configurations where a message acknowledged by the -SyncProducer can still sometimes be lost. - -To consume messages, use the Consumer. Note that Sarama's Consumer implementation does not currently support automatic -consumer-group rebalancing and offset tracking. For Zookeeper-based tracking (Kafka 0.8.2 and earlier), the -https://github.com/wvanbergen/kafka library builds on Sarama to add this support. For Kafka-based tracking (Kafka 0.9 -and later), the https://github.com/bsm/sarama-cluster library builds on Sarama to add this support. - -For lower-level needs, the Broker and Request/Response objects permit precise control over each connection -and message sent on the wire; the Client provides higher-level metadata management that is shared between -the producers and the consumer. The Request/Response objects and properties are mostly undocumented, as they line up -exactly with the protocol fields documented by Kafka at -https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol - -Metrics are exposed through https://github.com/rcrowley/go-metrics library in a local registry. - -Broker related metrics: - - +----------------------------------------------+------------+---------------------------------------------------------------+ - | Name | Type | Description | - +----------------------------------------------+------------+---------------------------------------------------------------+ - | incoming-byte-rate | meter | Bytes/second read off all brokers | - | incoming-byte-rate-for-broker- | meter | Bytes/second read off a given broker | - | outgoing-byte-rate | meter | Bytes/second written off all brokers | - | outgoing-byte-rate-for-broker- | meter | Bytes/second written off a given broker | - | request-rate | meter | Requests/second sent to all brokers | - | request-rate-for-broker- | meter | Requests/second sent to a given broker | - | request-size | histogram | Distribution of the request size in bytes for all brokers | - | request-size-for-broker- | histogram | Distribution of the request size in bytes for a given broker | - | request-latency-in-ms | histogram | Distribution of the request latency in ms for all brokers | - | request-latency-in-ms-for-broker- | histogram | Distribution of the request latency in ms for a given broker | - | response-rate | meter | Responses/second received from all brokers | - | response-rate-for-broker- | meter | Responses/second received from a given broker | - | response-size | histogram | Distribution of the response size in bytes for all brokers | - | response-size-for-broker- | histogram | Distribution of the response size in bytes for a given broker | - +----------------------------------------------+------------+---------------------------------------------------------------+ - -Note that we do not gather specific metrics for seed brokers but they are part of the "all brokers" metrics. - -Producer related metrics: - - +-------------------------------------------+------------+--------------------------------------------------------------------------------------+ - | Name | Type | Description | - +-------------------------------------------+------------+--------------------------------------------------------------------------------------+ - | batch-size | histogram | Distribution of the number of bytes sent per partition per request for all topics | - | batch-size-for-topic- | histogram | Distribution of the number of bytes sent per partition per request for a given topic | - | record-send-rate | meter | Records/second sent to all topics | - | record-send-rate-for-topic- | meter | Records/second sent to a given topic | - | records-per-request | histogram | Distribution of the number of records sent per request for all topics | - | records-per-request-for-topic- | histogram | Distribution of the number of records sent per request for a given topic | - | compression-ratio | histogram | Distribution of the compression ratio times 100 of record batches for all topics | - | compression-ratio-for-topic- | histogram | Distribution of the compression ratio times 100 of record batches for a given topic | - +-------------------------------------------+------------+--------------------------------------------------------------------------------------+ - -*/ -package sarama - -import ( - "io/ioutil" - "log" -) - -// Logger is the instance of a StdLogger interface that Sarama writes connection -// management events to. By default it is set to discard all log messages via ioutil.Discard, -// but you can set it to redirect wherever you want. -var Logger StdLogger = log.New(ioutil.Discard, "[Sarama] ", log.LstdFlags) - -// StdLogger is used to log error messages. -type StdLogger interface { - Print(v ...interface{}) - Printf(format string, v ...interface{}) - Println(v ...interface{}) -} - -// PanicHandler is called for recovering from panics spawned internally to the library (and thus -// not recoverable by the caller's goroutine). Defaults to nil, which means panics are not recovered. -var PanicHandler func(interface{}) - -// MaxRequestSize is the maximum size (in bytes) of any request that Sarama will attempt to send. Trying -// to send a request larger than this will result in an PacketEncodingError. The default of 100 MiB is aligned -// with Kafka's default `socket.request.max.bytes`, which is the largest request the broker will attempt -// to process. -var MaxRequestSize int32 = 100 * 1024 * 1024 - -// MaxResponseSize is the maximum size (in bytes) of any response that Sarama will attempt to parse. If -// a broker returns a response message larger than this value, Sarama will return a PacketDecodingError to -// protect the client from running out of memory. Please note that brokers do not have any natural limit on -// the size of responses they send. In particular, they can send arbitrarily large fetch responses to consumers -// (see https://issues.apache.org/jira/browse/KAFKA-2063). -var MaxResponseSize int32 = 100 * 1024 * 1024 diff --git a/third/github.com/Shopify/sarama/sasl_handshake_request.go b/third/github.com/Shopify/sarama/sasl_handshake_request.go deleted file mode 100644 index fbbc8947b..000000000 --- a/third/github.com/Shopify/sarama/sasl_handshake_request.go +++ /dev/null @@ -1,33 +0,0 @@ -package sarama - -type SaslHandshakeRequest struct { - Mechanism string -} - -func (r *SaslHandshakeRequest) encode(pe packetEncoder) error { - if err := pe.putString(r.Mechanism); err != nil { - return err - } - - return nil -} - -func (r *SaslHandshakeRequest) decode(pd packetDecoder, version int16) (err error) { - if r.Mechanism, err = pd.getString(); err != nil { - return err - } - - return nil -} - -func (r *SaslHandshakeRequest) key() int16 { - return 17 -} - -func (r *SaslHandshakeRequest) version() int16 { - return 0 -} - -func (r *SaslHandshakeRequest) requiredVersion() KafkaVersion { - return V0_10_0_0 -} diff --git a/third/github.com/Shopify/sarama/sasl_handshake_request_test.go b/third/github.com/Shopify/sarama/sasl_handshake_request_test.go deleted file mode 100644 index 806e628fd..000000000 --- a/third/github.com/Shopify/sarama/sasl_handshake_request_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package sarama - -import "testing" - -var ( - baseSaslRequest = []byte{ - 0, 3, 'f', 'o', 'o', // Mechanism - } -) - -func TestSaslHandshakeRequest(t *testing.T) { - var request *SaslHandshakeRequest - - request = new(SaslHandshakeRequest) - request.Mechanism = "foo" - testRequest(t, "basic", request, baseSaslRequest) -} diff --git a/third/github.com/Shopify/sarama/sasl_handshake_response.go b/third/github.com/Shopify/sarama/sasl_handshake_response.go deleted file mode 100644 index ef290d4bc..000000000 --- a/third/github.com/Shopify/sarama/sasl_handshake_response.go +++ /dev/null @@ -1,38 +0,0 @@ -package sarama - -type SaslHandshakeResponse struct { - Err KError - EnabledMechanisms []string -} - -func (r *SaslHandshakeResponse) encode(pe packetEncoder) error { - pe.putInt16(int16(r.Err)) - return pe.putStringArray(r.EnabledMechanisms) -} - -func (r *SaslHandshakeResponse) decode(pd packetDecoder, version int16) error { - kerr, err := pd.getInt16() - if err != nil { - return err - } - - r.Err = KError(kerr) - - if r.EnabledMechanisms, err = pd.getStringArray(); err != nil { - return err - } - - return nil -} - -func (r *SaslHandshakeResponse) key() int16 { - return 17 -} - -func (r *SaslHandshakeResponse) version() int16 { - return 0 -} - -func (r *SaslHandshakeResponse) requiredVersion() KafkaVersion { - return V0_10_0_0 -} diff --git a/third/github.com/Shopify/sarama/sasl_handshake_response_test.go b/third/github.com/Shopify/sarama/sasl_handshake_response_test.go deleted file mode 100644 index 1fd4c79e0..000000000 --- a/third/github.com/Shopify/sarama/sasl_handshake_response_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package sarama - -import "testing" - -var ( - saslHandshakeResponse = []byte{ - 0x00, 0x00, - 0x00, 0x00, 0x00, 0x01, - 0x00, 0x03, 'f', 'o', 'o', - } -) - -func TestSaslHandshakeResponse(t *testing.T) { - var response *SaslHandshakeResponse - - response = new(SaslHandshakeResponse) - testVersionDecodable(t, "no error", response, saslHandshakeResponse, 0) - if response.Err != ErrNoError { - t.Error("Decoding error failed: no error expected but found", response.Err) - } - if response.EnabledMechanisms[0] != "foo" { - t.Error("Decoding error failed: expected 'foo' but found", response.EnabledMechanisms) - } -} diff --git a/third/github.com/Shopify/sarama/sync_group_request.go b/third/github.com/Shopify/sarama/sync_group_request.go deleted file mode 100644 index fe207080e..000000000 --- a/third/github.com/Shopify/sarama/sync_group_request.go +++ /dev/null @@ -1,100 +0,0 @@ -package sarama - -type SyncGroupRequest struct { - GroupId string - GenerationId int32 - MemberId string - GroupAssignments map[string][]byte -} - -func (r *SyncGroupRequest) encode(pe packetEncoder) error { - if err := pe.putString(r.GroupId); err != nil { - return err - } - - pe.putInt32(r.GenerationId) - - if err := pe.putString(r.MemberId); err != nil { - return err - } - - if err := pe.putArrayLength(len(r.GroupAssignments)); err != nil { - return err - } - for memberId, memberAssignment := range r.GroupAssignments { - if err := pe.putString(memberId); err != nil { - return err - } - if err := pe.putBytes(memberAssignment); err != nil { - return err - } - } - - return nil -} - -func (r *SyncGroupRequest) decode(pd packetDecoder, version int16) (err error) { - if r.GroupId, err = pd.getString(); err != nil { - return - } - if r.GenerationId, err = pd.getInt32(); err != nil { - return - } - if r.MemberId, err = pd.getString(); err != nil { - return - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - if n == 0 { - return nil - } - - r.GroupAssignments = make(map[string][]byte) - for i := 0; i < n; i++ { - memberId, err := pd.getString() - if err != nil { - return err - } - memberAssignment, err := pd.getBytes() - if err != nil { - return err - } - - r.GroupAssignments[memberId] = memberAssignment - } - - return nil -} - -func (r *SyncGroupRequest) key() int16 { - return 14 -} - -func (r *SyncGroupRequest) version() int16 { - return 0 -} - -func (r *SyncGroupRequest) requiredVersion() KafkaVersion { - return V0_9_0_0 -} - -func (r *SyncGroupRequest) AddGroupAssignment(memberId string, memberAssignment []byte) { - if r.GroupAssignments == nil { - r.GroupAssignments = make(map[string][]byte) - } - - r.GroupAssignments[memberId] = memberAssignment -} - -func (r *SyncGroupRequest) AddGroupAssignmentMember(memberId string, memberAssignment *ConsumerGroupMemberAssignment) error { - bin, err := encode(memberAssignment, nil) - if err != nil { - return err - } - - r.AddGroupAssignment(memberId, bin) - return nil -} diff --git a/third/github.com/Shopify/sarama/sync_group_request_test.go b/third/github.com/Shopify/sarama/sync_group_request_test.go deleted file mode 100644 index 3f537ef9f..000000000 --- a/third/github.com/Shopify/sarama/sync_group_request_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package sarama - -import "testing" - -var ( - emptySyncGroupRequest = []byte{ - 0, 3, 'f', 'o', 'o', // Group ID - 0x00, 0x01, 0x02, 0x03, // Generation ID - 0, 3, 'b', 'a', 'z', // Member ID - 0, 0, 0, 0, // no assignments - } - - populatedSyncGroupRequest = []byte{ - 0, 3, 'f', 'o', 'o', // Group ID - 0x00, 0x01, 0x02, 0x03, // Generation ID - 0, 3, 'b', 'a', 'z', // Member ID - 0, 0, 0, 1, // one assignment - 0, 3, 'b', 'a', 'z', // Member ID - 0, 0, 0, 3, 'f', 'o', 'o', // Member assignment - } -) - -func TestSyncGroupRequest(t *testing.T) { - var request *SyncGroupRequest - - request = new(SyncGroupRequest) - request.GroupId = "foo" - request.GenerationId = 66051 - request.MemberId = "baz" - testRequest(t, "empty", request, emptySyncGroupRequest) - - request = new(SyncGroupRequest) - request.GroupId = "foo" - request.GenerationId = 66051 - request.MemberId = "baz" - request.AddGroupAssignment("baz", []byte("foo")) - testRequest(t, "populated", request, populatedSyncGroupRequest) -} diff --git a/third/github.com/Shopify/sarama/sync_group_response.go b/third/github.com/Shopify/sarama/sync_group_response.go deleted file mode 100644 index 194b382b4..000000000 --- a/third/github.com/Shopify/sarama/sync_group_response.go +++ /dev/null @@ -1,41 +0,0 @@ -package sarama - -type SyncGroupResponse struct { - Err KError - MemberAssignment []byte -} - -func (r *SyncGroupResponse) GetMemberAssignment() (*ConsumerGroupMemberAssignment, error) { - assignment := new(ConsumerGroupMemberAssignment) - err := decode(r.MemberAssignment, assignment) - return assignment, err -} - -func (r *SyncGroupResponse) encode(pe packetEncoder) error { - pe.putInt16(int16(r.Err)) - return pe.putBytes(r.MemberAssignment) -} - -func (r *SyncGroupResponse) decode(pd packetDecoder, version int16) (err error) { - kerr, err := pd.getInt16() - if err != nil { - return err - } - - r.Err = KError(kerr) - - r.MemberAssignment, err = pd.getBytes() - return -} - -func (r *SyncGroupResponse) key() int16 { - return 14 -} - -func (r *SyncGroupResponse) version() int16 { - return 0 -} - -func (r *SyncGroupResponse) requiredVersion() KafkaVersion { - return V0_9_0_0 -} diff --git a/third/github.com/Shopify/sarama/sync_group_response_test.go b/third/github.com/Shopify/sarama/sync_group_response_test.go deleted file mode 100644 index 6fb708858..000000000 --- a/third/github.com/Shopify/sarama/sync_group_response_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package sarama - -import ( - "reflect" - "testing" -) - -var ( - syncGroupResponseNoError = []byte{ - 0x00, 0x00, // No error - 0, 0, 0, 3, 0x01, 0x02, 0x03, // Member assignment data - } - - syncGroupResponseWithError = []byte{ - 0, 27, // ErrRebalanceInProgress - 0, 0, 0, 0, // No member assignment data - } -) - -func TestSyncGroupResponse(t *testing.T) { - var response *SyncGroupResponse - - response = new(SyncGroupResponse) - testVersionDecodable(t, "no error", response, syncGroupResponseNoError, 0) - if response.Err != ErrNoError { - t.Error("Decoding Err failed: no error expected but found", response.Err) - } - if !reflect.DeepEqual(response.MemberAssignment, []byte{0x01, 0x02, 0x03}) { - t.Error("Decoding MemberAssignment failed, found:", response.MemberAssignment) - } - - response = new(SyncGroupResponse) - testVersionDecodable(t, "no error", response, syncGroupResponseWithError, 0) - if response.Err != ErrRebalanceInProgress { - t.Error("Decoding Err failed: ErrRebalanceInProgress expected but found", response.Err) - } - if !reflect.DeepEqual(response.MemberAssignment, []byte{}) { - t.Error("Decoding MemberAssignment failed, found:", response.MemberAssignment) - } -} diff --git a/third/github.com/Shopify/sarama/sync_producer.go b/third/github.com/Shopify/sarama/sync_producer.go deleted file mode 100644 index 021c5a010..000000000 --- a/third/github.com/Shopify/sarama/sync_producer.go +++ /dev/null @@ -1,149 +0,0 @@ -package sarama - -import "sync" - -// SyncProducer publishes Kafka messages, blocking until they have been acknowledged. It routes messages to the correct -// broker, refreshing metadata as appropriate, and parses responses for errors. You must call Close() on a producer -// to avoid leaks, it may not be garbage-collected automatically when it passes out of scope. -// -// The SyncProducer comes with two caveats: it will generally be less efficient than the AsyncProducer, and the actual -// durability guarantee provided when a message is acknowledged depend on the configured value of `Producer.RequiredAcks`. -// There are configurations where a message acknowledged by the SyncProducer can still sometimes be lost. -// -// For implementation reasons, the SyncProducer requires `Producer.Return.Errors` and `Producer.Return.Successes` to -// be set to true in its configuration. -type SyncProducer interface { - - // SendMessage produces a given message, and returns only when it either has - // succeeded or failed to produce. It will return the partition and the offset - // of the produced message, or an error if the message failed to produce. - SendMessage(msg *ProducerMessage) (partition int32, offset int64, err error) - - // SendMessages produces a given set of messages, and returns only when all - // messages in the set have either succeeded or failed. Note that messages - // can succeed and fail individually; if some succeed and some fail, - // SendMessages will return an error. - SendMessages(msgs []*ProducerMessage) error - - // Close shuts down the producer and waits for any buffered messages to be - // flushed. You must call this function before a producer object passes out of - // scope, as it may otherwise leak memory. You must call this before calling - // Close on the underlying client. - Close() error -} - -type syncProducer struct { - producer *asyncProducer - wg sync.WaitGroup -} - -// NewSyncProducer creates a new SyncProducer using the given broker addresses and configuration. -func NewSyncProducer(addrs []string, config *Config) (SyncProducer, error) { - if config == nil { - config = NewConfig() - config.Producer.Return.Successes = true - } - - if err := verifyProducerConfig(config); err != nil { - return nil, err - } - - p, err := NewAsyncProducer(addrs, config) - if err != nil { - return nil, err - } - return newSyncProducerFromAsyncProducer(p.(*asyncProducer)), nil -} - -// NewSyncProducerFromClient creates a new SyncProducer using the given client. It is still -// necessary to call Close() on the underlying client when shutting down this producer. -func NewSyncProducerFromClient(client Client) (SyncProducer, error) { - if err := verifyProducerConfig(client.Config()); err != nil { - return nil, err - } - - p, err := NewAsyncProducerFromClient(client) - if err != nil { - return nil, err - } - return newSyncProducerFromAsyncProducer(p.(*asyncProducer)), nil -} - -func newSyncProducerFromAsyncProducer(p *asyncProducer) *syncProducer { - sp := &syncProducer{producer: p} - - sp.wg.Add(2) - go withRecover(sp.handleSuccesses) - go withRecover(sp.handleErrors) - - return sp -} - -func verifyProducerConfig(config *Config) error { - if !config.Producer.Return.Errors { - return ConfigurationError("Producer.Return.Errors must be true to be used in a SyncProducer") - } - if !config.Producer.Return.Successes { - return ConfigurationError("Producer.Return.Successes must be true to be used in a SyncProducer") - } - return nil -} - -func (sp *syncProducer) SendMessage(msg *ProducerMessage) (partition int32, offset int64, err error) { - expectation := make(chan *ProducerError, 1) - msg.expectation = expectation - sp.producer.Input() <- msg - - if err := <-expectation; err != nil { - return -1, -1, err.Err - } - - return msg.Partition, msg.Offset, nil -} - -func (sp *syncProducer) SendMessages(msgs []*ProducerMessage) error { - expectations := make(chan chan *ProducerError, len(msgs)) - go func() { - for _, msg := range msgs { - expectation := make(chan *ProducerError, 1) - msg.expectation = expectation - sp.producer.Input() <- msg - expectations <- expectation - } - close(expectations) - }() - - var errors ProducerErrors - for expectation := range expectations { - if err := <-expectation; err != nil { - errors = append(errors, err) - } - } - - if len(errors) > 0 { - return errors - } - return nil -} - -func (sp *syncProducer) handleSuccesses() { - defer sp.wg.Done() - for msg := range sp.producer.Successes() { - expectation := msg.expectation - expectation <- nil - } -} - -func (sp *syncProducer) handleErrors() { - defer sp.wg.Done() - for err := range sp.producer.Errors() { - expectation := err.Msg.expectation - expectation <- err - } -} - -func (sp *syncProducer) Close() error { - sp.producer.AsyncClose() - sp.wg.Wait() - return nil -} diff --git a/third/github.com/Shopify/sarama/sync_producer_test.go b/third/github.com/Shopify/sarama/sync_producer_test.go deleted file mode 100644 index cb97548db..000000000 --- a/third/github.com/Shopify/sarama/sync_producer_test.go +++ /dev/null @@ -1,199 +0,0 @@ -package sarama - -import ( - "log" - "sync" - "testing" -) - -func TestSyncProducer(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader := NewMockBroker(t, 2) - - metadataResponse := new(MetadataResponse) - metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) - metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataResponse) - - prodSuccess := new(ProduceResponse) - prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError) - for i := 0; i < 10; i++ { - leader.Returns(prodSuccess) - } - - producer, err := NewSyncProducer([]string{seedBroker.Addr()}, nil) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < 10; i++ { - msg := &ProducerMessage{ - Topic: "my_topic", - Value: StringEncoder(TestMessage), - Metadata: "test", - } - - partition, offset, err := producer.SendMessage(msg) - - if partition != 0 || msg.Partition != partition { - t.Error("Unexpected partition") - } - if offset != 0 || msg.Offset != offset { - t.Error("Unexpected offset") - } - if str, ok := msg.Metadata.(string); !ok || str != "test" { - t.Error("Unexpected metadata") - } - if err != nil { - t.Error(err) - } - } - - safeClose(t, producer) - leader.Close() - seedBroker.Close() -} - -func TestSyncProducerBatch(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader := NewMockBroker(t, 2) - - metadataResponse := new(MetadataResponse) - metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) - metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataResponse) - - prodSuccess := new(ProduceResponse) - prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError) - leader.Returns(prodSuccess) - - config := NewConfig() - config.Producer.Flush.Messages = 3 - config.Producer.Return.Successes = true - producer, err := NewSyncProducer([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - err = producer.SendMessages([]*ProducerMessage{ - { - Topic: "my_topic", - Value: StringEncoder(TestMessage), - Metadata: "test", - }, - { - Topic: "my_topic", - Value: StringEncoder(TestMessage), - Metadata: "test", - }, - { - Topic: "my_topic", - Value: StringEncoder(TestMessage), - Metadata: "test", - }, - }) - - if err != nil { - t.Error(err) - } - - safeClose(t, producer) - leader.Close() - seedBroker.Close() -} - -func TestConcurrentSyncProducer(t *testing.T) { - seedBroker := NewMockBroker(t, 1) - leader := NewMockBroker(t, 2) - - metadataResponse := new(MetadataResponse) - metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) - metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError) - seedBroker.Returns(metadataResponse) - - prodSuccess := new(ProduceResponse) - prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError) - leader.Returns(prodSuccess) - - config := NewConfig() - config.Producer.Flush.Messages = 100 - config.Producer.Return.Successes = true - producer, err := NewSyncProducer([]string{seedBroker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - wg := sync.WaitGroup{} - - for i := 0; i < 100; i++ { - wg.Add(1) - go func() { - msg := &ProducerMessage{Topic: "my_topic", Value: StringEncoder(TestMessage)} - partition, _, err := producer.SendMessage(msg) - if partition != 0 { - t.Error("Unexpected partition") - } - if err != nil { - t.Error(err) - } - wg.Done() - }() - } - wg.Wait() - - safeClose(t, producer) - leader.Close() - seedBroker.Close() -} - -func TestSyncProducerToNonExistingTopic(t *testing.T) { - broker := NewMockBroker(t, 1) - - metadataResponse := new(MetadataResponse) - metadataResponse.AddBroker(broker.Addr(), broker.BrokerID()) - metadataResponse.AddTopicPartition("my_topic", 0, broker.BrokerID(), nil, nil, ErrNoError) - broker.Returns(metadataResponse) - - config := NewConfig() - config.Metadata.Retry.Max = 0 - config.Producer.Retry.Max = 0 - config.Producer.Return.Successes = true - - producer, err := NewSyncProducer([]string{broker.Addr()}, config) - if err != nil { - t.Fatal(err) - } - - metadataResponse = new(MetadataResponse) - metadataResponse.AddTopic("unknown", ErrUnknownTopicOrPartition) - broker.Returns(metadataResponse) - - _, _, err = producer.SendMessage(&ProducerMessage{Topic: "unknown"}) - if err != ErrUnknownTopicOrPartition { - t.Error("Uxpected ErrUnknownTopicOrPartition, found:", err) - } - - safeClose(t, producer) - broker.Close() -} - -// This example shows the basic usage pattern of the SyncProducer. -func ExampleSyncProducer() { - producer, err := NewSyncProducer([]string{"localhost:9092"}, nil) - if err != nil { - log.Fatalln(err) - } - defer func() { - if err := producer.Close(); err != nil { - log.Fatalln(err) - } - }() - - msg := &ProducerMessage{Topic: "my_topic", Value: StringEncoder("testing 123")} - partition, offset, err := producer.SendMessage(msg) - if err != nil { - log.Printf("FAILED to send message: %s\n", err) - } else { - log.Printf("> message sent to partition %d at offset %d\n", partition, offset) - } -} diff --git a/third/github.com/Shopify/sarama/timestamp.go b/third/github.com/Shopify/sarama/timestamp.go deleted file mode 100644 index 372278d0b..000000000 --- a/third/github.com/Shopify/sarama/timestamp.go +++ /dev/null @@ -1,40 +0,0 @@ -package sarama - -import ( - "fmt" - "time" -) - -type Timestamp struct { - *time.Time -} - -func (t Timestamp) encode(pe packetEncoder) error { - timestamp := int64(-1) - - if !t.Before(time.Unix(0, 0)) { - timestamp = t.UnixNano() / int64(time.Millisecond) - } else if !t.IsZero() { - return PacketEncodingError{fmt.Sprintf("invalid timestamp (%v)", t)} - } - - pe.putInt64(timestamp) - return nil -} - -func (t Timestamp) decode(pd packetDecoder) error { - millis, err := pd.getInt64() - if err != nil { - return err - } - - // negative timestamps are invalid, in these cases we should return - // a zero time - timestamp := time.Time{} - if millis >= 0 { - timestamp = time.Unix(millis/1000, (millis%1000)*int64(time.Millisecond)) - } - - *t.Time = timestamp - return nil -} diff --git a/third/github.com/Shopify/sarama/tools/README.md b/third/github.com/Shopify/sarama/tools/README.md deleted file mode 100644 index 3464c4ad8..000000000 --- a/third/github.com/Shopify/sarama/tools/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Sarama tools - -This folder contains applications that are useful for exploration of your Kafka cluster, or instrumentation. -Some of these tools mirror tools that ship with Kafka, but these tools won't require installing the JVM to function. - -- [kafka-console-producer](./kafka-console-producer): a command line tool to produce a single message to your Kafka custer. -- [kafka-console-partitionconsumer](./kafka-console-partitionconsumer): (deprecated) a command line tool to consume a single partition of a topic on your Kafka cluster. -- [kafka-console-consumer](./kafka-console-consumer): a command line tool to consume arbitrary partitions of a topic on your Kafka cluster. - -To install all tools, run `go get github.com/Shopify/sarama/tools/...` diff --git a/third/github.com/Shopify/sarama/tools/kafka-console-consumer/.gitignore b/third/github.com/Shopify/sarama/tools/kafka-console-consumer/.gitignore deleted file mode 100644 index 67da9dfa9..000000000 --- a/third/github.com/Shopify/sarama/tools/kafka-console-consumer/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -kafka-console-consumer -kafka-console-consumer.test diff --git a/third/github.com/Shopify/sarama/tools/kafka-console-consumer/README.md b/third/github.com/Shopify/sarama/tools/kafka-console-consumer/README.md deleted file mode 100644 index 4e77f0b70..000000000 --- a/third/github.com/Shopify/sarama/tools/kafka-console-consumer/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# kafka-console-consumer - -A simple command line tool to consume partitions of a topic and print the -messages on the standard output. - -### Installation - - go get github.com/Shopify/sarama/tools/kafka-console-consumer - -### Usage - - # Minimum invocation - kafka-console-consumer -topic=test -brokers=kafka1:9092 - - # It will pick up a KAFKA_PEERS environment variable - export KAFKA_PEERS=kafka1:9092,kafka2:9092,kafka3:9092 - kafka-console-consumer -topic=test - - # You can specify the offset you want to start at. It can be either - # `oldest`, `newest`. The default is `newest`. - kafka-console-consumer -topic=test -offset=oldest - kafka-console-consumer -topic=test -offset=newest - - # You can specify the partition(s) you want to consume as a comma-separated - # list. The default is `all`. - kafka-console-consumer -topic=test -partitions=1,2,3 - - # Display all command line options - kafka-console-consumer -help diff --git a/third/github.com/Shopify/sarama/tools/kafka-console-consumer/kafka-console-consumer.go b/third/github.com/Shopify/sarama/tools/kafka-console-consumer/kafka-console-consumer.go deleted file mode 100644 index 4e9f2eaff..000000000 --- a/third/github.com/Shopify/sarama/tools/kafka-console-consumer/kafka-console-consumer.go +++ /dev/null @@ -1,145 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "log" - "os" - "os/signal" - "strconv" - "strings" - "sync" - - "github.com/gogf/gf/third/github.com/Shopify/sarama" -) - -var ( - brokerList = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The comma separated list of brokers in the Kafka cluster") - topic = flag.String("topic", "", "REQUIRED: the topic to consume") - partitions = flag.String("partitions", "all", "The partitions to consume, can be 'all' or comma-separated numbers") - offset = flag.String("offset", "newest", "The offset to start with. Can be `oldest`, `newest`") - verbose = flag.Bool("verbose", false, "Whether to turn on sarama logging") - bufferSize = flag.Int("buffer-size", 256, "The buffer size of the message channel.") - - logger = log.New(os.Stderr, "", log.LstdFlags) -) - -func main() { - flag.Parse() - - if *brokerList == "" { - printUsageErrorAndExit("You have to provide -brokers as a comma-separated list, or set the KAFKA_PEERS environment variable.") - } - - if *topic == "" { - printUsageErrorAndExit("-topic is required") - } - - if *verbose { - sarama.Logger = logger - } - - var initialOffset int64 - switch *offset { - case "oldest": - initialOffset = sarama.OffsetOldest - case "newest": - initialOffset = sarama.OffsetNewest - default: - printUsageErrorAndExit("-offset should be `oldest` or `newest`") - } - - c, err := sarama.NewConsumer(strings.Split(*brokerList, ","), nil) - if err != nil { - printErrorAndExit(69, "Failed to start consumer: %s", err) - } - - partitionList, err := getPartitions(c) - if err != nil { - printErrorAndExit(69, "Failed to get the list of partitions: %s", err) - } - - var ( - messages = make(chan *sarama.ConsumerMessage, *bufferSize) - closing = make(chan struct{}) - wg sync.WaitGroup - ) - - go func() { - signals := make(chan os.Signal, 1) - signal.Notify(signals, os.Kill, os.Interrupt) - <-signals - logger.Println("Initiating shutdown of consumer...") - close(closing) - }() - - for _, partition := range partitionList { - pc, err := c.ConsumePartition(*topic, partition, initialOffset) - if err != nil { - printErrorAndExit(69, "Failed to start consumer for partition %d: %s", partition, err) - } - - go func(pc sarama.PartitionConsumer) { - <-closing - pc.AsyncClose() - }(pc) - - wg.Add(1) - go func(pc sarama.PartitionConsumer) { - defer wg.Done() - for message := range pc.Messages() { - messages <- message - } - }(pc) - } - - go func() { - for msg := range messages { - fmt.Printf("Partition:\t%d\n", msg.Partition) - fmt.Printf("Offset:\t%d\n", msg.Offset) - fmt.Printf("Key:\t%s\n", string(msg.Key)) - fmt.Printf("Value:\t%s\n", string(msg.Value)) - fmt.Println() - } - }() - - wg.Wait() - logger.Println("Done consuming topic", *topic) - close(messages) - - if err := c.Close(); err != nil { - logger.Println("Failed to close consumer: ", err) - } -} - -func getPartitions(c sarama.Consumer) ([]int32, error) { - if *partitions == "all" { - return c.Partitions(*topic) - } - - tmp := strings.Split(*partitions, ",") - var pList []int32 - for i := range tmp { - val, err := strconv.ParseInt(tmp[i], 10, 32) - if err != nil { - return nil, err - } - pList = append(pList, int32(val)) - } - - return pList, nil -} - -func printErrorAndExit(code int, format string, values ...interface{}) { - fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...)) - fmt.Fprintln(os.Stderr) - os.Exit(code) -} - -func printUsageErrorAndExit(format string, values ...interface{}) { - fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...)) - fmt.Fprintln(os.Stderr) - fmt.Fprintln(os.Stderr, "Available command line options:") - flag.PrintDefaults() - os.Exit(64) -} diff --git a/third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/.gitignore b/third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/.gitignore deleted file mode 100644 index 5837fe8ca..000000000 --- a/third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -kafka-console-partitionconsumer -kafka-console-partitionconsumer.test diff --git a/third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/README.md b/third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/README.md deleted file mode 100644 index 646dd5f5c..000000000 --- a/third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# kafka-console-partitionconsumer - -NOTE: this tool is deprecated in favour of the more general and more powerful -`kafka-console-consumer`. - -A simple command line tool to consume a partition of a topic and print the messages -on the standard output. - -### Installation - - go get github.com/Shopify/sarama/tools/kafka-console-partitionconsumer - -### Usage - - # Minimum invocation - kafka-console-partitionconsumer -topic=test -partition=4 -brokers=kafka1:9092 - - # It will pick up a KAFKA_PEERS environment variable - export KAFKA_PEERS=kafka1:9092,kafka2:9092,kafka3:9092 - kafka-console-partitionconsumer -topic=test -partition=4 - - # You can specify the offset you want to start at. It can be either - # `oldest`, `newest`, or a specific offset number - kafka-console-partitionconsumer -topic=test -partition=3 -offset=oldest - kafka-console-partitionconsumer -topic=test -partition=2 -offset=1337 - - # Display all command line options - kafka-console-partitionconsumer -help diff --git a/third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/kafka-console-partitionconsumer.go b/third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/kafka-console-partitionconsumer.go deleted file mode 100644 index 0de9401c7..000000000 --- a/third/github.com/Shopify/sarama/tools/kafka-console-partitionconsumer/kafka-console-partitionconsumer.go +++ /dev/null @@ -1,102 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "log" - "os" - "os/signal" - "strconv" - "strings" - - "github.com/gogf/gf/third/github.com/Shopify/sarama" -) - -var ( - brokerList = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The comma separated list of brokers in the Kafka cluster") - topic = flag.String("topic", "", "REQUIRED: the topic to consume") - partition = flag.Int("partition", -1, "REQUIRED: the partition to consume") - offset = flag.String("offset", "newest", "The offset to start with. Can be `oldest`, `newest`, or an actual offset") - verbose = flag.Bool("verbose", false, "Whether to turn on sarama logging") - - logger = log.New(os.Stderr, "", log.LstdFlags) -) - -func main() { - flag.Parse() - - if *brokerList == "" { - printUsageErrorAndExit("You have to provide -brokers as a comma-separated list, or set the KAFKA_PEERS environment variable.") - } - - if *topic == "" { - printUsageErrorAndExit("-topic is required") - } - - if *partition == -1 { - printUsageErrorAndExit("-partition is required") - } - - if *verbose { - sarama.Logger = logger - } - - var ( - initialOffset int64 - offsetError error - ) - switch *offset { - case "oldest": - initialOffset = sarama.OffsetOldest - case "newest": - initialOffset = sarama.OffsetNewest - default: - initialOffset, offsetError = strconv.ParseInt(*offset, 10, 64) - } - - if offsetError != nil { - printUsageErrorAndExit("Invalid initial offset: %s", *offset) - } - - c, err := sarama.NewConsumer(strings.Split(*brokerList, ","), nil) - if err != nil { - printErrorAndExit(69, "Failed to start consumer: %s", err) - } - - pc, err := c.ConsumePartition(*topic, int32(*partition), initialOffset) - if err != nil { - printErrorAndExit(69, "Failed to start partition consumer: %s", err) - } - - go func() { - signals := make(chan os.Signal, 1) - signal.Notify(signals, os.Kill, os.Interrupt) - <-signals - pc.AsyncClose() - }() - - for msg := range pc.Messages() { - fmt.Printf("Offset:\t%d\n", msg.Offset) - fmt.Printf("Key:\t%s\n", string(msg.Key)) - fmt.Printf("Value:\t%s\n", string(msg.Value)) - fmt.Println() - } - - if err := c.Close(); err != nil { - logger.Println("Failed to close consumer: ", err) - } -} - -func printErrorAndExit(code int, format string, values ...interface{}) { - fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...)) - fmt.Fprintln(os.Stderr) - os.Exit(code) -} - -func printUsageErrorAndExit(format string, values ...interface{}) { - fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...)) - fmt.Fprintln(os.Stderr) - fmt.Fprintln(os.Stderr, "Available command line options:") - flag.PrintDefaults() - os.Exit(64) -} diff --git a/third/github.com/Shopify/sarama/tools/kafka-console-producer/.gitignore b/third/github.com/Shopify/sarama/tools/kafka-console-producer/.gitignore deleted file mode 100644 index 2b9e563a1..000000000 --- a/third/github.com/Shopify/sarama/tools/kafka-console-producer/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -kafka-console-producer -kafka-console-producer.test diff --git a/third/github.com/Shopify/sarama/tools/kafka-console-producer/README.md b/third/github.com/Shopify/sarama/tools/kafka-console-producer/README.md deleted file mode 100644 index 6b3a65f21..000000000 --- a/third/github.com/Shopify/sarama/tools/kafka-console-producer/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# kafka-console-producer - -A simple command line tool to produce a single message to Kafka. - -### Installation - - go get github.com/Shopify/sarama/tools/kafka-console-producer - - -### Usage - - # Minimum invocation - kafka-console-producer -topic=test -value=value -brokers=kafka1:9092 - - # It will pick up a KAFKA_PEERS environment variable - export KAFKA_PEERS=kafka1:9092,kafka2:9092,kafka3:9092 - kafka-console-producer -topic=test -value=value - - # It will read the value from stdin by using pipes - echo "hello world" | kafka-console-producer -topic=test - - # Specify a key: - echo "hello world" | kafka-console-producer -topic=test -key=key - - # Partitioning: by default, kafka-console-producer will partition as follows: - # - manual partitioning if a -partition is provided - # - hash partitioning by key if a -key is provided - # - random partioning otherwise. - # - # You can override this using the -partitioner argument: - echo "hello world" | kafka-console-producer -topic=test -key=key -partitioner=random - - # Display all command line options - kafka-console-producer -help diff --git a/third/github.com/Shopify/sarama/tools/kafka-console-producer/kafka-console-producer.go b/third/github.com/Shopify/sarama/tools/kafka-console-producer/kafka-console-producer.go deleted file mode 100644 index 3bb951f16..000000000 --- a/third/github.com/Shopify/sarama/tools/kafka-console-producer/kafka-console-producer.go +++ /dev/null @@ -1,124 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "io/ioutil" - "log" - "os" - "strings" - - "github.com/gogf/gf/third/github.com/Shopify/sarama" - "github.com/gogf/gf/third/github.com/rcrowley/go-metrics" -) - -var ( - brokerList = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The comma separated list of brokers in the Kafka cluster. You can also set the KAFKA_PEERS environment variable") - topic = flag.String("topic", "", "REQUIRED: the topic to produce to") - key = flag.String("key", "", "The key of the message to produce. Can be empty.") - value = flag.String("value", "", "REQUIRED: the value of the message to produce. You can also provide the value on stdin.") - partitioner = flag.String("partitioner", "", "The partitioning scheme to use. Can be `hash`, `manual`, or `random`") - partition = flag.Int("partition", -1, "The partition to produce to.") - verbose = flag.Bool("verbose", false, "Turn on sarama logging to stderr") - showMetrics = flag.Bool("metrics", false, "Output metrics on successful publish to stderr") - silent = flag.Bool("silent", false, "Turn off printing the message's topic, partition, and offset to stdout") - - logger = log.New(os.Stderr, "", log.LstdFlags) -) - -func main() { - flag.Parse() - - if *brokerList == "" { - printUsageErrorAndExit("no -brokers specified. Alternatively, set the KAFKA_PEERS environment variable") - } - - if *topic == "" { - printUsageErrorAndExit("no -topic specified") - } - - if *verbose { - sarama.Logger = logger - } - - config := sarama.NewConfig() - config.Producer.RequiredAcks = sarama.WaitForAll - config.Producer.Return.Successes = true - - switch *partitioner { - case "": - if *partition >= 0 { - config.Producer.Partitioner = sarama.NewManualPartitioner - } else { - config.Producer.Partitioner = sarama.NewHashPartitioner - } - case "hash": - config.Producer.Partitioner = sarama.NewHashPartitioner - case "random": - config.Producer.Partitioner = sarama.NewRandomPartitioner - case "manual": - config.Producer.Partitioner = sarama.NewManualPartitioner - if *partition == -1 { - printUsageErrorAndExit("-partition is required when partitioning manually") - } - default: - printUsageErrorAndExit(fmt.Sprintf("Partitioner %s not supported.", *partitioner)) - } - - message := &sarama.ProducerMessage{Topic: *topic, Partition: int32(*partition)} - - if *key != "" { - message.Key = sarama.StringEncoder(*key) - } - - if *value != "" { - message.Value = sarama.StringEncoder(*value) - } else if stdinAvailable() { - bytes, err := ioutil.ReadAll(os.Stdin) - if err != nil { - printErrorAndExit(66, "Failed to read data from the standard input: %s", err) - } - message.Value = sarama.ByteEncoder(bytes) - } else { - printUsageErrorAndExit("-value is required, or you have to provide the value on stdin") - } - - producer, err := sarama.NewSyncProducer(strings.Split(*brokerList, ","), config) - if err != nil { - printErrorAndExit(69, "Failed to open Kafka producer: %s", err) - } - defer func() { - if err := producer.Close(); err != nil { - logger.Println("Failed to close Kafka producer cleanly:", err) - } - }() - - partition, offset, err := producer.SendMessage(message) - if err != nil { - printErrorAndExit(69, "Failed to produce message: %s", err) - } else if !*silent { - fmt.Printf("topic=%s\tpartition=%d\toffset=%d\n", *topic, partition, offset) - } - if *showMetrics { - metrics.WriteOnce(config.MetricRegistry, os.Stderr) - } -} - -func printErrorAndExit(code int, format string, values ...interface{}) { - fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...)) - fmt.Fprintln(os.Stderr) - os.Exit(code) -} - -func printUsageErrorAndExit(message string) { - fmt.Fprintln(os.Stderr, "ERROR:", message) - fmt.Fprintln(os.Stderr) - fmt.Fprintln(os.Stderr, "Available command line options:") - flag.PrintDefaults() - os.Exit(64) -} - -func stdinAvailable() bool { - stat, _ := os.Stdin.Stat() - return (stat.Mode() & os.ModeCharDevice) == 0 -} diff --git a/third/github.com/Shopify/sarama/txn_offset_commit_request.go b/third/github.com/Shopify/sarama/txn_offset_commit_request.go deleted file mode 100644 index 71e95b814..000000000 --- a/third/github.com/Shopify/sarama/txn_offset_commit_request.go +++ /dev/null @@ -1,126 +0,0 @@ -package sarama - -type TxnOffsetCommitRequest struct { - TransactionalID string - GroupID string - ProducerID int64 - ProducerEpoch int16 - Topics map[string][]*PartitionOffsetMetadata -} - -func (t *TxnOffsetCommitRequest) encode(pe packetEncoder) error { - if err := pe.putString(t.TransactionalID); err != nil { - return err - } - if err := pe.putString(t.GroupID); err != nil { - return err - } - pe.putInt64(t.ProducerID) - pe.putInt16(t.ProducerEpoch) - - if err := pe.putArrayLength(len(t.Topics)); err != nil { - return err - } - for topic, partitions := range t.Topics { - if err := pe.putString(topic); err != nil { - return err - } - if err := pe.putArrayLength(len(partitions)); err != nil { - return err - } - for _, partition := range partitions { - if err := partition.encode(pe); err != nil { - return err - } - } - } - - return nil -} - -func (t *TxnOffsetCommitRequest) decode(pd packetDecoder, version int16) (err error) { - if t.TransactionalID, err = pd.getString(); err != nil { - return err - } - if t.GroupID, err = pd.getString(); err != nil { - return err - } - if t.ProducerID, err = pd.getInt64(); err != nil { - return err - } - if t.ProducerEpoch, err = pd.getInt16(); err != nil { - return err - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - t.Topics = make(map[string][]*PartitionOffsetMetadata) - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - - m, err := pd.getArrayLength() - if err != nil { - return err - } - - t.Topics[topic] = make([]*PartitionOffsetMetadata, m) - - for j := 0; j < m; j++ { - partitionOffsetMetadata := new(PartitionOffsetMetadata) - if err := partitionOffsetMetadata.decode(pd, version); err != nil { - return err - } - t.Topics[topic][j] = partitionOffsetMetadata - } - } - - return nil -} - -func (a *TxnOffsetCommitRequest) key() int16 { - return 28 -} - -func (a *TxnOffsetCommitRequest) version() int16 { - return 0 -} - -func (a *TxnOffsetCommitRequest) requiredVersion() KafkaVersion { - return V0_11_0_0 -} - -type PartitionOffsetMetadata struct { - Partition int32 - Offset int64 - Metadata *string -} - -func (p *PartitionOffsetMetadata) encode(pe packetEncoder) error { - pe.putInt32(p.Partition) - pe.putInt64(p.Offset) - if err := pe.putNullableString(p.Metadata); err != nil { - return err - } - - return nil -} - -func (p *PartitionOffsetMetadata) decode(pd packetDecoder, version int16) (err error) { - if p.Partition, err = pd.getInt32(); err != nil { - return err - } - if p.Offset, err = pd.getInt64(); err != nil { - return err - } - if p.Metadata, err = pd.getNullableString(); err != nil { - return err - } - - return nil -} diff --git a/third/github.com/Shopify/sarama/txn_offset_commit_request_test.go b/third/github.com/Shopify/sarama/txn_offset_commit_request_test.go deleted file mode 100644 index fe3bba548..000000000 --- a/third/github.com/Shopify/sarama/txn_offset_commit_request_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package sarama - -import "testing" - -var ( - txnOffsetCommitRequest = []byte{ - 0, 3, 't', 'x', 'n', - 0, 7, 'g', 'r', 'o', 'u', 'p', 'i', 'd', - 0, 0, 0, 0, 0, 0, 31, 64, // producer ID - 0, 1, // producer epoch - 0, 0, 0, 1, // 1 topic - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 0, 0, 1, // 1 partition - 0, 0, 0, 2, // partition no 2 - 0, 0, 0, 0, 0, 0, 0, 123, - 255, 255, // no meta data - } -) - -func TestTxnOffsetCommitRequest(t *testing.T) { - req := &TxnOffsetCommitRequest{ - TransactionalID: "txn", - GroupID: "groupid", - ProducerID: 8000, - ProducerEpoch: 1, - Topics: map[string][]*PartitionOffsetMetadata{ - "topic": []*PartitionOffsetMetadata{{ - Offset: 123, - Partition: 2, - }}, - }, - } - - testRequest(t, "", req, txnOffsetCommitRequest) -} diff --git a/third/github.com/Shopify/sarama/txn_offset_commit_response.go b/third/github.com/Shopify/sarama/txn_offset_commit_response.go deleted file mode 100644 index 6c980f406..000000000 --- a/third/github.com/Shopify/sarama/txn_offset_commit_response.go +++ /dev/null @@ -1,83 +0,0 @@ -package sarama - -import ( - "time" -) - -type TxnOffsetCommitResponse struct { - ThrottleTime time.Duration - Topics map[string][]*PartitionError -} - -func (t *TxnOffsetCommitResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(t.ThrottleTime / time.Millisecond)) - if err := pe.putArrayLength(len(t.Topics)); err != nil { - return err - } - - for topic, e := range t.Topics { - if err := pe.putString(topic); err != nil { - return err - } - if err := pe.putArrayLength(len(e)); err != nil { - return err - } - for _, partitionError := range e { - if err := partitionError.encode(pe); err != nil { - return err - } - } - } - - return nil -} - -func (t *TxnOffsetCommitResponse) decode(pd packetDecoder, version int16) (err error) { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - t.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - t.Topics = make(map[string][]*PartitionError) - - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - - m, err := pd.getArrayLength() - if err != nil { - return err - } - - t.Topics[topic] = make([]*PartitionError, m) - - for j := 0; j < m; j++ { - t.Topics[topic][j] = new(PartitionError) - if err := t.Topics[topic][j].decode(pd, version); err != nil { - return err - } - } - } - - return nil -} - -func (a *TxnOffsetCommitResponse) key() int16 { - return 28 -} - -func (a *TxnOffsetCommitResponse) version() int16 { - return 0 -} - -func (a *TxnOffsetCommitResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/third/github.com/Shopify/sarama/txn_offset_commit_response_test.go b/third/github.com/Shopify/sarama/txn_offset_commit_response_test.go deleted file mode 100644 index b275265c6..000000000 --- a/third/github.com/Shopify/sarama/txn_offset_commit_response_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package sarama - -import ( - "testing" - "time" -) - -var ( - txnOffsetCommitResponse = []byte{ - 0, 0, 0, 100, - 0, 0, 0, 1, // 1 topic - 0, 5, 't', 'o', 'p', 'i', 'c', - 0, 0, 0, 1, // 1 partition response - 0, 0, 0, 2, // partition number 2 - 0, 47, // err - } -) - -func TestTxnOffsetCommitResponse(t *testing.T) { - resp := &TxnOffsetCommitResponse{ - ThrottleTime: 100 * time.Millisecond, - Topics: map[string][]*PartitionError{ - "topic": []*PartitionError{{ - Partition: 2, - Err: ErrInvalidProducerEpoch, - }}, - }, - } - - testResponse(t, "", resp, txnOffsetCommitResponse) -} diff --git a/third/github.com/Shopify/sarama/utils.go b/third/github.com/Shopify/sarama/utils.go deleted file mode 100644 index 7dcbf034f..000000000 --- a/third/github.com/Shopify/sarama/utils.go +++ /dev/null @@ -1,220 +0,0 @@ -package sarama - -import ( - "bufio" - "fmt" - "net" - "regexp" -) - -type none struct{} - -// make []int32 sortable so we can sort partition numbers -type int32Slice []int32 - -func (slice int32Slice) Len() int { - return len(slice) -} - -func (slice int32Slice) Less(i, j int) bool { - return slice[i] < slice[j] -} - -func (slice int32Slice) Swap(i, j int) { - slice[i], slice[j] = slice[j], slice[i] -} - -func dupInt32Slice(input []int32) []int32 { - ret := make([]int32, 0, len(input)) - for _, val := range input { - ret = append(ret, val) - } - return ret -} - -func withRecover(fn func()) { - defer func() { - handler := PanicHandler - if handler != nil { - if err := recover(); err != nil { - handler(err) - } - } - }() - - fn() -} - -func safeAsyncClose(b *Broker) { - tmp := b // local var prevents clobbering in goroutine - go withRecover(func() { - if connected, _ := tmp.Connected(); connected { - if err := tmp.Close(); err != nil { - Logger.Println("Error closing broker", tmp.ID(), ":", err) - } - } - }) -} - -// Encoder is a simple interface for any type that can be encoded as an array of bytes -// in order to be sent as the key or value of a Kafka message. Length() is provided as an -// optimization, and must return the same as len() on the result of Encode(). -type Encoder interface { - Encode() ([]byte, error) - Length() int -} - -// make strings and byte slices encodable for convenience so they can be used as keys -// and/or values in kafka messages - -// StringEncoder implements the Encoder interface for Go strings so that they can be used -// as the Key or Value in a ProducerMessage. -type StringEncoder string - -func (s StringEncoder) Encode() ([]byte, error) { - return []byte(s), nil -} - -func (s StringEncoder) Length() int { - return len(s) -} - -// ByteEncoder implements the Encoder interface for Go byte slices so that they can be used -// as the Key or Value in a ProducerMessage. -type ByteEncoder []byte - -func (b ByteEncoder) Encode() ([]byte, error) { - return b, nil -} - -func (b ByteEncoder) Length() int { - return len(b) -} - -// bufConn wraps a net.Conn with a buffer for reads to reduce the number of -// reads that trigger syscalls. -type bufConn struct { - net.Conn - buf *bufio.Reader -} - -func newBufConn(conn net.Conn) *bufConn { - return &bufConn{ - Conn: conn, - buf: bufio.NewReader(conn), - } -} - -func (bc *bufConn) Read(b []byte) (n int, err error) { - return bc.buf.Read(b) -} - -// KafkaVersion instances represent versions of the upstream Kafka broker. -type KafkaVersion struct { - // it's a struct rather than just typing the array directly to make it opaque and stop people - // generating their own arbitrary versions - version [4]uint -} - -func newKafkaVersion(major, minor, veryMinor, patch uint) KafkaVersion { - return KafkaVersion{ - version: [4]uint{major, minor, veryMinor, patch}, - } -} - -// IsAtLeast return true if and only if the version it is called on is -// greater than or equal to the version passed in: -// V1.IsAtLeast(V2) // false -// V2.IsAtLeast(V1) // true -func (v KafkaVersion) IsAtLeast(other KafkaVersion) bool { - for i := range v.version { - if v.version[i] > other.version[i] { - return true - } else if v.version[i] < other.version[i] { - return false - } - } - return true -} - -// Effective constants defining the supported kafka versions. -var ( - V0_8_2_0 = newKafkaVersion(0, 8, 2, 0) - V0_8_2_1 = newKafkaVersion(0, 8, 2, 1) - V0_8_2_2 = newKafkaVersion(0, 8, 2, 2) - V0_9_0_0 = newKafkaVersion(0, 9, 0, 0) - V0_9_0_1 = newKafkaVersion(0, 9, 0, 1) - V0_10_0_0 = newKafkaVersion(0, 10, 0, 0) - V0_10_0_1 = newKafkaVersion(0, 10, 0, 1) - V0_10_1_0 = newKafkaVersion(0, 10, 1, 0) - V0_10_1_1 = newKafkaVersion(0, 10, 1, 1) - V0_10_2_0 = newKafkaVersion(0, 10, 2, 0) - V0_10_2_1 = newKafkaVersion(0, 10, 2, 1) - V0_11_0_0 = newKafkaVersion(0, 11, 0, 0) - V0_11_0_1 = newKafkaVersion(0, 11, 0, 1) - V0_11_0_2 = newKafkaVersion(0, 11, 0, 2) - V1_0_0_0 = newKafkaVersion(1, 0, 0, 0) - V1_1_0_0 = newKafkaVersion(1, 1, 0, 0) - V1_1_1_0 = newKafkaVersion(1, 1, 1, 0) - V2_0_0_0 = newKafkaVersion(2, 0, 0, 0) - V2_0_1_0 = newKafkaVersion(2, 0, 1, 0) - V2_1_0_0 = newKafkaVersion(2, 1, 0, 0) - - SupportedVersions = []KafkaVersion{ - V0_8_2_0, - V0_8_2_1, - V0_8_2_2, - V0_9_0_0, - V0_9_0_1, - V0_10_0_0, - V0_10_0_1, - V0_10_1_0, - V0_10_1_1, - V0_10_2_0, - V0_10_2_1, - V0_11_0_0, - V0_11_0_1, - V0_11_0_2, - V1_0_0_0, - V1_1_0_0, - V1_1_1_0, - V2_0_0_0, - V2_0_1_0, - V2_1_0_0, - } - MinVersion = V0_8_2_0 - MaxVersion = V2_1_0_0 -) - -func ParseKafkaVersion(s string) (KafkaVersion, error) { - if len(s) < 5 { - return MinVersion, fmt.Errorf("invalid version `%s`", s) - } - var major, minor, veryMinor, patch uint - var err error - if s[0] == '0' { - err = scanKafkaVersion(s, `^0\.\d+\.\d+\.\d+$`, "0.%d.%d.%d", [3]*uint{&minor, &veryMinor, &patch}) - } else { - err = scanKafkaVersion(s, `^\d+\.\d+\.\d+$`, "%d.%d.%d", [3]*uint{&major, &minor, &veryMinor}) - } - if err != nil { - return MinVersion, err - } - return newKafkaVersion(major, minor, veryMinor, patch), nil -} - -func scanKafkaVersion(s string, pattern string, format string, v [3]*uint) error { - if !regexp.MustCompile(pattern).MatchString(s) { - return fmt.Errorf("invalid version `%s`", s) - } - _, err := fmt.Sscanf(s, format, v[0], v[1], v[2]) - return err -} - -func (v KafkaVersion) String() string { - if v.version[0] == 0 { - return fmt.Sprintf("0.%d.%d.%d", v.version[1], v.version[2], v.version[3]) - } - - return fmt.Sprintf("%d.%d.%d", v.version[0], v.version[1], v.version[2]) -} diff --git a/third/github.com/Shopify/sarama/utils_test.go b/third/github.com/Shopify/sarama/utils_test.go deleted file mode 100644 index 013620e55..000000000 --- a/third/github.com/Shopify/sarama/utils_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package sarama - -import "testing" - -func TestVersionCompare(t *testing.T) { - if V0_8_2_0.IsAtLeast(V0_8_2_1) { - t.Error("0.8.2.0 >= 0.8.2.1") - } - if !V0_8_2_1.IsAtLeast(V0_8_2_0) { - t.Error("! 0.8.2.1 >= 0.8.2.0") - } - if !V0_8_2_0.IsAtLeast(V0_8_2_0) { - t.Error("! 0.8.2.0 >= 0.8.2.0") - } - if !V0_9_0_0.IsAtLeast(V0_8_2_1) { - t.Error("! 0.9.0.0 >= 0.8.2.1") - } - if V0_8_2_1.IsAtLeast(V0_10_0_0) { - t.Error("0.8.2.1 >= 0.10.0.0") - } -} - -func TestVersionParsing(t *testing.T) { - validVersions := []string{"0.8.2.0", "0.8.2.1", "0.9.0.0", "0.10.2.0", "1.0.0"} - for _, s := range validVersions { - v, err := ParseKafkaVersion(s) - if err != nil { - t.Errorf("could not parse valid version %s: %s", s, err) - } - if v.String() != s { - t.Errorf("version %s != %s", v.String(), s) - } - } - - invalidVersions := []string{"0.8.2-4", "0.8.20", "1.19.0.0", "1.0.x"} - for _, s := range invalidVersions { - if _, err := ParseKafkaVersion(s); err == nil { - t.Errorf("invalid version %s parsed without error", s) - } - } -} diff --git a/third/github.com/Shopify/sarama/vagrant/boot_cluster.sh b/third/github.com/Shopify/sarama/vagrant/boot_cluster.sh deleted file mode 100755 index 95e47dde4..000000000 --- a/third/github.com/Shopify/sarama/vagrant/boot_cluster.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh - -set -ex - -# Launch and wait for toxiproxy -${REPOSITORY_ROOT}/vagrant/run_toxiproxy.sh & -while ! nc -q 1 localhost 2181 ${KAFKA_INSTALL_ROOT}/zookeeper-${ZK_PORT}/myid -done diff --git a/third/github.com/Shopify/sarama/vagrant/kafka.conf b/third/github.com/Shopify/sarama/vagrant/kafka.conf deleted file mode 100644 index 25101df5a..000000000 --- a/third/github.com/Shopify/sarama/vagrant/kafka.conf +++ /dev/null @@ -1,9 +0,0 @@ -start on started zookeeper-ZK_PORT -stop on stopping zookeeper-ZK_PORT - -# Use a script instead of exec (using env stanza leaks KAFKA_HEAP_OPTS from zookeeper) -script - sleep 2 - export KAFKA_HEAP_OPTS="-Xmx320m" - exec /opt/kafka-KAFKAID/bin/kafka-server-start.sh /opt/kafka-KAFKAID/config/server.properties -end script diff --git a/third/github.com/Shopify/sarama/vagrant/provision.sh b/third/github.com/Shopify/sarama/vagrant/provision.sh deleted file mode 100755 index 13a8d5623..000000000 --- a/third/github.com/Shopify/sarama/vagrant/provision.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -set -ex - -apt-get update -yes | apt-get install default-jre - -export KAFKA_INSTALL_ROOT=/opt -export KAFKA_HOSTNAME=192.168.100.67 -export KAFKA_VERSION=1.0.0 -export REPOSITORY_ROOT=/vagrant - -sh /vagrant/vagrant/install_cluster.sh -sh /vagrant/vagrant/setup_services.sh -sh /vagrant/vagrant/create_topics.sh diff --git a/third/github.com/Shopify/sarama/vagrant/run_toxiproxy.sh b/third/github.com/Shopify/sarama/vagrant/run_toxiproxy.sh deleted file mode 100755 index e52c00e7b..000000000 --- a/third/github.com/Shopify/sarama/vagrant/run_toxiproxy.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh - -set -ex - -${KAFKA_INSTALL_ROOT}/toxiproxy -port 8474 -host 0.0.0.0 & -PID=$! - -while ! nc -q 1 localhost 8474 - -# The number of threads handling network requests -num.network.threads=2 - -# The number of threads doing disk I/O -num.io.threads=8 - -# The send buffer (SO_SNDBUF) used by the socket server -socket.send.buffer.bytes=1048576 - -# The receive buffer (SO_RCVBUF) used by the socket server -socket.receive.buffer.bytes=1048576 - -# The maximum size of a request that the socket server will accept (protection against OOM) -socket.request.max.bytes=104857600 - - -############################# Log Basics ############################# - -# A comma seperated list of directories under which to store log files -log.dirs=KAFKA_DATADIR - -# The default number of log partitions per topic. More partitions allow greater -# parallelism for consumption, but this will also result in more files across -# the brokers. -num.partitions=2 - -# Create new topics with a replication factor of 2 so failover can be tested -# more easily. -default.replication.factor=2 - -auto.create.topics.enable=false -delete.topic.enable=true - -############################# Log Flush Policy ############################# - -# Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. -# There are a few important trade-offs here: -# 1. Durability: Unflushed data may be lost if you are not using replication. -# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. -# The settings below allow one to configure the flush policy to flush data after a period of time or -# every N messages (or both). This can be done globally and overridden on a per-topic basis. - -# The number of messages to accept before forcing a flush of data to disk -#log.flush.interval.messages=10000 - -# The maximum amount of time a message can sit in a log before we force a flush -#log.flush.interval.ms=1000 - -############################# Log Retention Policy ############################# - -# The following configurations control the disposal of log segments. The policy can -# be set to delete segments after a period of time, or after a given size has accumulated. -# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens -# from the end of the log. - -# The minimum age of a log file to be eligible for deletion -log.retention.hours=168 - -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. -log.retention.bytes=268435456 - -# The maximum size of a log segment file. When this size is reached a new log segment will be created. -log.segment.bytes=268435456 - -# The interval at which log segments are checked to see if they can be deleted according -# to the retention policies -log.retention.check.interval.ms=60000 - -# By default the log cleaner is disabled and the log retention policy will default to just delete segments after their retention expires. -# If log.cleaner.enable=true is set the cleaner will be enabled and individual logs can then be marked for log compaction. -log.cleaner.enable=false - -############################# Zookeeper ############################# - -# Zookeeper connection string (see zookeeper docs for details). -# This is a comma separated host:port pairs, each corresponding to a zk -# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". -# You can also append an optional chroot string to the urls to specify the -# root directory for all kafka znodes. -zookeeper.connect=localhost:ZK_PORT - -# Timeout in ms for connecting to zookeeper -zookeeper.session.timeout.ms=3000 -zookeeper.connection.timeout.ms=3000 diff --git a/third/github.com/Shopify/sarama/vagrant/setup_services.sh b/third/github.com/Shopify/sarama/vagrant/setup_services.sh deleted file mode 100755 index 81d8ea05d..000000000 --- a/third/github.com/Shopify/sarama/vagrant/setup_services.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh - -set -ex - -stop toxiproxy || true -cp ${REPOSITORY_ROOT}/vagrant/toxiproxy.conf /etc/init/toxiproxy.conf -cp ${REPOSITORY_ROOT}/vagrant/run_toxiproxy.sh ${KAFKA_INSTALL_ROOT}/ -start toxiproxy - -for i in 1 2 3 4 5; do - ZK_PORT=`expr $i + 2180` - KAFKA_PORT=`expr $i + 9090` - - stop zookeeper-${ZK_PORT} || true - - # set up zk service - cp ${REPOSITORY_ROOT}/vagrant/zookeeper.conf /etc/init/zookeeper-${ZK_PORT}.conf - sed -i s/KAFKAID/${KAFKA_PORT}/g /etc/init/zookeeper-${ZK_PORT}.conf - - # set up kafka service - cp ${REPOSITORY_ROOT}/vagrant/kafka.conf /etc/init/kafka-${KAFKA_PORT}.conf - sed -i s/KAFKAID/${KAFKA_PORT}/g /etc/init/kafka-${KAFKA_PORT}.conf - sed -i s/ZK_PORT/${ZK_PORT}/g /etc/init/kafka-${KAFKA_PORT}.conf - - start zookeeper-${ZK_PORT} -done - -# Wait for the last kafka node to finish booting -while ! nc -q 1 localhost 29095