代码拉取完成,页面将自动刷新
同步操作将从 Gitee 极速下载/goreplay 强制同步,此操作会覆盖自 Fork 仓库以来所做的任何修改,且无法恢复!!!
确定后同步将在后台操作,完成时将刷新页面,请耐心等待。
package main
import (
"encoding/json"
"log"
"strings"
"github.com/Shopify/sarama"
"github.com/Shopify/sarama/mocks"
)
// KafkaInput is used for receiving Kafka messages and
// transforming them into HTTP payloads.
type KafkaInput struct {
config *InputKafkaConfig
consumers []sarama.PartitionConsumer
messages chan *sarama.ConsumerMessage
quit chan struct{}
}
// NewKafkaInput creates instance of kafka consumer client with TLS config
func NewKafkaInput(address string, config *InputKafkaConfig, tlsConfig *KafkaTLSConfig) *KafkaInput {
c := NewKafkaConfig(tlsConfig)
var con sarama.Consumer
if mock, ok := config.consumer.(*mocks.Consumer); ok && mock != nil {
con = config.consumer
} else {
var err error
con, err = sarama.NewConsumer(strings.Split(config.Host, ","), c)
if err != nil {
log.Fatalln("Failed to start Sarama(Kafka) consumer:", err)
}
}
partitions, err := con.Partitions(config.Topic)
if err != nil {
log.Fatalln("Failed to collect Sarama(Kafka) partitions:", err)
}
i := &KafkaInput{
config: config,
consumers: make([]sarama.PartitionConsumer, len(partitions)),
messages: make(chan *sarama.ConsumerMessage, 256),
quit: make(chan struct{}),
}
for index, partition := range partitions {
consumer, err := con.ConsumePartition(config.Topic, partition, sarama.OffsetNewest)
if err != nil {
log.Fatalln("Failed to start Sarama(Kafka) partition consumer:", err)
}
go func(consumer sarama.PartitionConsumer) {
defer consumer.Close()
for message := range consumer.Messages() {
i.messages <- message
}
}(consumer)
go i.ErrorHandler(consumer)
i.consumers[index] = consumer
}
return i
}
// ErrorHandler should receive errors
func (i *KafkaInput) ErrorHandler(consumer sarama.PartitionConsumer) {
for err := range consumer.Errors() {
Debug(1, "Failed to read access log entry:", err)
}
}
// PluginRead a reads message from this plugin
func (i *KafkaInput) PluginRead() (*Message, error) {
var message *sarama.ConsumerMessage
var msg Message
select {
case <-i.quit:
return nil, ErrorStopped
case message = <-i.messages:
}
msg.Data = message.Value
if i.config.UseJSON {
var kafkaMessage KafkaMessage
json.Unmarshal(message.Value, &kafkaMessage)
var err error
msg.Data, err = kafkaMessage.Dump()
if err != nil {
Debug(1, "[INPUT-KAFKA] failed to decode access log entry:", err)
return nil, err
}
}
// does it have meta
if isOriginPayload(msg.Data) {
msg.Meta, msg.Data = payloadMetaWithBody(msg.Data)
}
return &msg, nil
}
func (i *KafkaInput) String() string {
return "Kafka Input: " + i.config.Host + "/" + i.config.Topic
}
// Close closes this plugin
func (i *KafkaInput) Close() error {
close(i.quit)
return nil
}
此处可能存在不合适展示的内容,页面不予展示。您可通过相关编辑功能自查并修改。
如您确认内容无涉及 不当用语 / 纯广告导流 / 暴力 / 低俗色情 / 侵权 / 盗版 / 虚假 / 无价值内容或违法国家有关法律法规的内容,可点击提交进行申诉,我们将尽快为您处理。