Packetbeat version 7.9 and kafka version is 2.5
#----------------------------- Kafka output --------------------------------
output.kafka:
hosts: ["kafka1:9092","kafka2:9092","kafka3:9092"]
topic: "packetbeat"
required_acks: -1
worker: 2
#compression: gzip
max_message_bytes: 100000000
packetbeat logs
2020-11-23T19:27:20.376+0530 INFO [publisher_pipeline_output] pipeline/output.go:151 Connection to kafka(192.168.0.114:9092,192.168.0.114:9092,192.168.0.116:9092,192.168.0.116:9092,192.168.0.118:9092,192.168.0.118:9092) established
2020-11-23T19:27:20.376+0530 INFO [publisher] pipeline/retry.go:219 retryer: send unwait signal to consumer
2020-11-23T19:27:20.377+0530 INFO [publisher] pipeline/retry.go:223 done
2020-11-23T19:27:40.529+0530 INFO [publisher] pipeline/retry.go:159 Drop batch
2020-11-23T19:27:41.553+0530 INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":421,"time":{"ms":421}},"total":{"ticks":811,"time":{"ms":811},"value":811},"user":{"ticks":390,"time":{"ms":390}}},"handles":{"open":233},"info":{"ephemeral_id":"708ee657-277f-4250-b5ad-f2eb7a1e48d4","uptime":{"ms":30609}},"memstats":{"gc_next":38736784,"memory_alloc":26090032,"memory_total":32935672,"rss":70897664},"runtime":{"goroutines":61}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"active":53,"batches":3,"failed":1,"total":54},"type":"kafka"},"outputs":{"kafka":{"bytes_read":2264,"bytes_write":288}},"pipeline":{"clients":15,"events":{"active":53,"dropped":1,"published":54,"retry":1,"total":54},"queue":{"acked":1}}},"system":{"cpu":{"cores":4}}}}}
2020-11-23T19:27:43.368+0530 INFO beater/packetbeat.go:247 Packetbeat send stop signal
2020-11-23T19:27:43.736+0530 INFO flows/util.go:64 flows worker loop stopped
2020-11-23T19:27:43.762+0530 INFO [monitoring] log/log.go:153 Total non-zero metrics {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":421,"time":{"ms":421}},"total":{"ticks":811,"time":{"ms":811},"value":811},"user":{"ticks":390,"time":{"ms":390}}},"handles":{"open":231},"info":{"ephemeral_id":"708ee657-277f-4250-b5ad-f2eb7a1e48d4","uptime":{"ms":32791}},"memstats":{"gc_next":38736784,"memory_alloc":27265240,"memory_total":34110880,"rss":66940928},"runtime":{"goroutines":39}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"active":53,"batches":3,"failed":1,"total":54},"type":"kafka"},"outputs":{"kafka":{"bytes_read":2547,"bytes_write":324}},"pipeline":{"clients":15,"events":{"active":104,"dropped":1,"published":105,"retry":1,"total":105},"queue":{"acked":1}}},"system":{"cpu":{"cores":4}}}}}
2020-11-23T19:27:43.763+0530 INFO [monitoring] log/log.go:154 Uptime: 32.8190235s
2020-11-23T19:27:43.764+0530 INFO [monitoring] log/log.go:131 Stopping metrics logging.
2020-11-23T19:27:43.765+0530 INFO instance/beat.go:456 packetbeat stopped.
when i run single node kafka it works fine but does not work in case of kafka multi node
and yes when i write version: '2.5' in kafka output(packetbeat.yml) packetbeat can't identify version.