I am capturing netflow packets and displaying the information on kibana dashboard .I am able to do so but around 30-40 % packets are being dropped . I am verfiying the packets in tcp dump and the some packets not able to see in kibana .
I have created following pipeline : Filebeat >>>logstash >>Elastic .
Following is the Filbeat's netflow configuration .
netflow.yml
- module: netflow
log:
enabled: true
var:
netflow_host: 0.0.0.0
netflow_port: 2055
queue_size: 8192
I see dropped packets in filbeat log
{"monitoring": {"metrics": {"beat":{"cgroup":{"memory":{"mem":{"usage":{"bytes":12787712}}}},"cpu":{"system":{"ticks":11398950,"time":{"ms":5422}},"total":{"ticks":178189080,"time":{"ms":84037},"value":178189080},"user":{"ticks":166790130,"time":{"ms":78615}}},"handles":{"limit":{"hard":262144,"soft":1024},"open":19},"info":{"ephemeral_id":"1105d091-ec9a-426d-9eb8-9829f3970212","uptime":{"ms":64110088}},"memstats":{"gc_next":589458544,"memory_alloc":381015256,"memory_total":24917757135216,"rss":720912384},"runtime":{"goroutines":98}},"filebeat":{"events":{"active":-2010,"added":552998,"done":555008},"harvester":{"open_files":0,"running":0},"input":{"netflow":{"flows":552984,"packets":{"dropped":35469,"received":23605}}}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":555008,"active":2048,"batches":270,"total":552960}},"outputs":{"kafka":{"bytes_read":10941726,"bytes_write":877756729}},"pipeline":{"clients":1,"events":{"active":2619,"published":552998,"total":552997},"queue":{"acked":555008}}},"registrar":{"states":{"current":0}},"system":{"load":{"1":64.24,"15":63.77,"5":63.24,"norm":{"1":2.0075,"15":1.9928,"5":1.9763}}}}}}