Filebeat version: 6.7.2
Background: Filebeat deployed as DaemonSet on Kubernetes with about a hundred nodes, sending logs to logstash via proxy; the workload is about 20-40 k messages/minutes.
I noticed that the Stats API of my Filebeat would sometimes show more "done" events than "added" events, and "active" would be a negative number.
Edit: I should've said metrics logs instead of Stats API since the numbers actually come from the logs. But they should be the same thing.
Here is an example:
|2019-08-28T00:58:53.852Z|INFO|[monitoring]|log/log.go:144|Non-zero metrics in the last 30s|{"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":499880,"time":{"ms":94}},"total":{"ticks":3173750,"time":{"ms":614},"value":3173750},"user":{"ticks":2673870,"time":{"ms":520}}},"handles":{"limit":{"hard":65536,"soft":65536},"open":28},"info":{"ephemeral_id":"fc90d8ac-a638-459e-91a6-905747e9d0f9","uptime":{"ms":117180012}},"memstats":{"gc_next":70346848,"memory_alloc":60027848,"memory_total":815532071664}},"filebeat":{"events":{"active":7,"added":5583,"done":5576},"harvester":{"open_files":8,"running":8}},"libbeat":{"config":{"module":{"running":0},"reloads":3},"output":{"events":{"acked":5362,"active":-444,"batches":6,"total":4918},"read":{"bytes":54},"write":{"bytes":1652167}},"pipeline":{"clients":18,"events":{"active":4121,"filtered":220,"published":5362,"total":5583},"queue":{"acked":5362}}},"registrar":{"states":{"current":74,"update":5576},"writes":{"success":6,"total":6}},"system":{"load":{"1":0.33,"15":0.37,"5":0.33,"norm":{"1":0.0206,"15":0.0231,"5":0.0206}}}}}}|
|2019-08-28T00:59:23.853Z|INFO|[monitoring]|log/log.go:144|Non-zero metrics in the last 30s|{"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":499960,"time":{"ms":80}},"total":{"ticks":3174290,"time":{"ms":542},"value":3174290},"user":{"ticks":2674330,"time":{"ms":462}}},"handles":{"limit":{"hard":65536,"soft":65536},"open":28},"info":{"ephemeral_id":"fc90d8ac-a638-459e-91a6-905747e9d0f9","uptime":{"ms":117210012}},"memstats":{"gc_next":77255488,"memory_alloc":49008504,"memory_total":815661003840}},"filebeat":{"events":{"active":-5,"added":4291,"done":4296},"harvester":{"open_files":8,"running":8}},"libbeat":{"config":{"module":{"running":0},"reloads":3},"output":{"events":{"acked":4138,"active":103,"batches":5,"total":4241},"read":{"bytes":42},"write":{"bytes":1520469}},"pipeline":{"clients":18,"events":{"active":4120,"filtered":154,"published":4138,"total":4291},"queue":{"acked":4138}}},"registrar":{"states":{"current":74,"update":4296},"writes":{"success":5,"total":5}},"system":{"load":{"1":0.32,"15":0.36,"5":0.33,"norm":{"1":0.02,"15":0.0225,"5":0.0206}}}}}}|
|2019-08-28T00:59:53.853Z|INFO|[monitoring]|log/log.go:144|Non-zero metrics in the last 30s|{"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":500010,"time":{"ms":52}},"total":{"ticks":3174730,"time":{"ms":437},"value":3174730},"user":{"ticks":2674720,"time":{"ms":385}}},"handles":{"limit":{"hard":65536,"soft":65536},"open":28},"info":{"ephemeral_id":"fc90d8ac-a638-459e-91a6-905747e9d0f9","uptime":{"ms":117240012}},"memstats":{"gc_next":80716448,"memory_alloc":41189344,"memory_total":815756643752}},"filebeat":{"events":{"active":-15,"added":3216,"done":3231},"harvester":{"open_files":8,"running":8}},"libbeat":{"config":{"module":{"running":0},"reloads":3},"output":{"events":{"acked":3117,"active":306,"batches":4,"total":3423},"read":{"bytes":48},"write":{"bytes":1142452}},"pipeline":{"clients":18,"events":{"active":4117,"filtered":102,"published":3117,"total":3216},"queue":{"acked":3117}}},"registrar":{"states":{"current":74,"update":3231},"writes":{"success":4,"total":4}},"system":{"load":{"1":0.27,"15":0.36,"5":0.31,"norm":{"1":0.0169,"15":0.0225,"5":0.0194}}}}}}|