I am running filebeat 7.6.2 in docker sending files to logstash. The output performance seems to be limited to around 150 events per second, but the file is being written to around 400 events per second. I am looking at the filebeat logs, and can see that the read_offset does not increase continually like i would expect. What is the read_offset, and what does it mean when it decreases?
filebeat_1 | 2021-07-13T10:54:44.261Z INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":6800,"time":{"ms":50}},"total":{"ticks":89170,"time":{"ms":528},"value":89170},"user":{"ticks":82370,"time":{"ms":478}}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":12},"info":{"ephemeral_id":"625fd760-cbee-49e9-8cbb-f3fcdd3bd517","uptime":{"ms":4770167}},"memstats":{"gc_next":158356816,"memory_alloc":108289608,"memory_total":20251047864},"runtime":{"goroutines":29}},"filebeat":{"events":{"added":2048,"done":2048},"harvester":{"files":{"7e9ce597-6c87-48c8-9a67-4bcdb5b5a1d3":{"last_event_published_time":"2021-07-13T10:54:43.424Z","last_event_timestamp":"2021-07-13T10:54:43.424Z","read_offset":7092454,"size":22018694}},"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":2048,"batches":1,"total":2048},"read":{"bytes":78},"write":{"bytes":1827909}},"pipeline":{"clients":1,"events":{"active":4117,"published":2048,"total":2048},"queue":{"acked":2048}}},"registrar":{"states":{"current":1,"update":2048},"writes":{"success":1,"total":1}},"system":{"load":{"1":0.62,"15":1.5,"5":0.96,"norm":{"1":0.31,"15":0.75,"5":0.48}}}}}}
filebeat_1 | 2021-07-13T10:55:14.261Z INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":6850,"time":{"ms":50}},"total":{"ticks":90140,"time":{"ms":968},"value":90140},"user":{"ticks":83290,"time":{"ms":918}}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":12},"info":{"ephemeral_id":"625fd760-cbee-49e9-8cbb-f3fcdd3bd517","uptime":{"ms":4800166}},"memstats":{"gc_next":154244640,"memory_alloc":108296368,"memory_total":20483329488,"rss":5570560},"runtime":{"goroutines":29}},"filebeat":{"events":{"added":4096,"done":4096},"harvester":{"files":{"7e9ce597-6c87-48c8-9a67-4bcdb5b5a1d3":{"last_event_published_time":"2021-07-13T10:55:05.898Z","last_event_timestamp":"2021-07-13T10:55:05.898Z","read_offset":13957422,"size":24380042}},"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":4096,"batches":2,"total":4096},"read":{"bytes":78},"write":{"bytes":3632400}},"pipeline":{"clients":1,"events":{"active":4117,"published":4096,"total":4096},"queue":{"acked":4096}}},"registrar":{"states":{"current":1,"update":4096},"writes":{"success":2,"total":2}},"system":{"load":{"1":0.43,"15":1.46,"5":0.88,"norm":{"1":0.215,"15":0.73,"5":0.44}}}}}}
filebeat_1 | 2021-07-13T10:55:44.261Z INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":6850,"time":{"ms":2}},"total":{"ticks":90140,"time":{"ms":3},"value":90140},"user":{"ticks":83290,"time":{"ms":1}}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":12},"info":{"ephemeral_id":"625fd760-cbee-49e9-8cbb-f3fcdd3bd517","uptime":{"ms":4830167}},"memstats":{"gc_next":154244640,"memory_alloc":108548992,"memory_total":20483582112},"runtime":{"goroutines":29}},"filebeat":{"harvester":{"files":{"7e9ce597-6c87-48c8-9a67-4bcdb5b5a1d3":{"size":25768913}},"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"output":{"read":{"bytes":72}},"pipeline":{"clients":1,"events":{"active":4117}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.38,"15":1.42,"5":0.83,"norm":{"1":0.19,"15":0.71,"5":0.415}}}}}}
filebeat_1 | 2021-07-13T10:56:14.261Z INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":6900,"time":{"ms":43}},"total":{"ticks":90670,"time":{"ms":525},"value":90670},"user":{"ticks":83770,"time":{"ms":482}}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":12},"info":{"ephemeral_id":"625fd760-cbee-49e9-8cbb-f3fcdd3bd517","uptime":{"ms":4860166}},"memstats":{"gc_next":165079008,"memory_alloc":94973272,"memory_total":20601269384},"runtime":{"goroutines":29}},"filebeat":{"events":{"added":2048,"done":2048},"harvester":{"files":{"7e9ce597-6c87-48c8-9a67-4bcdb5b5a1d3":{"last_event_published_time":"2021-07-13T10:56:08.858Z","last_event_timestamp":"2021-07-13T10:56:08.858Z","read_offset":7073555,"size":34068172}},"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":2048,"batches":1,"total":2048},"read":{"bytes":72},"write":{"bytes":1828186}},"pipeline":{"clients":1,"events":{"active":4117,"published":2048,"total":2048},"queue":{"acked":2048}}},"registrar":{"states":{"current":1,"update":2048},"writes":{"success":1,"total":1}},"system":{"load":{"1":0.29,"15":1.38,"5":0.77,"norm":{"1":0.145,"15":0.69,"5":0.385}}}}}}
filebeat_1 | 2021-07-13T10:56:44.261Z INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":6950,"time":{"ms":53}},"total":{"ticks":91180,"time":{"ms":506},"value":91180},"user":{"ticks":84230,"time":{"ms":453}}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":12},"info":{"ephemeral_id":"625fd760-cbee-49e9-8cbb-f3fcdd3bd517","uptime":{"ms":4890166}},"memstats":{"gc_next":147148960,"memory_alloc":123008952,"memory_total":20719541856},"runtime":{"goroutines":29}},"filebeat":{"events":{"added":2048,"done":2048},"harvester":{"files":{"7e9ce597-6c87-48c8-9a67-4bcdb5b5a1d3":{"last_event_published_time":"2021-07-13T10:56:29.230Z","last_event_timestamp":"2021-07-13T10:56:29.230Z","read_offset":7084044,"size":28417536}},"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":2048,"batches":1,"total":2048},"read":{"bytes":78},"write":{"bytes":1824147}},"pipeline":{"clients":1,"events":{"active":4117,"published":2048,"total":2048},"queue":{"acked":2048}}},"registrar":{"states":{"current":1,"update":2048},"writes":{"success":1,"total":1}},"system":{"load":{"1":0.75,"15":1.38,"5":0.83,"norm":{"1":0.375,"15":0.69,"5":0.415}}}}}}
Thanks in advance