My log writing rate is about 3M/s, single log input, my output is kafka, I checked the metrics, pipeline.events.active keeps 4118,
{"monitoring": {"metrics": {"beat":{"cgroup":{"cpuacct":{"total":{"ns":1024565234}},"memory":{"mem":{"usage":{"bytes":6688768}}}},"cpu":{"system":{"ticks":40780,"time":{"ms":187}},"total":{"ticks":198550,"time":{"ms":1021},"value":198550},"user":{"ticks":157770,"time":{"ms":834}}},"handles":{"limit":{"hard":65536,"soft":65536},"open":19},"info":{"ephemeral_id":"24740809-0670-4333-a62e-9dc7e610a53e","uptime":{"ms":9780025},"version":"8.0.0"},"memstats":{"gc_next":46045104,"memory_alloc":30617304,"memory_total":52362771736,"rss":101363712},"runtime":{"goroutines":185}},"filebeat":{"events":{"active":1,"added":14338,"done":14337},"harvester":{"closed":1,"open_files":5,"running":5,"started":1}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":14336,"active":4096,"batches":7,"total":14336}},"outputs":{"kafka":{"bytes_write":23093414}},"pipeline":{"clients":4,"events":{"active":4118,"filtered":2,"published":14336,"total":14338},"queue":{"acked":14336}}},"registrar":{"states":{"current":6,"update":14337},"writes":{"success":10,"total":10}},"system":{"load":{"1":1.76,"15":1.77,"5":1.75,"norm":{"1":0.22,"15":0.2213,"5":0.2188}}}}}}
{"monitoring": {"metrics": {"beat":{"cgroup":{"cpuacct":{"total":{"ns":2024577969}},"memory":{"mem":{"usage":{"bytes":3981312}}}},"cpu":{"system":{"ticks":41170,"time":{"ms":397}},"total":{"ticks":200360,"time":{"ms":1817},"value":200360},"user":{"ticks":159190,"time":{"ms":1420}}},"handles":{"limit":{"hard":65536,"soft":65536},"open":20},"info":{"ephemeral_id":"24740809-0670-4333-a62e-9dc7e610a53e","uptime":{"ms":9840022},"version":"8.0.0"},"memstats":{"gc_next":40162960,"memory_alloc":30439384,"memory_total":52823686160,"rss":94916608},"runtime":{"goroutines":185}},"filebeat":{"events":{"added":24578,"done":24578},"harvester":{"open_files":6,"running":5}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":24576,"active":4096,"batches":12,"total":24576}},"outputs":{"kafka":{"bytes_write":36580967}},"pipeline":{"clients":4,"events":{"active":4118,"filtered":1,"published":24576,"total":24577},"queue":{"acked":24576}}},"registrar":{"states":{"cleanup":1,"current":5,"update":24578},"writes":{"success":14,"total":14}},"system":{"load":{"1":1.44,"15":1.73,"5":1.64,"norm":{"1":0.18,"15":0.2163,"5":0.205}}}}}}
{"monitoring": {"metrics": {"beat":{"cgroup":{"cpuacct":{"total":{"ns":1491092108}},"memory":{"mem":{"usage":{"bytes":9986048}}}},"cpu":{"system":{"ticks":41440,"time":{"ms":262}},"total":{"ticks":201830,"time":{"ms":1466},"value":201830},"user":{"ticks":160390,"time":{"ms":1204}}},"handles":{"limit":{"hard":65536,"soft":65536},"open":20},"info":{"ephemeral_id":"24740809-0670-4333-a62e-9dc7e610a53e","uptime":{"ms":9900023},"version":"8.0.0"},"memstats":{"gc_next":56618240,"memory_alloc":29087608,"memory_total":53209701696,"rss":102318080},"runtime":{"goroutines":190}},"filebeat":{"events":{"added":20484,"done":20484},"harvester":{"closed":1,"open_files":6,"running":6,"started":2}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":20480,"active":4096,"batches":10,"total":20480}},"outputs":{"kafka":{"bytes_write":29112060}},"pipeline":{"clients":4,"events":{"active":4118,"filtered":4,"published":20480,"total":20484},"queue":{"acked":20480}}},"registrar":{"states":{"cleanup":1,"current":6,"update":20484},"writes":{"success":14,"total":14}},"system":{"load":{"1":1.93,"15":1.78,"5":1.79,"norm":{"1":0.2413,"15":0.2225,"5":0.2238}}}}}}
I set GOMAXPROCS=4, but It still can't use more cpu, does kafka output affect the collection? My filebeat version is 8.0.0