Hello,
Recently, we've encountered significant challenges with Filebeat's memory usage and performance, specifically after integrating additional netflow shippers. This led to Filebeat running out of memory just minutes after startup, with a hard cap of 6 GB set in the service configuration. Adjustments like reducing the harvester_limit from 100 to 20 provided temporary relief. However, the issue persists, pointing towards possible performance issues with syslog file reading in the Fortinet or Cisco modules. Disabling the Fortinet module has momentarily resolved the issue.
Generally, I think it is related to Elevated Memory Utilization and Errors in Filebeat When Integrating External MISP CTI Log Source · Issue #38053 · elastic/beats · GitHub
When Filebeat, specifically with the Fortinet module enabled, processes a high volume of logs, there is a noticeable and rapid increase in memory usage. This phenomenon is accompanied by log entries indicating the establishment of a connection to Elasticsearch, suggesting that the performance bottleneck may be related to the handling of output connections in environments with heavy log traffic.
log.logger":"publisher_pipeline_output","log.origin":{"file.name":"pipeline/client_worker.go","file.line":145},"message":"Connec
tion to backoff(elasticsearch(https://someurl:9200)) established","service.name":"filebeat","ecs.version":"1.6.0"}
Can you suggest a resolution via filebeat configuration for quicker handling of documents?
CONTEX:
OS:EuroLinux 8.9 (Monaco)
Filebeat: 8.11.3
Elasticserach: 8.11.3
CPU: 2
MEM:12 GB
Modules: Netflow, Fortinet, System, Cisco,
CONFIG:
#=========================== Filebeat inputs =============================
filebeat.inputs:
# Each - is an input. Most options can be set at the input level, so
# you can use different inputs for various configurations.
# Below are the input specific configurations.
- type: log
scan_frequency: 10s
harvester_limit: 20
backoff: 2s
max_backoff: 10s
backoff_factor: 2
...
worker: 20
flush.interval: 1s
bulk_max_size: 20480
# compression_level: 5
indices:
- index: "filebeat-%{[agent.version]}-netflow-%{+yyyy.MM.dd}"
when.equals:
event.module: "netflow"
- index: "filebeat-%{[agent.version]}-cisco-%{+xxxx.ww}"
when.equals:
event.module: "cisco"
- index: "filebeat-%{[agent.version]}-system-%{+xxxx.ww}"
when.equals:
event.module: "system"
- index: "filebeat-%{[agent.version]}-elasticsearch-%{+xxxx.ww}"
when.equals:
event.module: "elasticsearch"
- index: "filebeat-%{[agent.version]}-fortinet-%{+yyyy.MM.dd}"
when.equals:
event.module: "fortinet"
...
processors:
- add_host_metadata: ~
# - add_cloud_metadata: ~
# - add_docker_metadata: ~
# - add_kubernetes_metadata: ~
- add_fields:
target: ''
fields:
observer.name: "GCP"
when.equals:
event.dataset: "gcp.firewall"
- drop_event:
when.and:
- equals:
event.module: "netflow"
- equals:
network.transport: "icmp"
...
queue.mem:
events: 2048
flush.min_events: 512
flush.timeout: 1s
Error on elasticsearch side
[2024-03-07T11:04:03,413][WARN ][o.e.h.AbstractHttpServerTransport] [node-1] caught exception while handling client http traffic, closin g connection Netty4HttpChannel{localAddress=/10.zzz.3:9200, remoteAddress=/10.xxx21:56212}
io.netty.handler.codec.PrematureChannelClosureException: Channel closed while still aggregating message
at io.netty.handler.codec.MessageAggregator.channelInactive(MessageAggregator.java:436) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[?:?]
at io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[?:?]
at io.netty.handler.codec.http.HttpContentDecoder.channelInactive(HttpContentDecoder.java:235) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[?:?]
at io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[?:?]
at org.elasticsearch.http.netty4.Netty4HttpHeaderValidator.channelInactive(Netty4HttpHeaderValidator.java:186) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[?:?]
at io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:411) ~[?:?]
at io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:376) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[?:?]
at io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[?:?]
at org.elasticsearch.transport.netty4.Netty4WriteThrottlingHandler.channelInactive(Netty4WriteThrottlingHandler.java:109) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[?:?]
at io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:411) ~[?:?]
at io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:376) ~[?:?]
at io.netty.handler.ssl.SslHandler.channelInactive(SslHandler.java:1085) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[?:?]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[?:?]
at io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) ~[?:?]
at io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:813) ~[?:?]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[?:?]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[?:?]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[?:?]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:566) ~[?:?]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[?:?]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]
at java.lang.Thread.run(Thread.java:1583) ~[?:?]
