Hi!
I have the following setup to ingest logs from s3.
- filebeat pulls data from s3 and sends it to logstash.
- logstash ingests into elastic.
The problem is after a while (usually a day or two) filebeat consumes all server memory until oom gets triggered and the kernel kills filebeat. Box has 128gb memory with around 64gb dedicated to filebeat...
Bellow is filebeat and logstash configuration. Obscuring keys, endpoints and domains.
Please advise how to approach this.
Thanks!
filebeat
monitoring:
enabled: true
cluster_uuid: "_7KKV20eSemBTs_BxXfs6A"
elasticsearch:
hosts: "elasticsearch:9200"
username: elastic
password: changeme
filebeat.queue.mem:
events: 4096
flush.min_events: 512
flush.timeout: 30s
filebeat.queue.disk:
max_size: 100GB
filebeat.inputs:
- type: aws-s3
bucket_arn: arn
number_of_workers: 5
endpoint: https://endpoint
access_key_id: key
secret_access_key: key
tags: [ "stage" ]
- type: aws-s3
bucket_arn: arn
ignore_older: 3h
clean_inactive: 4h
close_inactive: 1m
scan_frequency: 2m
harvester_limit: 5
number_of_workers: 8
endpoint: https://endpoint.com
access_key_id: key
secret_access_key: key
bucket_list_prefix: "other-domain.com/"
file_selectors:
- regex: "other-domain.com/202302"
- regex: "other-domain.com/202303"
- regex: "other-domain.com/202304"
- regex: "other-domain.com/202305"
- regex: "other-domain.com/202306"
- regex: "other-domain.com/202307"
- regex: "other-domain.com/202308"
- regex: "other-domain.com/202309"
- regex: "other-domain.com/202310"
- regex: "other-domain.com/202311"
- regex: "other-domain.com/202312"
- regex: "other-domain.com/2024"
tags: [ "other" ]
- type: aws-s3
bucket_arn: arn
ignore_older: 3h
clean_inactive: 4h
close_inactive: 1m
scan_frequency: 2m
harvester_limit: 5
number_of_workers: 8
endpoint: https://endpoint.com
access_key_id: key
secret_access_key: key
bucket_list_prefix: "domain.com/"
file_selectors:
- regex: "domain.com/202302"
- regex: "domain.com/202303"
- regex: "domain.com/202304"
- regex: "domain.com/202305"
- regex: "domain.com/202306"
- regex: "domain.com/202307"
- regex: "domain.com/202308"
- regex: "domain.com/202309"
- regex: "domain.com/202310"
- regex: "domain.com/202311"
- regex: "domain.com/202312"
tags: [ "domain" ]
- type: aws-s3
bucket_arn: arn
ignore_older: 3h
clean_inactive: 4h
close_inactive: 1m
scan_frequency: 2m
harvester_limit: 5
number_of_workers: 8
endpoint: https://endpoint.com
access_key_id: key
secret_access_key: key
bucket_list_prefix: "more-domain.com/"
file_selectors:
- regex: "more-domain.com/202302"
- regex: "more-domain.com/202303"
- regex: "more-domain.com/202304"
- regex: "more-domain.com/202305"
- regex: "more-domain.com/202306"
- regex: "more-domain.com/202307"
- regex: "more-domain.com/202308"
- regex: "more-domain.com/202309"
- regex: "more-domain.com/202310"
- regex: "more-domain.com/202311"
- regex: "more-domain.com/202312"
tags: [ "more" ]
processors:
- decode_json_fields:
fields: [ "message" ]
process_array: false
max_depth: 10
target: ""
overwrite_keys: true
add_error_key: false
- fingerprint:
fields: [ "message" ]
target_field: "@metadata._id"
setup.kibana.host: "http://kibana:5601"
output.logstash:
hosts: [ "logstash:5044" ]
logstash
input {
beats {
port => 5044
include_codec_tag => false
client_inactivity_timeout => 120
}
}
## Add your filters / logstash plugins configuration here
filter {
date {
match => ["EdgeStartTimestamp", "ISO8601", "UNIX", "UNIX_MS"]
}
useragent {
# ecs_compatibility => v8
source => "ClientRequestUserAgent"
target => "ParsedUserAgent"
}
grok {
match => {"ClientRequestPath" => "/%{WORD:api}/%{WORD:api_version}/%{WORD:functionality}/%{WORD:sub_functionality}"}
}
mutate {
convert => {"EdgeResponseCompressionRatio"=>"float" }
}
if "_grokparsefailure" in [tags] {
mutate {
remove_tag => ["_grokparsefailure"]
add_field => {"_grokparsefailure" => "true"}
}
}
}
output {
if "other" in [tags] {
elasticsearch {
hosts => "elasticsearch:9200"
user => "elastic"
index => "domain-logs-%{+yyyy-MM}"
password => "changeme"
ecs_compatibility => disabled
}
} else if "other" in [tags] {
elasticsearch {
hosts => "elasticsearch:9200"
user => "elastic"
index => "other-logs-%{+yyyy-MM}"
password => "changeme"
ecs_compatibility => disabled
}
} else if "another" in [tags] {
elasticsearch {
hosts => "elasticsearch:9200"
user => "elastic"
index => "another-logs-%{+yyyy-MM}"
password => "changeme"
ecs_compatibility => disabled
}
} else if "stage" in [tags] {
elasticsearch {
hosts => "elasticsearch:9200"
user => "elastic"
index => "stage-logs"
password => "changeme"
ecs_compatibility => disabled
}
}
}
docker stats
CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS
3764f82c7c4c filebeat 147.32% 53.78GiB / 125.7GiB 42.78% 49.8GB / 6.74GB 34.6MB / 833GB 40
1aeb8e2c27da nginx 0.00% 29.63MiB / 125.7GiB 0.02% 29.3MB / 35.5MB 51.7MB / 20.5kB 33
671370e801fb zotaelk_logstash_1 0.46% 24.21GiB / 125.7GiB 19.26% 71.9GB / 598GB 3.55TB / 112MB 218
d1cc2eafd36e zotaelk_kibana_1 1.79% 403.2MiB / 125.7GiB 0.31% 1.45GB / 1.7GB 660GB / 4.1kB 12
296ea56bd26f vouch-proxy 0.00% 16.98MiB / 125.7GiB 0.01% 7.5MB / 1.08MB 186GB / 0B 27
f87849d812f6 cloudflared 0.14% 32.68MiB / 125.7GiB 0.03% 93.6MB / 105MB 1.65TB / 0B 37
80d4d5e9955d zotaelk_elasticsearch_1 3.09% 35.19GiB / 125.7GiB 27.99% 599GB / 3.07GB 4.16TB / 1.62TB 271
