Hi All, I have ben using filebeat 7.5.2 to forward custom application logs to logstash, recently i noticed a sharp rise in number of events being received by logstash and also realised that i was now getting duplicate logs in Kibana.
Nothing was changed recently that could have caused this.
here is filebeat filter for the logs i am getting duplicates for
"type": "log"
- "close_inactive": "30s"
"enabled": true
"ignore_older": "24h"
"paths":
- "/mnt/logs/*-syslog-*/*/*/log/er/PP.*"
- "/mnt/logs/*-syslog-*/*/*/*/log/er/PP.*"
"processors":
- "dissect":
"field": "log.file.path"
"tokenizer": "/mnt/logs/%{log_pod}-syslog-%{log_zone}/%{app_path}/log/er/%{log_file}"
- "script":
"id": "er_log_details"
"lang": "javascript"
"params":
"prefix": "er"
"source": "${vars.log_details_script}"
"recursive_glob":
"enabled": false
"scan_frequency": "5s"
"tags":
- "ER"
only one of these directories actually contains content -
- "/mnt/logs/*-syslog-*/*/*/log/er/PP.*"
- "/mnt/logs/*-syslog-*/*/*/*/log/er/PP.*"
last monitoring metrics from filebeat
{"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":1839840,"time":{"ms":1171}},"total":{"ticks":6090060,"time":{"ms":2721},"value":6090060},"user":{"ticks":4250220,"time":{"ms":1550}}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":28},"info":{"ephemeral_id":"46884eef-b8ec-44d3-a7f8-c261c22adb51","uptime":{"ms":56040029}},"memstats":{"gc_next":30571616,"memory_alloc":22194928,"memory_total":711015165376},"runtime":{"goroutines":201}},"filebeat":{"events":{"active":65,"added":1775,"done":1710},"harvester":{"open_files":16,"running":16}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":1710,"active":79,"batches":23,"total":1789},"read":{"bytes":138},"write":{"bytes":177429}},"pipeline":{"clients":16,"events":{"active":321,"published":1775,"total":1775},"queue":{"acked":1710}}},"registrar":{"states":{"current":3940,"update":1710},"writes":{"success":23,"total":23}},"system":{"load":{"1":4.23,"15":3.5,"5":3.97,"norm":{"1":0.1322,"15":0.1094,"5":0.1241}}}}}}