Hi Team!
I, like many, have a problem with the filebeat. Data is not reaching to logstash.
Before opening the request, I read many posts, but did not find a clear solution to this problem? Pocheiu it doesn't work?
Checked on several Windows servers in Linux - the problem is similar.
"File is inactive: /var/log/test/20201219.AUD. Closing because close_inactive of 5m0s reached."
close_inactive: 15m -- does not bring results.
config fielbeat:
# ============================== Filebeat inputs ===============================
filebeat.inputs:
# Each - is an input. Most options can be set at the input level, so
# you can use different inputs for various configurations.
# Below are the input specific configurations.
- type: log
# Change to true to enable this input configuration.
enabled: true
close_inactive: 15m
tags: ["test-audit"]
encoding: UTF-16LE
# Paths that should be crawled and fetched. Glob based paths.
paths:
- /var/log/test/*.AUD
level: debug
# ============================== Filebeat modules ==============================
filebeat.config.modules:
# Glob pattern for configuration loading
path: ${path.config}/modules.d/*.yml
# Set to true to enable config reloading
reload.enabled: true
# ------------------------------ Logstash Output -------------------------------
output.logstash:
# The Logstash hosts
#hosts: ["localhost:5044"]
hosts: ["10.0.0.1:5053"]
index: "filebeat-test"
# Optional SSL. By default is off.
# List of root certificates for HTTPS server verifications
#ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
# Certificate for SSL client authentication
#ssl.certificate: "/etc/pki/client/cert.pem"
# Client Certificate Key
#ssl.key: "/etc/pki/client/cert.key"
# ================================= Processors =================================
processors:
- add_host_metadata:
when.not.contains.tags: forwarded
- add_cloud_metadata: ~
- add_docker_metadata: ~
- add_kubernetes_metadata: ~
logs filebeat:
2021-04-23T11:56:06.570+0300 INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cgroup":{"memory":{"id":"filebeat.service","mem":{"limit":{"bytes":9223372036854771712},"usage":{"bytes":36253696}}}},"cpu":{"system":{"ticks":20,"time":{"ms":28}},"total":{"ticks":120,"time":{"ms":134},"value":0},"user":{"ticks":100,"time":{"ms":106}}},"handles":{"limit":{"hard":262144,"soft":1024},"open":13},"info":{"ephemeral_id":"ee3216d1-5d72-4a2d-af7d-a57edef34e36","uptime":{"ms":30116}},"memstats":{"gc_next":26481088,"memory_alloc":14882664,"memory_sys":74793984,"memory_total":46069064,"rss":79257600},"runtime":{"goroutines":30}},"filebeat":{"events":{"added":2,"done":2},"harvester":{"open_files":1,"running":1,"started":1}},"libbeat":{"config":{"module":{"running":0},"reloads":1,"scans":2},"output":{"events":{"active":0},"type":"logstash"},"pipeline":{"clients":1,"events":{"active":0,"filtered":2,"total":2}}},"registrar":{"states":{"current":2,"update":2},"writes":{"success":2,"total":2}},"system":{"cpu":{"cores":1},"load":{"1":0.09,"15":0.09,"5":0.1,"norm":{"1":0.09,"15":0.09,"5":0.1}}}}}}
........
filebeat[29168]: 2021-04-23T12:10:06.569+0300 INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cgroup":{"memory":{"mem":{"usage":{"bytes":8192}}}},"cpu":{"system":{"ticks":130,"time":{"ms":3}},"total":{"ticks":400,"time":{"ms":15},"value":400},"user":{"ticks":270,"time":{"ms":12}}},"handles":{"limit":{"hard":262144,"soft":1024},"open":13},"info":{"ephemeral_id":"ee3216d1-5d72-4a2d-af7d-a57edef34e36","uptime":{"ms":870113}},"memstats":{"gc_next":18961040,"memory_alloc":9609000,"memory_total":59226808,"rss":80748544},"runtime":{"goroutines":30}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0},"scans":3},"output":{"events":{"active":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":2}},"system":{"load":{"1":0.24,"15":0.08,"5":0.1,"norm":{"1":0.24,"15":0.08,"5":0.1}}}}}}
filebeat[29168]: 2021-04-23T12:10:36.568+0300 INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cgroup":{"memory":{"mem":{"usage":{"bytes":131072}}}},"cpu":{"system":{"ticks":130,"time":{"ms":3}},"total":{"ticks":400,"time":{"ms":7},"value":400},"user":{"ticks":270,"time":{"ms":4}}},"handles":{"limit":{"hard":262144,"soft":1024},"open":13},"info":{"ephemeral_id":"ee3216d1-5d72-4a2d-af7d-a57edef34e36","uptime":{"ms":900112}},"memstats":{"gc_next":18961040,"memory_alloc":9990432,"memory_total":59608240,"rss":80748544},"runtime":{"goroutines":30}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0},"scans":3},"output":{"events":{"active":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":2}},"system":{"load":{"1":0.14,"15":0.07,"5":0.08,"norm":{"1":0.14,"15":0.07,"5":0.08}}}}}}
Apr 23 12:10:41 v-kgo-FTP-centos filebeat[29168]: 2021-04-23T12:10:41.594+0300 INFO log/harvester.go:333 File is inactive: /var/log/test/112.AUD. Closing because close_inactive of 15m0s reached.
2021-04-23T12:10:46.602+0300 INFO log/harvester.go:302 Harvester started for file: /var/log/test/112.AUD
filebeat[29168]: 2021-04-23T12:11:06.568+0300 INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cgroup":{"memory":{"mem":{"usage":{"bytes":8192}}}},"cpu":{"system":{"ticks":130,"time":{"ms":5}},"total":{"ticks":400,"time":{"ms":8},"value":400},"user":{"ticks":270,"time":{"ms":3}}},"handles":{"limit":{"hard":262144,"soft":1024},"open":13},"info":{"ephemeral_id":"ee3216d1-5d72-4a2d-af7d-a57edef34e36","uptime":{"ms":930112}},"memstats":{"gc_next":18961040,"memory_alloc":10546512,"memory_total":60164320,"rss":80748544},"runtime":{"goroutines":30}},"filebeat":{"events":{"added":2,"done":2},"harvester":{"closed":1,"open_files":1,"running":1,"started":1}},"libbeat":{"config":{"module":{"running":0},"scans":3},"output":{"events":{"active":0}},"pipeline":{"clients":1,"events":{"active":0,"filtered":2,"total":2}}},"registrar":{"states":{"current":2,"update":2},"writes":{"success":2,"total":2}},"system":{"load":{"1":0.09,"15":0.07,"5":0.08,"norm":{"1":0.09,"15":0.07,"5":0.08}}}}}}