Hi,
An instance of Filebeat 7.13.0 (running on Openshift) stops publishing logs to Kafka after some time.
The source logs are read from a NFS volume and are rotated every 5 minutes (a timestamp is added to the filename). My paths are not reading the rotated files.
On this instance of the problem, the last event was published at 2021-08-05T20:43:33 and then all I have in Filebeat log are repeated messages as below.
Could you please help me to understand and resolve the issue?
2021-08-05T20:43:34.445-0300 DEBUG [input] input/input.go:139 Run input
2021-08-05T20:43:34.445-0300 DEBUG [input] log/input.go:205 Start next scan
2021-08-05T20:43:34.509-0300 DEBUG [input] log/input.go:439 Check file for harvesting: /mydir/grp-int_traffic-int-7-fr556.log
2021-08-05T20:43:34.509-0300 DEBUG [input] log/input.go:530 Update existing file for harvesting: /mydir/grp-int_traffic-int-7-fr556.log, offset: 20798
2021-08-05T20:43:34.509-0300 DEBUG [input] log/input.go:583 Harvester for file is still running: /mydir/grp-int_traffic-int-7-fr556.log
2021-08-05T20:43:34.509-0300 DEBUG [input] log/input.go:439 Check file for harvesting: /mydir/grp-int_traffic-int-7-wq99k.log
2021-08-05T20:43:34.509-0300 DEBUG [input] log/input.go:530 Update existing file for harvesting: /mydir/grp-int_traffic-int-7-wq99k.log, offset: 20785
2021-08-05T20:43:34.509-0300 DEBUG [input] log/input.go:583 Harvester for file is still running: /mydir/grp-int_traffic-int-7-wq99k.log
2021-08-05T20:43:34.509-0300 DEBUG [input] log/input.go:226 input states cleaned up. Before: 2, After: 2, Pending: 2
...
2021-08-05T20:45:13.924-0300 DEBUG [harvester] log/harvester.go:587 Setting offset for file based on seek: /mydir/grp-ext_traffic-ext-16-mpvkf.log
2021-08-05T20:45:13.924-0300 DEBUG [harvester] log/harvester.go:573 Setting offset for file: /mydir/grp-ext_traffic-ext-16-mpvkf.log. Offset: 0
2021-08-05T20:45:13.924-0300 DEBUG [harvester] log/harvester.go:648 newLogFileReader with config.MaxBytes: 10485760
2021-08-05T20:45:13.924-0300 DEBUG [harvester] log/harvester.go:207 Harvester setup successful. Line terminator: 1
2021-08-05T20:45:13.928-0300 DEBUG [harvester] log/harvester.go:587 Setting offset for file based on seek: /mydir/grp-int_apimgr-int-5-t4qbr.log
2021-08-05T20:45:13.928-0300 DEBUG [harvester] log/harvester.go:573 Setting offset for file: /mydir/grp-int_apimgr-int-5-t4qbr.log. Offset: 0
2021-08-05T20:45:13.928-0300 DEBUG [harvester] log/harvester.go:648 newLogFileReader with config.MaxBytes: 10485760
2021-08-05T20:45:13.928-0300 DEBUG [harvester] log/harvester.go:207 Harvester setup successful. Line terminator: 1
2021-08-05T20:45:28.936-0300 INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cgroup":{"cpu":{"stats":{"periods":170}},"cpuacct":{"total":{"ns":153842109}},"memory":{"mem":{"usage":{"bytes":-4096}}}},"cpu":{"system":{"ticks":17690,"time":{"ms":100}},"total":{"ticks":38930,"time":{"ms":156},"value":38930},"user":{"ticks":21240,"time":{"ms":56}}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":23},"info":{"ephemeral_id":"27776998-c358-4387-9cb3-5ec119bfd865","uptime":{"ms":5250245}},"memstats":{"gc_next":76964864,"memory_alloc":49597128,"memory_total":2146758680,"rss":179548160},"runtime":{"goroutines":88}},"filebeat":{"events":{"active":4,"added":4},"harvester":{"open_files":10,"running":6}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"active":0}},"pipeline":{"clients":4,"events":{"active":4115}}},"registrar":{"states":{"current":1797}},"system":{"load":{"1":1.98,"15":3.37,"5":2.82,"norm":{"1":0.2475,"15":0.4213,"5":0.3525}}}}}}
filebeat.yml:
#=========================== Filebeat inputs =============================
filebeat.inputs:
- type: log
enabled: true
paths:
- /mydir/grp-int_traffic-int-?-?????.log
- /mydir/grp-int_traffic-int-??-?????.log
- /mydir/grp-int_traffic-int-???-?????.log
- /mydir/grp-int_traffic-int-????-?????.log
fields:
ambiente: gtw-hml
rede: interna
componente: traffic
topic_log: MY_TOPIC
scan_frequency: 10s
backoff: 1s
close_inactive: 1m
ignore_older: 2h
clean_inactive: 25h
- type: log
enabled: true
paths:
- /mydir/grp-ext_traffic-ext-?-?????.log
- /mydir/grp-ext_traffic-ext-??-?????.log
- /mydir/grp-ext_traffic-ext-???-?????.log
- /mydir/grp-ext_traffic-ext-????-?????.log
fields:
ambiente: gtw-hml
rede: externa
componente: traffic
topic_log: MY_TOPIC
scan_frequency: 10s
backoff: 1s
close_inactive: 1m
ignore_older: 2h
clean_inactive: 25h
- type: log
enabled: true
paths:
- /mydir/grp-int_apimgr-int-?-?????.log
- /mydir/grp-int_apimgr-int-??-?????.log
- /mydir/grp-int_apimgr-int-???-?????.log
- /mydir/grp-int_apimgr-int-????-?????.log
fields:
ambiente: gtw-hml
rede: interna
componente: apimgr
topic_log: MY_TOPIC
scan_frequency: 10s
backoff: 1s
close_inactive: 1m
ignore_older: 2h
clean_inactive: 25h
- type: log
enabled: true
paths:
- /mydir/grp-ext_apimgr-ext-?-?????.log
- /mydir/grp-ext_apimgr-ext-??-?????.log
- /mydir/grp-ext_apimgr-ext-???-?????.log
- /mydir/grp-ext_apimgr-ext-????-?????.log
fields:
ambiente: gtw-hml
rede: externa
componente: apimgr
topic_log: MY_TOPIC
scan_frequency: 10s
backoff: 1s
close_inactive: 1m
ignore_older: 2h
clean_inactive: 25h
#============================= Filebeat modules ===============================
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
#================================ General =====================================
# The name of the shipper that publishes the network data. It can be used to group
# all the transactions sent by a single shipper in the web interface.
name: gtw-hml
#================================ Outputs =====================================
# Configure what output to use when sending the data collected by the beat.
#------------------------------- Kafka output ----------------------------------
output.kafka:
enabled: true
hosts: ["mykafka1:9092", "mykafka2:9092", "mykafka3:9092"]
topic: '%{[fields.topic_log]}'
codec.json:
pretty: false
#================================ Processors =====================================
# Configure processors to enhance or manipulate events generated by the beat.
processors:
- add_host_metadata: ~
- add_id: ~
#================================ Logging =====================================
# Sets log level. The default log level is info.
# Available log levels are: error, warning, info, debug
logging.level: debug
keepfiles: 7
# At debug level, you can selectively enable logging only for some components.
# To enable all selectors use ["*"]. Examples of other selectors are "beat",
# "publish", "service".
logging.selectors: ["*"]
#======================= Filebeat Global Options ==============================
filebeat.registry.flush: 5s