Hello, we are seeing these logs continuously being repeated in our filebeat container logs, it can be up to 20 logs per second constantly.
The "input ticker" stops and then logs about configured paths, it seems like filebeat is re-creating inputs for every log file, but the logs are being collected as expected.
Does anyone have any guidance on what could be causing this situation? Thanks in advance
Version 8.9.0 of filebeat is being used
2024-03-14T13:57:57.158513883Z stderr F {"log.level":"info","@timestamp":"2024-03-14T13:57:57.158Z","log.logger":"input","log.origin":{"file.name":"log/input.go","file.line":174},"message":"Configured paths: [/var/log/containers/*-e3225b9b366f5ef232c7c34b40fe9526980ea1c257b17385e382a26a63345590.log]","service.name":"filebeat","input_id":"afd3915b-8827-4f36-9762-ea5e29cc3935","ecs.version":"1.6.0"}
2024-03-14T13:57:57.168371256Z stderr F {"log.level":"info","@timestamp":"2024-03-14T13:57:57.168Z","log.origin":{"file.name":"input/input.go","file.line":134},"message":"input ticker stopped","service.name":"filebeat","ecs.version":"1.6.0"}
2024-03-14T13:57:57.377214158Z stderr F {"log.level":"info","@timestamp":"2024-03-14T13:57:57.377Z","log.logger":"input","log.origin":{"file.name":"log/input.go","file.line":174},"message":"Configured paths: [/var/log/containers/*-eb1da453064b0c1779dac130acc3a0cfae64a2d655c18cab48a10f34c1f9f38e.log]","service.name":"filebeat","input_id":"2b1ec611-2533-4b37-a4e0-22d1052f9573","ecs.version":"1.6.0"}
2024-03-14T13:57:57.380497543Z stderr F {"log.level":"info","@timestamp":"2024-03-14T13:57:57.380Z","log.logger":"input","log.origin":{"file.name":"log/input.go","file.line":174},"message":"Configured paths: [/var/log/containers/*-0a59d8571622b1625f7409f26740fd9cb91f94481cda43e8f4b539fb9ec717b8.log]","service.name":"filebeat","input_id":"c05e19cb-4de4-40a2-bca9-e204b002d081","ecs.version":"1.6.0"}
<snip>
2024-03-14T13:58:01.22407091Z stderr F {"log.level":"info","@timestamp":"2024-03-14T13:58:01.224Z","log.logger":"input","log.origin":{"file.name":"log/input.go","file.line":174},"message":"Configured paths: [/var/log/containers/*-c796c54f31771083919d27a21c577a4ad06aa75764616e63b7288edc6e7b54cc.log]","service.name":"filebeat","input_id":"0f58d900-e1d5-46a7-8c91-0230c524387d","ecs.version":"1.6.0"}
2024-03-14T13:58:01.233190848Z stderr F {"log.level":"info","@timestamp":"2024-03-14T13:58:01.233Z","log.logger":"input","log.origin":{"file.name":"log/input.go","file.line":174},"message":"Configured paths: [/var/log/containers/*-208900ecdafd7a2d1ba4e23bc3dbf472dc141c99c7ac526e1be2639e156c8f02.log]","service.name":"filebeat","input_id":"34a76382-1ba1-465b-a5d9-57b7a05fe409","ecs.version":"1.6.0"}
2024-03-14T13:58:01.236490936Z stderr F {"log.level":"info","@timestamp":"2024-03-14T13:58:01.236Z","log.logger":"input","log.origin":{"file.name":"log/input.go","file.line":174},"message":"Configured paths: [/var/log/containers/*-5629e2232f2bdb9956095470d8a20d62f8c416661c31febd312f5f25504e5ea1.log]","service.name":"filebeat","input_id":"669581de-a419-4c88-971c-31b42d652a7b","ecs.version":"1.6.0"}
2024-03-14T13:58:04.102197517Z stderr F {"log.level":"info","@timestamp":"2024-03-14T13:58:04.102Z","log.logger":"input.harvester","log.origin":{"file.name":"log/harvester.go","file.line":311},"message":"Harvester started for paths: [/var/log/containers/*-3d89a68f1d04704de8e310688bb1e766c77656c23c1bf4f5acbfcd321186f617.log]","service.name":"filebeat","input_id":"0f863f34-35d0-420e-a0d8-cca22a2b3371","source_file":"/var/log/containers/rbd-provisioner-76b686c967-6xrkn_kube-system_csi-rbdplugin-controller-3d89a68f1d04704de8e310688bb1e766c77656c23c1bf4f5acbfcd321186f617.log","state_id":"native::260375-64770","finished":false,"os_id":"260375-64770","old_source":"/var/log/containers/rbd-provisioner-76b686c967-6xrkn_kube-system_csi-rbdplugin-controller-3d89a68f1d04704de8e310688bb1e766c77656c23c1bf4f5acbfcd321186f617.log","old_finished":true,"old_os_id":"260375-64770","harvester_id":"7edd7e0f-df1b-49f1-9057-f3d5d5e28bc0","ecs.version":"1.6.0"}
2024-03-14T13:58:04.893634044Z stderr F {"log.level":"info","@timestamp":"2024-03-14T13:58:04.893Z","log.logger":"input","log.origin":{"file.name":"log/input.go","file.line":174},"message":"Configured paths: [/var/log/containers/*-591c67767d9bd2dad3f74345a5acbd33ea988419e760f1e663baf489e8be131c.log]","service.name":"filebeat","input_id":"702bc55f-eb03-4c24-93c8-c775cca2ce4a","ecs.version":"1.6.0"}
2024-03-14T13:58:04.896924757Z stderr F {"log.level":"info","@timestamp":"2024-03-14T13:58:04.896Z","log.logger":"input","log.origin":{"file.name":"log/input.go","file.line":174},"message":"Configured paths: [/var/log/containers/*-9749c89ffeefb189d695841ac5ada9ee2c234fa85db4d08e9ebfca0fd16c5b81.log]","service.name":"filebeat","input_id":"4efd669b-9fc2-4885-8f1d-812d43631ed2","ecs.version":"1.6.0"}
These are the config values used:
fields_under_root: true
filebeat.autodiscover:
providers:
- add_resource_metadata:
cronjob: false
deployment: false
hints.default_config:
close_renamed: true
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
type: container
hints.enabled: true
host: ${NODE_NAME}
type: kubernetes
filebeat.inputs:
- close_timeout: 5m
enabled: true
exclude_files:
- ^/var/log/containers/
- ^/var/log/pods/
paths:
- /var/log/*.log
- /var/log/messages
- /var/log/syslog
- /var/log/**/*.log
type: log
http.port: 5066
monitoring:
cluster_uuid: ${CLUSTER_UUID}
elasticsearch:
hosts:
- https://mon-elasticsearch-client:9200
password: ${beats_system_monitoring_password}
ssl.certificate_authorities:
- /usr/share/filebeat/ext-ca.crt
username: ${beats_system_monitoring_user}
enabled: ${BEAT_MONITORING_ENABLED}
name: ${NODE_NAME}
output.elasticsearch:
enabled: false
host: ${NODE_NAME}
hosts:
- https://mon-elasticsearch-client:9200
ilm.pattern: '000001'
index: ${INDEX_NAME}-%{+yyyy.MM.dd}
password: ${ELASTICSEARCH_PASSWORD}
protocol: https
ssl.certificate_authorities:
- /usr/share/filebeat/ext-ca.crt
username: ${ELASTICSEARCH_USERNAME}
output.file:
enabled: false
output.logstash:
enabled: true
hosts:
- mon-logstash:5044
ssl.certificate: /usr/share/filebeat/config/instance/filebeat.crt
ssl.certificate_authorities:
- /usr/share/filebeat/ca.crt
- /usr/share/filebeat/previous/ca.crt
- /usr/share/filebeat/next/ca.crt
ssl.key: /usr/share/filebeat/config/instance/filebeat.key
timeout: 9
setup.dashboards:
enabled: false
setup.kibana:
host: mon-kibana:5601
password: ${filebeat_password}
protocol: https
ssl.certificate_authorities:
- /usr/share/filebeat/ext-ca.crt
ssl.verification_mode: none
username: ${filebeat_user}
setup.template:
name: ${INDEX_NAME}
pattern: ${INDEX_PATTERN}