Here is an example of one of our filebeat prospectors. In case any shines any clues.
---
filebeat:
prospectors:
- input_type: log
paths:
- /var/log/app_name/web/*.log
- /var/log/app_name/web/*.err
encoding: plain
fields:
environment: production
client: production
product: app_name
role: web
team: undef
log_type: rails
fields_under_root: true
document_type: app_name
scan_frequency: 10s
harvester_buffer_size: 16384
max_bytes: 10485760
tail_files: false
# Experimental: If symlinks is enabled, symlinks are opened and harvested. The harvester is openening the
# original for harvesting but will report the symlink name as source.
#symlinks: false
backoff: 1s
max_backoff: 10s
backoff_factor: 2
# Experimental: Max number of harvesters that are started in parallel.
# Default is 0 which means unlimited
#harvester_limit: 0
### Harvester closing options
# Close inactive closes the file handler after the predefined period.
# The period starts when the last line of the file was, not the file ModTime.
# Time strings like 2h (2 hours), 5m (5 minutes) can be used.
close_inactive: 5m
# Close renamed closes a file handler when the file is renamed or rotated.
# Note: Potential data loss. Make sure to read and understand the docs for this option.
close_renamed: false
# When enabling this option, a file handler is closed immediately in case a file can't be found
# any more. In case the file shows up again later, harvesting will continue at the last known position
# after scan_frequency.
close_removed: true
# Closes the file handler as soon as the harvesters reaches the end of the file.
# By default this option is disabled.
# Note: Potential data loss. Make sure to read and understand the docs for this option.
close_eof: false
### State options
# Files for the modification data is older then clean_inactive the state from the registry is removed
# By default this is disabled.
clean_inactive: 0
# Removes the state for file which cannot be found on disk anymore immediately
clean_removed: true
# Close timeout closes the harvester after the predefined time.
# This is independent if the harvester did finish reading the file or not.
# By default this option is disabled.
# Note: Potential data loss. Make sure to