To anyone interested the below is my working config now.
conf.d/to_confluent.conf
input {
beats {
port => 5044
client_inactivity_timeout => 300
}
}
filebeat.yml
filebeat.inputs:
- type: filestream
id: access-logs
enabled: true
paths:
- /mnt/*/log/access.log
prospector.scanner.fingerprint:
enabled: true
offset: 0
length: 64
file_identity.fingerprint: ~
clean_removed: false
processors:
- add_host_metadata:
when.not.contains.tags: forwarded
output.logstash:
hosts: ["xxx.xxx.xxx.xxx:5044"]
bulk_max_size: 1
timeout: 300
pipelining: 0