use filebeat-6.1.2 to collect logs to kafka, but part of logs in kafka are Garbled.
filebeat.yml
path.data: ${FILEBEAT_HOME}/data
path.logs: ${FILEBEAT_HOME}/logs
filebeat.config.prospectors:
enabled: true
path: ${FILEBEAT_HOME}/config/*/*.yml
reload.enabled: true
reload.period: 6s
output.kafka:
hosts: [...]
topic: '%{[fields.kafka_topic]}'
partition.round_robin:
reachable_only: false
required_acks: 1
compression: snappy
max_message_bytes: 1048576
bulk_max_size: 2048
codec.format:
string: '%{[message]}'
max_procs: 2
logging.level: info
logging.to_files: true
logging.to_syslog: false
logging.files.rotateeverybytes: 524288000
logging.files:
path: ${FILEBEAT_HOME}/logs
name: filebeat.log
keepfiles: 20
permissions: 0644
prospectors.yml
- type: log
paths:
- /channel/dt/data/*
fields:
kafka_topic: ...
scan_frequency: 10s
harvester_buffer_size: 10485760
multiline: null
logs sample
logs in kafka sample