Running Filebeat 8.5.3 on Ubuntu 20.04. I am trying to send some logs to Kafka using FIlebeat. This is my filebeat.yml
filebeat.inputs:
- type: filestream
id: jenkinsfilestream
enabled: true
paths:
- "/var/log/jenkins/*"
fields:
topic: "jenkins"
- type: filestream
id: mysqlfilestream
enabled: true
paths:
- "/var/log/mysql/*.log"
fields:
topic: "mysqllog"
- type: filestream
id: redisfilestream
enabled: true
paths:
- "/var/log/redis/*.log"
fields:
topic: "redislog"
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.ilm.overwrite: false
setup.template.enabled: false
setup.template.overwrite: false
setup.ilm.enabled: false
ilm.enabled: false
output.kafka:
hosts: ["xxx:9092", "xxx:9093", "xxx:9094"]
topic: '%{[fields.topic]}'
partition.round_robin:
reachable_only: false
required_acks: 1
compression: gzip
max_message_bytes: 1000000
processors:
- add_host_metadata:
when.not.contains.tags: forwarded
- add_cloud_metadata: ~
- add_docker_metadata: ~
- add_kubernetes_metadata: ~
logging.level: info
logging.to_files: true
logging.files:
path: /var/log/filebeat
name: filebeat
keepfiles: 7
permissions: 0640
And the log:
{"log.level":"info","@timestamp":"2022-12-11T19:28:13.733+0800","log.origin":{"file.name":"instance/beat.go","file.line":708},"message":"Home path: [/usr/share/filebeat] Config path: [/etc/filebeat] Data path: [/var/lib/filebeat] Logs path: [/var/log/filebeat]","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:13.734+0800","log.origin":{"file.name":"instance/beat.go","file.line":716},"message":"Beat ID: 57550900-e1fd-4712-9fbd-7a2e3c662024","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"warn","@timestamp":"2022-12-11T19:28:16.735+0800","log.logger":"add_cloud_metadata","log.origin":{"file.name":"add_cloud_metadata/provider_aws_ec2.go","file.line":81},"message":"read token request for getting IMDSv2 token returns empty: Put \"http://169.254.169.254/latest/api/token\": context deadline exceeded (Client.Timeout exceeded while awaiting headers). No token in the metadata request will be used.","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.736+0800","log.logger":"seccomp","log.origin":{"file.name":"seccomp/seccomp.go","file.line":124},"message":"Syscall filter successfully installed","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.736+0800","log.logger":"beat","log.origin":{"file.name":"instance/beat.go","file.line":1082},"message":"Beat info","service.name":"filebeat","system_info":{"beat":{"path":{"config":"/etc/filebeat","data":"/var/lib/filebeat","home":"/usr/share/filebeat","logs":"/var/log/filebeat"},"type":"filebeat","uuid":"57550900-e1fd-4712-9fbd-7a2e3c662024"},"ecs.version":"1.6.0"}}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.737+0800","log.logger":"beat","log.origin":{"file.name":"instance/beat.go","file.line":1091},"message":"Build info","service.name":"filebeat","system_info":{"build":{"commit":"6d03209df870c63ef9d59d609268c11dfdc835dd","libbeat":"8.5.3","time":"2022-12-04T04:51:48.000Z","version":"8.5.3"},"ecs.version":"1.6.0"}}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.737+0800","log.logger":"beat","log.origin":{"file.name":"instance/beat.go","file.line":1094},"message":"Go runtime info","service.name":"filebeat","system_info":{"go":{"os":"linux","arch":"amd64","max_procs":2,"version":"go1.18.7"},"ecs.version":"1.6.0"}}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.737+0800","log.logger":"beat","log.origin":{"file.name":"instance/beat.go","file.line":1098},"message":"Host info","service.name":"filebeat","system_info":{"host":{"architecture":"x86_64","boot_time":"2022-12-10T21:31:58+08:00","containerized":false,"name":"iZuf62bas62gq2dhkizlgeZ","ip":["127.0.0.1/8","::1/128","172.17.224.102/20","fe80::216:3eff:fe31:607c/64"],"kernel_version":"5.4.0-135-generic","mac":["00:16:3e:31:60:7c"],"os":{"type":"linux","family":"debian","platform":"ubuntu","name":"Ubuntu","version":"20.04.5 LTS (Focal Fossa)","major":20,"minor":4,"patch":5,"codename":"focal"},"timezone":"CST","timezone_offset_sec":28800,"id":"f60897fec40e480a804ce0f452a4c686"},"ecs.version":"1.6.0"}}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.737+0800","log.logger":"beat","log.origin":{"file.name":"instance/beat.go","file.line":1127},"message":"Process info","service.name":"filebeat","system_info":{"process":{"capabilities":{"inheritable":null,"permitted":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read"],"effective":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read"],"bounding":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read"],"ambient":null},"cwd":"/","exe":"/usr/share/filebeat/bin/filebeat","name":"filebeat","pid":6433,"ppid":1,"seccomp":{"mode":"filter","no_new_privs":true},"start_time":"2022-12-11T19:28:12.830+0800"},"ecs.version":"1.6.0"}}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.738+0800","log.origin":{"file.name":"instance/beat.go","file.line":294},"message":"Setup Beat: filebeat; Version: 8.5.3","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.741+0800","log.logger":"publisher","log.origin":{"file.name":"pipeline/module.go","file.line":113},"message":"Beat name: iZuf62bas62gq2dhkizlgeZ","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.742+0800","log.logger":"modules","log.origin":{"file.name":"fileset/modules.go","file.line":120},"message":"Enabled modules/filesets: ","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"warn","@timestamp":"2022-12-11T19:28:16.742+0800","log.origin":{"file.name":"beater/filebeat.go","file.line":162},"message":"Filebeat is unable to load the ingest pipelines for the configured modules because the Elasticsearch output is not configured/enabled. If you have already loaded the ingest pipelines or are using Logstash pipelines, you can ignore this warning.","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.742+0800","log.logger":"monitoring","log.origin":{"file.name":"log/log.go","file.line":144},"message":"Starting metrics logging every 30s","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.742+0800","log.origin":{"file.name":"instance/beat.go","file.line":471},"message":"filebeat start running.","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.743+0800","log.origin":{"file.name":"memlog/store.go","file.line":134},"message":"Finished loading transaction log file for '/var/lib/filebeat/registry/filebeat'. Active transaction id=35","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"warn","@timestamp":"2022-12-11T19:28:16.743+0800","log.origin":{"file.name":"beater/filebeat.go","file.line":288},"message":"Filebeat is unable to load the ingest pipelines for the configured modules because the Elasticsearch output is not configured/enabled. If you have already loaded the ingest pipelines or are using Logstash pipelines, you can ignore this warning.","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.743+0800","log.logger":"registrar","log.origin":{"file.name":"registrar/registrar.go","file.line":109},"message":"States Loaded from registrar: 3","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.743+0800","log.logger":"crawler","log.origin":{"file.name":"beater/crawler.go","file.line":71},"message":"Loading Inputs: 3","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.743+0800","log.logger":"crawler","log.origin":{"file.name":"beater/crawler.go","file.line":117},"message":"starting input, keys present on the config: [filebeat.inputs.0.enabled filebeat.inputs.0.fields.topic filebeat.inputs.0.id filebeat.inputs.0.paths.0 filebeat.inputs.0.type]","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.743+0800","log.logger":"crawler","log.origin":{"file.name":"beater/crawler.go","file.line":148},"message":"Starting input (ID: 15323006513889051607)","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.743+0800","log.logger":"crawler","log.origin":{"file.name":"beater/crawler.go","file.line":117},"message":"starting input, keys present on the config: [filebeat.inputs.1.enabled filebeat.inputs.1.fields.topic filebeat.inputs.1.id filebeat.inputs.1.paths.0 filebeat.inputs.1.type]","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.744+0800","log.logger":"crawler","log.origin":{"file.name":"beater/crawler.go","file.line":148},"message":"Starting input (ID: 16598860088261641133)","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.744+0800","log.logger":"crawler","log.origin":{"file.name":"beater/crawler.go","file.line":117},"message":"starting input, keys present on the config: [filebeat.inputs.2.enabled filebeat.inputs.2.fields.topic filebeat.inputs.2.id filebeat.inputs.2.paths.0 filebeat.inputs.2.type]","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.744+0800","log.logger":"crawler","log.origin":{"file.name":"beater/crawler.go","file.line":148},"message":"Starting input (ID: 3792330016103154583)","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.744+0800","log.logger":"crawler","log.origin":{"file.name":"beater/crawler.go","file.line":106},"message":"Loading and starting Inputs completed. Enabled inputs: 3","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.744+0800","log.logger":"input.filestream","log.origin":{"file.name":"compat/compat.go","file.line":113},"message":"Input 'filestream' starting","service.name":"filebeat","id":"jenkinsfilestream","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.744+0800","log.logger":"input.filestream","log.origin":{"file.name":"compat/compat.go","file.line":113},"message":"Input 'filestream' starting","service.name":"filebeat","id":"mysqlfilestream","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.744+0800","log.logger":"input.filestream","log.origin":{"file.name":"compat/compat.go","file.line":113},"message":"Input 'filestream' starting","service.name":"filebeat","id":"redisfilestream","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.744+0800","log.origin":{"file.name":"cfgfile/reload.go","file.line":164},"message":"Config reloader started","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:16.745+0800","log.origin":{"file.name":"cfgfile/reload.go","file.line":224},"message":"Loading of config files completed.","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:19.736+0800","log.logger":"add_cloud_metadata","log.origin":{"file.name":"add_cloud_metadata/add_cloud_metadata.go","file.line":102},"message":"add_cloud_metadata: hosting provider type not detected.","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:19.773+0800","log.logger":"publisher_pipeline_output","log.origin":{"file.name":"pipeline/client_worker.go","file.line":139},"message":"Connecting to kafka(xxx:9092,xxx:9093,xxx:9094)","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:19.774+0800","log.logger":"publisher_pipeline_output","log.origin":{"file.name":"pipeline/client_worker.go","file.line":147},"message":"Connection to kafka(xxx:9092,xxx:9093,xxx:9094) established","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-12-11T19:28:46.745+0800","log.logger":"monitoring","log.origin":{"file.name":"log/log.go","file.line":186},"message":"Non-zero metrics in the last 30s","service.name":"filebeat","monitoring":{"metrics":{"beat":{"cgroup":{"memory":{"id":"filebeat.service","mem":{"limit":{"bytes":9223372036854771712},"usage":{"bytes":52260864}}}},"cpu":{"system":{"ticks":60,"time":{"ms":60}},"total":{"ticks":400,"time":{"ms":400},"value":400},"user":{"ticks":340,"time":{"ms":340}}},"handles":{"limit":{"hard":65536,"soft":65536},"open":19},"info":{"ephemeral_id":"1b653de2-89a9-4a44-95e1-4ad74e8fa303","name":"filebeat","uptime":{"ms":33097},"version":"8.5.3"},"memstats":{"gc_next":51202520,"memory_alloc":31162536,"memory_sys":45958152,"memory_total":88737992,"rss":123760640},"runtime":{"goroutines":68}},"filebeat":{"events":{"active":3571,"added":3571},"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0},"reloads":1,"scans":1},"output":{"events":{"active":3551,"batches":2,"total":3551},"type":"kafka"},"outputs":{"kafka":{"bytes_read":464,"bytes_write":101}},"pipeline":{"clients":5,"events":{"active":3571,"published":3571,"retry":2048,"total":3571},"queue":{"max_events":4096}}},"registrar":{"states":{"current":0}},"system":{"cpu":{"cores":2},"load":{"1":0.06,"15":0.08,"5":0.13,"norm":{"1":0.03,"15":0.04,"5":0.065}}}},"ecs.version":"1.6.0"}}
No error is showed in the log.
I am sure Filebeat connected to Kafka successfully, because the three topic are created in Kafka. However, there's no data in these topics when I use kafka-console-consumer to check (while I can manually produce data using kafka-console-producer).
And there are log generated in the input files.
How can I fix it? Thanks.