Filebeat is not reading logs

root@ub2204elk:/etc/elasticsearch# grep -v "#" elasticsearch.yml |uniq

path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: localhost

xpack.security.enabled: true

xpack.security.enrollment.enabled: true

xpack.security.http.ssl:
  enabled: false
  keystore.path: certs/http.p12

xpack.security.transport.ssl:
  enabled: true
  verification_mode: certificate
  keystore.path: certs/transport.p12
  truststore.path: certs/transport.p12
cluster.initial_master_nodes: ["ub2204elk"]

http.host: 0.0.0.0

root@ub2204elk:/etc/filebeat# l grep -v "#" /etc/filebeat/filebeat.yml |uniq

filebeat.inputs:

- type: filestream
  id: my-filestream-id
  enabled: false
  paths:
    - /var/log/sys1/*.log

filebeat.config.modules:
  path: /etc/filebeat/modules.d/modules.d/*.yml
  reload.enabled: false

setup.template.settings:
  index.number_of_shards: 1

setup.kibana:
  host: "localhost:5601"

output.elasticsearch:
  hosts: ["localhost:9200"]

  username: "elastic"
  password: "oRPweOskO3ODRgI6Hik-"

processors:
  - add_host_metadata:
      when.not.contains.tags: forwarded
  - add_cloud_metadata: ~
  - add_docker_metadata: ~
  - add_kubernetes_metadata: ~

rsyslog is doing its thing correctly both remote and local elk/rsyslog server. The remote server's logs are in the elk/rsyslog server's /var/log/{remote.servername}.

These are the logs I am most interested in.
root@ub2204elk:/etc/filebeat# ls -l /var/log/sys1/*.log

-rw-r----- 1 syslog adm   270 Dec 21 18:34 /var/log/sys1/anacron.log
-rw-r----- 1 syslog adm   245 Dec 21 17:20 /var/log/sys1/avahi-daemon.log
-rw-r----- 1 syslog adm    63 Dec 21 17:24 /var/log/sys1/cron.log
-rw-r----- 1 syslog adm 36846 Dec 21 18:53 /var/log/sys1/CRON.log
-rw-r----- 1 syslog adm   298 Dec 21 17:23 /var/log/sys1/crontab.log
-rw-r----- 1 syslog adm  2837 Dec 21 18:52 /var/log/sys1/dbus-daemon.log
-rw-r----- 1 syslog adm   172 Dec 21 18:48 /var/log/sys1/fwupd.log
-rw-r----- 1 syslog adm  2249 Dec 21 17:38 /var/log/sys1/kernel.log
-rw-r----- 1 syslog adm   132 Dec 21 18:41 /var/log/sys1/NetworkManager.log
-rw-r----- 1 syslog adm  1119 Dec 21 18:40 /var/log/sys1/nordvpnd.log
-rw-r----- 1 syslog adm   181 Dec 21 17:20 /var/log/sys1/ntpd.log
-rw-r----- 1 syslog adm  4792 Dec 21 18:51 /var/log/sys1/org.freedesktop.thumbnails.Thumb.log
-rw-r----- 1 syslog adm   259 Dec 21 18:14 /var/log/sys1/root.log
-rw-r----- 1 syslog adm  3957 Dec 21 17:32 /var/log/sys1/rsyslogd.log
-rw-r----- 1 syslog adm 24552 Dec 21 18:51 /var/log/sys1/rtkit-daemon.log
-rw-r----- 1 syslog adm   593 Dec 21 17:57 /var/log/sys1/sudo.log
-rw-r----- 1 syslog adm 32889 Dec 21 18:53 /var/log/sys1/systemd.log

But, in http://192.168.1.139:5601 | discover | Logs | Streams does not show any logs.

Am I looking in the correct place in Kibana for the remote server's logs? If so, Why isn't filebeats putting the logs into ealsticsearch?

(please let me know if you need a peak at any other conf files)

the filestream input isn't enabled you have:

enabled: false

change that to:

enabled: true
1 Like

I owe you a beer! As soon as I made that change, the logs showed up in streaming!

(I intentionally have smbd flapping on sys1 every min (via cron) to generate logs for testing)

1 Like

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.