Filebeat-logstash issue

mustafa.husny · December 21, 2020, 2:33pm

I am using ELK 7.6.2
I have three piplines
first for the heartbeat
second for the tcp to forward logs from QRadar
Third is for filebeat
I am receiving logs from heartbeat and qradar, but when I installed filebeat on windows machine I can not receive logs from it
my filebeat config

#=========================== Filebeat inputs =============================

filebeat.inputs:

Each - is an input. Most options can be set at the input level, so

you can use different inputs for various configurations.

Below are the input specific configurations.

type: log

Change to true to enable this input configuration.

enabled: true

Paths that should be crawled and fetched. Glob based paths.

paths:

'c:\Program Files\IBM\WebSphere\AppServer\profiles\AppSrv01\logs\activity.log'

#============================= Filebeat modules ===============================

filebeat.config.modules:

Glob pattern for configuration loading

path: ${path.config}/modules.d/*.yml

Set to true to enable config reloading

reload.enabled: false

Period on which files under path should be checked for changes

#reload.period: 10s

#----------------------------- Logstash output --------------------------------
output.logstash:

The Logstash hosts

hosts: ["ELK:5044"]

logstash pipline for filebeat

input {
tcp {
port => 5000
codec => json
}
}

filter {
date {
match => [ "timeMillis", "UNIX_MS" ]
}
}

output {

stdout { codec => rubydebug }

elasticsearch {
hosts => "ip:9200"
user => "elastic"
password => "mypassword"
index => "qradar-%{+YYYY.MM.dd}"
}
}

piplines.conf

pipeline.id: main
path.config: "/etc/logstash/conf.d/filebeat.conf"

pipeline.id: heartbeat
path.config: "/etc/logstash/conf.d/heartbeat.conf"

pipeline.id: qradar
path.config: "/etc/logstash/conf.d/qradar.conf"

I start the service using systemctl

I can telnet from my windows machine to ELK machine on port 5044

sample of filebeat logs

2020-12-21T15:10:46.218+0200 INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":311,"time":{"ms":16},"value":311},"user":{"ticks":249,"time":{"ms":16}}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1382417}},"memstats":{"gc_next":9814304,"memory_alloc":4971112,"memory_total":15764888,"rss":8192},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}

2020-12-21T15:11:16.219+0200 INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":327,"value":327},"user":{"ticks":265}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1412417}},"memstats":{"gc_next":9814304,"memory_alloc":5010584,"memory_total":15804360},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}

2020-12-21T15:11:46.219+0200 INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":327,"value":327},"user":{"ticks":265}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1442416}},"memstats":{"gc_next":9814304,"memory_alloc":5055768,"memory_total":15849544,"rss":4096},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}

2020-12-21T15:12:16.219+0200 INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":327,"value":327},"user":{"ticks":265}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1472416}},"memstats":{"gc_next":9819616,"memory_alloc":4908352,"memory_total":15898536,"rss":-126976},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}

2020-12-21T15:12:46.219+0200 INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":327,"value":327},"user":{"ticks":265}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1502416}},"memstats":{"gc_next":9819616,"memory_alloc":4963368,"memory_total":15953552},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}

2020-12-21T15:13:16.219+0200 INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":327,"value":327},"user":{"ticks":265}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1532416}},"memstats":{"gc_next":9819616,"memory_alloc":5005096,"memory_total":15995280},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}

2020-12-21T15:13:46.220+0200 INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":327,"value":327},"user":{"ticks":265}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1562418}},"memstats":{"gc_next":9819616,"memory_alloc":5055416,"memory_total":16045600},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}

2020-12-21T15:14:16.220+0200 INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":342,"time":{"ms":15},"value":342},"user":{"ticks":280,"time":{"ms":15}}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1592416}},"memstats":{"gc_next":9814304,"memory_alloc":4911088,"memory_total":16093632,"rss":4096},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}

2020-12-21T15:14:46.220+0200 INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":342,"value":342},"user":{"ticks":280}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1622417}},"memstats":{"gc_next":9814304,"memory_alloc":4961272,"memory_total":16143816},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}

2020-12-21T15:15:16.220+0200 INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":342,"value":342},"user":{"ticks":280}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1652416}},"memstats":{"gc_next":9814304,"memory_alloc":5007304,"memory_total":16189848},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}

I can not find index in kibana to create index pattern
Tried to check traffic using tcpdump utility but I can not receive any traffic

please please advice

leandrojmp · December 21, 2020, 2:48pm

I think you shared the wrong config file in your question, the pipeline you shared seems to be the qradar pipeline.

Share the config of your filebeat pipeline.

mustafa.husny · December 21, 2020, 6:32pm

Sorry my bad

input {
beats {
port => 5044
ssl => false
}
}

output {
elasticsearch {
hosts => ["http://elk:9200"]
index => "logs-%{[host][name]}-%{IP}-%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
user => "elastic"
password => "mypassword"
}
}

I can not put the configuration with its indentation

mustafa.husny · December 23, 2020, 3:16am

Any idea

system · January 20, 2021, 3:16am

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.

2020-12-21T15:10:46.218+0200	INFO	[monitoring]	log/log.go:145	Non-zero metrics in the last 30s	{"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":311,"time":{"ms":16},"value":311},"user":{"ticks":249,"time":{"ms":16}}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1382417}},"memstats":{"gc_next":9814304,"memory_alloc":4971112,"memory_total":15764888,"rss":8192},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}
2020-12-21T15:11:16.219+0200	INFO	[monitoring]	log/log.go:145	Non-zero metrics in the last 30s	{"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":327,"value":327},"user":{"ticks":265}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1412417}},"memstats":{"gc_next":9814304,"memory_alloc":5010584,"memory_total":15804360},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}
2020-12-21T15:11:46.219+0200	INFO	[monitoring]	log/log.go:145	Non-zero metrics in the last 30s	{"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":327,"value":327},"user":{"ticks":265}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1442416}},"memstats":{"gc_next":9814304,"memory_alloc":5055768,"memory_total":15849544,"rss":4096},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}
2020-12-21T15:12:16.219+0200	INFO	[monitoring]	log/log.go:145	Non-zero metrics in the last 30s	{"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":327,"value":327},"user":{"ticks":265}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1472416}},"memstats":{"gc_next":9819616,"memory_alloc":4908352,"memory_total":15898536,"rss":-126976},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}
2020-12-21T15:12:46.219+0200	INFO	[monitoring]	log/log.go:145	Non-zero metrics in the last 30s	{"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":327,"value":327},"user":{"ticks":265}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1502416}},"memstats":{"gc_next":9819616,"memory_alloc":4963368,"memory_total":15953552},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}
2020-12-21T15:13:16.219+0200	INFO	[monitoring]	log/log.go:145	Non-zero metrics in the last 30s	{"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":327,"value":327},"user":{"ticks":265}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1532416}},"memstats":{"gc_next":9819616,"memory_alloc":5005096,"memory_total":15995280},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}
2020-12-21T15:13:46.220+0200	INFO	[monitoring]	log/log.go:145	Non-zero metrics in the last 30s	{"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":327,"value":327},"user":{"ticks":265}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1562418}},"memstats":{"gc_next":9819616,"memory_alloc":5055416,"memory_total":16045600},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}
2020-12-21T15:14:16.220+0200	INFO	[monitoring]	log/log.go:145	Non-zero metrics in the last 30s	{"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":342,"time":{"ms":15},"value":342},"user":{"ticks":280,"time":{"ms":15}}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1592416}},"memstats":{"gc_next":9814304,"memory_alloc":4911088,"memory_total":16093632,"rss":4096},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}
2020-12-21T15:14:46.220+0200	INFO	[monitoring]	log/log.go:145	Non-zero metrics in the last 30s	{"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":342,"value":342},"user":{"ticks":280}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1622417}},"memstats":{"gc_next":9814304,"memory_alloc":4961272,"memory_total":16143816},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}
2020-12-21T15:15:16.220+0200	INFO	[monitoring]	log/log.go:145	Non-zero metrics in the last 30s	{"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":62},"total":{"ticks":342,"value":342},"user":{"ticks":280}},"handles":{"open":169},"info":{"ephemeral_id":"fc74c7bf-bf32-4fc8-ae14-8bd53d923764","uptime":{"ms":1652416}},"memstats":{"gc_next":9814304,"memory_alloc":5007304,"memory_total":16189848},"runtime":{"goroutines":26}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":0}}}}}