I have installed Elasticstash 6.2.4 with one filebeat node and a node with elasticsearch, kibana and logstash on the same node. Now i am trying to send logs of the a server using filebeat to logstash using the below config file located in /etc/logstash/conf.d
input {
beats {
port => "5044"
host => "xxxxxxxx"
}
}
filter {
if [type] == "syslog" {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
}
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
}
}
output {
elasticsearch { hosts => ["xxxxxxxx:9200"]
hosts => "xxxxxxxxx:9200"
user => "elastic"
password => "changeme"
manage_template => false
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}
I am getting no errors in any of the service logs and filebeat is also sending logs to my logstash. But i am not able to see any indexes getting created by elasticsearch
http://xxxxxxxxx:9200/_cat/indices?v
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
Below are my logstash logs and
[INFO ] 2018-11-26 06:24:47.959 [[main]-pipeline-manager] beats - Beats inputs: Starting input listener {:address=>"10.1.20.140:5044"}
[INFO ] 2018-11-26 06:24:48.025 [Ruby-0-Thread-1: /usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:22] pipeline - Pipeline started successfully {:pipeline_id=>"main", :thread=>"#<Thread:0x2cb14725@/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:247 sleep>"}
[INFO ] 2018-11-26 06:24:48.031 [[main]<beats] Server - Starting server on port: 5044
[INFO ] 2018-11-26 06:24:48.047 [Ruby-0-Thread-1: /usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:22] agent - Pipelines running {:count=>1, :pipelines=>["main"]}
filebeat logs
2018-11-26T12:29:20.164Z INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":10,"time":20},"total":{"ticks":20,"time":36,"value":20},"user":{"ticks":10,"time":16}},"info":{"ephemeral_id":"58504565-c112-4c4e-81c8-f1d29149bbf8","uptime":{"ms":240008}},"memstats":{"gc_next":4194304,"memory_alloc":1624952,"memory_total":4349504}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":0,"events":{"active":0}}},"registrar":{"states":{"current":0}},"system":{"load":{"1":16.88,"15":17.41,"5":17.2,"norm":{"1":2.11,"15":2.1763,"5":2.15}}}}}}
2018-11-26T12:29:50.165Z INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":20,"time":22},"total":{"ticks":30,"time":38,"value":30},"user":{"ticks":10,"time":16}},"info":{"ephemeral_id":"58504565-c112-4c4e-81c8-f1d29149bbf8","uptime":{"ms":270007}},"memstats":{"gc_next":4194304,"memory_alloc":1732568,"memory_total":4457120}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":0,"events":{"active":0}}},"registrar":{"states":{"current":0}},"system":{"load":{"1":17.26,"15":17.42,"5":17.26,"norm":{"1":2.1575,"15":2.1775,"5":2.1575}}}}}}
Something wrong with the logstash conf file? Am i missing something?