Hi,
I am setting up one port for all my beats input, including redis & haproxy logs, where haproxy logs work but redis logs do not get into elastic search (after adding Date plugin).
if I remove the Data plugin in the redis stream, it works (redis logs get into ES);
it does not append any dateparsefailure tag, so I assume the parse is ok;
even I turned on config.debug = true and log.level = debug, did not see any error or exception;
I have tried to rename the "timestamp" to "timestamp_redis" in redis stream, but it did not help.
Any suggestion?
config.yml:
input {
beats {
port => 5001
}
}
## Add your filters / logstash plugins configuration here
filter {
if [fileset][module] == "redis" {
grok {
match => { "message" => ["(\[)?%{POSINT:pid}(\])?(:)?%{NOTSPACE:[redis][log][role]} %{REDISTIMESTAMP:timestamp} %{NOTSPACE:[redis][log][level]} %{GREEDYDATA:what}"] }
}
date {
match => ["timestamp", "dd MMM HH:mm:ss.SSS"]
}
}
else if [source] == "/opt/logs/haproxy/haproxy.log" {
grok {
match => { "message" => ["%{HAPROXYTCP}"] }
}
date {
match => ["timestamp8601", "ISO8601"]
}
}
}
output {
elasticsearch {
hosts => "elasticsearch:9200"
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
}
stdout {
codec => "rubydebug"
}
}
rubydebug:
{
"timestamp" => "28 Apr 15:14:38.239",
"source" => "/opt/data/redis_cluster/logs/5162.1060",
"tags" => [
[0] "beats_input_codec_plain_applied"
],
"offset" => 249810163,
"what" => "DB 0: 4725 keys (4395 volatile) in 8192 slots HT.",
"pid" => "124455",
"@timestamp" => 2018-04-28T15:14:38.239Z,
"prospector" => {
"type" => "log"
},
"@version" => "1",
"beat" => {
"name" => "yz-8-111",
"version" => "6.2.2",
"hostname" => "yz-8-111"
},
"fileset" => {
"name" => "log",
"module" => "redis"
},
"redis" => {
"log" => {
"level" => "-",
"role" => "M"
}
},
"host" => "yz-8-111",
"message" => "124455:M 28 Apr 15:14:38.239 - DB 0: 4725 keys (4395 volatile) in 8192 slots HT."
}