While using redis as the in-memory database the logstash config file is not working fine whereas when in place redis if I use the file input plugin it does not fail at all. I am not able to understand what is wrong with my config file where I use redis as the input plugin.
Redis forwader
input {
file {
path => "/logs/app_/.log"
exclude => [
"access*.log*",
"fe*.log*",
"stat*.log*",
"gc*.log*",
"dump.log*"
]
sincedb_path => "/dev/null"
type => "app"
codec => multiline {
pattern => "^%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}"
negate => true
what => "previous"
}
}
}
filter {
grok {
match => {
"path" => "/logs/(?<app>[^/]+)/[^/]+/logs/(?<instance>[^/]+)/[^/]+"
}
overwrite => [ "host" ]
}
}
output {
stdout { codec => rubydebug }
redis {
host => "gauapt13"
data_type => "list"
key => "logstash-app"
}
}
redis indexer
input {
redis {
host => "gauapt13"
data_type => "list"
key => "logstash-app"
codec => json
threads => 2
codec => multiline {
pattern => "^%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}"
negate => true
what => "previous"
}
}
}
filter {
if [type] == "app" {
grok {
match => {
# "message" => "%{APT_TIMESTAMP:timestamp} \| %{HOSTNAME:hostname} \| %{DATA:application} \| %{APT_LOGLEVEL:loglevel} *\| 1-%{DATA:thread:int} \| %{DATA:class} *\| %{DATA:correlationId} \| %{GREEDYDATA:msg}"
"message" => "%{APT_TIMESTAMP:timestamp} \| \S+ \| \S+ \| %{APT_LOGLEVEL:loglevel} *\| 1-%{DATA:thread:int} \| %{DATA:class} *\| %{DATA:correlationId} \| %{GREEDYDATA:msg}"
}
patterns_dir => "/logs/pattern"
remove_field => [ "message" ]
}
}
if "_grokparsefailure" not in [tags] {
date {
match => [ "timestamp", "YYYY/MM/dd HH:mm:ss.SSS", "YYYY/MM/dd HH:mm:ss,SSS", "dd/MMM/YYYY:HH:mm:ss +0000", "EEE MMM dd HH:mm:ss YYYY" ]
timezone => "UTC"
}
if [type] == "web_access" or [type] == "web_error" {
mutate {
gsub => [
"referrer","\"","",
"agent","\"","",
"JSESSIONID","\"","",
"APT_SESSIONID","\"","",
"correlationId","\"","",
"transactionOriginator","\"","",
"customerId","\"",""
]
remove_field => [ "logline", "timestamp", "BASE10NUM", "INT", "HOSTNAME", "IPV4", "day", "month", "monthday", "time", "year"]
}
}
}
}
output {
stdout{ codec => rubydebug}
if [app] == "aaa" or [app] == "amga" or [type] == "fem" or [type] == "sepal" or [type] == "web_access" {
elasticsearch {
hosts => "gauapt13"
index => "apt-%{+YYYY.MM.dd}"
}
}
else {
elasticsearch {
hosts => "gauapt13"
index => "apt-%{+YYYY.MM.dd}"
}
}
}