Hello All,
I have written 2 conf files for Log stash and placed under the path /etc/logstash/conf.d
1st conf file
[root@blrvmelk-01 conf.d]# cat logstash-Error1.conf
input {
file {
path => "/advdata/tmp/Error_Trace_blrhpbl_SC1*"
}
}
filter {
grok {
match => { "message" => "%{DATE_EU:date}-%{TIME:time}.[0-9]+" }
}
mutate {
gsub => [ "date","/","-" ]
add_field => { "timestamp" => "20%{date} %{time}" }
}
date {
match => [ "timestamp", "YYYY-MM-dd HH:mm:ss" ]
timezone => "Asia/Calcutta"
locale => "en"
}
}
output {
elasticsearch {
hosts => ["localhost:9200"]
user => "elastic"
password => "changeme"
}
stdout { codec => rubydebug }
}
[root@blrvmelk-01 conf.d]#
Second Conf File
[root@blrvmelk-01 conf.d]# cat logstash-Error2..conf
input {
file {
path => "/advdata/tmp/Error_Trace_blrhpbl_SC2*"
}
}
filter {
grok {
match => { "message" => "%{DATE_EU:date}-%{TIME:time}.[0-9]+" }
}
mutate {
gsub => [ "date","/","-" ]
add_field => { "timestamp" => "20%{date} %{time}" }
}
date {
match => [ "timestamp", "YYYY-MM-dd HH:mm:ss" ]
timezone => "Asia/Calcutta"
locale => "en"
}
}
output {
elasticsearch {
hosts => ["localhost:9200"]
user => "elastic"
password => "changeme"
}
stdout { codec => rubydebug }
}
[root@blrvmelk-01 conf.d]#
Now in Kibana, when i Visualise the data, i see duplicate entries, example as below (see time, timestamp,)
November 27th 2016, 08:04:45.000 message:blrhpbl001-bl06:2016/11/27-08:04:45.319319-36501-38250-IMS_G_PCS_CORE_290- > Connection details are not available for the peer : GGSN_A date:16-11-27, 16-11-27 path:/advdata/tmp/Error_Trace_blrhpbl_SC1_27112016.txt @timestamp:November 27th 2016, 08:04:45.000 @version:1 host:blrvmelk-01 time:08:04:45, 08:04:45 tags:_dateparsefailure timestamp:2016-11-27 08:04:45, 2016-11-27,16-11-27 08:04:45,08:04:45 _id:AVips3fM-9_iqxSYx-LJ _type:logs _index:logstash-2016.11.27 _score: -
This was observed after i added the second conf file. Even though the Input files are different for the 2 conf files, i expect Kibana not to duplicate the tags. How do i avoid this ? Any suggestions for improvement in the Conf files ?