I am new to logstash ,
I am trying to convert an logstash 2.4 conf file to latest 5.x release but I am getting strange behavior . It does create the elastic index but that index is not making sense to me .
why is he creating two indexes ?
and I am also not understanding why in first index the field "message" is not populated ?
my conf file
input {
file {
path => "/home/admin/a.log"
start_position => "beginning"
codec => multiline {
pattern => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
negate => true
what => "previous"
}
}}
filter {Create new field: oradb_status: starting,running,shutdown
if [message] =~ /Starting ORACLE instance/ {
mutate {
add_field => [ "oradb_status", "starting" ]
}
} else if [message] =~ /Instance shutdown complete/ {
mutate {
add_field => [ "oradb_status", "shutdown" ]
}
} else {
mutate {
add_field => [ "oradb_status", "running" ]
}
}Search for ORA- and create field if match
if [message] =~ /ORA-/ {
grok {
match => [ "message","(?ORA-[0-9]*)" ]
}
}Extract the date and the rest from the message
grok {
match => [ "message","%{DAY:day} %{MONTH:month} %{MONTHDAY:monthday} %{TIME:time} %{YEAR:year}(?<log_message>.*$)" ]
}mutate {
add_field => {
"timestamp" => "%{year} %{month} %{monthday} %{time}"
}
}replace the timestamp by the one coming from the alert.log
date {
locale => "en"
match => [ "timestamp" , "yyyy MMM dd HH:mm:ss" ]
}replace the message (remove the date)
mutate { replace => [ "message", "%{log_message}" ] }
mutate {
remove_field => [ "time" ,"month","monthday","year","timestamp","day","log_message"]
}}
output {
elasticsearch {
hosts => ["hadoop5:9200"]
index => "oracle-%{+YYYY.MM.dd}"
}
}
my input log file
[root@hadoop1 bin]# more /home/admin/a.log
==> alert_patron1.log <==
Tue Feb 16 10:34:08 2016
Thread 1 advanced to log sequence 189275 (LGWR switch)
Current log# 6 seq# 189275 mem# 0: +PATRON_SYS/patron/onlinelog/group_6.272.872756301
Tue Feb 16 10:34:10 2016
LNS: Standby redo logfile selected for thread 1 sequence 189275 for destination LOG_ARCHIVE_DEST_2
Tue Feb 16 10:35:03 2016
Archived Log entry 135628 added for thread 1 sequence 189274 ID 0xffffffffad76223d dest 1:
Tue Feb 16 10:48:05 2016
Emon ping encountered error 24347
Tue Feb 16 11:28:04 2016
[root@hadoop1 bin]#
indexes created
[root@hadoop5 ~]# curl 'hadoop5:9200/_cat/indices?v'
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
green open oracle-2016.02.16 xbXrNwxjTwGn_muGtqxkFw 5 1 4 0 47.2kb 23.6kb
green open oracle-2017.08.12 -MVsunq1RSie64o5Wjgj_g 5 1 1 0 15.4kb 7.7kb[root@hadoop5 ~]# curl -XGET 'hadoop5:9200/oracle-2017.08.12/_search?pretty' -d '
{
"query" : {
"match_all" : {}
}
}'
{
"took" : 1,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"failed" : 0
},
"hits" : {
"total" : 1,
"max_score" : 1.0,
"hits" : [
{
"_index" : "oracle-2017.08.12",
"_type" : "logs",
"_id" : "AV3T3Zja9FciQA_yg8mt",
"_score" : 1.0,
"_source" : {
"path" : "/home/admin/a.log",
"@timestamp" : "2017-08-12T00:35:01.693Z",
"@version" : "1",
"host" : "hadoop1.tolls.dot.state.fl.us",
"oradb_status" : "running",
"message" : "%{log_message}",
"tags" : [
"_grokparsefailure",
"_dateparsefailure"
]
}
}
]
}
}[root@hadoop5 ~]# curl -XGET 'hadoop5:9200/oracle-2016.02.16/_search?pretty' -d '
{
"query" : {
"match_all" : {}
}
}'
{
"took" : 3,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"failed" : 0
},
"hits" : {
"total" : 4,
"max_score" : 1.0,
"hits" : [
{
"_index" : "oracle-2016.02.16",
"_type" : "logs",
"_id" : "AV3T3Zja9FciQA_yg8mx",
"_score" : 1.0,
"_source" : {
"oradb_status" : "running",
"message" : "\nEmon ping encountered error 24347",
"tags" : [
"multiline"
],
"path" : "/home/admin/a.log",
"@timestamp" : "2016-02-16T15:48:05.000Z",
"@version" : "1",
"host" : "hadoop1.tolls.dot.state.fl.us"
}
},
{
"_index" : "oracle-2016.02.16",
"_type" : "logs",
"_id" : "AV3T3Zja9FciQA_yg8mu",
"_score" : 1.0,
"_source" : {
"oradb_status" : "running",
"message" : "\nThread 1 advanced to log sequence 189275 (LGWR switch)\n Current log# 6 seq# 189275 mem# 0: +PATRON_SYS/patron/onlinelog/group_6.272.872756301",
"tags" : [
"multiline"
],
"path" : "/home/admin/a.log",
"@timestamp" : "2016-02-16T15:34:08.000Z",
"@version" : "1",
"host" : "hadoop1.tolls.dot.state.fl.us"
}
},
{
"_index" : "oracle-2016.02.16",
"_type" : "logs",
"_id" : "AV3T3Zja9FciQA_yg8mv",
"_score" : 1.0,
"_source" : {
"oradb_status" : "running",
"message" : "\nLNS: Standby redo logfile selected for thread 1 sequence 189275 for destination LOG_ARCHIVE_DEST_2",
"tags" : [
"multiline"
],
"path" : "/home/admin/a.log",
"@timestamp" : "2016-02-16T15:34:10.000Z",
"@version" : "1",
"host" : "hadoop1.tolls.dot.state.fl.us"
}
},
{
"_index" : "oracle-2016.02.16",
"_type" : "logs",
"_id" : "AV3T3Zja9FciQA_yg8mw",
"_score" : 1.0,
"_source" : {
"oradb_status" : "running",
"message" : "\nArchived Log entry 135628 added for thread 1 sequence 189274 ID 0xffffffffad76223d dest 1:",
"tags" : [
"multiline"
],
"path" : "/home/admin/a.log",
"@timestamp" : "2016-02-16T15:35:03.000Z",
"@version" : "1",
"host" : "hadoop1.tolls.dot.state.fl.us"
}
}
]
}
}