Sending Logstash logs to D:/Hexa_Software/ELK/logstash-6.4.2/logs which is now configured via log4j2.properties
[2018-10-11T11:51:02,241][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2018-10-11T11:51:02,780][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"6.4.2"}
[2018-10-11T11:51:03,253][ERROR][logstash.agent ] Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_id:main, :exception=>"LogStash::ConfigurationError", :message=>"Expected one of #, { at line 10, column 10 (byte 152) after filter {\n \n# Join lines based on the time\nmultiline", :backtrace=>["D:/Hexa_Software/ELK/logstash-6.4.2/logstash-core/lib/logstash/compiler.rb:41:in compile_imperative'", "D:/Hexa_Software/ELK/logstash-6.4.2/logstash-core/lib/logstash/compiler.rb:49:incompile_graph'", "D:/Hexa_Software/ELK/logstash-6.4.2/logstash-core/lib/logstash/compiler.rb:11:in block in compile_sources'", "org/jruby/RubyArray.java:2486:inmap'", "D:/Hexa_Software/ELK/logstash-6.4.2/logstash-core/lib/logstash/compiler.rb:10:in compile_sources'", "org/logstash/execution/AbstractPipelineExt.java:149:ininitialize'", "D:/Hexa_Software/ELK/logstash-6.4.2/logstash-core/lib/logstash/pipeline.rb:22:in initialize'", "D:/Hexa_Software/ELK/logstash-6.4.2/logstash-core/lib/logstash/pipeline.rb:90:ininitialize'", "D:/Hexa_Software/ELK/logstash-6.4.2/logstash-core/lib/logstash/pipeline_action/create.rb:38:in execute'", "D:/Hexa_Software/ELK/logstash-6.4.2/logstash-core/lib/logstash/agent.rb:309:inblock in converge_state'"]}
[2018-10-11T11:51:03,543][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
====================== Config File================
input {
file {
path => "D:/Hexa_Software/ELK/logstash-6.4.2/bin/Test/alert_spf.log"
}
}
filter {
Join lines based on the time
multiline {
pattern => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
negate => true
what => "previous"
}
Create new field: oradb_status: starting,running,shutdown
if [message] =~ /Starting ORACLE instance/ {
mutate {
add_field => [ "oradb_status", "starting" ]
}
} else if [message] =~ /Instance shutdown complete/ {
mutate {
add_field => [ "oradb_status", "shutdown" ]
}
} else {
mutate {
add_field => [ "oradb_status", "running" ]
}
}
Search for ORA- and create field if match
if [message] =~ /ORA-/ {
grok {
match => [ "message","(?ORA-[0-9]*)" ]
}
}
Extract the date and the rest from the message
grok {
match => [ "message","%{DAY:day} %{MONTH:month} %{MONTHDAY:monthday} %{TIME:time} %{YEAR:year}(?<log_message>.*$)" ]
}
mutate {
add_field => {
"timestamp" => "%{year} %{month} %{monthday} %{time}"
}
}
replace the timestamp by the one coming from the alert.log
date {
locale => "en"
match => [ "timestamp" , "yyyy MMM dd HH:mm:ss" ]
}
replace the message (remove the date)
mutate { replace => [ "message", "%{log_message}" ] }
mutate {
remove_field => [ "time" ,"month","monthday","year","timestamp","day","log_message"]
}
}
output {
elasticsearch {
hosts => ["http://XXXXX:9200"]
index => "oracle-%{+YYYY.MM.dd}"
}
stdout { codec => rubydebug }
}