Hello,
we are having an issue with filter configuration in the logstash. we have upgraded the logstash form 5 to logstash 6.8.1 version. the filter configuration was working fine in logstash 5.4 version, but not in the logstash 6.8.1. if we remove filter configuration the logstash is working fine, but logstash is failing after adding the filter. Can you please help me here?
logstash.conf:
input {
syslog {
port => 5514
type => "syslog"
}
beats {
port => 5044
}
}
filter {
if [source] =~ "app1" {
grok {
tag_on_failure => ["_grokdateparsefailure"]
match => { "message" => "%{TIMESTAMP_ISO8601:logdate}.*"}
}
date {
match => [ "logdate", "YYYY-MM-dd HH:mm:ss,SSS" ]
target => "logdateTime"
timezone => "America/New_York"
remove_field => ["logdate"]
}
if [message] =~ "JSON:" {
grok {
patterns_dir => ["/appdata/config/logstash/patterns"]
tag_on_failure => ["_jsonparsefailure" ]
break_on_match => true
keep_empty_captures => false
match => { "message" => ".*JSON:%{JSON:requestJSON}.*"}
}
json{
source => "requestJSON"
target => "doc"
remove_field => ["requestJSON"]
}
}
}
if [source] =~ "app2" {
if [message] =~ "\[.*\]" {
grok {
patterns_dir => ["/appdata/config/logstash/patterns"]
break_on_match => true
tag_on_failure => ["_grokfixparsefailure"]
match => { "message" => ".*\[%{GREEDYDATA:FIXMESSAGE}\]"}
}
kv {
value_split => "="
field_split => "\001"
source => "APPMESSAGE"
}
}
grok {
patterns_dir => ["/appdata/config/logstash/patterns"]
break_on_match => true
tag_on_failure => ["_grokdateparsefailure"]
match => { "message" => "%{APPDATETIMESTAMP:logdate}.*"}
}
date {
match => [ "logdate", "YYYY/MM/dd HH:mm:ss:SSS" ]
target => "logdateTime"
timezone => "America/New_York"
remove_field => ["logdate"]
remove_field => ["APPMESSAGE"]
}
}
}
output {
if [type] == "syslog" {
elasticsearch {
hosts => ["http://localhost:9200"]
user => "xxxxxx"
password => "xxxxxxxx"
index => "syslog-%{+YYYY.MM.dd}"
ssl => false
ssl_certificate_verification => false
ilm_enabled => false
}
}
else {
elasticsearch {
hosts => ["http://localhost:9200"]
manage_template => false
user => "xxxxxxxxx"
password => "xxxxxxxxxx"
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
ssl => false
ssl_certificate_verification => false
ilm_enabled => false
}
}
}
Logstash Logs:
2020-06-11T10:50:20.586-04:00
[2020-06-11T14:50:20,586][ERROR][logstash.pipeline ] Error registering plugin {:pipeline_id=>"main", :plugin=>"#LogStash::FilterDelegator:0x10981ad5", :error=>"pattern %{JSON:requestJSON} not defined", :thread=>"#<Thread:0x11cdb9f2 run>"}
2020-06-11T10:50:20.740-04:00
[2020-06-11T14:50:20,739][ERROR][logstash.pipeline ] Pipeline aborted due to error {:pipeline_id=>"main", :exception=>#<Grok::PatternError: pattern %{JSON:requestJSON} not defined>, :backtrace=>["/etc/logstash/vendor/bundle/jruby/2.5.0/gems/jls-grok-0.11.5/lib/grok-pure.rb:123:in block in compile'", "org/jruby/RubyKernel.java:1425:in loop'", "/etc/logstash/vendor/bundle/jruby/2.5.0/gems/jls-grok-0.11.5/lib/grok-pure.rb:93:in compile'", "/etc/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-grok-4.0.4/lib/logstash/filters/grok.rb:281:in block in register'", "org/jruby/RubyArray.java:1792:in each'", "/etc/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-grok-4.0.4/lib/logstash/filters/grok.rb:275:in block in register'", "org/jruby/RubyHash.java:1419:in each'", "/etc/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-grok-4.0.4/lib/logstash/filters/grok.rb:270:in register'", "org/logstash/config/ir/compiler/AbstractFilterDelegatorExt.java:56:in register'", "/etc/logstash/logstash-core/lib/logstash/pipeline.rb:259:in register_plugin'", "/etc/logstash/logstash-core/lib/logstash/pipeline.rb:270:in block in register_plugins'", "org/jruby/RubyArray.java:1792:in each'", "/etc/logstash/logstash-core/lib/logstash/pipeline.rb:270:in register_plugins'", "/etc/logstash/logstash-core/lib/logstash/pipeline.rb:612:in maybe_setup_out_plugins'", "/etc/logstash/logstash-core/lib/logstash/pipeline.rb:280:in start_workers'", "/etc/logstash/logstash-core/lib/logstash/pipeline.rb:217:in run'", "/etc/logstash/logstash-core/lib/logstash/pipeline.rb:176:in `block in start'"], :thread=>"#<Thread:0x11cdb9f2 run>"}
2020-06-11T10:50:20.756-04:00
[2020-06-11T14:50:20,756][ERROR][logstash.agent ] Failed to execute action {:id=>:main, :action_type=>LogStash::ConvergeResult::FailedAction, :message=>"Could not execute action: PipelineAction::Create, action_result: false", :backtrace=>nil}