Hello,
Recently I upgraded ELK from 6.2.2 to 6.3.2.
I have translate filter plugin for my logs processing filter which was working great in 6.2.2 but not in 6.3.2. I have 4 different conf files out of 4, 2 are working and the conf which has translate filter is not working
I updated the syntax as per the new version of translate plugin ver 3.1.0..
but logstash throws following Error message.
[ERROR][logstash.agent ] Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_id:syserr, :exception=>"LogStash::ConfigurationError", :message=>"Expected one of #, {, } at line 30, column 30 (byte 1175) after filter{\r\n\tgrok {\r\n\t\tpatterns_dir => ["./appl/grokpattern"]\r\n\t\tmatch => ["message", "\[%{TimeDate:loggingTime}\] %{BASE16NUM:ThreadId} %{WORD:ComponentName}%{SPACE}%{WORD:loglevel}%{SPACE}%{GREEDYDATA:logMessage}"]\r\n\t\toverwrite => [ "message" ]\r\n\t}\t\r\n\tdate {\r\n\t\tmatch => ["loggingTime", "M/dd/yy HH:mm:ss:SSS z", "MM/d/yy HH:mm:ss:SSS z", "M/d/yy HH:mm:ss:SSS z", "MM/dd/yy H:mm:ss:SSS z", "M/d/yy H:mm:ss:SSS z", "MM/d/yy H:mm:ss:SSS z", "M/dd/yy H:mm:ss:SSS z"]\r\n\t\ttarget => "@timestamp"\r\n \t}\r\n\ttranslate {\r\n\t\tfield => "loglevel"\r\n\t\toverride => true\r\n\t\tdestination => "loglevel"\r\n\t\tdictionary => {"F"=>"FATAL"", :backtrace=>["/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:42:in compile_imperative'", "/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:50:incompile_graph'", "/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:12:in block in compile_sources'", "org/jruby/RubyArray.java:2486:inmap'", "/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:11:in compile_sources'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:49:ininitialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:167:in initialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline_action/create.rb:40:inexecute'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:305:in `block in converge_state'"]}
for your reference I have the conf file content here.. To me every thing syntax wise looks good. but not sure why it is not working..
input {
file {
path => ["PATH to files"]
start_position => "beginning"
sincedb_path => "/path/sincedbfile_SysErr.txt"
codec => multiline {
pattern => "^\["
negate => true
what => "previous"
}
exclude => ["*.gz","native_std*.log"]
type => "WAS_SysErr_logs"
tags => "WAS_SysErr_logs"
}
}
filter{
grok {
patterns_dir => ["./Path/grokpattern"]
match => ["message", "\[%{TimeDate:loggingTime}\] %{BASE16NUM:ThreadId} %{WORD:ComponentName}%{SPACE}%{WORD:loglevel}%{SPACE}%{GREEDYDATA:logMessage}"]
overwrite => [ "message" ]
}
date {
match => ["loggingTime", "M/dd/yy HH:mm:ss:SSS z", "MM/d/yy HH:mm:ss:SSS z", "M/d/yy HH:mm:ss:SSS z", "MM/dd/yy H:mm:ss:SSS z", "M/d/yy H:mm:ss:SSS z", "MM/d/yy H:mm:ss:SSS z", "M/dd/yy H:mm:ss:SSS z"]
target => "@timestamp"
}
translate {
field => "loglevel"
override => true
destination => "loglevel"
dictionary => {"F"=>"FATAL","W"=>"WARNING","I"=>"INFO","D"=>"DETAIL","E"=>"ERROR","A"=>"AUDIT","C"=>"CONFIG"}
}
mutate{
remove_field => ["TimeDate","TIME","DATE", "TIMEZONE"]
}
fingerprint{
id => "WASSysErrLogs"
source => ["@timestamp","message"]
target => "fingerprint"
key => "35353535"
method => "SHA1"
concatenate_sources => true
}
}
output {
elasticsearch {
hosts => ["Server1:9200","Server2:9200"]
document_id => "%{fingerprint}"
index => "was-galc-logs-%{+YYYY.MM.dd}"
}
}
Could anyone take a look and whats wrong in the configuration?
Thanks in Advance.
Fredrick