Hey all.
This is my first post in these forums, so- hey everyone
I am parsing few different input file sources using grok filters.
if there is no match on any of them I have created a "fail safe" while adding a tag "no_match".
if filter "ccc" have had a hit I want to add tag "ccc".
I have managed to see matches both from the "failsafe" and the "ccc" file, but no tag were added.
I do see "multiline" tag some times, but none of my manual tags are being added.
Please see my part of my Logstash configuration below, I have removed some data for privacy.
Thank you.
Edit: I am using Logstash and ES version 6.1.1
input {
file {
path => "aaa.log*"
start_position => "beginning"
codec => multiline {
pattern => "^%{TIMESTAMP_ISO8601}"
negate => true
what => "previous"
max_lines => 1000
}
type => "aaa"
}
file {
path => "bbb.log*"
start_position => "beginning"
codec => multiline {
pattern => "^%{TIMESTAMP_ISO8601}"
negate => true
what => "previous"
max_lines => 1000
}
type => "bbb"
}
file {
path => "ccc.log*"
start_position => "beginning"
codec => multiline {
pattern => "^%{TIMESTAMP_ISO8601}"
negate => true
what => "previous"
max_lines => 1000
}
type => "ccc"
}
}
filter {
if [type] == "aaa" {
grok {
break_on_match => true
#With exception stacktrace:
match => {
"message" => "%{TIMESTAMP_ISO8601:event_timestamp}%{SPACE}%{DATA:log_level}%{SPACE}\[%{DATA:thread_name}\]%{SPACE}\[%{DATA:session}\]%{SPACE}\[%{DATA:trx}\]%{SPACE}\[%{DATA:java_class}\]%{SPACE}\[%{DATA:correlation_id}\]%{SPACE}\[%{DATA:host}\]%{SPACE}\[%{DATA:tenant}\]%{SPACE}\[%{DATA:organization}\]%{SPACE}\[%{DATA:user}\]%{SPACE}-((%{SPACE}<)%{GREEDYDATA:text}>(\n|\r\n)(?m)%{GREEDYDATA:exception})"
"path" => "%{GREEDYDATA:filename}"
}
#Without stacktrace:
match => {
"message" => "%{TIMESTAMP_ISO8601:event_timestamp}%{SPACE}%{DATA:log_level}%{SPACE}\[%{DATA:thread_name}\]%{SPACE}\[%{DATA:session}\]%{SPACE}\[%{DATA:trx}\]%{SPACE}\[%{DATA:java_class}\]%{SPACE}\[%{DATA:correlation_id}\]%{SPACE}\[%{DATA:host}\]%{SPACE}\[%{DATA:tenant}\]%{SPACE}\[%{DATA:organization}\]%{SPACE}\[%{DATA:user}\]%{SPACE}-%{SPACE}<%=@lt%>{GREEDYDATA:text}>"
"path" => "%{GREEDYDATA:filename}"
}
#failsafe
match => {
"message" => "%{GREEDYDATA:text}"
"path" => "%{GREEDYDATA:filename}"
add_tag => ["no_match"]
}
}
}
if [type] == "bbb" {
grok {
break_on_match => true
#With exception stacktrace:
match => {
"message" => "%{TIMESTAMP_ISO8601:event_timestamp}%{SPACE}%{DATA:log_level}%{SPACE}\[%{DATA:thread_name}\]%{SPACE}\[%{DATA:host}\]%{SPACE}\[%{DATA:session}\]%{SPACE}\[%{DATA:trx}\]%{SPACE}\[%{DATA:java_class}\]%{SPACE}-((%{SPACE}<)%{GREEDYDATA:text}>(\n|\r\n)(?m)%{GREEDYDATA:exception})"
"path" => "%{GREEDYDATA:filename}"
}
#Without stacktrace:
match => {
"message" => "%{TIMESTAMP_ISO8601:event_timestamp}%{SPACE}%{DATA:log_level}%{SPACE}\[%{DATA:thread_name}\]%{SPACE}\[%{DATA:host}\]%{SPACE}\[%{DATA:session}\]%{SPACE}\[%{DATA:trx}\]%{SPACE}\[%{DATA:java_class}\]%{SPACE}-%{SPACE}<%=@lt%>{GREEDYDATA:text}>"
"path" => "%{GREEDYDATA:filename}"
}
#failsafe
match => {
"message" => "%{GREEDYDATA:text}"
"path" => "%{GREEDYDATA:filename}"
add_tag => ["no_match"]
}
}
}
if [type] == "ccc" {
grok {
break_on_match => true
match => {
"message" => "%{TIMESTAMP_ISO8601:event_timestamp},%{DATA:tenant},%{DATA:organization},%{DATA:request},%{DATA:correlation_id},%{NUMBER:time:int}(,,|,|)(%{SPACE}\[%{GREEDYDATA:text}\]|)"
"path" => "%{GREEDYDATA:filename}"
add_tag => ["ccc"]
}
#failsafe
match => {
"message" => "%{GREEDYDATA:text}"
"path" => "%{GREEDYDATA:filename}"
add_tag => ["no_match"]
}
}
}
date {
match => [ "event_timestamp", "YYYY-MM-dd HH:mm:ss.SSS" ]
timezone => "Etc/GMT"
locale => "en"
}
output { ElasticSearch info}