Sorry due to max character limit, couldn't post my filter section. Here it is:
filter {
if [type] == "sometype" {
if "prio" in [message]
{
if ("multiline" in [tags]) {
grok {
match => {
"message" => "(?m)%{DATA}\| %{YEAR:year}/%{MONTHNUM2:month}/%{MONTHDAY:day} %{TIME:time} \|%{DATA}nid=%{BASE16NUM:nid_hex}%{DATA}java.lang.Thread.State: %{DATA:thread_state}((\n)|(\r))%{DATA}"
}
}
}
else {
grok {
match => {
"message" => "(?m)%{DATA}\| %{YEAR:year}/%{MONTHNUM2:month}/%{MONTHDAY:day} %{TIME:time} \|%{DATA}nid=%{BASE16NUM:nid_hex}%{DATA}"
}
}
}
ruby {
code => "event.set('nid_int', event.get('nid_hex').hex)"
}
mutate {
add_field => {
"timestamp" => "%{year}-%{month}-%{day} %{time}"
}
}
date {
match => ["timestamp", "YYYY-MM-dd HH:mm:ss"]
remove_field => ["timestamp","year","month","day","time"]
#locale => "en"
# timezone => "Asia/Kolkata"
timezone => "Etc/GMT"
}
}
else if "Exception" in [message] {
# EXCEPTION LINES
# GET EXCEPTIONS
if "Caused by" in [message] {
# trace
grok {
# break_on_match => "false"
match => { "message" => "(?m)%{DATA}\| %{YEAR:year}/%{MONTHNUM2:month}/%{MONTHDAY:day} %{TIME:time} \|%{DATA}\.(?<exception>[a-zA-Z]+)Exception:%{DATA}Caused by:%{DATA}\.(?<caused_by>[a-zA-Z]+)Exception%{GREEDYDATA}" }
add_field => {
"timestamp" => "%{year}-%{month}-%{day} %{time}"
}
}
}
else if "Exception" in [message] {
grok {
match => { "message" => "(?m)%{DATA}\| %{YEAR:year}/%{MONTHNUM2:month}/%{MONTHDAY:day} %{TIME:time} \|%{DATA}\.(?<exception>[a-zA-Z]+)Exception:%{GREEDYDATA}" }
add_field => {
"timestamp" => "%{year}-%{month}-%{day} %{time}"
}
}
}
date {
match => ["timestamp", "YYYY-MM-dd HH:mm:ss"]
remove_field => ["timestamp","year","month","day","time"]
#locale => "en"
timezone => "Etc/GMT"
}
}
else
{
#non threaddump lines
#drop {}
grok {
match => {
"message" => "%{DATA:loglevel}%{SPACE}\|%{SPACE}%{DATA:field}%{SPACE}\| %{YEAR:year}/%{MONTHNUM2:month}/%{MONTHDAY:day} %{TIME:time} \| %{GREEDYDATA:details}"
}
}
mutate {
add_field => {
"timestamp" => "%{year}-%{month}-%{day} %{time}"
}
}
date {
match => ["timestamp", "YYYY-MM-dd HH:mm:ss"]
remove_field => ["timestamp","year","month","day","time","message"]
#locale => "en"
# timezone => "Asia/Kolkata"
timezone => "Etc/GMT"
}
if ("Total time for which application threads were stopped" in [details]) {
#drop {}
grok {
match => { "details" => "%{NUMBER:collection_time:float}: Total time for which application threads were stopped: %{NUMBER:app_thread_stopped:float} seconds, Stopping threads took: %{NUMBER:stop_threads:float} seconds"
}
}
}
else if("PSYoungGen" in [details] and "ParOldGen" not in [details])
{
# drop {}
grok { #-- [PSYoungGen: 2538481K->2538481K(2538496K)] 7869259K->8009199K(8009216K), 0.8418280 secs] [Times: user=1.67 sys=0.47, real=0.84 secs]
match => { "details" => "%{DATA}\[PSYoungGen: %{NUMBER:young_gen_before:int}K->%{NUMBER:young_gen_after:int}K\(%{NUMBER:young_gen_total:int}K\)\] %{NUMBER:overall_before:int}K->%{NUMBER:overall_after:int}K\(%{NUMBER:overall_total:int}K\), %{NUMBER:gc_time:float} secs\] \[Times: user=%{NUMBER:user_time:float} sys=%{NUMBER:sys_time:float}, real=%{NUMBER:real_time:float} secs\]"
}
}
mutate {
add_field => {
"linetype" => "gc"
}
}
}
else if ("ParOldGen" in [details]){
# drop {}
grok { # [PSYoungGen: 83718K->0K(2549248K)] [ParOldGen: 5400973K->1594715K(5470208K)] 5484691K->1594715K(8019456K) [PSPermGen: 230277K->230277K(230400K)], 2.4420490 secs] [Times: user=7.85 sys=0.00, real=2.44 secs]
match => { "details" => "%{SPACE}\[PSYoungGen: %{NUMBER:young_gen_before:int}K->%{NUMBER:young_gen_after:int}K\(%{NUMBER:young_gen_total:int}K\)\] \[ParOldGen: %{NUMBER:old_gen_before:int}K->%{NUMBER:old_gen_after:int}K\(%{NUMBER:old_gen_total:int}K\)\] %{NUMBER:overall_before:int}K->%{NUMBER:overall_after:int}K\(%{NUMBER:overall_total:int}K\) \[PSPermGen: %{NUMBER:perm_gen_before:int}K->%{NUMBER:perm_gen_after:int}K\(%{NUMBER:perm_gen_total:int}K\)\], %{NUMBER:gc_time:float} secs\] \[Times: user=%{NUMBER:user_time:float} sys=%{NUMBER:sys_time:float}, real=%{NUMBER:real_time:float} secs\]"
}
}
mutate {
add_field => {
"linetype" => "fullgc"
}
}
}
else if ("Cannot allocate memory" in [details]) {
mutate {
add_field => {
"linetype" => "outofmemory"
}
}
}
else {
drop {}
}
}
}
}