Hi Team,
I am trying to parse multiple log files where log syntax are not same and extract differnt lines which is coming between 2 lines of log data. Below is the sample data
Feb 12 18:24:26 localhost.localdomain: TC START SampleTC()"
Feb 10 18:24:34 localhost.localdomain kibana[697]: WARNING in TestCase2()
Feb 10 18:24:40 localhost.localdomain kibana[697]: FAILED in TestCase2()
Feb 10 18:24:42 localhost.localdomain kibana[697]: FAILED String1 Strng2 in TestCase2()
Feb 12 18:26:24 localhost.localdomain: TC COMPLETED SampleTC()
I have to extract all the failure message ( like warning , fail, error) between SAMPLE Start and SAMPLE complete lines which is coming the log file and maintain the order of data.
Currently I am able to extract releavent log lines by using multiple grok.
To implement the condition i am trying to implement below filter but i am not sure whether this approach is correct. Also I want to add TCNAME which i added in first grok in other grok also, but i am not able to achieve this.
filter {
grok
{
match => {"message" => "%{SYSLOGTIMESTAMP:Time} %{GREEDYDATA:host} (?%{WORD})[[0-9]*]: TC START %{GREEDYDATA:DataString1}"}
add_field => { "[TCNAME]" => "%{DataString1}" }
remove_tag => ["_grokparsefailure"]
}
if [message] != "TESTCASE COMPLETED %{DataString}"
{
if "_grokparsefailure" in [tags] {
grok
{
match => {"message" => "%{SYSLOGTIMESTAMP:Time} %{HOSTNAME:HOST} (?<LOGINTYPE>%{WORD})\[[0-9]*\]\: (?<DataString>WARNING %{GREEDYDATA})"}
add_field => { "[TCNAME]" => "%{DataString1}" }
remove_tag => ["_grokparsefailure"]
}
}
if "_grokparsefailure" in [tags] {
grok
{
match => {"message" => "%{SYSLOGTIMESTAMP:Time} %{HOSTNAME:HOST} (?<LOGINTYPE>%{WORD})\[[0-9]*\]\: (?<DataString>FAILED %{GREEDYDATA})"}
add_field => { "[TCNAME]" => "%{DataString1}" }
remove_tag => ["_grokparsefailure"]
}
}
}
if "_grokparsefailure" in [tags] {
grok
{
match => {"message" => "%{SYSLOGTIMESTAMP:Time} %{HOSTNAME:HOST} (?<LOGINTYPE>%{WORD})\[[0-9]*\]\: TC COMPLETED %{GREEDYDATA:DataString}"}
add_field => { "[TCNAME]" => "%{DataString}"}
remove_tag => ["_grokparsefailure"]
}
}
if "_grokparsefailure" in [tags]
{
drop{}
}
mutate{
remove_field => ["@version", "host", "tags", "offset", "type", "input_type", "errordata"]
}
}