Gork regex

Elastic Stack Elasticsearch
1

Hi
I use this logstash gork:
%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:loglevel} %{DATA:id} [%{DATA}] %{DATA:jboss_errors}(?=:|$)

here is the log:

2023-06-30 09:09:55,941 ERROR CUS.InEP-AAAA-123194144 [invocation] WFLYEJB0034: EJB Invocation failed on component Processor
2023-06-30 00:08:40,326 ERROR CUS.InEP-BBBB-121117180 [ExceptionLogger] FATAL: exception in processMessage: common.exception.Exception: [QQ_111] Packet Failed >> WFLYJPA0060: Packet is required to perform this operation (either use a packet or extended persistence context)

here is current output:

WFLYEJB0034
FATAL

here is expected output:

WFLYEJB0034
WFLYJPA0060

Any idea?
Thanks

2

Correct square brackets
%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:loglevel} %{DATA:id} \[%{DATA}\] %{DATA:jboss_errors}(?=:|$)

3

@Rios what is the different you mention exact gork filter that i mention, did you change any thing?

4

Sorry, corrected. The editor removed \, \[%{DATA}\]

5

@Rios still same result:

here is current output:
WFLYEJB0034
FATAL

here is expected output:
WFLYEJB0034
WFLYJPA0060

Any idea?

6

The problem is in the log records, normally should be something like this: timestamp,loglevel, id, class, jboss_error or jboss error ode, logmsg. Your code is moved inside logmsg.
If error code always in format: *>> code: * this should work.

filter {
    grok {
      match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\s+%{LOGLEVEL:loglevel}\s+%{DATA:id}\s+\[%{DATA:class}\]\s+%{DATA:jboss_errors}:\s+%{GREEDYDATA:logmsg}" 
      }
	}

	if [jboss_errors] in ["FATAL", "CRITICAL"]{
		grok {
		  match => { "logmsg" => ">>\s+%{DATA:[@metadata][msgerror]}:\s+" 
		  }
		}
		mutate { copy => { "[@metadata][msgerror]" => "jboss_errors" } }
	}

 	date {
        match => ["timestamp", "ISO8601"]
		timezone => "Europe/Berlin"
		target=> "timestamp" 
      }

}

If this is JBoss EAP log, you can use the dictionary plugin for codes.
Result:

{
          "loglevel" => "ERROR",
           "message" => "2023-06-30 09:09:55,941 ERROR CUS.InEP-AAAA-123194144 [invocation] WFLYEJB0034: EJB Invocation failed on component Processor\r",
        "@timestamp" => 2023-07-05T13:38:17.734780400Z,
         "timestamp" => 2023-06-30T07:09:55.941Z,
    "jboss_err_desc" => "EJB Invocation failed on component %s for method %s",
      "jboss_errors" => "WFLYEJB0034",
                "id" => "CUS.InEP-AAAA-123194144",
            "logmsg" => "EJB Invocation failed on component Processor\r",
             "class" => "invocation"
}
{
          "loglevel" => "ERROR",
           "message" => "2023-06-30 10:08:40,326 ERROR CUS.InEP-BBBB-121117180 [ExceptionLogger] FATAL: exception in processMessage: common.exception.Exception: [QQ_111] Packet Failed >> WFLYJPA0060: Packet is required to perform this operation (either use a packet or extended persistence context)\r",
        "@timestamp" => 2023-07-05T13:38:17.736775700Z,
         "timestamp" => 2023-06-30T08:08:40.326Z,
    "jboss_err_desc" => "Transaction is required to perform this operation (either use a transaction or extended persistence context)",
      "jboss_errors" => "WFLYJPA0060",
                "id" => "CUS.InEP-BBBB-121117180",
            "logmsg" => "exception in processMessage: common.exception.Exception: [QQ_111] Packet Failed >> WFLYJPA0060: Packet is required to perform this operation (either use a packet or extended persistence context)\r",
             "class" => "ExceptionLogger"
}
7

@Rios the issue with the gork that you mentioned is, there lines with different severity like Fatal, error, trace, debug …

I think is the best way is to do this like this (without define exact pattern of lines, just search for keywords):

1- search for this keywords. (AMQ*, ARJUNA*, …)

https://docs.wildfly.org/19/wildscribe/log-message-reference.html

2- store them as a field.

Any idea?
Thanks

8

This should work. Not sure does FATAL exist always or jboss_errors field will always exist. If does, then in 2nd grok use: logmsg instead message field

      grok {
      match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\s+%{LOGLEVEL:loglevel}\s+%{DATA:id}\s+\[%{DATA:class}\]\s+%{DATA:jboss_errors}:\s+%{GREEDYDATA:logmsg}" 
      }
	}

#if [jboss_errors] in ["FATAL", "CRITICAL"]{
		grok { # or logmsg 
		  match => { "message" => "(?<jbosserrorcode>\b(AMQ|ARJUNA|EJBCLIENT-|ELY|HCANN|HHH|HSEARCH|HV|IJ|ISNPHIB|ISPN|JB*|JIPI|JNDIWFHTTP|MODCLUSTER|MSC|PBOX|PROBE-|RESTEASY|TXNWFHTTP|UT|UTJS|VFS|WELD-|WF.*|XN.*|jlibaio)\d+\b):" 
		  }
		}

	#}
9

After several workarounds here is the solution:

input {
  file {
    path => "/opt/app/log/*"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
}


filter {
  if [message] =~ /\[SqlExceptionHelper\] SQL (Error|Warning Code):/ {
    grok {
      match => {
        "message" => [
          "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:loglevel} %{DATA:thread} \[SqlExceptionHelper\] SQL Error: -%{INT:db_errorcode1}, SQLState: %{WORD:sql_state1}",
          "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:loglevel} %{DATA:thread} \[SqlExceptionHelper\] SQL Warning Code: %{INT:db_errorcode2}, SQLState: %{WORD:sql_state2}"
        ]
      }
    }

    date {
      match => ["timestamp", "YYYY-MM-dd HH:mm:ss,SSS"]
    }
  } else if [message] =~ /(AMQ|ARJUNA|COM|EJBCLIENT|ELY|HCANN|HHH|HSEARCH|HV|IJ|ISNPHIB|ISPN|JBERET|JBREM|JBTHR|JBWEB|JBWS|JIPI|JNDIWFHTTP|MODCLUSTER|MSC|PBOX|PROBE|RESTEASY|TXNWFHTTP|UT|UTJS|VFS|WELD|WFCMTOOL|WFHTTP|WFHTTPEJB|WFLY|WFMIGRCLI|WFNAM|WFSM|WFTXN|XNIO|jlibaio)/ {
    grok {
      match => { "message" => ".*\b(?<jboss_errors>(?:AMQ\w*|ARJUNA\w*|COM\w*|EJBCLIENT\w*|ELY\w*|HCANN\w*|HHH\w*|HSEARCH\w*|HV\w*|IJ\w*|ISNPHIB\w*|ISPN\w*|JBERET\w*|JBREM\w*|JBTHR\w*|JBWEB\w*|JBWS\w*|JIPI\w*|JNDIWFHTTP\w*|MODCLUSTER\w*|MSC\w*|PBOX\w*|PROBE\w*|RESTEASY\w*|TXNWFHTTP\w*|UT\w*|UTJS\w*|VFS\w*|WELD\w*|WFCMTOOL\w*|WFHTTP\w*|WFHTTPEJB\w*|WFLY\w*|WFMIGRCLI\w*|WFNAM\w*|WFSM\w*|WFTXN\w*|XNIO\w*|jlibaio\w*)\b).*" }
    }
    grok {
      match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\s+%{LOGLEVEL:loglevel}\s+%{DATA:id}\s+\[%{DATA:class}\]" }
    }
    if "_grokparsefailure" in [tags] {
      drop { }
    }
    mutate {
      remove_field => ["message", "@version", "event"] # Optionally remove unnecessary fields
    }
    date {
      match => ["timestamp", "YYYY-MM-dd HH:mm:ss,SSS"]
    }
  }
  else {
    drop {}
  }
}


output {
  elasticsearch {
    hosts => "localhost:9200"
    index => "input_file_jboss"
    user => "elastic"
    password => "$PASS"
  }
}
1 Like
10

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.