Hello, I am trying to add multiline to handle javaexception in our logs but still having issue :
This is my pattern :
paths:
- /var/log/tomcat10/*
multiline.type: pattern
multiline.pattern: '^\d{2}-\w{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d{3}'
multiline.negate: true
my logstash config looks like:
input {
beats {
port => 5044
}
}
filter {
if [log][file][path] == "/var/log/web.stdout.log" {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:log_timestamp} %{IPORHOST:log_host} %{WORD:log_program}\[%{POSINT:log_pid}\]: %{GREEDYDATA:log_details}" }
}
date {
match => [ "log_timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
target => "log_timestamp"
}
mutate {
remove_field => [ "log_host", "log_pid" ]
}
} else if [log][file][path] =~ /^\/var\/log\/tomcat10\/localhost_access_log\.txt/ {
grok {
match => { "message" => "%{IPORHOST:clientip} - - \[%{HTTPDATE:log_timestamp}\] \"%{GREEDYDATA:log_details}" }
}
date {
match => [ "log_timestamp", "dd-MMM-yyyy HH:mm:ss.SSS" ]
target => "log_timestamp"
}
mutate {
remove_field => [ "clientip" ]
}
} else if [log][file][path] =~ /^\/var\/log\/tomcat10\/catalina\.\d{4}-\d{2}-\d{2}\.log/ {
grok {
match => {
"message" => "%{MONTHDAY:day}-%{MONTH:month}-%{YEAR:year} %{HOUR:hour}:%{MINUTE:minute}:%{SECOND:second}.%{NONNEGINT:millisecond} %{LOGLEVEL:loglevel} \[%{DATA:thread}\] %{JAVACLASS:class} %{GREEDYDATA:log_details}"
}
}
mutate {
remove_field => [ "thread", "class" ]
add_field => { "log_timestamp" => "%{day}-%{month}-%{year} %{hour}:%{minute}:%{second}.%{millisecond}" }
}
date {
match => [ "log_timestamp", "dd-MMM-yyyy HH:mm:ss.SSS" ]
target => "log_timestamp"
}
} else if [log][file][path] =~ /^\/var\/log\/tomcat10\/localhost\.\d{4}-\d{2}-\d{2}\.log/ {
grok {
match => {
"message" => "%{MONTHDAY:day}-%{MONTH:month}-%{YEAR:year} %{HOUR:hour}:%{MINUTE:minute}:%{SECOND:second}.%{NONNEGINT:millisecond} %{LOGLEVEL:loglevel} \[%{DATA:thread}\] %{JAVACLASS:class} %{GREEDYDATA:log_details}"
}
}
mutate {
remove_field => [ "thread", "class" ]
gsub => [ "log_details", "V2MessageServlet:", "" ]
add_field => { "log_timestamp" => "%{day}-%{month}-%{year} %{hour}:%{minute}:%{second}.%{millisecond}" }
}
date {
match => [ "log_timestamp", "dd-MMM-yyyy HH:mm:ss.SSS" ]
target => "log_timestamp"
}
}
}
output {
elasticsearch {
hosts => ["http://XXXX:9200"]
user => "XXX"
password => "XXXXX"
index => "nlogs"
}
if "_grokparsefailure" in [tags] or "_dateparsefailure" in [tags] {
stdout { codec => rubydebug }
}
}
this is my sample data :
15-Nov-2023 21:55:19.476 INFO [http-nio-8080-exec-2] org.apache.catalina.core.ApplicationContext.log V2MessageServlet: Error handling message [300434068633270/2170]
15-Nov-2023 21:55:19.496 INFO [http-nio-8080-exec-2] org.apache.catalina.core.ApplicationContext.log V2MessageServlet: Parse error handling message: 300434068633270/2170
15-Nov-2023 21:58:45.864 SEVERE [http-nio-8080-exec-1] org.apache.catalina.core.StandardWrapperValve.invoke Servlet.service() for servlet [ErrorCheckServlet] in context with path [] threw exception [💥 Test Exception! 💥] with root cause
jakarta.servlet.ServletException: 💥 Test Exception! 💥
at net.boggroup.buoyserver.ErrorCheckServlet.doGet(ErrorCheckServlet.java:20)
at jakarta.servlet.http.HttpServlet.service(HttpServlet.java:564)
at jakarta.servlet.http.HttpServlet.service(HttpServlet.java:658)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:205)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:149)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:51)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:174)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:149)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:167)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:90)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:482)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:115)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:93)
at org.apache.catalina.valves.RemoteIpValve.invoke(RemoteIpValve.java:735)
at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:673)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:340)
at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:391)
at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:63)
at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:896)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1744)
at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:52)
at org.apache.tomcat.util.threads.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1191)
at org.apache.tomcat.util.threads.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:659)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.base/java.lang.Thread.run(Thread.java:833)
When I tested my setting with the Gork debugger is shows the result as expected same when I tried to match my multiline pattern in GOplaygound also everything is matched but still logs in kibana doesn't show as expected and they come as separate messages.