I am having a json data in the following manne:-
{"@timestamp":"2021-06-04T09:57:25.141Z","@metadata":{"beat":"filebeat","type":"_doc","version":"7.6.3"},"log":{"offset":10413931,"file":{"path":""}},"message":"[ERROR] 2021-10-18T22:36:04.672 [http-nio2-8080-exec-48] [FTDS] deployment-75bf886778-gj8hv - [i.i.i.a.e.f.s.ManageForm] :: Exception is caught with error code: I-EXEC and DevMessage: null and UserMessage: Due to some technical error not able to process.Please check corresponding class
and error code I-EXEC and stacktrace: java.lang.NullPointerException
at in.it.ic.api.efiling.form.FormData.validateAndSet(FormData.java:858)
at in.it.ic.api.efiling.form.FormData.setFeilds(FormData.java:822)
at form.service.ManageForm.populateDraftDetail(ManageForm.java:1082)
at jdk.internal.reflect.GeneratedMethodAccessor304.invoke(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:344)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:198)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
at org.springframework.transaction.interceptor.TransactionInterceptor$$Lambda$510/00000000046D7C60.proceedWithInvocation(Unknown Source)
at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:366)
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:99)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:212)
at com.sun.proxy.$Proxy135.serve(Unknown Source)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:190)
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:138)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:106)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:888)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:793)
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1040)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:943)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:909)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:652)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:733)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.apache.catalina.filters.HttpHeaderSecurityFilter.doFilter(HttpHeaderSecurityFilter.java:126)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:541)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:690)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343)
at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:373)
at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868)
at org.apache.tomcat.util.net.Nio2Endpoint$SocketProcessor.doRun(Nio2Endpoint.java:1674)
at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
at org.apache.tomcat.util.net.AbstractEndpoint.processSocket(AbstractEndpoint.java:1104)
at org.apache.tomcat.util.net.Nio2Endpoint.setSocketOptions(Nio2Endpoint.java:335)
at org.apache.tomcat.util.net.Nio2Endpoint$Nio2Acceptor.completed(Nio2Endpoint.java:436)
at org.apache.tomcat.util.net.Nio2Endpoint$Nio2Acceptor.completed(Nio2Endpoint.java:391)
at java.base/sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:127)
at java.base/sun.nio.ch.Invoker$2.run(Invoker.java:219)
at java.base/sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.base/java.lang.Thread.run(Thread.java:836)
and the possible root cause is java.lang.NullPointerException at 1634576764672 with objectarray values asnull[#]","input":{"type":"log"},"ecs":{"version":"1.4.0"},"host":{"mac":["/"],"hostname":"dcplicpboot","architecture":"ppc64le","os":{"codename":"Maipo","platform":"rhel","version":"7.6 (Maipo)","family":"redhat","name":"Red Hat Enterprise Linux Server","kernel":"4.14.0-115.13.1.el7a.ppc64le"},"id":"4994b06b9b4248dd81e0c113f2221e54","name":"dcplicpboot","containerized":false,"ip":["17.0.1"]},"agent":{"version":"7.6.3","type":"filebeat","ephemeral_id":"e249728c-21f3-4ea6-be0f-b5ce5ff7447f","hostname":"","id":"6bd6e68b-3f3e-4985-8738-08b3488fffd7"}}
out of this I want to parse only message field, In some logs stack trace are there in some logs java stack trace is not present.
I am using the following grok parser, I am not getting any error but I am not even getting any data for the stack trace logs:-
input {
file {
id => "itdapp1"
path => "D:/app.logs"
start_position => "beginning"
sincedb_path =>"NUL"
codec => multiline {
pattern => "%{DATA:loglevel}"
negate => false
what => "previous"
}
}
}
filter {
json {
source => "message"
target => "doc"
}
mutate {
copy => { "[doc][message]" => "mesg" }
copy => { "[doc][log][file][path]" => "logpath" }
remove_field => [ "[doc]" ]
}
if ( "/prodlogsfs/" not in [logpath] ) {
drop { }
}
if [logpath] {
if ("-" in [logpath]){
dissect {
mapping => {
"logpath" => "%{deployment}-%{?id}-%{?extra}"
}
}
}
else{
dissect {
mapping => {
"logpath" => "%{deployment}.%{?extra}"
}
}
}
}
mutate {
#copy => { "[logmesg][log]" => "mesg" }
remove_field => [ "[logmesg]" ]
remove_field => [ "message" ]
remove_field => [ "@version" ]
remove_field => [ "path" ]
#remove_field => [ "host" ]
remove_field => [ "longid" ]
remove_field => [ "extra" ]
}
#This is prod tomcat log format
grok { match => { "mesg" => [ "(?m)^\s?(\[%{DATA:loglevel}\] )?%{TIMESTAMP_ISO8601:logts} \[%{DATA:threadname}\] (\[%{DATA:formname}\] )?%{DATA:podname} %{DATA:filler1} \[%{DATA:classname}\] %{GREEDYDATA:fullmesg}",
"(?m)(\s)+(?<exception>%{DATA}Exception)[:\s]+(?<trace>(%{DATA}$)+)"
]
} }
if [fullmesg]=~"Exception"
{
grok { match => { "fullmesg" => ["(?m)(?%{DATA}?!.*Exception)[:\s]+(?(%{DATA}$))"
]
} }
}
if [logts] {
date {
match => [ "logts", "ISO8601" ]
target => ["@timestamp"]
remove_field => [ "logts" ]
}
aggregate {
task_id => "%{logpath}"
code => "map['tmplogts'] = event.get('@timestamp')"
}
}
output {
stdout { codec => rubydebug }
Elasticsearch {
hosts => ["http://localhost:9200"]
index => "logs-%{+YYYY.MM.dd}"
}
}