APM Java Agent error java.lang.StackOverflowError

Elastic Observability APM
1

We have implemented Java Agent with our springboot applicatiions

Springboot : 1.5.10
Java: 1.8.172
APM Server : 6.3
APM Agent : 0.6.2

We are getting the below error in our logs

      Exception in Token validation filter  with exception {} 
    org.springframework.web.util.NestedServletException: Handler dispatch failed; nested exception is java.lang.StackOverflowError
    	at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:982)
    	at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901)
    	at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
    	at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:861)
    	at javax.servlet.http.HttpServlet.service(HttpServlet.java:635)
    	at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
    	at javax.servlet.http.HttpServlet.service(HttpServlet.java:742)
    ...................

org.springframework.boot.web.filter.ApplicationContextHeaderFilter.doFilterInternal(ApplicationContextHeaderFilter.java:55)

.........................

org.springframework.boot.actuate.trace.WebRequestTraceFilter.doFilterInternal(WebRequestTraceFilter.java:110)
    	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
    	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    	at com.paytmmoney.filter.RequestResponseFilter.doFilterInternal(RequestResponseFilter.java:75)
    	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
    	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    	at com.paytmmoney.filter.TokenValidationFilter.doFilterInternal(TokenValidationFilter.java:101)
    	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
    	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    	at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:99)
    	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
    	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    	at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:108)
    	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
    	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    	at org.springframework.web.filter.HiddenHttpMethodFilter.doFilterInternal(HiddenHttpMethodFilter.java:81)
    	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
    	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    	at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:197)
    	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
    	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    	at org.springframework.boot.actuate.autoconfigure.MetricsFilter.doFilterInternal(MetricsFilter.java:106)
    	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
    	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    	at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:199)
    	at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
    	at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:504)
    	at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:140)
    	at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:81)
    	at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87)
    	at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:342)
    	at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:803)
    	at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)
    	at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:790)
    	at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1459)
    	at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
    	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    	at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
    	at java.lang.Thread.run(Thread.java:748)
    Caused by: java.lang.StackOverflowError: null
    	at co.elastic.apm.impl.ElasticApmTracer.startManualSpan(ElasticApmTracer.java:185)
    	at co.elastic.apm.impl.ElasticApmTracer.startSpan(ElasticApmTracer.java:170)
    	at co.elastic.apm.jdbc.JdbcHelper.createJdbcSpan(JdbcHelper.java:82)
    	at co.elastic.apm.jdbc.StatementInstrumentation.onBeforeExecute(StatementInstrumentation.java:62)
    	at com.mysql.jdbc.StatementImpl.executeQuery(StatementImpl.java:1305)
    	at com.mysql.jdbc.DatabaseMetaData.getUserName(DatabaseMetaData.java:6274)
 .........
 .............

What could be the cause?

2

Hi and thanks for the bug report!

That seems like a bug in the JDBC instrumentation. Which Database, connection pool and JDBC driver (incl. versions) are you using?

Also, could you add some more lines to the Caused by: java.lang.StackOverflowError: null?

3

The error is too long that cant paste here

We use

HikariCP', version: '3.1.0'
'mysql-connector-java', version:'5.1.43'
'hibernate-jpa-2.1-api', version: '1.0.2.Final'

build.gradle

plugins {
id 'java'
id 'groovy'
id 'org.springframework.boot' version '1.5.10.RELEASE'
id 'nebula.lint' version '9.1.0'
id "org.flywaydb.flyway" version "5.0.7"

id "com.gorylenko.gradle-git-properties" version "1.5.1"
}

ext {
springBootVersion = "1.5.10.RELEASE"

prometheusVersion = '0.4.0'
logbookVersion = "1.8.1"
}

allprojects {
apply plugin: 'nebula.lint'
gradleLint.rules = [''] // add as many rules here as you'd like
}


group = 'appname'
version = '0.0.1-SNAPSHOT'

description = """App Name"""

sourceCompatibility = 1.8
targetCompatibility = 1.8


tasks.withType(JavaCompile) {
options.encoding = 'UTF-8'
}

jar {
baseName = 'appname'
version = '0.0.1-SNAPSHOT'
}

repositories {
mavenCentral()
maven { url "http://repo.maven.apache.org/maven2" }

}
}

configurations.all {

resolutionStrategy.cacheChangingModulesFor 0, 'seconds'
}


dependencies {

compile group: 'org.springframework.boot', name: 'spring-boot-starter-web', version:springBootVersion
compile group: 'org.springframework.boot', name: 'spring-boot-starter-data-jpa', version:springBootVersion
//compile group: 'org.springframework.boot', name: 'spring-boot-starter-data-mongodb', version:springBootVersion
compile group: 'org.springframework.boot', name: 'spring-boot-starter-actuator', version: springBootVersion
compile group: 'org.springframework.boot', name: 'spring-boot-starter-data-redis', version: springBootVersion
compile group: 'org.springframework.boot', name: 'spring-boot-starter-data-solr', version: springBootVersion

compile group: 'org.springframework.boot', name: 'spring-boot-starter-aop', version: springBootVersion
compile group: 'org.springframework.retry', name: 'spring-retry', version: '1.2.2.RELEASE'

compile group: 'org.springframework.kafka', name: 'spring-kafka', version:'1.3.3.RELEASE'
compile group: 'org.modelmapper', name: 'modelmapper', version:'1.1.0'
compile group: 'com.google.guava', name: 'guava', version: '23.5-jre'

compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-xml', version: '2.9.4'
compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-csv', version: '2.9.4'

compile group: "de.codecentric", name :'spring-boot-admin-starter-client', version: '1.5.7'
compile "io.prometheus:simpleclient:${prometheusVersion}"
compile "io.prometheus:simpleclient_hotspot:${prometheusVersion}"
compile "io.prometheus:simpleclient_spring_boot:${prometheusVersion}"
compile group: 'io.prometheus', name: 'simpleclient_servlet', version: prometheusVersion


compile group: 'org.springframework.boot', name: 'spring-boot-starter-actuator', version: springBootVersion

compile group: 'com.squareup.retrofit2', name: 'retrofit', version: '2.3.0'
compile group: 'com.squareup.retrofit2', name: 'converter-gson', version: '2.3.0'
compile group: 'com.squareup.retrofit2', name: 'converter-simplexml', version: '2.3.0'
compile group: 'net.adamcin.granite', name: 'granite-client-packman', version: '0.8.0'

compile group: 'io.springfox', name: 'springfox-swagger2', version: '2.8.0'


compile group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.4'


	compile group: 'org.kie', name: 'kie-api', version: '7.8.0.Final'

	
	compile group: 'org.drools', name: 'drools-core', version: '7.8.0.Final'
	compile group: 'org.drools', name: 'drools-compiler', version: '7.8.0.Final'

compile group: 'com.hierynomus', name: 'sshj', version: '0.23.0'
compile group: 'org.jsoup', name: 'jsoup', version: '1.11.2'

compile group: 'commons-codec', name: 'commons-codec', version: '1.11'
compile files('./src/main/resources/lib/sunjce_provider.jar')

compile group: 'com.sun.jersey.contribs.jersey-oauth', name: 'oauth-signature', version: '1.19.4'
compile group: 'org.hibernate.javax.persistence', name: 'hibernate-jpa-2.1-api', version: '1.0.2.Final'
compile group: 'com.github.albfernandez', name: 'javadbf', version: '1.9.2'

compile('javax.mail:javax.mail-api:1.6.1')
compile('com.sun.mail:mailapi:1.6.1')
compile('com.sun.mail:imap:1.6.1')

compile group: 'net.lingala.zip4j', name: 'zip4j', version: '1.3.2'
compile group: 'javax.mail', name: 'javax.mail-api', version: '1.6.1'
compile group: 'javax.mail', name: 'javax.mail-api', version: '1.6.1'
compile group: 'com.sun.mail', name: 'imap', version: '1.6.1'
compile group: 'ch.qos.logback', name: 'logback-access', version: '1.2.3'
compile group: 'net.logstash.logback', name: 'logstash-logback-encoder', version: '5.1'
compile (group: 'com.zaxxer', name: 'HikariCP', version: '3.1.0'){
    exclude group: 'org.hibernate', module: 'hibernate-core'
}
	
	compile group: 'ch.qos.logback', name: 'logback-access', version: '0.9.26'
compile group: 'joda-time', name: 'joda-time', version: '2.9.9'


	compile group: 'org.hibernate', name: 'hibernate-envers'

compile group: 'commons-codec', name: 'commons-codec', version: '1.11'
compile group: 'com.amazonaws', name: 'aws-java-sdk', version: '1.11.341'
compile group: 'org.zalando', name: 'logbook-httpclient', version:logbookVersion
	compile group: 'org.zalando', name: 'logbook-core', version:logbookVersion
	compile group: 'org.zalando', name: 'logbook-servlet', version:logbookVersion
	compile group: 'org.zalando', name: 'logbook-spring-boot-starter', version:logbookVersion

compile files('./src/main/resources/lib/sunjce_provider.jar')


compile group: 'uk.co.jemos.podam', name: 'podam', version: '7.1.0.RELEASE'
runtime group: 'mysql', name: 'mysql-connector-java', version:'5.1.43'
testCompile(group: 'org.springframework.boot', name: 'spring-boot-starter-test', version:springBootVersion) {
    exclude(module: 'commons-logging')
}

compile group: 'com.ibm.icu', name: 'icu4j', version: '62.1'

compile group: 'org.zalando', name: 'logbook-httpclient', version:logbookVersion
	compile group: 'org.zalando', name: 'logbook-core', version:logbookVersion
	compile group: 'org.zalando', name: 'logbook-servlet', version:logbookVersion
	compile group: 'org.zalando', name: 'logbook-spring-boot-starter', version:logbookVersion
	
testCompile group: 'org.springframework.restdocs', name: 'spring-restdocs-mockmvc', version:'1.1.3.RELEASE'
}



build.dependsOn copyJar

task wrapper(type: Wrapper) {
gradleVersion = '4.7'
}
4

Actually, there should be a guard agains this error. Not sure yet why it does not work in your case. Could you try the latest snapshot to see if the error is still present there?

5

I have downloaded and deployed the snapshot. I will observe and get back

6

This topic was automatically closed 20 days after the last reply. New replies are no longer allowed.