APM Agent language and version: apm-java-agent 1.18.1
I have a problem with java apm agent and tomcat with multiple apps since version 1.18.0. New migrated plugins to indy aren't loaded correctly. It seems that the issue is related to classloader. It gives me errors like that:
[main] ERROR co.elastic.apm.agent.bci.IndyBootstrap - no such method: co.elastic.apm.agent.springwebmvc.SpringServiceNameInstrumentation$SpringServiceNameAdvice.afterInitPropertySources(WebApplicationContext)void/invokeStatic
java.lang.IllegalAccessException: no such method: co.elastic.apm.agent.springwebmvc.SpringServiceNameInstrumentation$SpringServiceNameAdvice.afterInitPropertySources(WebApplicationContext)void/invokeStatic
at java.lang.invoke.MemberName.makeAccessException(MemberName.java:959) ~[?:?]
at java.lang.invoke.MemberName$Factory.resolveOrFail(MemberName.java:1101) ~[?:?]
at java.lang.invoke.MethodHandles$Lookup.resolveOrFail(MethodHandles.java:2030) ~[?:?]
at java.lang.invoke.MethodHandles$Lookup.findStatic(MethodHandles.java:1102) ~[?:?]
at co.elastic.apm.agent.bci.IndyBootstrap.bootstrap(IndyBootstrap.java:289) [?:?]
at jdk.internal.reflect.GeneratedMethodAccessor8.invoke(Unknown Source) ~[?:?]
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
at java.lang.reflect.Method.invoke(Method.java:566) ~[?:?]
at java.lang.IndyBootstrapDispatcher.bootstrap(IndyBootstrapDispatcher.java:31) [?:?]
at java.lang.invoke.BootstrapMethodInvoker.invoke(BootstrapMethodInvoker.java:138) [?:?]
at java.lang.invoke.CallSite.makeSite(CallSite.java:307) [?:?]
at java.lang.invoke.MethodHandleNatives.linkCallSiteImpl(MethodHandleNatives.java:258) [?:?]
at java.lang.invoke.MethodHandleNatives.linkCallSite(MethodHandleNatives.java:248) [?:?]
at org.springframework.web.context.support.GenericWebApplicationContext.initPropertySources(GenericWebApplicationContext.java:196) [spring-web-5.2.9.RELEASE.jar:5.2.9.RELEASE]
at org.springframework.context.support.AbstractApplicationContext.prepareRefresh(AbstractApplicationContext.java:601) [spring-context-5.2.9.RELEASE.jar:5.2.9.RELEASE]
at org.springframework.boot.web.servlet.context.AnnotationConfigServletWebServerApplicationContext.prepareRefresh(AnnotationConfigServletWebServerApplicationContext.java:195) [spring-boot-2.3.4.RELEASE.jar:2.3.4.RELEASE]
at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:520) [spring-context-5.2.9.RELEASE.jar:5.2.9.RELEASE]
at org.springframework.boot.web.servlet.context.ServletWebServerApplicationContext.refresh(ServletWebServerApplicationContext.java:143) [spring-boot-2.3.4.RELEASE.jar:2.3.4.RELEASE]
at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:758) [spring-boot-2.3.4.RELEASE.jar:2.3.4.RELEASE]
at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:750) [spring-boot-2.3.4.RELEASE.jar:2.3.4.RELEASE]
at org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:397) [spring-boot-2.3.4.RELEASE.jar:2.3.4.RELEASE]
at org.springframework.boot.SpringApplication.run(SpringApplication.java:315) [spring-boot-2.3.4.RELEASE.jar:2.3.4.RELEASE]
at org.springframework.boot.web.servlet.support.SpringBootServletInitializer.run(SpringBootServletInitializer.java:173) [spring-boot-2.3.4.RELEASE.jar:2.3.4.RELEASE]
at org.springframework.boot.web.servlet.support.SpringBootServletInitializer.createRootApplicationContext(SpringBootServletInitializer.java:153) [spring-boot-2.3.4.RELEASE.jar:2.3.4.RELEASE]
at org.springframework.boot.web.servlet.support.SpringBootServletInitializer.onStartup(SpringBootServletInitializer.java:95) [spring-boot-2.3.4.RELEASE.jar:2.3.4.RELEASE]
at org.springframework.web.SpringServletContainerInitializer.onStartup(SpringServletContainerInitializer.java:172) [spring-web-5.2.9.RELEASE.jar:5.2.9.RELEASE]
at org.apache.catalina.core.StandardContext.startInternal(StandardContext.java:5128) [catalina.jar:9.0.37]
at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:183) [catalina.jar:9.0.37]
at org.apache.catalina.core.ContainerBase.addChildInternal(ContainerBase.java:717) [catalina.jar:9.0.37]
at org.apache.catalina.core.ContainerBase.addChild(ContainerBase.java:690) [catalina.jar:9.0.37]
at org.apache.catalina.core.StandardHost.addChild(StandardHost.java:705) [catalina.jar:9.0.37]
at org.apache.catalina.startup.HostConfig.deployWAR(HostConfig.java:978) [catalina.jar:9.0.37]
at org.apache.catalina.startup.HostConfig$DeployWar.run(HostConfig.java:1848) [catalina.jar:9.0.37]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at org.apache.tomcat.util.threads.InlineExecutorService.execute(InlineExecutorService.java:75) [tomcat-util.jar:9.0.37]
at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:118) [?:?]
at org.apache.catalina.startup.HostConfig.deployWARs(HostConfig.java:773) [catalina.jar:9.0.37]
at org.apache.catalina.startup.HostConfig.deployApps(HostConfig.java:427) [catalina.jar:9.0.37]
at org.apache.catalina.startup.HostConfig.start(HostConfig.java:1576) [catalina.jar:9.0.37]
at org.apache.catalina.startup.HostConfig.lifecycleEvent(HostConfig.java:309) [catalina.jar:9.0.37]
at org.apache.catalina.util.LifecycleBase.fireLifecycleEvent(LifecycleBase.java:123) [catalina.jar:9.0.37]
at org.apache.catalina.util.LifecycleBase.setStateInternal(LifecycleBase.java:423) [catalina.jar:9.0.37]
at org.apache.catalina.util.LifecycleBase.setState(LifecycleBase.java:366) [catalina.jar:9.0.37]
at org.apache.catalina.core.ContainerBase.startInternal(ContainerBase.java:936) [catalina.jar:9.0.37]
at org.apache.catalina.core.StandardHost.startInternal(StandardHost.java:841) [catalina.jar:9.0.37]
at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:183) [catalina.jar:9.0.37]
at org.apache.catalina.core.ContainerBase$StartChild.call(ContainerBase.java:1384) [catalina.jar:9.0.37]
at org.apache.catalina.core.ContainerBase$StartChild.call(ContainerBase.java:1374) [catalina.jar:9.0.37]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at org.apache.tomcat.util.threads.InlineExecutorService.execute(InlineExecutorService.java:75) [tomcat-util.jar:9.0.37]
at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:140) [?:?]
at org.apache.catalina.core.ContainerBase.startInternal(ContainerBase.java:909) [catalina.jar:9.0.37]
at org.apache.catalina.core.StandardEngine.startInternal(StandardEngine.java:262) [catalina.jar:9.0.37]
at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:183) [catalina.jar:9.0.37]
at org.apache.catalina.core.StandardService.startInternal(StandardService.java:421) [catalina.jar:9.0.37]
at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:183) [catalina.jar:9.0.37]
at org.apache.catalina.core.StandardServer.startInternal(StandardServer.java:930) [catalina.jar:9.0.37]
at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:183) [catalina.jar:9.0.37]
at org.apache.catalina.startup.Catalina.start(Catalina.java:738) [catalina.jar:9.0.37]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:?]
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
at java.lang.reflect.Method.invoke(Method.java:566) ~[?:?]
at org.apache.catalina.startup.Bootstrap.start(Bootstrap.java:342) [bootstrap.jar:9.0.37]
at org.apache.catalina.startup.Bootstrap.main(Bootstrap.java:473) [bootstrap.jar:9.0.37]
Caused by: java.lang.LinkageError: bad method type alias: (WebApplicationContext)void not visible from class co.elastic.apm.agent.springwebmvc.SpringServiceNameInstrumentation$SpringServiceNameAdvice
at java.lang.invoke.MemberName.checkForTypeAlias(MemberName.java:872) ~[?:?]
at java.lang.invoke.MemberName$Factory.resolve(MemberName.java:1074) ~[?:?]
at java.lang.invoke.MemberName$Factory.resolveOrFail(MemberName.java:1098) ~[?:?]
... 64 more
or
[registrationTask1] ERROR co.elastic.apm.agent.bci.IndyBootstrap - no such method: co.elastic.apm.agent.httpclient.ApacheHttpClientInstrumentation$ApacheHttpClientAdvice.onBeforeExecute(HttpRoute,HttpRequestWrapper)Object/invokeStatic
java.lang.IllegalAccessException: no such method: co.elastic.apm.agent.httpclient.ApacheHttpClientInstrumentation$ApacheHttpClientAdvice.onBeforeExecute(HttpRoute,HttpRequestWrapper)Object/invokeStatic
at java.lang.invoke.MemberName.makeAccessException(MemberName.java:959) ~[?:?]
at java.lang.invoke.MemberName$Factory.resolveOrFail(MemberName.java:1101) ~[?:?]
at java.lang.invoke.MethodHandles$Lookup.resolveOrFail(MethodHandles.java:2030) ~[?:?]
at java.lang.invoke.MethodHandles$Lookup.findStatic(MethodHandles.java:1102) ~[?:?]
at co.elastic.apm.agent.bci.IndyBootstrap.bootstrap(IndyBootstrap.java:289) ~[?:?]
at jdk.internal.reflect.GeneratedMethodAccessor8.invoke(Unknown Source) ~[?:?]
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
at java.lang.reflect.Method.invoke(Method.java:566) ~[?:?]
at java.lang.IndyBootstrapDispatcher.bootstrap(IndyBootstrapDispatcher.java:31) ~[?:?]
at java.lang.invoke.BootstrapMethodInvoker.invoke(BootstrapMethodInvoker.java:138) ~[?:?]
at java.lang.invoke.CallSite.makeSite(CallSite.java:307) ~[?:?]
at java.lang.invoke.MethodHandleNatives.linkCallSiteImpl(MethodHandleNatives.java:258) ~[?:?]
at java.lang.invoke.MethodHandleNatives.linkCallSite(MethodHandleNatives.java:248) ~[?:?]
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:155) ~[httpclient-4.5.9.jar:4.5.9]
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) ~[httpclient-4.5.9.jar:4.5.9]
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) ~[httpclient-4.5.9.jar:4.5.9]
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) ~[httpclient-4.5.9.jar:4.5.9]
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) ~[httpclient-4.5.9.jar:4.5.9]
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) ~[httpclient-4.5.9.jar:4.5.9]
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56) ~[httpclient-4.5.9.jar:4.5.9]
at org.springframework.http.client.HttpComponentsClientHttpRequest.executeInternal(HttpComponentsClientHttpRequest.java:87) ~[spring-web-5.1.14.RELEASE.jar:5.1.14.RELEASE]
at org.springframework.http.client.AbstractBufferingClientHttpRequest.executeInternal(AbstractBufferingClientHttpRequest.java:48) ~[spring-web-5.1.14.RELEASE.jar:5.1.14.RELEASE]
at org.springframework.http.client.AbstractClientHttpRequest.execute(AbstractClientHttpRequest.java:53) ~[spring-web-5.1.14.RELEASE.jar:5.1.14.RELEASE]
at ...
I did not have these problems with version 1.17.0. Also, error with SpringServiceNameInstrumentation wasn't there with version 1.18.0, where the plugin was not migrated yet.
Steps to reproduce:
- download https://github.com/aizerin/java-apm-bug
- run
docker-compose up - see whats in
elastic-apm.log
NOTE: you can reproduce only the first error with SpringServiceNameInstrumentation. I was not able to simulate the second one with ApacheHttpClientInstrumentation which is from my company. But I think that the issue will identical.