ElasticSeach java client ocasionally throws NoClassDefFoundError


(Andreano24) #1

We're using ES java client 1.5.2 to communicate with our ES cluster(cluster obviously has the same version). Several times in week we observe following errors:

Exception in thread "elasticsearch[waElasticSearchNode][management][T#3]" Exception in thread "elasticsearch[waElasticSearchNode][management][T#4]" java.lang.NoClassDefFoundError: org/elasticsearch/common/netty/channel/Channels$7
    at org.elasticsearch.common.netty.channel.Channels.fireExceptionCaughtLater(Channels.java:496)
    at org.elasticsearch.common.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:46)
    at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.notifyHandlerException(DefaultChannelPipeline.java:658)
    at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:577)
    at org.elasticsearch.common.netty.channel.Channels.close(Channels.java:812)
    at org.elasticsearch.common.netty.channel.AbstractChannel.close(AbstractChannel.java:206)
    at org.elasticsearch.transport.netty.NettyTransport$NodeChannels.caloseChannelsAndWait(NettyTransport.java:1098)
    at org.elasticsearch.transport.netty.NettyTransport$NodeChannels.close(NettyTransport.java:1084)
    at org.elasticsearch.transport.netty.NettyTransport.disconnectFromNode(NettyTransport.java:873)
    at org.elasticsearch.transport.TransportService.disconnectFromNode(TransportService.java:224)
    at org.elasticsearch.client.transport.TransportClientNodesService$SniffNodesSampler$1.run(TransportClientNodesService.java:477)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:745)
java.lang.NoClassDefFoundError: org/elasticsearch/common/netty/channel/Channels$7
    at org.elasticsearch.common.netty.channel.Channels.fireExceptionCaughtLater(Channels.java:496)
    at org.elasticsearch.common.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:46)
    at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.notifyHandlerException(DefaultChannelPipeline.java:658)
    at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:577)
    at org.elasticsearch.common.netty.channel.Channels.close(Channels.java:812)
    at org.elasticsearch.common.netty.channel.AbstractChannel.close(AbstractChannel.java:206)
    at org.elasticsearch.transport.netty.NettyTransport$NodeChannels.closeChannelsAndWait(NettyTransport.java:1098)
    at org.elasticsearch.transport.netty.NettyTransport$NodeChannels.close(NettyTransport.java:1084)
    at org.elasticsearch.transport.netty.NettyTransport.disconnectFromNode(NettyTransport.java:873)
    at org.elasticsearch.transport.TransportService.disconnectFromNode(TransportService.java:224)
    at org.elasticsearch.client.transport.TransportClientNodesService$SniffNodesSampler$1.run(TransportClientNodesService.java:477)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:745)
Exception in thread "elasticsearch[waElasticSearchNode][management][T#1]" java.lang.NoClassDefFoundError: org/elasticsearch/common/netty/channel/Channels$7
    at org.elasticsearch.common.netty.channel.Channels.fireExceptionCaughtLater(Channels.java:496)
    at org.elasticsearch.common.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:46)
    at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.notifyHandlerException(DefaultChannelPipeline.java:658)
    at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:577)
    at org.elasticsearch.common.netty.channel.Channels.close(Channels.java:812)
    at org.elasticsearch.common.netty.channel.AbstractChannel.close(AbstractChannel.java:206)
    at org.elasticsearch.transport.netty.NettyTransport$NodeChannels.closeChannelsAndWait(NettyTransport.java:1098)
    at org.elasticsearch.transport.netty.NettyTransport$NodeChannels.close(NettyTransport.java:1084)
    at org.elasticsearch.transport.netty.NettyTransport.disconnectFromNode(NettyTransport.java:873)
    at org.elasticsearch.transport.TransportService.disconnectFromNode(TransportService.java:224)
    at org.elasticsearch.client.transport.TransportClientNodesService$SniffNodesSampler$1.run(TransportClientNodesService.java:477)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
Exception in thread "elasticsearch[waElasticSearchNode][transport_client_worker][T#7]"     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:745)
java.lang.NullPointerException
    at org.elasticsearch.transport.netty.NettyInternalESLogger.warn(NettyInternalESLogger.java:83)
    at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:340)
Exception in thread "elasticsearch[waElasticSearchNode][transport_client_worker][T#13]"     at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
    at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
    at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
    at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:745)
java.lang.NullPointerException
    at org.elasticsearch.transport.netty.NettyInternalESLogger.warn(NettyInternalESLogger.java:83)
    at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:340)
    at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
    at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
Exception in thread "elasticsearch[waElasticSearchNode][transport_client_worker][T#26]"     at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
    at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:745)
java.lang.NullPointerException
    at org.elasticsearch.transport.netty.NettyInternalESLogger.warn(NettyInternalESLogger.java:83)
    at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:340)
    at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
    at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
    at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
    at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:745)

We have no solid idea why it happens, any suggestions would be appreciated


(system) #2