We're using ES java client 1.5.2 to communicate with our ES cluster(cluster obviously has the same version). Several times in week we observe following errors:
Exception in thread "elasticsearch[waElasticSearchNode][management][T#3]" Exception in thread "elasticsearch[waElasticSearchNode][management][T#4]" java.lang.NoClassDefFoundError: org/elasticsearch/common/netty/channel/Channels$7
at org.elasticsearch.common.netty.channel.Channels.fireExceptionCaughtLater(Channels.java:496)
at org.elasticsearch.common.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:46)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.notifyHandlerException(DefaultChannelPipeline.java:658)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:577)
at org.elasticsearch.common.netty.channel.Channels.close(Channels.java:812)
at org.elasticsearch.common.netty.channel.AbstractChannel.close(AbstractChannel.java:206)
at org.elasticsearch.transport.netty.NettyTransport$NodeChannels.caloseChannelsAndWait(NettyTransport.java:1098)
at org.elasticsearch.transport.netty.NettyTransport$NodeChannels.close(NettyTransport.java:1084)
at org.elasticsearch.transport.netty.NettyTransport.disconnectFromNode(NettyTransport.java:873)
at org.elasticsearch.transport.TransportService.disconnectFromNode(TransportService.java:224)
at org.elasticsearch.client.transport.TransportClientNodesService$SniffNodesSampler$1.run(TransportClientNodesService.java:477)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
java.lang.NoClassDefFoundError: org/elasticsearch/common/netty/channel/Channels$7
at org.elasticsearch.common.netty.channel.Channels.fireExceptionCaughtLater(Channels.java:496)
at org.elasticsearch.common.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:46)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.notifyHandlerException(DefaultChannelPipeline.java:658)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:577)
at org.elasticsearch.common.netty.channel.Channels.close(Channels.java:812)
at org.elasticsearch.common.netty.channel.AbstractChannel.close(AbstractChannel.java:206)
at org.elasticsearch.transport.netty.NettyTransport$NodeChannels.closeChannelsAndWait(NettyTransport.java:1098)
at org.elasticsearch.transport.netty.NettyTransport$NodeChannels.close(NettyTransport.java:1084)
at org.elasticsearch.transport.netty.NettyTransport.disconnectFromNode(NettyTransport.java:873)
at org.elasticsearch.transport.TransportService.disconnectFromNode(TransportService.java:224)
at org.elasticsearch.client.transport.TransportClientNodesService$SniffNodesSampler$1.run(TransportClientNodesService.java:477)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Exception in thread "elasticsearch[waElasticSearchNode][management][T#1]" java.lang.NoClassDefFoundError: org/elasticsearch/common/netty/channel/Channels$7
at org.elasticsearch.common.netty.channel.Channels.fireExceptionCaughtLater(Channels.java:496)
at org.elasticsearch.common.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:46)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.notifyHandlerException(DefaultChannelPipeline.java:658)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:577)
at org.elasticsearch.common.netty.channel.Channels.close(Channels.java:812)
at org.elasticsearch.common.netty.channel.AbstractChannel.close(AbstractChannel.java:206)
at org.elasticsearch.transport.netty.NettyTransport$NodeChannels.closeChannelsAndWait(NettyTransport.java:1098)
at org.elasticsearch.transport.netty.NettyTransport$NodeChannels.close(NettyTransport.java:1084)
at org.elasticsearch.transport.netty.NettyTransport.disconnectFromNode(NettyTransport.java:873)
at org.elasticsearch.transport.TransportService.disconnectFromNode(TransportService.java:224)
at org.elasticsearch.client.transport.TransportClientNodesService$SniffNodesSampler$1.run(TransportClientNodesService.java:477)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
Exception in thread "elasticsearch[waElasticSearchNode][transport_client_worker][T#7]" at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
java.lang.NullPointerException
at org.elasticsearch.transport.netty.NettyInternalESLogger.warn(NettyInternalESLogger.java:83)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:340)
Exception in thread "elasticsearch[waElasticSearchNode][transport_client_worker][T#13]" at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
java.lang.NullPointerException
at org.elasticsearch.transport.netty.NettyInternalESLogger.warn(NettyInternalESLogger.java:83)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:340)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
Exception in thread "elasticsearch[waElasticSearchNode][transport_client_worker][T#26]" at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
java.lang.NullPointerException
at org.elasticsearch.transport.netty.NettyInternalESLogger.warn(NettyInternalESLogger.java:83)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:340)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
We have no solid idea why it happens, any suggestions would be appreciated