RejectedExecutionException while indexing the data

Shirsh_Bansal · November 8, 2012, 4:27am

I am using elastic search to store a large amount of data. Currently there
is only one node and one shard. The size of the machine I am using is
around 250GB. I am mainly using the default settings of elastic search. My
operations are quite a lot write specific. I got the following error on my
machine. Please do tell if some other settings related information is
required.

java.util.concurrent.RejectedExecutionException
at
java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:1956)
at
java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:816)
at
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1337)
at
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.start(AbstractNioWorker.java:184)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:330)
at
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:35)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:313)
at
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:35)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioChannelSink.execute(AbstractNioChannelSink.java:34)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.execute(DefaultChannelPipeline.java:637)
at
org.elasticsearch.common.netty.channel.Channels.fireExceptionCaughtLater(Channels.java:504)
at
org.elasticsearch.common.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:47)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.notifyHandlerException(DefaultChannelPipeline.java:659)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendDownstream(DefaultChannelPipeline.java:783)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:733)
at
org.elasticsearch.common.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:65)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:592)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:583)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:712)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:679)
at
org.elasticsearch.common.netty.channel.AbstractChannel.write(AbstractChannel.java:246)
at
org.elasticsearch.http.netty.NettyHttpChannel.sendResponse(NettyHttpChannel.java:150)
at
org.elasticsearch.rest.action.index.RestIndexAction$1.onFailure(RestIndexAction.java:139)
at
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$3.onClose(TransportShardReplicationOperationAction.java:497)
at
org.elasticsearch.cluster.service.InternalClusterService.doStop(InternalClusterService.java:125)
at
org.elasticsearch.common.component.AbstractLifecycleComponent.stop(AbstractLifecycleComponent.java:105)
at
org.elasticsearch.node.internal.InternalNode.stop(InternalNode.java:234)
at
org.elasticsearch.node.internal.InternalNode.close(InternalNode.java:254)
at org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:72)
[2012-11-08 08:26:49,679][WARN ][netty.channel.DefaultChannelPipeline] An
exception was thrown by an exception handler.
java.util.concurrent.RejectedExecutionException
at
java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:1956)
at
java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:816)
at
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1337)
at
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.start(AbstractNioWorker.java:184)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:330)
at
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:35)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:313)
at
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:35)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioChannelSink.execute(AbstractNioChannelSink.java:34)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.execute(DefaultChannelPipeline.java:637)
at
org.elasticsearch.common.netty.channel.Channels.fireExceptionCaughtLater(Channels.java:504)
at
org.elasticsearch.common.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:47)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.notifyHandlerException(DefaultChannelPipeline.java:659)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendDownstream(DefaultChannelPipeline.java:783)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:733)
at
org.elasticsearch.common.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:65)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:592)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:583)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:712)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:679)
at
org.elasticsearch.common.netty.channel.AbstractChannel.write(AbstractChannel.java:246)
at
org.elasticsearch.http.netty.NettyHttpChannel.sendResponse(NettyHttpChannel.java:150)
at
org.elasticsearch.rest.action.index.RestIndexAction$1.onFailure(RestIndexAction.java:139)
at
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$3.onClose(TransportShardReplicationOperationAction.java:497)
at
org.elasticsearch.cluster.service.InternalClusterService.doStop(InternalClusterService.java:125)
at
org.elasticsearch.common.component.AbstractLifecycleComponent.stop(AbstractLifecycleComponent.java:105)
at
org.elasticsearch.node.internal.InternalNode.stop(InternalNode.java:234)
at
org.elasticsearch.node.internal.InternalNode.close(InternalNode.java:254)
at org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:72)
[2012-11-08 08:26:49,680][WARN ][netty.channel.DefaultChannelPipeline] An
exception was thrown by an exception handler.
java.util.concurrent.RejectedExecutionException
at
java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:1956)
at
java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:816)
at
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1337)
at
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.start(AbstractNioWorker.java:184)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:330)
at
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:35)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:313)
at
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:35)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioChannelSink.execute(AbstractNioChannelSink.java:34)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.execute(DefaultChannelPipeline.java:637)
at
org.elasticsearch.common.netty.channel.Channels.fireExceptionCaughtLater(Channels.java:504)
at
org.elasticsearch.common.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:47)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.notifyHandlerException(DefaultChannelPipeline.java:659)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendDownstream(DefaultChannelPipeline.java:783)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:733)
at
org.elasticsearch.common.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:65)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:592)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:583)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:712)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:679)
at
org.elasticsearch.common.netty.channel.AbstractChannel.write(AbstractChannel.java:246)
at
org.elasticsearch.http.netty.NettyHttpChannel.sendResponse(NettyHttpChannel.java:150)
at
org.elasticsearch.rest.action.index.RestIndexAction$1.onFailure(RestIndexAction.java:139)
at
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$3.onClose(TransportShardReplicationOperationAction.java:497)
at
org.elasticsearch.cluster.service.InternalClusterService.doStop(InternalClusterService.java:125)
at
org.elasticsearch.common.component.AbstractLifecycleComponent.stop(AbstractLifecycleComponent.java:105)
at
org.elasticsearch.node.internal.InternalNode.stop(InternalNode.java:234)
at
org.elasticsearch.node.internal.InternalNode.close(InternalNode.java:254)
at org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:72)

--

mvg · November 8, 2012, 2:08pm

From the stacktrace it seems that you were trying to stop elastisearch.
What do you mean 250GB? RAM or disk space available? You wrote that
you have a lot of writes? Can you elaborate a bit more on this (how
large are your documents, what is the concurrent update rate)?

Martijn

On 8 November 2012 05:27, shirsh bansal shirshban@gmail.com wrote:

I am using Elasticsearch to store a large amount of data. Currently there
is only one node and one shard. The size of the machine I am using is around
250GB. I am mainly using the default settings of Elasticsearch. My
operations are quite a lot write specific. I got the following error on my
machine. Please do tell if some other settings related information is
required.

java.util.concurrent.RejectedExecutionException
at
java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:1956)
at
java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:816)
at
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1337)
at
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.start(AbstractNioWorker.java:184)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:330)
at
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:35)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:313)
at
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:35)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioChannelSink.execute(AbstractNioChannelSink.java:34)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.execute(DefaultChannelPipeline.java:637)
at
org.elasticsearch.common.netty.channel.Channels.fireExceptionCaughtLater(Channels.java:504)
at
org.elasticsearch.common.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:47)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.notifyHandlerException(DefaultChannelPipeline.java:659)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendDownstream(DefaultChannelPipeline.java:783)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:733)
at
org.elasticsearch.common.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:65)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:592)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:583)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:712)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:679)
at
org.elasticsearch.common.netty.channel.AbstractChannel.write(AbstractChannel.java:246)
at
org.elasticsearch.http.netty.NettyHttpChannel.sendResponse(NettyHttpChannel.java:150)
at
org.elasticsearch.rest.action.index.RestIndexAction$1.onFailure(RestIndexAction.java:139)
at
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$3.onClose(TransportShardReplicationOperationAction.java:497)
at
org.elasticsearch.cluster.service.InternalClusterService.doStop(InternalClusterService.java:125)
at
org.elasticsearch.common.component.AbstractLifecycleComponent.stop(AbstractLifecycleComponent.java:105)
at
org.elasticsearch.node.internal.InternalNode.stop(InternalNode.java:234)
at
org.elasticsearch.node.internal.InternalNode.close(InternalNode.java:254)
at org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:72)
[2012-11-08 08:26:49,679][WARN ][netty.channel.DefaultChannelPipeline] An
exception was thrown by an exception handler.
java.util.concurrent.RejectedExecutionException
at
java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:1956)
at
java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:816)
at
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1337)
at
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.start(AbstractNioWorker.java:184)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:330)
at
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:35)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:313)
at
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:35)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioChannelSink.execute(AbstractNioChannelSink.java:34)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.execute(DefaultChannelPipeline.java:637)
at
org.elasticsearch.common.netty.channel.Channels.fireExceptionCaughtLater(Channels.java:504)
at
org.elasticsearch.common.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:47)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.notifyHandlerException(DefaultChannelPipeline.java:659)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendDownstream(DefaultChannelPipeline.java:783)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:733)
at
org.elasticsearch.common.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:65)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:592)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:583)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:712)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:679)
at
org.elasticsearch.common.netty.channel.AbstractChannel.write(AbstractChannel.java:246)
at
org.elasticsearch.http.netty.NettyHttpChannel.sendResponse(NettyHttpChannel.java:150)
at
org.elasticsearch.rest.action.index.RestIndexAction$1.onFailure(RestIndexAction.java:139)
at
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$3.onClose(TransportShardReplicationOperationAction.java:497)
at
org.elasticsearch.cluster.service.InternalClusterService.doStop(InternalClusterService.java:125)
at
org.elasticsearch.common.component.AbstractLifecycleComponent.stop(AbstractLifecycleComponent.java:105)
at
org.elasticsearch.node.internal.InternalNode.stop(InternalNode.java:234)
at
org.elasticsearch.node.internal.InternalNode.close(InternalNode.java:254)
at org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:72)
[2012-11-08 08:26:49,680][WARN ][netty.channel.DefaultChannelPipeline] An
exception was thrown by an exception handler.
java.util.concurrent.RejectedExecutionException
at
java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:1956)
at
java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:816)
at
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1337)
at
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.start(AbstractNioWorker.java:184)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:330)
at
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:35)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:313)
at
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:35)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioChannelSink.execute(AbstractNioChannelSink.java:34)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.execute(DefaultChannelPipeline.java:637)
at
org.elasticsearch.common.netty.channel.Channels.fireExceptionCaughtLater(Channels.java:504)
at
org.elasticsearch.common.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:47)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.notifyHandlerException(DefaultChannelPipeline.java:659)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendDownstream(DefaultChannelPipeline.java:783)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:733)
at
org.elasticsearch.common.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:65)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:592)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:583)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:712)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:679)
at
org.elasticsearch.common.netty.channel.AbstractChannel.write(AbstractChannel.java:246)
at
org.elasticsearch.http.netty.NettyHttpChannel.sendResponse(NettyHttpChannel.java:150)
at
org.elasticsearch.rest.action.index.RestIndexAction$1.onFailure(RestIndexAction.java:139)
at
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$3.onClose(TransportShardReplicationOperationAction.java:497)
at
org.elasticsearch.cluster.service.InternalClusterService.doStop(InternalClusterService.java:125)
at
org.elasticsearch.common.component.AbstractLifecycleComponent.stop(AbstractLifecycleComponent.java:105)
at
org.elasticsearch.node.internal.InternalNode.stop(InternalNode.java:234)
at
org.elasticsearch.node.internal.InternalNode.close(InternalNode.java:254)
at org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:72)

--

--
Met vriendelijke groet,

Martijn van Groningen

--

otisg · November 8, 2012, 4:50pm

Hi,

Check your ES settings. I think in elasticsearch.yml you'll see settings
for various threadpools, including one for indexing. You can play with
their sizes - increase them in your case if your writes are high (how high
are they? 100 docs/second? 1000 docs/second? More? Big docs or small docs?
Just curious). This is also where you can tell ES how to behave when
limits are reached.

See Elasticsearch Platform — Find real-time answers at scale | Elastic

Next release of SPM for ES is scheduled for next week and will have info
about various network connections exposed/graphed, so you'll be able to see
this sort of stuff in pretty graphs.

Otis

Search Analytics - Cloud Monitoring Tools & Services | Sematext
Performance Monitoring - Sematext Monitoring | Infrastructure Monitoring Service

On Wednesday, November 7, 2012 11:27:32 PM UTC-5, shirsh bansal wrote:

I am using Elasticsearch to store a large amount of data. Currently there
is only one node and one shard. The size of the machine I am using is
around 250GB. I am mainly using the default settings of Elasticsearch. My
operations are quite a lot write specific. I got the following error on my
machine. Please do tell if some other settings related information is
required.

java.util.concurrent.RejectedExecutionException
at
java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:1956)
at
java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:816)
at
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1337)
at
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.start(AbstractNioWorker.java:184)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:330)
at
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:35)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:313)
at
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:35)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioChannelSink.execute(AbstractNioChannelSink.java:34)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.execute(DefaultChannelPipeline.java:637)
at
org.elasticsearch.common.netty.channel.Channels.fireExceptionCaughtLater(Channels.java:504)
at
org.elasticsearch.common.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:47)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.notifyHandlerException(DefaultChannelPipeline.java:659)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendDownstream(DefaultChannelPipeline.java:783)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:733)
at
org.elasticsearch.common.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:65)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:592)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:583)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:712)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:679)
at
org.elasticsearch.common.netty.channel.AbstractChannel.write(AbstractChannel.java:246)
at
org.elasticsearch.http.netty.NettyHttpChannel.sendResponse(NettyHttpChannel.java:150)
at
org.elasticsearch.rest.action.index.RestIndexAction$1.onFailure(RestIndexAction.java:139)
at
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$3.onClose(TransportShardReplicationOperationAction.java:497)
at
org.elasticsearch.cluster.service.InternalClusterService.doStop(InternalClusterService.java:125)
at
org.elasticsearch.common.component.AbstractLifecycleComponent.stop(AbstractLifecycleComponent.java:105)
at
org.elasticsearch.node.internal.InternalNode.stop(InternalNode.java:234)
at
org.elasticsearch.node.internal.InternalNode.close(InternalNode.java:254)
at org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:72)
[2012-11-08 08:26:49,679][WARN ][netty.channel.DefaultChannelPipeline] An
exception was thrown by an exception handler.
java.util.concurrent.RejectedExecutionException
at
java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:1956)
at
java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:816)
at
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1337)
at
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.start(AbstractNioWorker.java:184)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:330)
at
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:35)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:313)
at
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:35)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioChannelSink.execute(AbstractNioChannelSink.java:34)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.execute(DefaultChannelPipeline.java:637)
at
org.elasticsearch.common.netty.channel.Channels.fireExceptionCaughtLater(Channels.java:504)
at
org.elasticsearch.common.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:47)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.notifyHandlerException(DefaultChannelPipeline.java:659)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendDownstream(DefaultChannelPipeline.java:783)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:733)
at
org.elasticsearch.common.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:65)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:592)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:583)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:712)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:679)
at
org.elasticsearch.common.netty.channel.AbstractChannel.write(AbstractChannel.java:246)
at
org.elasticsearch.http.netty.NettyHttpChannel.sendResponse(NettyHttpChannel.java:150)
at
org.elasticsearch.rest.action.index.RestIndexAction$1.onFailure(RestIndexAction.java:139)
at
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$3.onClose(TransportShardReplicationOperationAction.java:497)
at
org.elasticsearch.cluster.service.InternalClusterService.doStop(InternalClusterService.java:125)
at
org.elasticsearch.common.component.AbstractLifecycleComponent.stop(AbstractLifecycleComponent.java:105)
at
org.elasticsearch.node.internal.InternalNode.stop(InternalNode.java:234)
at
org.elasticsearch.node.internal.InternalNode.close(InternalNode.java:254)
at org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:72)
[2012-11-08 08:26:49,680][WARN ][netty.channel.DefaultChannelPipeline] An
exception was thrown by an exception handler.
java.util.concurrent.RejectedExecutionException
at
java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:1956)
at
java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:816)
at
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1337)
at
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.start(AbstractNioWorker.java:184)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:330)
at
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:35)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:313)
at
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:35)
at
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioChannelSink.execute(AbstractNioChannelSink.java:34)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.execute(DefaultChannelPipeline.java:637)
at
org.elasticsearch.common.netty.channel.Channels.fireExceptionCaughtLater(Channels.java:504)
at
org.elasticsearch.common.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:47)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.notifyHandlerException(DefaultChannelPipeline.java:659)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendDownstream(DefaultChannelPipeline.java:783)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:733)
at
org.elasticsearch.common.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:65)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:592)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:583)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:712)
at
org.elasticsearch.common.netty.channel.Channels.write(Channels.java:679)
at
org.elasticsearch.common.netty.channel.AbstractChannel.write(AbstractChannel.java:246)
at
org.elasticsearch.http.netty.NettyHttpChannel.sendResponse(NettyHttpChannel.java:150)
at
org.elasticsearch.rest.action.index.RestIndexAction$1.onFailure(RestIndexAction.java:139)
at
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$3.onClose(TransportShardReplicationOperationAction.java:497)
at
org.elasticsearch.cluster.service.InternalClusterService.doStop(InternalClusterService.java:125)
at
org.elasticsearch.common.component.AbstractLifecycleComponent.stop(AbstractLifecycleComponent.java:105)
at
org.elasticsearch.node.internal.InternalNode.stop(InternalNode.java:234)
at
org.elasticsearch.node.internal.InternalNode.close(InternalNode.java:254)
at org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:72)

--