Elasticsearch high cpu usage

Elastic Stack Elasticsearch
1

Hi,
i have elasticsearch installed on two clustered ec2 large nodes
today i found that elasticsearch process consume almost all cpu available on both nodes
my elasticsearch version 0.19.2
i'm using openjdk-6-jre version 6b22-1.10.6-0ubuntu1

jstack on node 2 =>
<nabble_embed>
root@ga2:/usr/lib/jvm/java-6-sun# bin/jstack -F 26556
Attaching to process ID 26556, please wait...
Debugger attached successfully.
Server compiler detected.
JVM version is 20.1-b02
Deadlock Detection:

No deadlocks found.

Thread 2692: (state = BLOCKED)

  • sun.misc.Unsafe.park(boolean, long) @bci=0 (Compiled frame; information may be imprecise)
  • java.util.concurrent.locks.LockSupport.parkNanos(java.lang.Object, long) @bci=20, line=198 (Compiled frame)
  • java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.util.concurrent.SynchronousQueue$TransferStack$SNode, boolean, long) @bci=174, line=424 (Compiled frame)
  • java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.lang.Object, boolean, long) @bci=102, line=323 (Interpreted frame)
  • java.util.concurrent.SynchronousQueue.poll(long, java.util.concurrent.TimeUnit) @bci=11, line=874 (Compiled frame)
  • java.util.concurrent.ThreadPoolExecutor.getTask() @bci=62, line=945 (Compiled frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=18, line=907 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 2089: (state = BLOCKED)

  • sun.misc.Unsafe.park(boolean, long) @bci=0 (Compiled frame; information may be imprecise)
  • java.util.concurrent.locks.LockSupport.parkNanos(java.lang.Object, long) @bci=20, line=198 (Compiled frame)
  • org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(org.elasticsearch.common.util
    .concurrent.jsr166y.LinkedTransferQueue$Node, org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQue
    ue$Node, java.lang.Object, boolean, long) @bci=180, line=702 (Compiled frame)
  • org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(java.lang.Object, boolean, int, long) @bci=286, line=615 (Compiled frame)
  • org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(long, java.util.concurrent.TimeUnit) @bci=9, line=1117 (Compiled frame)
  • java.util.concurrent.ThreadPoolExecutor.getTask() @bci=62, line=945 (Compiled frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=18, line=907 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 2088: (state = BLOCKED)

  • sun.misc.Unsafe.park(boolean, long) @bci=0 (Compiled frame; information may be imprecise)
  • java.util.concurrent.locks.LockSupport.parkNanos(java.lang.Object, long) @bci=20, line=198 (Compiled frame)
  • org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue$Node, org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue$Node, java.lang.Object, boolean, long) @bci=180, line=702 (Compiled frame)
  • org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(java.lang.Object, boolean, int, long) @bci=286, line=615 (Compiled frame)
  • org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(long, java.util.concurrent.TimeUnit) @bci=9, line=1117 (Compiled frame)
  • java.util.concurrent.ThreadPoolExecutor.getTask() @bci=62, line=945 (Compiled frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=18, line=907 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 30638: (state = BLOCKED)

  • sun.misc.Unsafe.park(boolean, long) @bci=0 (Compiled frame; information may be imprecise)
  • java.util.concurrent.locks.LockSupport.parkNanos(java.lang.Object, long) @bci=20, line=198 (Compiled frame)
  • org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue$Node, org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue$Node, java.lang.Object, boolean, long) @bci=180, line=702 (Compiled frame)
  • org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(java.lang.Object, boolean, int, long) @bci=286, line=615 (Compiled frame)
  • org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(long, java.util.concurrent.TimeUnit) @bci=9, line=1117 (Compiled frame)
  • java.util.concurrent.ThreadPoolExecutor.getTask() @bci=62, line=945 (Compiled frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=18, line=907 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 30610: (state = IN_NATIVE)

  • sun.nio.ch.EPollArrayWrapper.epollWait(long, int, long, int) @bci=0 (Compiled frame; information may be imprecise)
  • sun.nio.ch.EPollArrayWrapper.poll(long) @bci=18, line=210 (Compiled frame)
  • sun.nio.ch.EPollSelectorImpl.doSelect(long) @bci=28, line=65 (Compiled frame)
  • sun.nio.ch.SelectorImpl.lockAndDoSelect(long) @bci=37, line=69 (Compiled frame)
  • sun.nio.ch.SelectorImpl.select(long) @bci=30, line=80 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(java.nio.channels.Selector) @bci=4, line=33 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run() @bci=57, line=157 (Compiled frame)
  • org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run() @bci=55, line=102 (Interpreted frame)
  • org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run() @bci=14, line=42 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.runTask(java.lang.Runnable) @bci=59, line=886 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=28, line=908 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 30553: (state = BLOCKED)

  • sun.misc.Unsafe.park(boolean, long) @bci=0 (Compiled frame; information may be imprecise)
  • java.util.concurrent.locks.LockSupport.parkNanos(java.lang.Object, long) @bci=20, line=198 (Compiled frame)
  • org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue$Node, org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue$Node, java.lang.Object, boolean, long) @bci=180, line=702 (Compiled frame)
  • org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(java.lang.Object, boolean, int, long) @bci=286, line=615 (Compiled frame)
  • org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(long, java.util.concurrent.TimeUnit
    ) @bci=9, line=1117 (Compiled frame)
  • java.util.concurrent.ThreadPoolExecutor.getTask() @bci=62, line=945 (Compiled frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=18, line=907 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 30552: (state = IN_NATIVE)

  • sun.nio.ch.EPollArrayWrapper.epollWait(long, int, long, int) @bci=0 (Compiled frame; information may be imprecise)
  • sun.nio.ch.EPollArrayWrapper.poll(long) @bci=18, line=210 (Compiled frame)
  • sun.nio.ch.EPollSelectorImpl.doSelect(long) @bci=28, line=65 (Compiled frame)
  • sun.nio.ch.SelectorImpl.lockAndDoSelect(long) @bci=37, line=69 (Compiled frame)
  • sun.nio.ch.SelectorImpl.select(long) @bci=30, line=80 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(java.nio.channels.Selector) @bci=4, line=33 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run() @bci=57, line=157 (Compiled frame)
  • org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run() @bci=55, line=102 (Interpreted frame)
  • org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run() @bci=14, line=42 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.runTask(java.lang.Runnable) @bci=59, line=886 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=28, line=908 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 26557: (state = BLOCKED)

Thread 27616: (state = BLOCKED)

  • sun.misc.Unsafe.park(boolean, long) @bci=0 (Compiled frame; information may be imprecise)
  • java.util.concurrent.locks.LockSupport.park(java.lang.Object) @bci=14, line=158 (Interpreted frame)
  • java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt() @bci=1, line=811 (Interpreted frame)
  • java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(int) @bci=55, line=969 (Interpreted frame)
  • java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(int) @bci=24, line=1281 (Interpreted frame)
  • java.util.concurrent.CountDownLatch.await() @bci=5, line=207 (Interpreted frame)
  • org.elasticsearch.bootstrap.Bootstrap$3.run() @bci=3, line=222 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 27615: (state = IN_NATIVE)

  • sun.nio.ch.EPollArrayWrapper.epollWait(long, int, long, int) @bci=0 (Compiled frame; information may be imprecise)
  • sun.nio.ch.EPollArrayWrapper.poll(long) @bci=18, line=210 (Compiled frame)
  • sun.nio.ch.EPollSelectorImpl.doSelect(long) @bci=28, line=65 (Interpreted frame)
  • sun.nio.ch.SelectorImpl.lockAndDoSelect(long) @bci=37, line=69 (Interpreted frame)
  • sun.nio.ch.SelectorImpl.select(long) @bci=30, line=80 (Interpreted frame)
  • org.elasticsearch.common.netty.channel.socket.nio.NioServerSocketPipelineSink$Boss.run() @bci=23, line=232 (Interpreted frame)
  • org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run() @bci=55, line=102 (Interpreted frame)
  • org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run() @bci=14, line=42 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.runTask(java.lang.Runnable) @bci=59, line=886 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=28, line=908 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 27614: (state = BLOCKED)

  • sun.misc.Unsafe.park(boolean, long) @bci=0 (Compiled frame; information may be imprecise)
  • java.util.concurrent.locks.LockSupport.park(java.lang.Object) @bci=14, line=158 (Interpreted frame)
  • java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await() @bci=42, line=1987 (Interpreted frame)
  • java.util.concurrent.LinkedBlockingQueue.take() @bci=29, line=399 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor.getTask() @bci=78, line=947 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=18, line=907 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 27501: (state = IN_NATIVE)

  • sun.nio.ch.EPollArrayWrapper.epollWait(long, int, long, int) @bci=0 (Compiled frame; information may be imprecise)
  • sun.nio.ch.EPollArrayWrapper.poll(long) @bci=18, line=210 (Compiled frame)
  • sun.nio.ch.EPollSelectorImpl.doSelect(long) @bci=28, line=65 (Compiled frame)
  • sun.nio.ch.SelectorImpl.lockAndDoSelect(long) @bci=37, line=69 (Compiled frame)
  • sun.nio.ch.SelectorImpl.select(long) @bci=30, line=80 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(java.nio.channels.Selector) @bci=4, line=33 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run() @bci=57, line=157 (Compiled frame)
  • org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run() @bci=55, line=102 (Interpreted frame)
  • org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run() @bci=14, line=42 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.runTask(java.lang.Runnable) @bci=59, line=886 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=28, line=908 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 27500: (state = IN_NATIVE)

  • sun.nio.ch.EPollArrayWrapper.epollWait(long, int, long, int) @bci=0 (Compiled frame; information may be imprecise)
  • sun.nio.ch.EPollArrayWrapper.poll(long) @bci=18, line=210 (Compiled frame)
  • sun.nio.ch.EPollSelectorImpl.doSelect(long) @bci=28, line=65 (Compiled frame)
  • sun.nio.ch.SelectorImpl.lockAndDoSelect(long) @bci=37, line=69 (Compiled frame)
  • sun.nio.ch.SelectorImpl.select(long) @bci=30, line=80 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(java.nio.channels.Selector) @bci=4, line=33 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run() @bci=57, line=157 (Compiled frame)
  • org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run() @bci=55, line=102 (Interpreted frame)
  • org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run() @bci=14, line=42 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.runTask(java.lang.Runnable) @bci=59, line=886 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=28, line=908 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 27499: (state = IN_NATIVE)

  • sun.nio.ch.EPollArrayWrapper.epollWait(long, int, long, int) @bci=0 (Compiled frame; information may be imprecise)
  • sun.nio.ch.EPollArrayWrapper.poll(long) @bci=18, line=210 (Compiled frame)
  • sun.nio.ch.EPollSelectorImpl.doSelect(long) @bci=28, line=65 (Compiled frame)
  • sun.nio.ch.SelectorImpl.lockAndDoSelect(long) @bci=37, line=69 (Compiled frame)
  • sun.nio.ch.SelectorImpl.select(long) @bci=30, line=80 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(java.nio.channels.Selector) @bci=4, line=33 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run() @bci=57, line=157 (Compiled frame)
  • org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run() @bci=55, line=102 (Interpreted frame)
  • org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run() @bci=14, line=42 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.runTask(java.lang.Runnable) @bci=59, line=886 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=28, line=908 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 27498: (state = IN_NATIVE)

  • sun.nio.ch.EPollArrayWrapper.epollWait(long, int, long, int) @bci=0 (Compiled frame; information may be imprecise)
  • sun.nio.ch.EPollArrayWrapper.poll(long) @bci=18, line=210 (Compiled frame)
  • sun.nio.ch.EPollSelectorImpl.doSelect(long) @bci=28, line=65 (Compiled frame)
  • sun.nio.ch.SelectorImpl.lockAndDoSelect(long) @bci=37, line=69 (Compiled frame)
  • sun.nio.ch.SelectorImpl.select(long) @bci=30, line=80 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(java.nio.channels.Selector) @bci=4, line=33 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run() @bci=57, line=157 (Compiled frame)
  • org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run() @bci=55, line=102 (Interpreted frame)
  • org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run() @bci=14, line=42 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.runTask(java.lang.Runnable) @bci=59, line=886 (Interpreted fra
    me)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=28, line=908 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 27497: (state = IN_NATIVE)

  • sun.nio.ch.EPollArrayWrapper.epollWait(long, int, long, int) @bci=0 (Compiled frame; information may be imprecise)
  • sun.nio.ch.EPollArrayWrapper.poll(long) @bci=18, line=210 (Compiled frame)
  • sun.nio.ch.EPollSelectorImpl.doSelect(long) @bci=28, line=65 (Compiled frame)
  • sun.nio.ch.SelectorImpl.lockAndDoSelect(long) @bci=37, line=69 (Compiled frame)
  • sun.nio.ch.SelectorImpl.select(long) @bci=30, line=80 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(java.nio.channels.Selector) @bci=4, line=33 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run() @bci=57, line=157 (Compiled frame)
  • org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run() @bci=55, line=102 (Interpreted frame)
  • org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run() @bci=14, line=42 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.runTask(java.lang.Runnable) @bci=59, line=886 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=28, line=908 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 27496: (state = IN_NATIVE)

  • sun.nio.ch.EPollArrayWrapper.epollWait(long, int, long, int) @bci=0 (Compiled frame; information may be imprecise)
  • sun.nio.ch.EPollArrayWrapper.poll(long) @bci=18, line=210 (Compiled frame)
  • sun.nio.ch.EPollSelectorImpl.doSelect(long) @bci=28, line=65 (Compiled frame)
  • sun.nio.ch.SelectorImpl.lockAndDoSelect(long) @bci=37, line=69 (Compiled frame)
  • sun.nio.ch.SelectorImpl.select(long) @bci=30, line=80 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(java.nio.channels.Selector) @bci=4, line=33 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run() @bci=57, line=157 (Compiled frame)
  • org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run() @bci=55, line=102 (Interpreted frame)
  • org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run() @bci=14, line=42 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.runTask(java.lang.Runnable) @bci=59, line=886 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=28, line=908 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 27494: (state = IN_NATIVE)

  • sun.nio.ch.EPollArrayWrapper.epollWait(long, int, long, int) @bci=0 (Compiled frame; information may be imprecise)
  • sun.nio.ch.EPollArrayWrapper.poll(long) @bci=18, line=210 (Compiled frame)
  • sun.nio.ch.EPollSelectorImpl.doSelect(long) @bci=28, line=65 (Compiled frame)
  • sun.nio.ch.SelectorImpl.lockAndDoSelect(long) @bci=37, line=69 (Compiled frame)
  • sun.nio.ch.SelectorImpl.select(long) @bci=30, line=80 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(java.nio.channels.Selector) @bci=4, line=33 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run() @bci=57, line=157 (Compiled frame)
  • org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run() @bci=55, line=102 (Interpreted frame)
  • org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run() @bci=14, line=42 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.runTask(java.lang.Runnable) @bci=59, line=886 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=28, line=908 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 27487: (state = IN_NATIVE)

  • sun.nio.ch.EPollArrayWrapper.epollWait(long, int, long, int) @bci=0 (Compiled frame; information may be imprecise)
  • sun.nio.ch.EPollArrayWrapper.poll(long) @bci=18, line=210 (Compiled frame)
  • sun.nio.ch.EPollSelectorImpl.doSelect(long) @bci=28, line=65 (Compiled frame)
  • sun.nio.ch.SelectorImpl.lockAndDoSelect(long) @bci=37, line=69 (Compiled frame)
  • sun.nio.ch.SelectorImpl.select(long) @bci=30, line=80 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(java.nio.channels.Selector) @bci=4, line=33 (Compiled frame)
  • org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run() @bci=57, line=157 (Compiled frame)
  • org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run() @bci=55, line=102 (Interpreted frame)
  • org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run() @bci=14, line=42 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.runTask(java.lang.Runnable) @bci=59, line=886 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=28, line=908 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 27398: (state = IN_NATIVE)

  • sun.nio.ch.EPollArrayWrapper.epollWait(long, int, long, int) @bci=0 (Compiled frame; information may be imprecise)
  • sun.nio.ch.EPollArrayWrapper.poll(long) @bci=18, line=210 (Compiled frame)
  • sun.nio.ch.EPollSelectorImpl.doSelect(long) @bci=28, line=65 (Interpreted frame)
  • sun.nio.ch.SelectorImpl.lockAndDoSelect(long) @bci=37, line=69 (Interpreted frame)
  • sun.nio.ch.SelectorImpl.select(long) @bci=30, line=80 (Interpreted frame)
  • org.elasticsearch.common.netty.channel.socket.nio.NioServerSocketPipelineSink$Boss.run() @bci=23, line=232 (Interpreted frame)
  • org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run() @bci=55, line=102 (Interpreted frame)
  • org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run() @bci=14, line=42 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.runTask(java.lang.Runnable) @bci=59, line=886 (Interpreted frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=28, line=908 (Interpreted frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 27390: (state = BLOCKED)

  • java.lang.Thread.sleep(long) @bci=0 (Interpreted frame)
  • org.elasticsearch.indices.ttl.IndicesTTLService$PurgerThread.run() @bci=60, line=132 (Interpreted frame)

Thread 27232: (state = BLOCKED)

  • sun.misc.Unsafe.park(boolean, long) @bci=0 (Compiled frame; information may be imprecise)
  • java.util.concurrent.locks.LockSupport.parkNanos(java.lang.Object, long) @bci=20, line=198 (Compiled frame)
  • java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(long) @bci=68, line=2025 (Compiled frame)
  • java.util.concurrent.DelayQueue.take() @bci=57, line=164 (Compiled frame)
  • java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take() @bci=4, line=609 (Compiled frame)
  • java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take() @bci=1, line=602 (Compiled frame)
  • java.util.concurrent.ThreadPoolExecutor.getTask() @bci=78, line=947 (Compiled frame)
  • java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=18, line=907 (Compiled frame)
  • java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)

Thread 26806: (state = BLOCKED)

  • java.lang.Thread.sleep(long) @bci=0 (Interpreted frame)
  • org.elasticsearch.threadpool.ThreadPool$EstimatedTimeThread.run() @bci=18, line=356 (Interpreted frame)

Thread 26579: (state = BLOCKED)

Thread 26578: (state = BLOCKED)

Thread 26565: (state = BLOCKED)

  • java.lang.Object.wait(long) @bci=0 (Interpreted frame)
  • java.lang.ref.ReferenceQueue.remove(long) @bci=44, line=118 (Interpreted frame)
  • java.lang.ref.ReferenceQueue.remove() @bci=2, line=134 (Interpreted frame)
  • java.lang.ref.Finalizer$FinalizerThread.run() @bci=3, line=159 (Interpreted frame)

Thread 26564: (state = BLOCKED)

  • java.lang.Object.wait(long) @bci=0 (Interpreted frame)
  • java.lang.Object.wait() @bci=2, line=485 (Interpreted frame)
  • java.lang.ref.Reference$ReferenceHandler.run() @bci=46, line=116 (Interpreted frame)

</nabble_embed>