TransportClient hangs and is stuck (w/thread dump)

[David_Jensen_2]

Here's the code:
esClient = new TransportClient().addTransportAddress(new
InetSocketTransportAddress(server, port));

Here's the thread dump at the moment of getting stuck (this is the
format that IntelliJ gave me when I pressed "export threads"). I also
removed some, hopefully, irrelevant code. I left in the thread that
invoked the thing and all the other threads that included
"elasticsearch"

New I/O client worker #1-2@2885 daemon, priority=5, in group 'main',
status: 'RUNNING'
at
sun.nio.ch.KQueueArrayWrapper.kevent0(KQueueArrayWrapper.java:-1)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:
69)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at
org.elasticsearch.util.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:
38)
at
org.elasticsearch.util.netty.channel.socket.nio.NioWorker.run(NioWorker.java:
155)
at
org.elasticsearch.util.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:
108)
at
org.elasticsearch.util.netty.util.internal.IoWorkerRunnable.run(IoWorkerRunnable.java:
46)
at java.util.concurrent.ThreadPoolExecutor
$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:637)

New I/O client worker #1-1@2842 daemon, priority=5, in group 'main',
status: 'RUNNING'
at
sun.nio.ch.KQueueArrayWrapper.kevent0(KQueueArrayWrapper.java:-1)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:
69)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at
org.elasticsearch.util.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:
38)
at
org.elasticsearch.util.netty.channel.socket.nio.NioWorker.run(NioWorker.java:
155)
at
org.elasticsearch.util.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:
108)
at
org.elasticsearch.util.netty.util.internal.IoWorkerRunnable.run(IoWorkerRunnable.java:
46)
at java.util.concurrent.ThreadPoolExecutor
$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:637)

elasticsearch[Man-Bull]transportClientBoss-pool-4-thread-1@2835
daemon, priority=5, in group 'main', status: 'WAIT'
at sun.misc.Unsafe.park(Unsafe.java:-1)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
at java.util.concurrent.SynchronousQueue
$TransferStack.awaitFulfill(SynchronousQueue.java:424)
at java.util.concurrent.SynchronousQueue
$TransferStack.transfer(SynchronousQueue.java:323)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:
874)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:
945)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:637)

elasticsearch[Man-Bull][tp]-pool-1-thread-1@2738 daemon, priority=5,
in group 'main', status: 'WAIT'
at sun.misc.Unsafe.park(Unsafe.java:-1)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
at java.util.concurrent.SynchronousQueue
$TransferStack.awaitFulfill(SynchronousQueue.java:424)
at java.util.concurrent.SynchronousQueue
$TransferStack.transfer(SynchronousQueue.java:323)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:
874)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:
945)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:637)

elasticsearch[Man-Bull][sc]-pool-2-thread-2@2893 daemon, priority=5,
in group 'main', status: 'MONITOR'
waiting for main@1
at org.elasticsearch.client.transport.TransportClientNodesService
$ScheduledNodesSampler.run(TransportClientNodesService.java:178)
at java.util.concurrent.Executors
$RunnableAdapter.call(Executors.java:441)
at java.util.concurrent.FutureTask
$Sync.innerRunAndReset(FutureTask.java:317)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)
at java.util.concurrent.ScheduledThreadPoolExecutor
$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)
at java.util.concurrent.ScheduledThreadPoolExecutor
$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:181)
at java.util.concurrent.ScheduledThreadPoolExecutor
$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:205)
at java.util.concurrent.ThreadPoolExecutor
$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:637)

elasticsearch[Man-Bull][sc]-pool-2-thread-1@2602 daemon, priority=5,
in group 'main', status: 'WAIT'
at sun.misc.Unsafe.park(Unsafe.java:-1)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
at java.util.concurrent.locks.AbstractQueuedSynchronizer
$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1,963)
at java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
at java.util.concurrent.ScheduledThreadPoolExecutor
$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583)
at java.util.concurrent.ScheduledThreadPoolExecutor
$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:
947)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:637)

<... irrelevant proprietary code removed ...>

main@1, priority=5, in group 'main', status: 'WAIT'
blocks elasticsearch[Man-Bull][sc]-pool-2-thread-2@2893
at sun.misc.Unsafe.park(Unsafe.java:-1)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:
158)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:
747)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:
905)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:
1,217)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:
207)
at org.elasticsearch.client.transport.TransportClientNodesService
$ScheduledNodesSampler.run(TransportClientNodesService.java:211)
at
org.elasticsearch.client.transport.TransportClientNodesService.addTransportAddress(TransportClientNodesService.java:
117)
at
org.elasticsearch.client.transport.TransportClient.addTransportAddress(TransportClient.java:
184)
at
com.spokesoftware.platform.services.CloudSearchServiceImpl.start(CloudSearchServiceImpl.java:
66)
...
at
sun.reflect.NativeMethodAccessorImpl.invoke0(NativeMethodAccessorImpl.java:-1)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:
39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:
25)
at java.lang.reflect.Method.invoke(Method.java:597)
...
at com.spokesoftware.core.bootstrap.Main.main(Main.java:89)

[kimchy (Shay Banon)]

I don't see something that can help in the thread dump... (thats the problem
with async systems, thread dumps looses some of their usefulness...). When
does it get stuck, on a specific operation you perform or simply during the
initialization of the transport client?

-shay.banon

On Tue, Jul 20, 2010 at 3:10 AM, David Jensen djensen47@gmail.com wrote:

Here's the code:
esClient = new TransportClient().addTransportAddress(new
InetSocketTransportAddress(server, port));

Here's the thread dump at the moment of getting stuck (this is the
format that IntelliJ gave me when I pressed "export threads"). I also
removed some, hopefully, irrelevant code. I left in the thread that
invoked the thing and all the other threads that included
"elasticsearch"

New I/O client worker #1-2@2885 daemon, priority=5, in group 'main',
status: 'RUNNING'
at
sun.nio.ch.KQueueArrayWrapper.kevent0(KQueueArrayWrapper.java:-1)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:
69)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at

org.elasticsearch.util.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:
38)
at

org.elasticsearch.util.netty.channel.socket.nio.NioWorker.run(NioWorker.java:
155)
at

org.elasticsearch.util.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:
108)
at

org.elasticsearch.util.netty.util.internal.IoWorkerRunnable.run(IoWorkerRunnable.java:
46)
at java.util.concurrent.ThreadPoolExecutor
$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:637)

New I/O client worker #1-1@2842 daemon, priority=5, in group 'main',
status: 'RUNNING'
at
sun.nio.ch.KQueueArrayWrapper.kevent0(KQueueArrayWrapper.java:-1)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:
69)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at

org.elasticsearch.util.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:
38)
at

org.elasticsearch.util.netty.channel.socket.nio.NioWorker.run(NioWorker.java:
155)
at

org.elasticsearch.util.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:
108)
at

org.elasticsearch.util.netty.util.internal.IoWorkerRunnable.run(IoWorkerRunnable.java:
46)
at java.util.concurrent.ThreadPoolExecutor
$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:637)

elasticsearch[Man-Bull]transportClientBoss-pool-4-thread-1@2835
daemon, priority=5, in group 'main', status: 'WAIT'
at sun.misc.Unsafe.park(Unsafe.java:-1)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
at java.util.concurrent.SynchronousQueue
$TransferStack.awaitFulfill(SynchronousQueue.java:424)
at java.util.concurrent.SynchronousQueue
$TransferStack.transfer(SynchronousQueue.java:323)
at
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:
874)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:
945)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:637)

elasticsearch[Man-Bull][tp]-pool-1-thread-1@2738 daemon, priority=5,
in group 'main', status: 'WAIT'
at sun.misc.Unsafe.park(Unsafe.java:-1)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
at java.util.concurrent.SynchronousQueue
$TransferStack.awaitFulfill(SynchronousQueue.java:424)
at java.util.concurrent.SynchronousQueue
$TransferStack.transfer(SynchronousQueue.java:323)
at
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:
874)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:
945)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:637)

elasticsearch[Man-Bull][sc]-pool-2-thread-2@2893 daemon, priority=5,
in group 'main', status: 'MONITOR'
waiting for main@1
at org.elasticsearch.client.transport.TransportClientNodesService
$ScheduledNodesSampler.run(TransportClientNodesService.java:178)
at java.util.concurrent.Executors
$RunnableAdapter.call(Executors.java:441)
at java.util.concurrent.FutureTask
$Sync.innerRunAndReset(FutureTask.java:317)
at
java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)
at java.util.concurrent.ScheduledThreadPoolExecutor
$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)
at java.util.concurrent.ScheduledThreadPoolExecutor
$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:181)
at java.util.concurrent.ScheduledThreadPoolExecutor
$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:205)
at java.util.concurrent.ThreadPoolExecutor
$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:637)

elasticsearch[Man-Bull][sc]-pool-2-thread-1@2602 daemon, priority=5,
in group 'main', status: 'WAIT'
at sun.misc.Unsafe.park(Unsafe.java:-1)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
at java.util.concurrent.locks.AbstractQueuedSynchronizer
$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1,963)
at java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
at java.util.concurrent.ScheduledThreadPoolExecutor
$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583)
at java.util.concurrent.ScheduledThreadPoolExecutor
$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:
947)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:637)

<... irrelevant proprietary code removed ...>

main@1, priority=5, in group 'main', status: 'WAIT'
blocks elasticsearch[Man-Bull][sc]-pool-2-thread-2@2893
at sun.misc.Unsafe.park(Unsafe.java:-1)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:
158)
at

java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:
747)
at

java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:
905)
at

java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:
1,217)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:
207)
at org.elasticsearch.client.transport.TransportClientNodesService
$ScheduledNodesSampler.run(TransportClientNodesService.java:211)
at

org.elasticsearch.client.transport.TransportClientNodesService.addTransportAddress(TransportClientNodesService.java:
117)
at

org.elasticsearch.client.transport.TransportClient.addTransportAddress(TransportClient.java:
184)
at

com.spokesoftware.platform.services.CloudSearchServiceImpl.start(CloudSearchServiceImpl.java:
66)
...
at

sun.reflect.NativeMethodAccessorImpl.invoke0(NativeMethodAccessorImpl.java:-1)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:
39)
at

sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:
25)
at java.lang.reflect.Method.invoke(Method.java:597)
...
at com.spokesoftware.core.bootstrap.Main.main(Main.java:89)

[David_Jensen_2]

It happens at that line of code that I provided. I imagine that it
happens when .addTransportAddress(..) is invoked. I can separate the
calls and find out for sure.

I'm running this within a Jetty container. I don't know if that helps
at all.

On Jul 20, 12:31 am, Shay Banon shay.ba...@elasticsearch.com wrote:

I don't see something that can help in the thread dump... (thats the problem
with async systems, thread dumps looses some of their usefulness...). When
does it get stuck, on a specific operation you perform or simply during the
initialization of the transport client?

-shay.banon

On Tue, Jul 20, 2010 at 3:10 AM, David Jensen djense...@gmail.com wrote:

Here's the code:
esClient = new TransportClient().addTransportAddress(new
InetSocketTransportAddress(server, port));

Here's the thread dump at the moment of getting stuck (this is the
format that IntelliJ gave me when I pressed "export threads"). I also
removed some, hopefully, irrelevant code. I left in the thread that
invoked the thing and all the other threads that included
"elasticsearch"

New I/O client worker #1-2@2885 daemon, priority=5, in group 'main',
status: 'RUNNING'
at
sun.nio.ch.KQueueArrayWrapper.kevent0(KQueueArrayWrapper.java:-1)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:
69)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at

org.elasticsearch.util.netty.channel.socket.nio.SelectorUtil.select(Selecto rUtil.java:
38)
at

org.elasticsearch.util.netty.channel.socket.nio.NioWorker.run(NioWorker.jav a:
155)
at

org.elasticsearch.util.netty.util.ThreadRenamingRunnable.run(ThreadRenaming Runnable.java:
108)
at

org.elasticsearch.util.netty.util.internal.IoWorkerRunnable.run(IoWorkerRun nable.java:
46)
at java.util.concurrent.ThreadPoolExecutor
$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:637)

New I/O client worker #1-1@2842 daemon, priority=5, in group 'main',
status: 'RUNNING'
at
sun.nio.ch.KQueueArrayWrapper.kevent0(KQueueArrayWrapper.java:-1)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:
69)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at

org.elasticsearch.util.netty.channel.socket.nio.SelectorUtil.select(Selecto rUtil.java:
38)
at

org.elasticsearch.util.netty.channel.socket.nio.NioWorker.run(NioWorker.jav a:
155)
at

org.elasticsearch.util.netty.util.ThreadRenamingRunnable.run(ThreadRenaming Runnable.java:
108)
at

org.elasticsearch.util.netty.util.internal.IoWorkerRunnable.run(IoWorkerRun nable.java:
46)
at java.util.concurrent.ThreadPoolExecutor
$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:637)

elasticsearch[Man-Bull]transportClientBoss-pool-4-thread-1@2835
daemon, priority=5, in group 'main', status: 'WAIT'
at sun.misc.Unsafe.park(Unsafe.java:-1)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
at java.util.concurrent.SynchronousQueue
$TransferStack.awaitFulfill(SynchronousQueue.java:424)
at java.util.concurrent.SynchronousQueue
$TransferStack.transfer(SynchronousQueue.java:323)
at
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:
874)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:
945)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:637)

elasticsearch[Man-Bull][tp]-pool-1-thread-1@2738 daemon, priority=5,
in group 'main', status: 'WAIT'
at sun.misc.Unsafe.park(Unsafe.java:-1)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
at java.util.concurrent.SynchronousQueue
$TransferStack.awaitFulfill(SynchronousQueue.java:424)
at java.util.concurrent.SynchronousQueue
$TransferStack.transfer(SynchronousQueue.java:323)
at
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:
874)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:
945)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:637)

elasticsearch[Man-Bull][sc]-pool-2-thread-2@2893 daemon, priority=5,
in group 'main', status: 'MONITOR'
waiting for main@1
at org.elasticsearch.client.transport.TransportClientNodesService
$ScheduledNodesSampler.run(TransportClientNodesService.java:178)
at java.util.concurrent.Executors
$RunnableAdapter.call(Executors.java:441)
at java.util.concurrent.FutureTask
$Sync.innerRunAndReset(FutureTask.java:317)
at
java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)
at java.util.concurrent.ScheduledThreadPoolExecutor
$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)
at java.util.concurrent.ScheduledThreadPoolExecutor
$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:181)
at java.util.concurrent.ScheduledThreadPoolExecutor
$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:205)
at java.util.concurrent.ThreadPoolExecutor
$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:637)

elasticsearch[Man-Bull][sc]-pool-2-thread-1@2602 daemon, priority=5,
in group 'main', status: 'WAIT'
at sun.misc.Unsafe.park(Unsafe.java:-1)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
at java.util.concurrent.locks.AbstractQueuedSynchronizer
$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1,963)
at java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
at java.util.concurrent.ScheduledThreadPoolExecutor
$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583)
at java.util.concurrent.ScheduledThreadPoolExecutor
$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:
947)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:637)

<... irrelevant proprietary code removed ...>

main@1, priority=5, in group 'main', status: 'WAIT'
blocks elasticsearch[Man-Bull][sc]-pool-2-thread-2@2893
at sun.misc.Unsafe.park(Unsafe.java:-1)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:
158)
at

java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt (AbstractQueuedSynchronizer.java:
747)
at

java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterr uptibly(AbstractQueuedSynchronizer.java:
905)
at

java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterrup tibly(AbstractQueuedSynchronizer.java:
1,217)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:
207)
at org.elasticsearch.client.transport.TransportClientNodesService
$ScheduledNodesSampler.run(TransportClientNodesService.java:211)
at

org.elasticsearch.client.transport.TransportClientNodesService.addTransport Address(TransportClientNodesService.java:
117)
at

org.elasticsearch.client.transport.TransportClient.addTransportAddress(Tran sportClient.java:
184)
at

com.spokesoftware.platform.services.CloudSearchServiceImpl.start(CloudSearc hServiceImpl.java:
66)
...
at

sun.reflect.NativeMethodAccessorImpl.invoke0(NativeMethodAccessorImpl.java: -1)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:
39)
at

sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImp l.java:
25)
at java.lang.reflect.Method.invoke(Method.java:597)
...
at com.spokesoftware.core.bootstrap.Main.main(Main.java:89)

[kimchy (Shay Banon)]

Can you try and separate it? If its blocking on the addTransportAddress
call, then we should see the call in the thread dump...

-shay.banon

On Tue, Jul 20, 2010 at 8:12 PM, David Jensen djensen47@gmail.com wrote:

It happens at that line of code that I provided. I imagine that it
happens when .addTransportAddress(..) is invoked. I can separate the
calls and find out for sure.

I'm running this within a Jetty container. I don't know if that helps
at all.

On Jul 20, 12:31 am, Shay Banon shay.ba...@elasticsearch.com wrote:

I don't see something that can help in the thread dump... (thats the
problem
with async systems, thread dumps looses some of their usefulness...).
When
does it get stuck, on a specific operation you perform or simply during
the
initialization of the transport client?

-shay.banon

On Tue, Jul 20, 2010 at 3:10 AM, David Jensen djense...@gmail.com
wrote:

Here's the code:
esClient = new TransportClient().addTransportAddress(new
InetSocketTransportAddress(server, port));

Here's the thread dump at the moment of getting stuck (this is the
format that IntelliJ gave me when I pressed "export threads"). I also
removed some, hopefully, irrelevant code. I left in the thread that
invoked the thing and all the other threads that included
"elasticsearch"

New I/O client worker #1-2@2885 daemon, priority=5, in group 'main',
status: 'RUNNING'
at
sun.nio.ch.KQueueArrayWrapper.kevent0(KQueueArrayWrapper.java:-1)
at
sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
at
sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:
69)
at
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at

org.elasticsearch.util.netty.channel.socket.nio.SelectorUtil.select(Selecto
rUtil.java:

38.  at
    

org.elasticsearch.util.netty.channel.socket.nio.NioWorker.run(NioWorker.jav
a:

155. at
     

org.elasticsearch.util.netty.util.ThreadRenamingRunnable.run(ThreadRenaming
Runnable.java:

108. at
     

org.elasticsearch.util.netty.util.internal.IoWorkerRunnable.run(IoWorkerRun
nable.java:

46.  at java.util.concurrent.ThreadPoolExecutor
    

$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:637)

New I/O client worker #1-1@2842 daemon, priority=5, in group 'main',
status: 'RUNNING'
at
sun.nio.ch.KQueueArrayWrapper.kevent0(KQueueArrayWrapper.java:-1)
at
sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
at
sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:
69)
at
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at

org.elasticsearch.util.netty.channel.socket.nio.SelectorUtil.select(Selecto
rUtil.java:

38.  at
    

org.elasticsearch.util.netty.channel.socket.nio.NioWorker.run(NioWorker.jav
a:

155. at
     

org.elasticsearch.util.netty.util.ThreadRenamingRunnable.run(ThreadRenaming
Runnable.java:

108. at
     

org.elasticsearch.util.netty.util.internal.IoWorkerRunnable.run(IoWorkerRun
nable.java:

46.  at java.util.concurrent.ThreadPoolExecutor
    

$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:637)

elasticsearch[Man-Bull]transportClientBoss-pool-4-thread-1@2835
daemon, priority=5, in group 'main', status: 'WAIT'
at sun.misc.Unsafe.park(Unsafe.java:-1)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
at java.util.concurrent.SynchronousQueue
$TransferStack.awaitFulfill(SynchronousQueue.java:424)
at java.util.concurrent.SynchronousQueue
$TransferStack.transfer(SynchronousQueue.java:323)
at
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:
874)
at

java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:

945. at java.util.concurrent.ThreadPoolExecutor
     

$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:637)

elasticsearch[Man-Bull][tp]-pool-1-thread-1@2738 daemon, priority=5,
in group 'main', status: 'WAIT'
at sun.misc.Unsafe.park(Unsafe.java:-1)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
at java.util.concurrent.SynchronousQueue
$TransferStack.awaitFulfill(SynchronousQueue.java:424)
at java.util.concurrent.SynchronousQueue
$TransferStack.transfer(SynchronousQueue.java:323)
at
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:
874)
at

java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:

945. at java.util.concurrent.ThreadPoolExecutor
     

$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:637)

elasticsearch[Man-Bull][sc]-pool-2-thread-2@2893 daemon, priority=5,
in group 'main', status: 'MONITOR'
waiting for main@1
at
org.elasticsearch.client.transport.TransportClientNodesService
$ScheduledNodesSampler.run(TransportClientNodesService.java:178)
at java.util.concurrent.Executors
$RunnableAdapter.call(Executors.java:441)
at java.util.concurrent.FutureTask
$Sync.innerRunAndReset(FutureTask.java:317)
at
java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)
at java.util.concurrent.ScheduledThreadPoolExecutor
$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)
at java.util.concurrent.ScheduledThreadPoolExecutor
$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:181)
at java.util.concurrent.ScheduledThreadPoolExecutor
$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:205)
at java.util.concurrent.ThreadPoolExecutor
$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:637)

elasticsearch[Man-Bull][sc]-pool-2-thread-1@2602 daemon, priority=5,
in group 'main', status: 'WAIT'
at sun.misc.Unsafe.park(Unsafe.java:-1)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
at java.util.concurrent.locks.AbstractQueuedSynchronizer
$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1,963)
at java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
at java.util.concurrent.ScheduledThreadPoolExecutor
$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583)
at java.util.concurrent.ScheduledThreadPoolExecutor
$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576)
at

java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:

947. at java.util.concurrent.ThreadPoolExecutor
     

$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:637)

<... irrelevant proprietary code removed ...>

main@1, priority=5, in group 'main', status: 'WAIT'
blocks elasticsearch[Man-Bull][sc]-pool-2-thread-2@2893
at sun.misc.Unsafe.park(Unsafe.java:-1)
at
java.util.concurrent.locks.LockSupport.park(LockSupport.java:
158)
at

java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt
(AbstractQueuedSynchronizer.java:

747. at
     

java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterr
uptibly(AbstractQueuedSynchronizer.java:

905. at
     

java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterrup
tibly(AbstractQueuedSynchronizer.java:

1,217)
at
java.util.concurrent.CountDownLatch.await(CountDownLatch.java:
207)
at
org.elasticsearch.client.transport.TransportClientNodesService
$ScheduledNodesSampler.run(TransportClientNodesService.java:211)
at

org.elasticsearch.client.transport.TransportClientNodesService.addTransport
Address(TransportClientNodesService.java:

117. at
     

org.elasticsearch.client.transport.TransportClient.addTransportAddress(Tran
sportClient.java:

184. at
     

com.spokesoftware.platform.services.CloudSearchServiceImpl.start(CloudSearc
hServiceImpl.java:

...
at

sun.reflect.NativeMethodAccessorImpl.invoke0(NativeMethodAccessorImpl.java:
-1)

     at

sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:

39.  at
    

sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImp
l.java:

25.  at java.lang.reflect.Method.invoke(Method.java:597)
    

...
at com.spokesoftware.core.bootstrap.Main.main(Main.java:89)