Hi Team,
while creating new Index shard acknowledgement fails, due to this mapping is not updated to the Index and status is yellow. Indexing fails due to ReceiveTimeoutTransportException. Also, Repo creation fails due to the same Exception.
Cluster Details:
Data size - 60 GB
Node count - 1
Indices count - 94
Heap committed - 5 GB
Node Hot Threads:
0.0% (0s out of 500ms) cpu usage by thread 'Attach Listener'
unique snapshot
unique snapshot
unique snapshot
unique snapshot
unique snapshot
unique snapshot
unique snapshot
unique snapshot
unique snapshot
unique snapshot
0.0% (0s out of 500ms) cpu usage by thread 'Wrapper-Control-Event-Monitor'
10/10 snapshots sharing following 2 elements
java.lang.Thread.sleep(Native Method)
org.tanukisoftware.wrapper.WrapperManager$3.run(WrapperManager.java:1074)
0.0% (0s out of 500ms) cpu usage by thread 'Wrapper-Connection'
10/10 snapshots sharing following 9 elements
java.net.SocketInputStream.socketRead0(Native Method)
java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
java.net.SocketInputStream.read(SocketInputStream.java:171)
java.net.SocketInputStream.read(SocketInputStream.java:141)
java.net.SocketInputStream.read(SocketInputStream.java:224)
java.io.DataInputStream.readByte(DataInputStream.java:265)
org.tanukisoftware.wrapper.WrapperManager.handleBackend(WrapperManager.java:5891)
org.tanukisoftware.wrapper.WrapperManager.run(WrapperManager.java:6332)
java.lang.Thread.run(Thread.java:748)
0.0% (0s out of 500ms) cpu usage by thread 'DestroyJavaVM'
unique snapshot
unique snapshot
unique snapshot
unique snapshot
unique snapshot
unique snapshot
unique snapshot
unique snapshot
unique snapshot
unique snapshot
0.0% (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/5.6.4]'
10/10 snapshots sharing following 8 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:837)
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:999)
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1308)
java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:84)
java.lang.Thread.run(Thread.java:748)
0.0% (0s out of 500ms) cpu usage by thread 'elasticsearch[int_desktop-7jo7479][[timer]]'
10/10 snapshots sharing following 2 elements
java.lang.Thread.sleep(Native Method)
org.elasticsearch.threadpool.ThreadPool$CachedTimeThread.run(ThreadPool.java:536)
0.0% (0s out of 500ms) cpu usage by thread 'elasticsearch[int_desktop-7jo7479][clusterService#updateTask][T#1]'
10/10 snapshots sharing following 16 elements
org.elasticsearch.index.shard.IndexShard.updateRoutingEntry(IndexShard.java:380)
org.elasticsearch.indices.cluster.IndicesClusterStateService.updateShard(IndicesClusterStateService.java:557)
org.elasticsearch.indices.cluster.IndicesClusterStateService.createOrUpdateShards(IndicesClusterStateService.java:521)
org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:204)
org.elasticsearch.cluster.service.ClusterService.callClusterStateAppliers(ClusterService.java:814)
org.elasticsearch.cluster.service.ClusterService.publishAndApplyChanges(ClusterService.java:768)
org.elasticsearch.cluster.service.ClusterService.runTasks(ClusterService.java:587)
org.elasticsearch.cluster.service.ClusterService$ClusterServiceTaskBatcher.run(ClusterService.java:263)
org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150)
org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:569)
org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:247)
org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:210)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
0.0% (0s out of 500ms) cpu usage by thread 'threadDeathWatcher-2-1'
10/10 snapshots sharing following 4 elements
java.lang.Thread.sleep(Native Method)
io.netty.util.ThreadDeathWatcher$Watcher.run(ThreadDeathWatcher.java:152)
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138)
java.lang.Thread.run(Thread.java:748)
0.0% (0s out of 500ms) cpu usage by thread 'elasticsearch[int_desktop-7jo7479][generic][T#7]'
10/10 snapshots sharing following 31 elements
sun.nio.fs.WindowsNativeDispatcher.CreateFile0(Native Method)
sun.nio.fs.WindowsNativeDispatcher.CreateFile(WindowsNativeDispatcher.java:71)
sun.nio.fs.WindowsChannelFactory.open(WindowsChannelFactory.java:302)
sun.nio.fs.WindowsChannelFactory.newFileChannel(WindowsChannelFactory.java:162)
sun.nio.fs.WindowsFileSystemProvider.newFileChannel(WindowsFileSystemProvider.java:110)
java.nio.channels.FileChannel.open(FileChannel.java:287)
java.nio.channels.FileChannel.open(FileChannel.java:335)
org.elasticsearch.index.translog.Translog$$Lambda$1448/1714002180.open(Unknown Source)
org.elasticsearch.index.translog.ChannelFactory.open(ChannelFactory.java:33)
org.elasticsearch.index.translog.TranslogWriter.create(TranslogWriter.java:92)
org.elasticsearch.index.translog.Translog.createWriter(Translog.java:379)
org.elasticsearch.index.translog.Translog.<init>(Translog.java:203)
org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:272)
org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:160)
org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25)
org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1602)
org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1584)
org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:1027)
org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:987)
org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:360)
org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:90)
org.elasticsearch.index.shard.StoreRecovery$$Lambda$1444/521597373.run(Unknown Source)
org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:257)
org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:88)
org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1236)
org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$1(IndexShard.java:1484)
org.elasticsearch.index.shard.IndexShard$$Lambda$1443/1274477608.run(Unknown Source)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:569)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
0.0% (0s out of 500ms) cpu usage by thread 'elasticsearch[int_desktop-7jo7479][flush][T#1]'
10/10 snapshots sharing following 12 elements
org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3006)
org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3244)
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3207)
org.elasticsearch.index.engine.InternalEngine.commitIndexWriter(InternalEngine.java:1576)
org.elasticsearch.index.engine.InternalEngine.flush(InternalEngine.java:1062)
org.elasticsearch.index.shard.IndexShard.flush(IndexShard.java:777)
org.elasticsearch.index.shard.IndexShard$2.doRun(IndexShard.java:1748)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
0.0% (0s out of 500ms) cpu usage by thread 'elasticsearch[int_desktop-7jo7479][refresh][T#2]'
10/10 snapshots sharing following 41 elements
sun.nio.fs.WindowsNativeDispatcher.CreateFile0(Native Method)
sun.nio.fs.WindowsNativeDispatcher.CreateFile(WindowsNativeDispatcher.java:71)
sun.nio.fs.WindowsChannelFactory.open(WindowsChannelFactory.java:302)
sun.nio.fs.WindowsChannelFactory.newFileChannel(WindowsChannelFactory.java:162)
sun.nio.fs.WindowsFileSystemProvider.newByteChannel(WindowsFileSystemProvider.java:225)
java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434)
java.nio.file.Files.newOutputStream(Files.java:216)
org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:413)
org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:409)
org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:253)
org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40)
org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
org.apache.lucene.codecs.lucene53.Lucene53NormsConsumer.<init>(Lucene53NormsConsumer.java:46)
org.apache.lucene.codecs.lucene53.Lucene53NormsFormat.normsConsumer(Lucene53NormsFormat.java:77)
org.apache.lucene.index.DefaultIndexingChain.writeNorms(DefaultIndexingChain.java:299)
org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:136)
org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:444)
org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:539)
org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:653)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:445)
org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:291)
org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:266)
org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:256)
org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104)
org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:156)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:909)
org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:632)
org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:690)
org.elasticsearch.index.IndexService.access$400(IndexService.java:92)
org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:832)
org.elasticsearch.index.IndexService$BaseAsyncTask.run(IndexService.java:743)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:569)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
Please help us to identify the possible root cause.
Thanks and Regards
Dharun Kumar