Hi Team,
We are facing None of the configured nodes are available when system is loaded with bulk data.
During that period we have capture logs from yourkit profiler and it seems some threads are block, not sure if it is the cause of None of the configured nodes are available. Please check.
Also in some blogs i have read by making sniff setting true this issue might resolve, as this is our production environment thought to double check with you , please suggest.
elasticsearch[_vxzYq9][bulk][T#4] <--- Frozen for at least 1h 39m 39 sec
java.io.WinNTFileSystem.canonicalizeWithPrefix0(String, String) WinNTFileSystem.java (native)
java.io.WinNTFileSystem.canonicalizeWithPrefix(String, String) WinNTFileSystem.java:451
java.io.WinNTFileSystem.canonicalize(String) WinNTFileSystem.java:422
java.io.File.getCanonicalPath() File.java:618
java.io.FilePermission$1.run() FilePermission.java:215
java.io.FilePermission$1.run() FilePermission.java:203
java.security.AccessController.doPrivileged(PrivilegedAction) AccessController.java (native)
java.io.FilePermission.init(int) FilePermission.java:203
java.io.FilePermission.(String, String) FilePermission.java:277
java.lang.SecurityManager.checkRead(String) SecurityManager.java:888
sun.nio.fs.WindowsPath.checkRead() WindowsPath.java:792
sun.nio.fs.WindowsFileAttributeViews$Basic.readAttributes() WindowsFileAttributeViews.java:49
sun.nio.fs.WindowsFileAttributeViews$Basic.readAttributes() WindowsFileAttributeViews.java:38
sun.nio.fs.WindowsFileSystemProvider.readAttributes(Path, Class, LinkOption[]) WindowsFileSystemProvider.java:193
java.nio.file.Files.readAttributes(Path, Class, LinkOption[]) Files.java:1737
org.apache.lucene.store.NativeFSLockFactory$NativeFSLock.ensureValid() NativeFSLockFactory.java:177
org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(String, IOContext) LockValidatingDirectoryWrapper.java:43
org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(String, IOContext) TrackingDirectoryWrapper.java:43
org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.(Directory, SegmentInfo, String, IOContext, String, CompressionMode, int, int, int) CompressingStoredFieldsWriter.java:109
org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(Directory, SegmentInfo, IOContext) CompressingStoredFieldsFormat.java:128
org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Directory, SegmentInfo, IOContext) Lucene50StoredFieldsFormat.java:183
org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter() DefaultIndexingChain.java:83
org.apache.lucene.index.DefaultIndexingChain.startStoredFields() DefaultIndexingChain.java:331
org.apache.lucene.index.DefaultIndexingChain.processDocument() DefaultIndexingChain.java:368
org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(Iterable, Analyzer, Term) DocumentsWriterPerThread.java:272
org.apache.lucene.index.DocumentsWriter.updateDocuments(Iterable, Analyzer, Term) DocumentsWriter.java:433
org.apache.lucene.index.IndexWriter.updateDocuments(Term, Iterable) IndexWriter.java:1388
org.elasticsearch.index.engine.InternalEngine.update(Engine$Index, IndexWriter) InternalEngine.java:564
org.elasticsearch.index.engine.InternalEngine.innerIndex(Engine$Index) InternalEngine.java:522
org.elasticsearch.index.engine.InternalEngine.index(Engine$Index) InternalEngine.java:409
org.elasticsearch.index.shard.IndexShard.index(Engine, Engine$Index) IndexShard.java:556
org.elasticsearch.index.shard.IndexShard.index(Engine$Index) IndexShard.java:546
org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(IndexRequest, IndexShard, MappingUpdatedAction) TransportIndexAction.java:213
org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(BulkShardRequest, IndexRequest, IndexMetaData, IndexShard, boolean) TransportShardBulkAction.java:348
org.elasticsearch.action.bulk.TransportShardBulkAction.shardUpdateOperation(IndexMetaData, BulkShardRequest, UpdateRequest, IndexShard) TransportShardBulkAction.java:411
org.elasticsearch.action.bulk.TransportShardBulkAction.update(IndexMetaData, BulkShardRequest, IndexShard, long[], VersionType[], Translog$Location, int, BulkItemRequest) TransportShardBulkAction.java:233
org.elasticsearch.action.bulk.TransportShardBulkAction.handleItem(IndexMetaData, BulkShardRequest, IndexShard, long[], VersionType[], Translog$Location, int, BulkItemRequest) TransportShardBulkAction.java:138
org.elasticsearch.action.bulk.TransportShardBulkAction.onPrimaryShard(BulkShardRequest, IndexShard) TransportShardBulkAction.java:120
org.elasticsearch.action.bulk.TransportShardBulkAction.onPrimaryShard(ReplicatedWriteRequest, IndexShard) TransportShardBulkAction.java:73
org.elasticsearch.action.support.replication.TransportWriteAction.shardOperationOnPrimary(ReplicatedWriteRequest, IndexShard) TransportWriteAction.java:76
org.elasticsearch.action.support.replication.TransportWriteAction.shardOperationOnPrimary(ReplicationRequest, IndexShard) TransportWriteAction.java:49
org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(ReplicationRequest) TransportReplicationAction.java:914
org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(ReplicationRequest) TransportReplicationAction.java:884
org.elasticsearch.action.support.replication.ReplicationOperation.execute() ReplicationOperation.java:113
org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction$PrimaryShardReference) TransportReplicationAction.java:327
org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(Object) TransportReplicationAction.java:262
org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(Releasable) TransportReplicationAction.java:864
org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(Object) TransportReplicationAction.java:861
org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(ActionListener, String, boolean) IndexShardOperationsLock.java:147
org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationLock(ActionListener, String) IndexShard.java:1652
org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryShard