Hi,
I'm facing this behaviour frequently where a data node stops applying the cluster state but still continues to be in the cluster. I'm using ES 6.1.3. Sequence of events:
1- Node receives a cluster state, which deletes 3 indices.
2- It deletes 2 of them but no log print for the third.
3- It continues receiving cluster states from master but doesn't apply them.
4- After pending tasks list grows to 25, node starts dropping the states.
This goes on until i stopped the node. While stopping the node i get this log.
[o.e.i.CompositeIndexEventListener] [] [] failed to invoke after index removed callback
java.lang.IllegalStateException: java.lang.InterruptedException
at org.elasticsearch.common.cache.Cache$CacheSegment.remove(Cache.java:292) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.elasticsearch.common.cache.Cache.invalidate(Cache.java:479) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.onClose(IndicesFieldDataCache.java:181) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.apache.lucene.index.StandardDirectoryReader.notifyReaderClosedListeners(StandardDirectoryReader.java:500) ~[lucene-core-7.1.0.jar:7.1.0 84c90ad2c0218156c840e19a64d72b8a38550659 - ubuntu - 2017-10-13 16:12:42]
at org.apache.lucene.index.IndexReader.decRef(IndexReader.java:254) ~[lucene-core-7.1.0.jar:7.1.0 84c90ad2c0218156c840e19a64d72b8a38550659 - ubuntu - 2017-10-13 16:12:42]
at org.apache.lucene.index.IndexReader.close(IndexReader.java:393) ~[lucene-core-7.1.0.jar:7.1.0 84c90ad2c0218156c840e19a64d72b8a38550659 - ubuntu - 2017-10-13 16:12:42]
at org.apache.lucene.index.FilterDirectoryReader.doClose(FilterDirectoryReader.java:134) ~[lucene-core-7.1.0.jar:7.1.0 84c90ad2c0218156c840e19a64d72b8a38550659 - ubuntu - 2017-10-13 16:12:42]
at org.apache.lucene.index.IndexReader.decRef(IndexReader.java:247) ~[lucene-core-7.1.0.jar:7.1.0 84c90ad2c0218156c840e19a64d72b8a38550659 - ubuntu - 2017-10-13 16:12:42]
at org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.decRef(InternalEngine.java:322) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.decRef(InternalEngine.java:267) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.apache.lucene.search.ReferenceManager.release(ReferenceManager.java:274) ~[lucene-core-7.1.0.jar:7.1.0 84c90ad2c0218156c840e19a64d72b8a38550659 - ubuntu - 2017-10-13 16:12:42]
at org.elasticsearch.index.engine.EngineSearcher.close(EngineSearcher.java:59) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.apache.lucene.util.IOUtils.close(IOUtils.java:89) ~[lucene-core-7.1.0.jar:7.1.0 84c90ad2c0218156c840e19a64d72b8a38550659 - ubuntu - 2017-10-13 16:12:42]
at org.elasticsearch.common.lease.Releasables.close(Releasables.java:36) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.elasticsearch.common.lease.Releasables.close(Releasables.java:46) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.elasticsearch.common.lease.Releasables.close(Releasables.java:51) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.elasticsearch.search.DefaultSearchContext.doClose(DefaultSearchContext.java:185) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.elasticsearch.search.internal.SearchContext.closeInternal(SearchContext.java:111) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.elasticsearch.common.util.concurrent.AbstractRefCounted.decRef(AbstractRefCounted.java:65) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.elasticsearch.search.internal.SearchContext.close(SearchContext.java:102) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.elasticsearch.search.SearchService.freeContext(SearchService.java:635) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.elasticsearch.search.SearchService.freeAllContextForIndex(SearchService.java:619) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.elasticsearch.search.SearchService.afterIndexRemoved(SearchService.java:225) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.elasticsearch.index.CompositeIndexEventListener.afterIndexRemoved(CompositeIndexEventListener.java:195) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.elasticsearch.indices.IndicesService.removeIndex(IndicesService.java:553) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.elasticsearch.indices.cluster.IndicesClusterStateService.deleteIndices(IndicesClusterStateService.java:285) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:219) ~[elasticsearch-6.1.3.jar:6.1.3]
at org.elasticsearch.cluster.service.ClusterApplierService.lambda$callClusterStateAppliers$6(ClusterApplierService.java:495) ~[elasticsearch-6.1.3.jar:6.1.3]
at java.lang.Iterable.forEach(Iterable.java:75) [?:1.8.0_72]
at org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:492) [elasticsearch-6.1.3.jar:6.1.3]
at org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java)