Hi
Recently we are encountering exceptions in our elasticsearch cluster post upgrade between 5.0 to 5.4.0
ast messages accounting for this alert:
source: es-host5-1 | message: [NAME_v0.2_2017-04-05_fee4822921374b799717a5b9cc9f66e3][5], node[kBbwymsDSZGvyeYzvB10Og], [R], s[STARTED], a[id=4IsI5TlRTdeIXI-KpuGArQ]: Failed to execute [SearchRequest{searchType=QUERY_THEN_FETCH, indices=[tabigdata_huge], (...) { level: 7 | full_message: SendRequestTransportException[[es-host5-4][1.1.1.164:9300][indices:data/read/search[phase/query]]]; nested: ConcurrentModificationException;
at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:579)
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:502)
at org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:529)
at org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:520)
at org.elasticsearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:138)
at org.elasticsearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:74)
at org.elasticsearch.action.search.InitialSearchPhase.performPhaseOnShard(InitialSearchPhase.java:146)
at org.elasticsearch.action.search.InitialSearchPhase.onShardFailure(InitialSearchPhase.java:101)
at org.elasticsearch.action.search.InitialSearchPhase.access$100(InitialSearchPhase.java:47)
at org.elasticsearch.action.search.InitialSearchPhase$1.onFailure(InitialSearchPhase.java:155)
at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:51)
at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1041)
at org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:600)
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.util.ConcurrentModificationException
at java.util.HashMap$HashIterator.nextNode(HashMap.java:1437)
at java.util.HashMap$EntryIterator.next(HashMap.java:1471)
at java.util.HashMap$EntryIterator.next(HashMap.java:1469)
at org.elasticsearch.common.io.stream.StreamOutput.lambda$static$11(StreamOutput.java:568)
at org.elasticsearch.common.io.stream.StreamOutput.writeGenericValue(StreamOutput.java:653)
at org.elasticsearch.common.io.stream.StreamOutput.lambda$static$11(StreamOutput.java:570)
at org.elasticsearch.common.io.stream.StreamOutput.writeGenericValue(StreamOutput.java:653)
at org.elasticsearch.common.io.stream.StreamOutput.writeMap(StreamOutput.java:443)
at org.elasticsearch.script.Script.writeTo(Script.java:566)
at org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder.doWriteTo(ValuesSourceAggregationBuilder.java:141)
at org.elasticsearch.search.aggregations.AbstractAggregationBuilder.writeTo(AbstractAggregationBuilder.java:61)
at org.elasticsearch.common.io.stream.StreamOutput.writeNamedWriteable(StreamOutput.java:857)
at org.elasticsearch.search.aggregations.AggregatorFactories$Builder.writeTo(AggregatorFactories.java:288)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalWriteable(StreamOutput.java:732)
at org.elasticsearch.search.builder.SearchSourceBuilder.writeTo(SearchSourceBuilder.java:230)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalWriteable(StreamOutput.java:732)
at org.elasticsearch.search.internal.ShardSearchLocalRequest.innerWriteTo(ShardSearchLocalRequest.java:205)
at org.elasticsearch.search.internal.ShardSearchTransportRequest.writeTo(ShardSearchTransportRequest.java:145)
at org.elasticsearch.transport.TcpTransport.buildMessage(TcpTransport.java:1210)
at org.elasticsearch.transport.TcpTransport.sendRequestToChannel(TcpTransport.java:1062)
at org.elasticsearch.transport.TcpTransport.access$1100(TcpTransport.java:115)
at org.elasticsearch.transport.TcpTransport$NodeChannels.sendRequest(TcpTransport.java:438)
at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:569)
... 17 more | logger: org.elasticsearch.action.search.TransportSearchAction | streams: [58738d804ad44010037cc0b3] | gl2_source_node: e705c070-a39d-4388-8fcc-827e8e570107 | _id: ef8b7550-6615-11e7-969e-000c29b6480c | thread: elasticsearch[es-host5-1][generic][T#18] | gl2_source_input: 590582044ad440614f4bb8ff | timestamp: 2017-07-11T09:07:16.449Z }