org.elasticsearch.action.search.SearchPhaseExecutionException: all shards failed
It is said that all shards fail, what is happening?
The index in question shows that the health was green.
What do I need to do to avoid this warning?
I've looked at several similar questions in the past, but those don't seem to have been resolved.
$ tail -n 200 /var/log/elasticsearch/elasticsearch.log
[2021-07-12T14:32:45,923][WARN ][r.suppressed ] [MY-SERVER] path: /my-api-*/_search, params: {index=my-api-*}
org.elasticsearch.action.search.SearchPhaseExecutionException: all shards failed
at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:636) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:357) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:669) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:440) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:62) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:289) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.action.search.SearchExecutionStatsCollector.onFailure(SearchExecutionStatsCollector.java:62) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:405) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.transport.TransportService$6.handleException(TransportService.java:745) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1290) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.transport.TransportService$DirectResponseChannel.processException(TransportService.java:1399) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1373) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:50) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.transport.TransportChannel.sendErrorResponse(TransportChannel.java:45) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.action.support.ChannelActionListener.onFailure(ChannelActionListener.java:40) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.action.ActionRunnable.onFailure(ActionRunnable.java:77) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:28) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.12.0.jar:7.12.0]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
at java.lang.Thread.run(Thread.java:832) [?:?]
Caused by: org.elasticsearch.ElasticsearchException: java.util.concurrent.ExecutionException: org.elasticsearch.search.query.QueryPhase$TimeExceededException
at org.elasticsearch.index.fielddata.plain.AbstractIndexOrdinalsFieldData.load(AbstractIndexOrdinalsFieldData.java:87) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.index.fielddata.plain.AbstractIndexOrdinalsFieldData.load(AbstractIndexOrdinalsFieldData.java:33) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.index.mapper.IdFieldMapper$IdFieldType$1$1.load(IdFieldMapper.java:172) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$FieldData.bytesValues(ValuesSource.java:241) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.search.aggregations.bucket.terms.MapStringTermsAggregator$ValuesSourceCollectorSource.getLeafCollector(MapStringTermsAggregator.java:170) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.search.aggregations.bucket.terms.MapStringTermsAggregator.getLeafCollector(MapStringTermsAggregator.java:88) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.search.aggregations.AggregatorBase.getLeafCollector(AggregatorBase.java:166) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.search.aggregations.AggregatorBase.getLeafCollector(AggregatorBase.java:31) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.apache.lucene.search.MultiCollector.getLeafCollector(MultiCollector.java:124) ~[lucene-core-8.8.0.jar:8.8.0 b10659f0fc18b58b90929cfdadde94544d202c4a - noble - 2021-01-25 19:07:45]
at org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:203) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:188) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445) ~[lucene-core-8.8.0.jar:8.8.0 b10659f0fc18b58b90929cfdadde94544d202c4a - noble - 2021-01-25 19:07:45]
at org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:332) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:287) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:139) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.indices.IndicesService.lambda$loadIntoContext$25(IndicesService.java:1412) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.indices.IndicesService.lambda$cacheShardLevelResult$26(IndicesService.java:1475) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.indices.IndicesRequestCache$Loader.load(IndicesRequestCache.java:164) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.indices.IndicesRequestCache$Loader.load(IndicesRequestCache.java:147) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:423) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.indices.IndicesRequestCache.getOrCompute(IndicesRequestCache.java:110) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.indices.IndicesService.cacheShardLevelResult(IndicesService.java:1481) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.indices.IndicesService.loadIntoContext(IndicesService.java:1406) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:364) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:431) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.search.SearchService.access$500(SearchService.java:135) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.search.SearchService$2.lambda$onResponse$0(SearchService.java:395) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:47) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:62) ~[elasticsearch-7.12.0.jar:7.12.0]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) ~[elasticsearch-7.12.0.jar:7.12.0]
... 6 more
$ curl -XGET 'localhost:9200/_cluster/health/my-api-*?pretty'
{
"cluster_name" : "elasticsearch",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 1,
"number_of_data_nodes" : 1,
"active_primary_shards" : 36,
"active_shards" : 36,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}