While running a query on an alias pointing to a single index, the query is to pull a single document. In the aggregations, we have a single aggs, a terms aggregation. We supply a missing value. This query returns a 500 error regarding an array_index_out_of_bounds_exception. What is frustrating is, this error is intermittent, and succeeds more than it fails. On an index that is queried thousands of times per week, we've tracked via Sentry 66 failed calls over the last 8 months.
The query can succeed if we:
- removing the
missingproperty - modify the
fieldto a field that does not exist
The query still fails if we:
- modifying the
missingvalue to a different value, or a blank value - modify the
sizeto a different number
The DSL query:
GET index_12345/_search
{
"aggs": {
"default_agg": {
"terms": {
"field": "opportunityStats.workingStatus.keyword",
"missing": "MissingValueHere",
"size": 100
}
}
},
"from": 0,
"query": {
... query pulls a single document
},
"size": 50
}
The index mapping:
{
"index_20221201_12345" : {
"mappings" : {
"properties" : {
...
"opportunityStats" : {
"properties" : {
...
"workingStatus" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"workingStatusId" : {
"type" : "integer"
}
}
},
...
}
}
}
}
There is an alias where index_12345 points to index_20221201_12345.
The error we receive:
Request failed to execute. Call: Status code 500 from: POST /index_12345/_search?pretty=true&error_trace=true&typed_keys=true. ServerError: Type: search_phase_execution_exception Reason: "all shards failed" CausedBy: "Type: array_index_out_of_bounds_exception Reason: "Index -1 out of bounds for length 1" CausedBy: "Type: array_index_out_of_bounds_exception Reason: "Index -1 out of bounds for length 1"""
The full exception:
[2022-12-19T15:50:35,036][WARN ][r.suppressed ] [master-1] path: /index_12345/_search, params: {pretty=true, index=index_12345}
org.elasticsearch.action.search.SearchPhaseExecutionException: all shards failed
at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:716) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:403) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:748) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:500) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:334) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:651) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.transport.TransportService$4.handleException(TransportService.java:724) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1349) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.transport.InboundHandler.lambda$handleException$3(InboundHandler.java:397) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:223) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.transport.InboundHandler.handleException(InboundHandler.java:395) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.transport.InboundHandler.handlerResponseError(InboundHandler.java:387) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.transport.InboundHandler.messageReceived(InboundHandler.java:140) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.transport.InboundHandler.inboundMessage(InboundHandler.java:94) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:796) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:149) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:121) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:86) [elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) [transport-netty4-8.1.2.jar:8.1.2]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) [netty-handler-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) [netty-handler-4.1.73.Final.jar:4.1.73.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) [netty-handler-4.1.73.Final.jar:4.1.73.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) [netty-handler-4.1.73.Final.jar:4.1.73.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:510) [netty-codec-4.1.73.Final.jar:4.1.73.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:449) [netty-codec-4.1.73.Final.jar:4.1.73.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:279) [netty-codec-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) [netty-common-4.1.73.Final.jar:4.1.73.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.73.Final.jar:4.1.73.Final]
at java.lang.Thread.run(Thread.java:833) [?:?]
Caused by: org.elasticsearch.ElasticsearchException$1: Index -1 out of bounds for length 1
at org.elasticsearch.ElasticsearchException.guessRootCauses(ElasticsearchException.java:638) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:401) [elasticsearch-8.1.2.jar:8.1.2]
... 51 more
Caused by: java.lang.ArrayIndexOutOfBoundsException: Index -1 out of bounds for length 1
at org.apache.lucene.util.packed.DirectMonotonicReader.get(DirectMonotonicReader.java:161) ~[lucene-core-9.0.0.jar:9.0.0 0b18b3b965cedaf5eb129aa41243a44c83ca826d - jpountz - 2021-12-01 14:23:49]
at org.apache.lucene.codecs.lucene90.Lucene90DocValuesProducer$TermsDict.getTermFromIndex(Lucene90DocValuesProducer.java:1125) ~[lucene-core-9.0.0.jar:9.0.0 0b18b3b965cedaf5eb129aa41243a44c83ca826d - jpountz - 2021-12-01 14:23:49]
at org.apache.lucene.codecs.lucene90.Lucene90DocValuesProducer$TermsDict.seekTermsIndex(Lucene90DocValuesProducer.java:1137) ~[lucene-core-9.0.0.jar:9.0.0 0b18b3b965cedaf5eb129aa41243a44c83ca826d - jpountz - 2021-12-01 14:23:49]
at org.apache.lucene.codecs.lucene90.Lucene90DocValuesProducer$TermsDict.seekBlock(Lucene90DocValuesProducer.java:1163) ~[lucene-core-9.0.0.jar:9.0.0 0b18b3b965cedaf5eb129aa41243a44c83ca826d - jpountz - 2021-12-01 14:23:49]
at org.apache.lucene.codecs.lucene90.Lucene90DocValuesProducer$TermsDict.seekCeil(Lucene90DocValuesProducer.java:1194) ~[lucene-core-9.0.0.jar:9.0.0 0b18b3b965cedaf5eb129aa41243a44c83ca826d - jpountz - 2021-12-01 14:23:49]
at org.apache.lucene.codecs.lucene90.Lucene90DocValuesProducer$BaseSortedDocValues.lookupTerm(Lucene90DocValuesProducer.java:987) ~[lucene-core-9.0.0.jar:9.0.0 0b18b3b965cedaf5eb129aa41243a44c83ca826d - jpountz - 2021-12-01 14:23:49]
at org.apache.lucene.index.SingletonSortedSetDocValues.lookupTerm(SingletonSortedSetDocValues.java:100) ~[lucene-core-9.0.0.jar:9.0.0 0b18b3b965cedaf5eb129aa41243a44c83ca826d - jpountz - 2021-12-01 14:23:49]
at org.elasticsearch.search.aggregations.support.MissingValues.replaceMissing(MissingValues.java:238) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.search.aggregations.support.MissingValues$6.ordinalsValues(MissingValues.java:210) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator$LowCardinality.getLeafCollector(GlobalOrdinalsStringTermsAggregator.java:306) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.search.aggregations.AggregatorBase.getLeafCollector(AggregatorBase.java:216) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.search.aggregations.MultiBucketCollector$1.getLeafCollector(MultiBucketCollector.java:94) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.search.aggregations.MultiBucketCollector$1.getLeafCollector(MultiBucketCollector.java:75) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.apache.lucene.search.MultiCollector.getLeafCollector(MultiCollector.java:124) ~[lucene-core-9.0.0.jar:9.0.0 0b18b3b965cedaf5eb129aa41243a44c83ca826d - jpountz - 2021-12-01 14:23:49]
at org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:182) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:167) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:541) ~[lucene-core-9.0.0.jar:9.0.0 0b18b3b965cedaf5eb129aa41243a44c83ca826d - jpountz - 2021-12-01 14:23:49]
at org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:232) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:187) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:88) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:462) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:625) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.search.SearchService.lambda$executeQueryPhase$2(SearchService.java:487) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:47) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:62) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:776) ~[elasticsearch-8.1.2.jar:8.1.2]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) ~[elasticsearch-8.1.2.jar:8.1.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
at java.lang.Thread.run(Thread.java:833) ~[?:?]