Hello, we've had a problem for several weeks.
We are overwriting our indexes on a weekly basis.
However, after indexing, within a day, we get the strangest errors. A search query starting with a specific letter doesn't work anymore. Here's the error we get
[2025-04-01T17:52:38,841][WARN ][r.suppressed ] [master-01] path: /MYINDEX-read/_search, params: {size=20, index=myindex-read, from=0, track_total_hits=true}
org.elasticsearch.action.search.SearchPhaseExecutionException: all shards failed
at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:729) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:419) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:761) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:513) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:350) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.action.ActionListenerImplementations.safeAcceptException(ActionListenerImplementations.java:62) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.action.ActionListenerImplementations.safeOnFailure(ActionListenerImplementations.java:73) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.action.DelegatingActionListener.onFailure(DelegatingActionListener.java:27) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:54) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:630) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.transport.TransportService$UnregisterChildTransportResponseHandler.handleException(TransportService.java:1707) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1424) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.transport.InboundHandler.doHandleException(InboundHandler.java:440) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.transport.InboundHandler.handleException(InboundHandler.java:427) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.transport.InboundHandler.handlerResponseError(InboundHandler.java:418) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.transport.InboundHandler.messageReceived(InboundHandler.java:141) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.transport.InboundHandler.inboundMessage(InboundHandler.java:96) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:831) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:124) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:96) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:61) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.transport.netty4.Netty4MessageInboundHandler.channelRead(Netty4MessageInboundHandler.java:48) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[?:?]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[?:?]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[?:?]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[?:?]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[?:?]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[?:?]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:689) ~[?:?]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:652) ~[?:?]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[?:?]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[?:?]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]
at java.lang.Thread.run(Thread.java:1623) ~[?:?]
Caused by: org.elasticsearch.ElasticsearchException$1: Index 0 out of bounds for length 0
at org.elasticsearch.ElasticsearchException.guessRootCauses(ElasticsearchException.java:667) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:417) ~[elasticsearch-8.10.0.jar:?]
... 39 more
Caused by: java.lang.ArrayIndexOutOfBoundsException: Index 0 out of bounds for length 0
at org.apache.lucene.store.ByteArrayDataInput.readVLong(ByteArrayDataInput.java:133) ~[lucene-core-9.7.0.jar:?]
at org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:239) ~[lucene-core-9.7.0.jar:?]
at org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:570) ~[lucene-core-9.7.0.jar:?]
at org.apache.lucene.index.FilterLeafReader$FilterTermsEnum.seekExact(FilterLeafReader.java:191) ~[lucene-core-9.7.0.jar:?]
at org.apache.lucene.index.FilterLeafReader$FilterTermsEnum.seekExact(FilterLeafReader.java:191) ~[lucene-core-9.7.0.jar:?]
at org.apache.lucene.index.TermStates.loadTermsEnum(TermStates.java:111) ~[lucene-core-9.7.0.jar:?]
at org.apache.lucene.index.TermStates.build(TermStates.java:96) ~[lucene-core-9.7.0.jar:?]
at org.apache.lucene.search.TermQuery.createWeight(TermQuery.java:227) ~[lucene-core-9.7.0.jar:?]
at org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:904) ~[lucene-core-9.7.0.jar:?]
at org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:250) ~[elasticsearch-8.10.0.jar:?]
at org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:59) ~[lucene-core-9.7.0.jar:?]
at org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:245) ~[lucene-core-9.7.0.jar:?]
at org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:904) ~[lucene-core-9.7.0.jar:?]
at org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:250) ~[elasticsearch-8.10.0.jar:?]
at org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:59) ~[lucene-core-9.7.0.jar:?]
at org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:245) ~[lucene-core-9.7.0.jar:?]
at org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:904) ~[lucene-core-9.7.0.jar:?]
at org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:250) ~[elasticsearch-8.10.0.jar:?]
at org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:59) ~[lucene-core-9.7.0.jar:?]
at org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:245) ~[lucene-core-9.7.0.jar:?]
at org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:904) ~[lucene-core-9.7.0.jar:?]
at org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:250) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:347) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.search.query.QueryPhase.addCollectorsAndSearch(QueryPhase.java:206) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.search.query.QueryPhase.executeQuery(QueryPhase.java:134) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:63) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:515) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:667) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.search.SearchService.lambda$executeQueryPhase$2(SearchService.java:540) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.action.ActionRunnable$2.accept(ActionRunnable.java:51) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.action.ActionRunnable$2.accept(ActionRunnable.java:48) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.action.ActionRunnable$3.doRun(ActionRunnable.java:73) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:983) ~[elasticsearch-8.10.0.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) ~[elasticsearch-8.10.0.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
at java.lang.Thread.run(Thread.java:1623) ~[?:?]
Then, if we reindex, the search works again for a while. Then again, the search is broken.
When I check, our shards are well assigned and none of our indexes seem to be corrupted.
We're using version 8.10.0