When using ES 7.7.0 the following query
curl -X POST -H "Content-Type:application/json" http://localhost:9200/bug\*/_search -d '{"size":1,"query":{"term":{"test":{"value":"test","boost":1.0}}},"sort":[{"@timestamp":{"order":"desc"}}]}'
The server sometimes responds with:
{
"_shards" : {
"failed" : 1,
"failures" : [
{
"index" : "bug-first",
"node" : "WJi6sSFOQdKQ6fmiJ8alxw",
"reason" : {
"reason" : "numHits must be > 0; please use TotalHitCountCollector if you just need the total hit count",
"type" : "illegal_argument_exception"
},
"shard" : 0
}
],
"skipped" : 0,
"successful" : 1,
"total" : 2
},
"hits" : {
"hits" : [
{
"_id" : "soJSYHIBhhdeGg3HEdIG",
"_index" : "bug-second",
"_score" : null,
"_source" : {
"@timestamp" : "2020-05-29T09:10:41.272235893Z",
"test" : "test"
},
"_type" : "_doc",
"sort" : [
1590743441272
]
}
],
"max_score" : null,
"total" : {
"relation" : "eq",
"value" : 1
}
},
"timed_out" : false,
"took" : 4
}
And logs this exception:
{"type": "server", "timestamp": "2020-05-29T12:23:47,774Z", "level": "DEBUG", "component": "o.e.a.s.TransportSearchAction", "cluster.name": "docker-cluster", "node.name": "9994b59cd16f", "message": "[bug-first][0], node[WJi6sSFOQdKQ6fmiJ8alxw], [P], s[STARTED], a[id=V2qDwV8OSciQPJ7KDYZOGg]: Failed to execute [SearchRequest{searchType=QUERY_THEN_FETCH, indices=[bug*], indicesOptions=IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true], types=[], routing='null', preference='null', requestCache=null, scroll=null, maxConcurrentShardRequests=0, batchedReduceSize=512, preFilterShardSize=null, allowPartialSearchResults=true, localClusterAlias=null, getOrCreateAbsoluteStartMillis=-1, ccsMinimizeRoundtrips=true, source={\"size\":1,\"query\":{\"term\":{\"test\":{\"value\":\"test\",\"boost\":1.0}}},\"sort\":[{\"@timestamp\":{\"order\":\"desc\"}}]}}]", "cluster.uuid": "vM6KAFOWTEa49nRmq-kM3A", "node.id": "WJi6sSFOQdKQ6fmiJ8alxw" ,
"stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [9994b59cd16f][10.0.9.3:9300][indices:data/read/search[phase/query]]",
"Caused by: org.elasticsearch.search.query.QueryPhaseExecutionException: Query Failed [Failed to execute main query]",
"at org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:323) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:151) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.indices.IndicesService.lambda$loadIntoContext$21(IndicesService.java:1344) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.indices.IndicesService.lambda$cacheShardLevelResult$22(IndicesService.java:1396) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.indices.IndicesRequestCache$Loader.load(IndicesRequestCache.java:176) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.indices.IndicesRequestCache$Loader.load(IndicesRequestCache.java:159) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.indices.IndicesRequestCache.getOrCompute(IndicesRequestCache.java:125) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.indices.IndicesService.cacheShardLevelResult(IndicesService.java:1402) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.indices.IndicesService.loadIntoContext(IndicesService.java:1341) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:359) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:434) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.search.SearchService.access$200(SearchService.java:135) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.search.SearchService$2.lambda$onResponse$0(SearchService.java:395) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.search.SearchService.lambda$runAsync$0(SearchService.java:411) [elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44) [elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692) [elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.7.0.jar:7.7.0]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:832) [?:?]",
"Caused by: java.lang.IllegalArgumentException: numHits must be > 0; please use TotalHitCountCollector if you just need the total hit count",
"at org.apache.lucene.search.TopFieldCollector.create(TopFieldCollector.java:464) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.search.TopFieldCollector$1.newCollector(TopFieldCollector.java:502) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.search.TopFieldCollector$1.newCollector(TopFieldCollector.java:495) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:166) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.search.query.QueryPhase.searchWithCollectorManager(QueryPhase.java:402) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:297) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:151) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.indices.IndicesService.lambda$loadIntoContext$21(IndicesService.java:1344) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.indices.IndicesService.lambda$cacheShardLevelResult$22(IndicesService.java:1396) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.indices.IndicesRequestCache$Loader.load(IndicesRequestCache.java:176) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.indices.IndicesRequestCache$Loader.load(IndicesRequestCache.java:159) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.indices.IndicesRequestCache.getOrCompute(IndicesRequestCache.java:125) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.indices.IndicesService.cacheShardLevelResult(IndicesService.java:1402) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.indices.IndicesService.loadIntoContext(IndicesService.java:1341) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:359) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:434) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.search.SearchService.access$200(SearchService.java:135) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.search.SearchService$2.lambda$onResponse$0(SearchService.java:395) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.search.SearchService.lambda$runAsync$0(SearchService.java:411) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) ~[?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) ~[?:?]",
"at java.lang.Thread.run(Thread.java:832) ~[?:?]"] }
To setup the system i've created two indices bug-first
and bug-second
. Further, i've added old documents to bug-first
through
for i in $(seq 1000); do echo '{ "@timestamp" : "2020-05-09T09:10:41.272235893Z","test":"test" }' | http post localhost:9200/bug-first/_doc; done
and a document to bug-second
> echo '{ "@timestamp" : "2020-05-29T09:10:41.272235893Z", "test":"test"}' | http post localhost:9200/bug-second/_doc;
Is this issue related to https://github.com/elastic/elasticsearch/issues/56923 ? Is there a known workaround?