I have often seen the following output in my elastic logs and i wondered if there is a way to look up the content of the search queue.
2018-04-06T07:32:47,717][WARN ][r.suppressed ] path: /.kibana/_search, params: {size=1000, index=.kibana, from=0}
org.elasticsearch.action.search.SearchPhaseExecutionException:
at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:272) ~[elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.action.search.FetchSearchPhase$1.onFailure(FetchSearchPhase.java:92) ~[elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.onRejection(AbstractRunnable.java:63) ~[elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.onRejection(ThreadContext.java:628) ~[elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor.doExecute(EsThreadPoolExecutor.java:100) ~[elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor.execute(EsThreadPoolExecutor.java:89) ~[elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.execute(AbstractSearchAsyncAction.java:287) ~[elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.action.search.FetchSearchPhase.run(FetchSearchPhase.java:81) ~[elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:144) ~[elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:138) ~[elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:241) ~[elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.action.search.InitialSearchPhase.successfulShardExecution(InitialSearchPhase.java:251) ~[elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.action.search.InitialSearchPhase.onShardResult(InitialSearchPhase.java:239) ~[elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.action.search.InitialSearchPhase.access$200(InitialSearchPhase.java:49) ~[elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.action.search.InitialSearchPhase$2.lambda$innerOnResponse$0(InitialSearchPhase.java:212) ~[elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.action.search.InitialSearchPhase.maybeFork(InitialSearchPhase.java:171) [elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.action.search.InitialSearchPhase.access$000(InitialSearchPhase.java:49) [elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.action.search.InitialSearchPhase$2.innerOnResponse(InitialSearchPhase.java:212) [elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:44) [elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:29) [elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:46) [elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1070) [elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.transport.TransportService$DirectResponseChannel.processResponse(TransportService.java:1144) [elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1134) [elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1123) [elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.transport.DelegatingTransportChannel.sendResponse(DelegatingTransportChannel.java:60) [elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.transport.RequestHandlerRegistry$TransportChannelWrapper.sendResponse(RequestHandlerRegistry.java:111) [elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.action.search.SearchTransportService$6.messageReceived(SearchTransportService.java:344) [elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.action.search.SearchTransportService$6.messageReceived(SearchTransportService.java:340) [elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69) [elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:654) [elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638) [elasticsearch-5.6.6.jar:5.6.6]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-5.6.6.jar:5.6.6]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_151]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_151]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_151]
Caused by: org.elasticsearch.common.util.concurrent.EsRejectedExecutionException: rejected execution of org.elasticsearch.action.search.FetchSearchPhase$1@7d2d5bb0 on EsThreadPoolExecutor[search, queue capacity = 1000, org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor@2319d6e0[Running, pool size = 7, active threads = 6, queued tasks = 1069, completed tasks = 15200567]]
at org.elasticsearch.common.util.concurrent.EsAbortPolicy.rejectedExecution(EsAbortPolicy.java:50) ~[elasticsearch-5.6.6.jar:5.6.6]
at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830) ~[?:1.8.0_151]
at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379) ~[?:1.8.0_151]
at org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor.doExecute(EsThreadPoolExecutor.java:94) ~[elasticsearch-5.6.6.jar:5.6.6]
... 31 more
With the help of the Kibana Dev Tools i also found out that i have alot of rejected searches:
