Searches timing out on certain indices

Elastic Stack Elasticsearch
1

Hi there,

We noticed a problem today that hasnt happened before. When querying some of our indices, our queries do not return or timeout. Some indices are working fine. I cant seem to find anything relevant in our logs, and I'm not sure where else to look.

Here is the output of _cluster/health?pretty=true

    {
"cluster_name": "es6-test",
"status": "green",
"timed_out": false,
"number_of_nodes": 4,
"number_of_data_nodes": 4,
"active_primary_shards": 326,
"active_shards": 652,
"relocating_shards": 0,
"initializing_shards": 0,
"unassigned_shards": 0,
"delayed_unassigned_shards": 0,
"number_of_pending_tasks": 0,
"number_of_in_flight_fetch": 0,
"task_max_waiting_in_queue_millis": 0,
"active_shards_percent_as_number": 100
}

This looks to be the only relevant thing I can find in the logs.

[2019-07-23T17:05:22,349][DEBUG][o.e.a.s.TransportSearchAction] [node-3] All shards failed for phase: [query]
[2019-07-23T17:05:22,349][WARN ][r.suppressed             ] path: /index/_search, params: {index=index}
org.elasticsearch.action.search.SearchPhaseExecutionException: all shards failed
	at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:274) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:132) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:243) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.action.search.InitialSearchPhase.onShardFailure(InitialSearchPhase.java:107) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.action.search.InitialSearchPhase.lambda$performPhaseOnShard$4(InitialSearchPhase.java:205) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.action.search.InitialSearchPhase$1.doRun(InitialSearchPhase.java:184) [elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:635) [elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41) [elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-6.2.1.jar:6.2.1]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]

[2019-07-23T17:04:33,569][WARN ][r.suppressed             ] path: /other_index/_search, params: {index=other_index}
org.elasticsearch.cluster.block.ClusterBlockException: blocked by: [SERVICE_UNAVAILABLE/1/state not recovered / initialized];
	at org.elasticsearch.cluster.block.ClusterBlocks.globalBlockedException(ClusterBlocks.java:165) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.cluster.block.ClusterBlocks.globalBlockedRaiseException(ClusterBlocks.java:151) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:286) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.action.search.TransportSearchAction.lambda$doExecute$4(TransportSearchAction.java:193) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:60) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:113) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:86) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:215) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:68) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:167) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:139) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:81) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:83) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:72) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:405) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.client.support.AbstractClient.search(AbstractClient.java:532) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:93) ~[elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:97) [elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:240) [elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:336) [elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:174) [elasticsearch-6.2.1.jar:6.2.1]
	at org.elasticsearch.http.netty4.Netty4HttpServerTransport.dispatchRequest(Netty4HttpServerTransport.java:500) [transport-netty4-6.2.1.jar:6.2.1]
	at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:80) [transport-netty4-6.2.1.jar:6.2.1]
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
	at org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler.channelRead(HttpPipeliningHandler.java:68) [transport-netty4-6.2.1.jar:6.2.1]
	at

Any ideas?

2

Interestingly, this query:

{
  "size": 10,
  "query": {
    "bool": {
      "must": [
        {
          "query_string": {
            "default_field": "text",
            "query": "\"tea\" ",
            "analyze_wildcard": "true",
            "minimum_should_match": "1"
          }
        }
      ],
      "filter": [
        {
          "range": {
            "message_time": {
              "gte": "2018-01-01",
              "lte": "2019-06-30||+1h/d",
              "time_zone": "America/Toronto"
            }
          }
        },
        {
          "bool": {
            "should": [
              {
                "query_string": {
                  "default_field": "text",
                  "query": "*",
                  "analyze_wildcard": "true",
                  "minimum_should_match": "1"
                }
              }
            ]
          }
        }
      ]
    }
  },
  "aggs": {
    "distinct_nodes": {
      "cardinality": {
        "field": "node"
      }
    },
    "days": {
      "date_histogram": {
        "field": "message_time",
        "interval": "month",
        "time_zone": "America/Toronto",
        "format": "yyyy-MM-dd",
        "offset": "+0d"
      },
      "aggs": {
        "distinct_nodes": {
          "cardinality": {
            "field": "node"
          }
        }
      }
    }
  }
}

Will execute fine if size is less than 500, for numbers greater than that it stalls and does not return anything.

3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.