Did I hit bug related to scroll and too many scroll contexts

Hi,

I've probably hit bug related to scrolling feature. I'm running ES 7.6.2 and have one process iterating over all entries from one of indices. It uses scroll feature and was working fine till yesterday. Now when I'm try to create even simple query with scroll on index with only two documents I'm getting 500 error "Too many scroll contexts". I've checked node stats and there are no opened contexts on any node.

health status index                                                           pri rep docs.count docs.deleted store.size pri.store.size iic segments.count mc       tm
green  open   test_ranking_urls_uk_en_201811                                    5   1          2            0       53kb         26.5kb   0              2  0   11.3kb
> curl 'localhost:9200/test_ranking_urls_uk_en_201811/_search?scroll=30s&size=1000&track_total_hits=true&pretty'
    {
      "error" : {
        "root_cause" : [
          {
            "type" : "exception",
            "reason" : "Trying to create too many scroll contexts. Must be less than or equal to: [5000]. This limit can be set by changing the [search.max_open_scroll_context] setting."
          }
        ],
        "type" : "search_phase_execution_exception",
        "reason" : "all shards failed",
        "phase" : "query",
        "grouped" : true,
        "failed_shards" : [
          {
            "shard" : 0,
            "index" : "test_ranking_urls_uk_en_201811",
            "node" : "ynRcMYz4QpWFeigKKOgmEw",
            "reason" : {
              "type" : "exception",
              "reason" : "Trying to create too many scroll contexts. Must be less than or equal to: [5000]. This limit can be set by changing the [search.max_open_scroll_context] setting."
            }
          }
        ]
      },
      "status" : 500
    }

and nodes stats:

{
  "_nodes": {
    "total": 8,
    "successful": 8,
    "failed": 0
  },
  "cluster_name": "my-cluster",
  "nodes": {
    "ynRcMYz4QpWFeigKKOgmEw": {
      "timestamp": 1588588454075,
      "name": "my-node-06",
      "transport_address": "10.10.92.61:9300",
      "host": "10.10.92.61",
      "ip": "10.10.92.61:9300",
      "roles": [
        "ingest",
        "master",
        "data",
        "ml"
      ],
      "attributes": {
        "ml.machine_memory": "67268014080",
        "ml.max_open_jobs": "20",
        "xpack.installed": "true"
      },
      "indices": {
        "search": {
          "open_contexts": 0,
          "query_total": 2351211152,
          "query_time_in_millis": 834118071,
          "query_current": 0,
          "fetch_total": 328136445,
          "fetch_time_in_millis": 204179719,
          "fetch_current": 0,
          "scroll_total": 370919,
          "scroll_time_in_millis": 13167130612,
          "scroll_current": 0,
          "suggest_total": 2671,
          "suggest_time_in_millis": 5056,
          "suggest_current": 0
        }
      }
    },
    "wbfLcFDLQtOPYdWgzmeoqw": {
      "timestamp": 1588588454075,
      "name": "my-node-02",
      "transport_address": "10.10.92.57:9300",
      "host": "10.10.92.57",
      "ip": "10.10.92.57:9300",
      "roles": [
        "ingest",
        "master",
        "data",
        "ml"
      ],
      "attributes": {
        "ml.machine_memory": "67267641344",
        "ml.max_open_jobs": "20",
        "xpack.installed": "true"
      },
      "indices": {
        "search": {
          "open_contexts": 0,
          "query_total": 2326908291,
          "query_time_in_millis": 848089731,
          "query_current": 0,
          "fetch_total": 312265694,
          "fetch_time_in_millis": 205204641,
          "fetch_current": 0,
          "scroll_total": 382989,
          "scroll_time_in_millis": 13254261535,
          "scroll_current": 0,
          "suggest_total": 2825,
          "suggest_time_in_millis": 6032,
          "suggest_current": 0
        }
      }
    },
    "xE2YxpAPRXaVISqUYT7LhQ": {
      "timestamp": 1588588454075,
      "name": "my-node-07",
      "transport_address": "10.10.92.62:9300",
      "host": "10.10.92.62",
      "ip": "10.10.92.62:9300",
      "roles": [
        "ingest",
        "master",
        "data",
        "ml"
      ],
      "attributes": {
        "ml.machine_memory": "67268014080",
        "ml.max_open_jobs": "20",
        "xpack.installed": "true"
      },
      "indices": {
        "search": {
          "open_contexts": 0,
          "query_total": 2347320943,
          "query_time_in_millis": 822398157,
          "query_current": 0,
          "fetch_total": 321806111,
          "fetch_time_in_millis": 210598357,
          "fetch_current": 0,
          "scroll_total": 248,
          "scroll_time_in_millis": 13031262472,
          "scroll_current": 0,
          "suggest_total": 125,
          "suggest_time_in_millis": 324,
          "suggest_current": 0
        }
      }
    },
    "2-ponLUDTk6MX85iJZAR_A": {
      "timestamp": 1588588454075,
      "name": "my-node-04",
      "transport_address": "10.10.92.59:9300",
      "host": "10.10.92.59",
      "ip": "10.10.92.59:9300",
      "roles": [
        "ingest",
        "master",
        "data",
        "ml"
      ],
      "attributes": {
        "ml.machine_memory": "67267641344",
        "ml.max_open_jobs": "20",
        "xpack.installed": "true"
      },
      "indices": {
        "search": {
          "open_contexts": 0,
          "query_total": 2390614786,
          "query_time_in_millis": 879188894,
          "query_current": 0,
          "fetch_total": 369614955,
          "fetch_time_in_millis": 229705336,
          "fetch_current": 0,
          "scroll_total": 261,
          "scroll_time_in_millis": 12249720463,
          "scroll_current": 0,
          "suggest_total": 308,
          "suggest_time_in_millis": 1417,
          "suggest_current": 0
        }
      }
    },
    "RyCKnzaYRHyOfA5IDKHx4Q": {
      "timestamp": 1588588454075,
      "name": "my-node-08",
      "transport_address": "10.10.92.63:9300",
      "host": "10.10.92.63",
      "ip": "10.10.92.63:9300",
      "roles": [
        "ingest",
        "master",
        "data",
        "ml"
      ],
      "attributes": {
        "ml.machine_memory": "67268014080",
        "xpack.installed": "true",
        "ml.max_open_jobs": "20"
      },
      "indices": {
        "search": {
          "open_contexts": 0,
          "query_total": 2344582494,
          "query_time_in_millis": 922696500,
          "query_current": 0,
          "fetch_total": 330034759,
          "fetch_time_in_millis": 254381920,
          "fetch_current": 0,
          "scroll_total": 252,
          "scroll_time_in_millis": 13248148143,
          "scroll_current": 0,
          "suggest_total": 2690,
          "suggest_time_in_millis": 5059,
          "suggest_current": 0
        }
      }
    },
    "PpE-6ks6SBKxtQmuIMLDNw": {
      "timestamp": 1588588454074,
      "name": "my-node-01",
      "transport_address": "10.10.92.56:9300",
      "host": "10.10.92.56",
      "ip": "10.10.92.56:9300",
      "roles": [
        "ingest",
        "master",
        "data",
        "ml"
      ],
      "attributes": {
        "ml.machine_memory": "67268087808",
        "ml.max_open_jobs": "20",
        "xpack.installed": "true"
      },
      "indices": {
        "search": {
          "open_contexts": 0,
          "query_total": 2408260027,
          "query_time_in_millis": 903387172,
          "query_current": 0,
          "fetch_total": 330369684,
          "fetch_time_in_millis": 223293819,
          "fetch_current": 0,
          "scroll_total": 262,
          "scroll_time_in_millis": 13566134547,
          "scroll_current": 0,
          "suggest_total": 300,
          "suggest_time_in_millis": 636,
          "suggest_current": 0
        }
      }
    },
    "qvExGprdQTu20IomInkWHw": {
      "timestamp": 1588588454075,
      "name": "my-node-05",
      "transport_address": "10.10.92.60:9300",
      "host": "10.10.92.60",
      "ip": "10.10.92.60:9300",
      "roles": [
        "ingest",
        "master",
        "data",
        "ml"
      ],
      "attributes": {
        "ml.machine_memory": "67268014080",
        "ml.max_open_jobs": "20",
        "xpack.installed": "true"
      },
      "indices": {
        "search": {
          "open_contexts": 0,
          "query_total": 2329556383,
          "query_time_in_millis": 853762105,
          "query_current": 0,
          "fetch_total": 332317606,
          "fetch_time_in_millis": 206486768,
          "fetch_current": 0,
          "scroll_total": 263,
          "scroll_time_in_millis": 13211918917,
          "scroll_current": 0,
          "suggest_total": 2669,
          "suggest_time_in_millis": 3735,
          "suggest_current": 0
        }
      }
    },
    "H7aqr3GYTDOCryxclYldRg": {
      "timestamp": 1588588454076,
      "name": "my-node-03",
      "transport_address": "10.10.92.58:9300",
      "host": "10.10.92.58",
      "ip": "10.10.92.58:9300",
      "roles": [
        "ingest",
        "master",
        "data",
        "ml"
      ],
      "attributes": {
        "ml.machine_memory": "67268022272",
        "ml.max_open_jobs": "20",
        "xpack.installed": "true"
      },
      "indices": {
        "search": {
          "open_contexts": 0,
          "query_total": 2409691347,
          "query_time_in_millis": 905375157,
          "query_current": 0,
          "fetch_total": 369057153,
          "fetch_time_in_millis": 234298151,
          "fetch_current": 0,
          "scroll_total": 253,
          "scroll_time_in_millis": 12049937747,
          "scroll_current": 0,
          "suggest_total": 340,
          "suggest_time_in_millis": 1351,
          "suggest_current": 0
        }
      }
    }
  }
}

Is this a bug or I missed anything?

Kind regards -
Krzysztof Księżyk

Hi,

Update... I've restarted our ES nodes one by one and after one of them it started working again and I can use queries with "scroll". As I've mentioned all my tests were done on almost idle cluster so I can say it wasn't effect of other queries running on this cluster.

Kind regards -
Krzysztof Księżyk