Hi,
I've probably hit bug related to scrolling feature. I'm running ES 7.6.2 and have one process iterating over all entries from one of indices. It uses scroll feature and was working fine till yesterday. Now when I'm try to create even simple query with scroll on index with only two documents I'm getting 500 error "Too many scroll contexts". I've checked node stats and there are no opened contexts on any node.
health status index pri rep docs.count docs.deleted store.size pri.store.size iic segments.count mc tm
green open test_ranking_urls_uk_en_201811 5 1 2 0 53kb 26.5kb 0 2 0 11.3kb
> curl 'localhost:9200/test_ranking_urls_uk_en_201811/_search?scroll=30s&size=1000&track_total_hits=true&pretty'
{
"error" : {
"root_cause" : [
{
"type" : "exception",
"reason" : "Trying to create too many scroll contexts. Must be less than or equal to: [5000]. This limit can be set by changing the [search.max_open_scroll_context] setting."
}
],
"type" : "search_phase_execution_exception",
"reason" : "all shards failed",
"phase" : "query",
"grouped" : true,
"failed_shards" : [
{
"shard" : 0,
"index" : "test_ranking_urls_uk_en_201811",
"node" : "ynRcMYz4QpWFeigKKOgmEw",
"reason" : {
"type" : "exception",
"reason" : "Trying to create too many scroll contexts. Must be less than or equal to: [5000]. This limit can be set by changing the [search.max_open_scroll_context] setting."
}
}
]
},
"status" : 500
}
and nodes stats:
{
"_nodes": {
"total": 8,
"successful": 8,
"failed": 0
},
"cluster_name": "my-cluster",
"nodes": {
"ynRcMYz4QpWFeigKKOgmEw": {
"timestamp": 1588588454075,
"name": "my-node-06",
"transport_address": "10.10.92.61:9300",
"host": "10.10.92.61",
"ip": "10.10.92.61:9300",
"roles": [
"ingest",
"master",
"data",
"ml"
],
"attributes": {
"ml.machine_memory": "67268014080",
"ml.max_open_jobs": "20",
"xpack.installed": "true"
},
"indices": {
"search": {
"open_contexts": 0,
"query_total": 2351211152,
"query_time_in_millis": 834118071,
"query_current": 0,
"fetch_total": 328136445,
"fetch_time_in_millis": 204179719,
"fetch_current": 0,
"scroll_total": 370919,
"scroll_time_in_millis": 13167130612,
"scroll_current": 0,
"suggest_total": 2671,
"suggest_time_in_millis": 5056,
"suggest_current": 0
}
}
},
"wbfLcFDLQtOPYdWgzmeoqw": {
"timestamp": 1588588454075,
"name": "my-node-02",
"transport_address": "10.10.92.57:9300",
"host": "10.10.92.57",
"ip": "10.10.92.57:9300",
"roles": [
"ingest",
"master",
"data",
"ml"
],
"attributes": {
"ml.machine_memory": "67267641344",
"ml.max_open_jobs": "20",
"xpack.installed": "true"
},
"indices": {
"search": {
"open_contexts": 0,
"query_total": 2326908291,
"query_time_in_millis": 848089731,
"query_current": 0,
"fetch_total": 312265694,
"fetch_time_in_millis": 205204641,
"fetch_current": 0,
"scroll_total": 382989,
"scroll_time_in_millis": 13254261535,
"scroll_current": 0,
"suggest_total": 2825,
"suggest_time_in_millis": 6032,
"suggest_current": 0
}
}
},
"xE2YxpAPRXaVISqUYT7LhQ": {
"timestamp": 1588588454075,
"name": "my-node-07",
"transport_address": "10.10.92.62:9300",
"host": "10.10.92.62",
"ip": "10.10.92.62:9300",
"roles": [
"ingest",
"master",
"data",
"ml"
],
"attributes": {
"ml.machine_memory": "67268014080",
"ml.max_open_jobs": "20",
"xpack.installed": "true"
},
"indices": {
"search": {
"open_contexts": 0,
"query_total": 2347320943,
"query_time_in_millis": 822398157,
"query_current": 0,
"fetch_total": 321806111,
"fetch_time_in_millis": 210598357,
"fetch_current": 0,
"scroll_total": 248,
"scroll_time_in_millis": 13031262472,
"scroll_current": 0,
"suggest_total": 125,
"suggest_time_in_millis": 324,
"suggest_current": 0
}
}
},
"2-ponLUDTk6MX85iJZAR_A": {
"timestamp": 1588588454075,
"name": "my-node-04",
"transport_address": "10.10.92.59:9300",
"host": "10.10.92.59",
"ip": "10.10.92.59:9300",
"roles": [
"ingest",
"master",
"data",
"ml"
],
"attributes": {
"ml.machine_memory": "67267641344",
"ml.max_open_jobs": "20",
"xpack.installed": "true"
},
"indices": {
"search": {
"open_contexts": 0,
"query_total": 2390614786,
"query_time_in_millis": 879188894,
"query_current": 0,
"fetch_total": 369614955,
"fetch_time_in_millis": 229705336,
"fetch_current": 0,
"scroll_total": 261,
"scroll_time_in_millis": 12249720463,
"scroll_current": 0,
"suggest_total": 308,
"suggest_time_in_millis": 1417,
"suggest_current": 0
}
}
},
"RyCKnzaYRHyOfA5IDKHx4Q": {
"timestamp": 1588588454075,
"name": "my-node-08",
"transport_address": "10.10.92.63:9300",
"host": "10.10.92.63",
"ip": "10.10.92.63:9300",
"roles": [
"ingest",
"master",
"data",
"ml"
],
"attributes": {
"ml.machine_memory": "67268014080",
"xpack.installed": "true",
"ml.max_open_jobs": "20"
},
"indices": {
"search": {
"open_contexts": 0,
"query_total": 2344582494,
"query_time_in_millis": 922696500,
"query_current": 0,
"fetch_total": 330034759,
"fetch_time_in_millis": 254381920,
"fetch_current": 0,
"scroll_total": 252,
"scroll_time_in_millis": 13248148143,
"scroll_current": 0,
"suggest_total": 2690,
"suggest_time_in_millis": 5059,
"suggest_current": 0
}
}
},
"PpE-6ks6SBKxtQmuIMLDNw": {
"timestamp": 1588588454074,
"name": "my-node-01",
"transport_address": "10.10.92.56:9300",
"host": "10.10.92.56",
"ip": "10.10.92.56:9300",
"roles": [
"ingest",
"master",
"data",
"ml"
],
"attributes": {
"ml.machine_memory": "67268087808",
"ml.max_open_jobs": "20",
"xpack.installed": "true"
},
"indices": {
"search": {
"open_contexts": 0,
"query_total": 2408260027,
"query_time_in_millis": 903387172,
"query_current": 0,
"fetch_total": 330369684,
"fetch_time_in_millis": 223293819,
"fetch_current": 0,
"scroll_total": 262,
"scroll_time_in_millis": 13566134547,
"scroll_current": 0,
"suggest_total": 300,
"suggest_time_in_millis": 636,
"suggest_current": 0
}
}
},
"qvExGprdQTu20IomInkWHw": {
"timestamp": 1588588454075,
"name": "my-node-05",
"transport_address": "10.10.92.60:9300",
"host": "10.10.92.60",
"ip": "10.10.92.60:9300",
"roles": [
"ingest",
"master",
"data",
"ml"
],
"attributes": {
"ml.machine_memory": "67268014080",
"ml.max_open_jobs": "20",
"xpack.installed": "true"
},
"indices": {
"search": {
"open_contexts": 0,
"query_total": 2329556383,
"query_time_in_millis": 853762105,
"query_current": 0,
"fetch_total": 332317606,
"fetch_time_in_millis": 206486768,
"fetch_current": 0,
"scroll_total": 263,
"scroll_time_in_millis": 13211918917,
"scroll_current": 0,
"suggest_total": 2669,
"suggest_time_in_millis": 3735,
"suggest_current": 0
}
}
},
"H7aqr3GYTDOCryxclYldRg": {
"timestamp": 1588588454076,
"name": "my-node-03",
"transport_address": "10.10.92.58:9300",
"host": "10.10.92.58",
"ip": "10.10.92.58:9300",
"roles": [
"ingest",
"master",
"data",
"ml"
],
"attributes": {
"ml.machine_memory": "67268022272",
"ml.max_open_jobs": "20",
"xpack.installed": "true"
},
"indices": {
"search": {
"open_contexts": 0,
"query_total": 2409691347,
"query_time_in_millis": 905375157,
"query_current": 0,
"fetch_total": 369057153,
"fetch_time_in_millis": 234298151,
"fetch_current": 0,
"scroll_total": 253,
"scroll_time_in_millis": 12049937747,
"scroll_current": 0,
"suggest_total": 340,
"suggest_time_in_millis": 1351,
"suggest_current": 0
}
}
}
}
}
Is this a bug or I missed anything?
Kind regards -
Krzysztof Księżyk