Hi team,
Recently, I upgraded my ES cluster from 7.4.2 to 7.6.2. Then I found "Too Many Scroll Context ..." error always occurred and scroll search is not available on my cluster.
I followed the exception tips and turned "search.max_open_scroll_context" settings to 10000, 100000. However, this issue still here. I used node stats api to check the context count on my cluster, but I found 0 scroll_current is active. Is this a bug or some mistake for my configuration?
Please help to check whether this is an opened ISSUE or how could I solve?
Thanks!
Detailed exception info:
GET /.kibana_1/_search?size=100&scroll=1m&pretty
{
"error" : {
"root_cause" : [
{
"type" : "exception",
"reason" : "Trying to create too many scroll contexts. Must be less than or equal to: [100000]. This limit can be set by changing the [search.max_open_scroll_context] setting."
}
],
"type" : "search_phase_execution_exception",
"reason" : "all shards failed",
"phase" : "query",
"grouped" : true,
"failed_shards" : [
{
"shard" : 0,
"index" : ".kibana_1",
"node" : "ggndr9-pQG2xpOSmljpcjg",
"reason" : {
"type" : "exception",
"reason" : "Trying to create too many scroll contexts. Must be less than or equal to: [100000]. This limit can be set by changing the [search.max_open_scroll_context] setting."
}
}
]
},
"status" : 500
}
Detailed nodes stat info:
GET /_nodes/stats/indices/search?pretty
"nodes" : {
"VQHhX9lxRDKjjgvb-zK-Vg" : {
"timestamp" : 1606274771999,
"name" : "node-1",
"indices" : {
"search" : {
"open_contexts" : 150,
"query_total" : 129395667,
"query_time_in_millis" : 46486283,
"query_current" : 7,
"fetch_total" : 41178198,
"fetch_time_in_millis" : 21162191,
"fetch_current" : 0,
"scroll_total" : 7035,
"scroll_time_in_millis" : 14025,
"scroll_current" : 0,
"suggest_total" : 0,
"suggest_time_in_millis" : 0,
"suggest_current" : 0
}
}
},
"QmDzWzslTf2w2Vy4rhsHJg" : {
"timestamp" : 1606274771997,
"name" : "node-2",
"indices" : {
"search" : {
"open_contexts" : 155,
"query_total" : 3186800204,
"query_time_in_millis" : 877948462,
"query_current" : 6,
"fetch_total" : 904340329,
"fetch_time_in_millis" : 379070859,
"fetch_current" : 3,
"scroll_total" : 11602,
"scroll_time_in_millis" : 30711,
"scroll_current" : 0,
"suggest_total" : 0,
"suggest_time_in_millis" : 0,
"suggest_current" : 0
}
}
},
"7LCXTuDrQBedpV4VNyWQVA" : {
"timestamp" : 1606274771996,
"name" : "node-3",
"indices" : {
"search" : {
"open_contexts" : 175,
"query_total" : 3345935363,
"query_time_in_millis" : 775661713,
"query_current" : 6,
"fetch_total" : 964843391,
"fetch_time_in_millis" : 353861601,
"fetch_current" : 3,
"scroll_total" : 1239,
"scroll_time_in_millis" : 1393,
"scroll_current" : 0,
"suggest_total" : 0,
"suggest_time_in_millis" : 0,
"suggest_current" : 0
}
}
},
"iPftIDNuRbaNUK9sS_TiuA" : {
"timestamp" : 1606274771997,
"name" : "node-4",
"indices" : {
"search" : {
"open_contexts" : 140,
"query_total" : 17843768,
"query_time_in_millis" : 7387810,
"query_current" : 11,
"fetch_total" : 5822120,
"fetch_time_in_millis" : 2959936,
"fetch_current" : 2,
"scroll_total" : 0,
"scroll_time_in_millis" : 0,
"scroll_current" : 0,
"suggest_total" : 0,
"suggest_time_in_millis" : 0,
"suggest_current" : 0
}
}
},
"v16oQZwaSSqbEMHHA_aSPw" : {
"timestamp" : 1606274771997,
"name" : "node-5",
"indices" : {
"search" : {
"open_contexts" : 136,
"query_total" : 3365704567,
"query_time_in_millis" : 799540998,
"query_current" : 2,
"fetch_total" : 967083736,
"fetch_time_in_millis" : 355821602,
"fetch_current" : 3,
"scroll_total" : 0,
"scroll_time_in_millis" : 0,
"scroll_current" : 0,
"suggest_total" : 0,
"suggest_time_in_millis" : 0,
"suggest_current" : 0
}
}
},
"UKVzTMVVSf2JKboCyovYGQ" : {
"timestamp" : 1606274771997,
"name" : "node-6",
"indices" : {
"search" : {
"open_contexts" : 0,
"query_total" : 0,
"query_time_in_millis" : 0,
"query_current" : 0,
"fetch_total" : 0,
"fetch_time_in_millis" : 0,
"fetch_current" : 0,
"scroll_total" : 0,
"scroll_time_in_millis" : 0,
"scroll_current" : 0,
"suggest_total" : 0,
"suggest_time_in_millis" : 0,
"suggest_current" : 0
}
}
},
"BamvBHzDR46-UJP3baSQrQ" : {
"timestamp" : 1606274771998,
"name" : "node-7",
"indices" : {
"search" : {
"open_contexts" : 172,
"query_total" : 10097236,
"query_time_in_millis" : 3679491,
"query_current" : 5,
"fetch_total" : 3154781,
"fetch_time_in_millis" : 1460012,
"fetch_current" : 2,
"scroll_total" : 0,
"scroll_time_in_millis" : 0,
"scroll_current" : 0,
"suggest_total" : 0,
"suggest_time_in_millis" : 0,
"suggest_current" : 0
}
}
},
"2NKZEoHgSouDpxg7hXinog" : {
"timestamp" : 1606274771997,
"name" : "node-8",
"indices" : {
"search" : {
"open_contexts" : 132,
"query_total" : 3288442014,
"query_time_in_millis" : 735619798,
"query_current" : 5,
"fetch_total" : 930825770,
"fetch_time_in_millis" : 336368026,
"fetch_current" : 3,
"scroll_total" : 1,
"scroll_time_in_millis" : 6,
"scroll_current" : 0,
"suggest_total" : 0,
"suggest_time_in_millis" : 0,
"suggest_current" : 0
}
}
},
"eou4gX6sTGiIoMPbJi0cLw" : {
"timestamp" : 1606274771998,
"name" : "node-9",
"indices" : {
"search" : {
"open_contexts" : 131,
"query_total" : 8981811,
"query_time_in_millis" : 3889592,
"query_current" : 3,
"fetch_total" : 2861921,
"fetch_time_in_millis" : 1514770,
"fetch_current" : 1,
"scroll_total" : 24,
"scroll_time_in_millis" : 296,
"scroll_current" : 0,
"suggest_total" : 0,
"suggest_time_in_millis" : 0,
"suggest_current" : 0
}
}
},
"aBRYWGpHQdywvPKW7yU-oQ" : {
"timestamp" : 1606274771997,
"name" : "node-10",
"indices" : {
"search" : {
"open_contexts" : 154,
"query_total" : 3258517489,
"query_time_in_millis" : 752056319,
"query_current" : 1,
"fetch_total" : 940633748,
"fetch_time_in_millis" : 338807924,
"fetch_current" : 2,
"scroll_total" : 471,
"scroll_time_in_millis" : 1251,
"scroll_current" : 0,
"suggest_total" : 0,
"suggest_time_in_millis" : 0,
"suggest_current" : 0
}
}
}
}