Getting es_rejected_execution_exception error
Do we need to increase queue size or uplift the data nodes?
also , why its showing task completed status true even after failures ?
This is production issue, any help will be greatly apreciated...
Task status output
Command : GET _tasks/Z0AwlH25T3ST7CG0HgBiHg:1189428949
{
"completed": true,
"task": {
"node": "Z0AwlH25T3ST7CG0BgBiHg",
"id": 1189428949,
"type": "transport",
"action": "indices:data/write/reindex",
"status": {
"total": 31631064,
"updated": 1055165,
"created": 9678835,
"deleted": 0,
"batches": 10734,
"version_conflicts": 0,
"noops": 0,
"retries": {
"bulk": 0,
"search": 0
},
"throttled_millis": 0,
"requests_per_second": -1,
"throttled_until_millis": 0
},
"description": "reindex from [i-100-2019-06-03, i-200-2019-06-24, i-200-2019-06-26, i-200-2019-06-25, i-200-2019-06-20, i-200-2019-06-21, i-600-2019-06-17, i-600-2019-06-18, i-600-2019-06-19, i-100-2019-06-07, i-200-2019-06-28, i-600-2019-06-12, i-100-2019-06-06, i-200-2019-06-27, i-600-2019-06-13, i-100-2019-06-05, i-600-2019-06-14, i-100-2019-06-04, i-600-2019-06-10, i-600-2019-06-11, i-100-2019-06-14, i-200-2019-06-13, i-100-2019-06-13, i-200-2019-06-12, i-100-2019-06-12, i-100-2019-06-11, i-200-2019-06-14, i-100-2019-06-10, i-200-2019-06-11, i-200-2019-06-10, i-600-2019-06-27, i-600-2019-06-28, i-100-2019-06-19, i-100-2019-06-18, i-200-2019-06-17, i-100-2019-06-17, i-600-2019-06-24, i-200-2019-06-19, i-600-2019-06-25, i-200-2019-06-18, i-600-2019-06-26, i-600-2019-06-20, i-600-2019-06-21, i-500-2019-06-19, i-500-2019-06-17, i-500-2019-06-18, i-500-2019-06-13, i-500-2019-06-14, i-500-2019-06-11, i-500-2019-06-12, i-500-2019-06-10, i-500-2019-06-06, i-500-2019-06-07, i-500-2019-06-04, i-500-2019-06-05, i-500-2019-06-03, i-400-2019-06-20, i-400-2019-06-21, i-400-2019-06-24, i-300-2019-06-04, i-400-2019-06-25, i-300-2019-06-05, i-400-2019-06-26, i-400-2019-06-27, i-300-2019-06-03, i-400-2019-06-28, i-300-2019-06-06, i-300-2019-06-07, i-300-2019-06-11, i-400-2019-06-10, i-300-2019-06-12, i-400-2019-06-11, i-400-2019-06-12, i-300-2019-06-10, i-400-2019-06-13, i-400-2019-06-14, i-300-2019-06-13, i-300-2019-06-14, i-400-2019-06-17, i-300-2019-06-19, i-400-2019-06-18, i-500-2019-06-28, i-400-2019-06-19, i-300-2019-06-17, i-500-2019-06-26, i-300-2019-06-18, i-500-2019-06-27, i-500-2019-06-24, i-500-2019-06-25, i-500-2019-06-20, i-500-2019-06-21, i-100-2019-06-25, i-100-2019-06-24, i-200-2019-06-04, i-300-2019-06-20, i-200-2019-06-03, i-300-2019-06-21, i-100-2019-06-21, i-300-2019-06-26, i-400-2019-06-03, i-100-2019-06-20, i-300-2019-06-27, i-400-2019-06-04, i-300-2019-06-24, i-400-2019-06-05, i-300-2019-06-25, i-400-2019-06-06, i-400-2019-06-07, i-300-2019-06-28, i-200-2019-06-06, i-100-2019-06-28, i-200-2019-06-05, i-100-2019-06-27, i-100-2019-06-26, i-200-2019-06-07, i-600-2019-06-05, i-600-2019-06-06, i-600-2019-06-07, i-600-2019-06-03, i-600-2019-06-04] to [data-2019-06]",
"start_time_in_millis": 1563238811197,
"running_time_in_nanos": 3896439564878,
"cancellable": true,
"headers": {}
},
"response": {
"took": 3896313,
"timed_out": false,
"total": 31631064,
"updated": 1055165,
"created": 9678835,
"deleted": 0,
"batches": 10734,
"version_conflicts": 0,
"noops": 0,
"retries": {
"bulk": 0,
"search": 0
},
"throttled_millis": 0,
"requests_per_second": -1,
"throttled_until_millis": 0,
"failures": [
{
"shard": -1,
"reason": {
"type": "es_rejected_execution_exception",
"reason": "rejected execution of org.elasticsearch.common.util.concurrent.TimedRunnable@795de3e0 on QueueResizingEsThreadPoolExecutor[name = p13-data/search, queue capacity = 1000, min queue capacity = 1000, max queue capacity = 1000, frame size = 2000, targeted response rate = 1s, task execution EWMA = 804nanos, adjustment amount = 50, org.elasticsearch.common.util.concurrent.QueueResizingEsThreadPoolExecutor@1af15a09[Running, pool size = 13, active threads = 13, queued tasks = 1050, completed tasks = 707224617]]"
.
.
.