Hi,
running 3 nodes cluster in prod workload, having problem with .kibana_task_manager index in red state, ran cluster/allocation api came to know 2 of the shards unassigned here is the response.
for workaround this can be delete and its get recreated when kibana restarted but looking for permanent solution which does not reoccur in future.
am trying reproduce this error in lower but we are unable do this. Can someone help me in this regard.
{
"index":".kibana_task_manager",
"shard":0,
"primary":true,
"current_state":"unassigned",
"unassigned_info":{
"reason":"NODE_LEFT",
"at":"2020-11-01T21:00:43.758Z",
"details":"node_left [pyMkwKrJR0-XXhVhR1D8Sw]",
"last_allocation_status":"no_valid_shard_copy"
},
"can_allocate":"no_valid_shard_copy",
"allocate_explanation":"cannot allocate because a previous copy of the primary shard existed but can no longer be found on the nodes in the cluster",
"node_allocation_decisions":[
{
"node_id":"prod3",
"node_name":"prod3",
"transport_address":"...:9300",
"node_attributes":{
"ml.machine_memory":"16170143744",
"xpack.installed":"true",
"ml.max_open_jobs":"20",
"ml.enabled":"true"
},
"node_decision":"no",
"store":{
"found":false
}
},
{
"node_id":"prod1",
"node_name":"prod1",
"transport_address":"...:9300",
"node_attributes":{
"ml.machine_memory":"16346312704",
"ml.max_open_jobs":"20",
"xpack.installed":"true",
"ml.enabled":"true"
},
"node_decision":"no",
"store":{
"found":false
}
},
{
"node_id":"prod2",
"node_name":"prod2",
"transport_address":"...:9300",
"node_attributes":{
"ml.machine_memory":"16170151936",
"ml.max_open_jobs":"20",
"xpack.installed":"true",
"ml.enabled":"true"
},
"node_decision":"no",
"store":{
"found":false
}
}
]
}