Please I need critical help with cluster in state RED after upgrade 7.4.2->7.9
during upgrade of 5th node of 15 it did not recover it health to 100% but 99%
all indices are green except 2 shards
index shard prirep state docs store ip node
ilm-history-2-000001 0 p UNASSIGNED
ilm-history-2-000001 0 r UNASSIGNED
primary shard for this replica is not yet active
{
"index" : "ilm-history-2-000001",
"shard" : 0,
"primary" : false,
"current_state" : "unassigned",
"unassigned_info" : {
"reason" : "REPLICA_ADDED",
"at" : "2020-08-31T13:49:47.872Z",
"last_allocation_status" : "no_attempt"
},
"can_allocate" : "no",
"allocate_explanation" : "cannot allocate because allocation is not permitted to any of the nodes",
"node_allocation_decisions" : [
{
"node_id" : "-u_CT2m7RKG2IM-31cobKA",
"node_name" : "tela17prahvl",
"transport_address" : "10.42.167.147:9300",
"node_attributes" : {
"ml.machine_memory" : "67386851328",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "replica_after_primary_active",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
},
{
"decider" : "throttling",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
}
]
},
...
{
"decider" : "throttling",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
}
]
},
{
"node_id" : "C1-cjmc1TByCPGLmcTTGiQ",
"node_name" : "tela19prahvl",
"transport_address" : "10.42.167.149:9300",
"node_attributes" : {
"ml.machine_memory" : "67386851328",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "replica_after_primary_active",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
},
{
"decider" : "throttling",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
}
]
},
{
"node_id" : "GnAJTEqpTeyawpDyUli4Aw",
"node_name" : "tela20prahvl",
"transport_address" : "10.42.167.150:9300",
"node_attributes" : {
"ml.machine_memory" : "67386851328",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "replica_after_primary_active",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
},
{
"decider" : "throttling",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
}
]
},
{
"node_id" : "J2zWjDl2T1mMkjP4hg0euA",
"node_name" : "cem-physical4",
"transport_address" : "10.199.90.84:9300",
"node_attributes" : {
"ml.machine_memory" : "67387641856",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "replica_after_primary_active",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
},
{
"decider" : "throttling",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
}
]
},
{
"node_id" : "LAb-NxmPR6uJ1AXZNEV_aQ",
"node_name" : "tela18prahvl",
"transport_address" : "10.42.167.148:9300",
"node_attributes" : {
"ml.machine_memory" : "67386851328",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "replica_after_primary_active",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
},
{
"decider" : "throttling",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
}
]
},
{
"node_id" : "NBIGTt5BQ3evOVvMEAvZfw",
"node_name" : "tela13prahvl",
"transport_address" : "10.42.167.143:9300",
"node_attributes" : {
"ml.machine_memory" : "67386851328",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "replica_after_primary_active",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
},
{
"decider" : "throttling",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
}
]
},
{
"node_id" : "NyLzcZ7xTNmjjkCBFQee2g",
"node_name" : "tela12prahvl",
"transport_address" : "10.42.167.142:9300",
"node_attributes" : {
"ml.machine_memory" : "67386851328",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "replica_after_primary_active",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
},
{
"decider" : "throttling",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
}
]
},
{
"node_id" : "Uk33_pfWSEaK8tm4bsQsfA",
"node_name" : "tela22prahvl",
"transport_address" : "10.42.167.152:9300",
"node_attributes" : {
"ml.machine_memory" : "67386851328",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "replica_after_primary_active",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
},
{
"decider" : "throttling",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
}
]
},
{
"node_id" : "YaZDrAFUReqf-X9nDaE3KQ",
"node_name" : "tela23prahvl",
"transport_address" : "10.42.167.153:9300",
"node_attributes" : {
"ml.machine_memory" : "67386851328",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "replica_after_primary_active",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
},
{
"decider" : "throttling",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
}
]
},
{
"node_id" : "e5PncesdSs-KblSq21Pdag",
"node_name" : "tela16prahvl",
"transport_address" : "10.42.167.146:9300",
"node_attributes" : {
"ml.machine_memory" : "67386851328",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "replica_after_primary_active",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
},
{
"decider" : "throttling",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
}
]
},
{
"node_id" : "fGqLCiOvTOm11NjB-HkvuA",
"node_name" : "tela11prahvl",
"transport_address" : "10.42.167.141:9300",
"node_attributes" : {
"ml.machine_memory" : "67386851328",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "replica_after_primary_active",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
},
{
"decider" : "throttling",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
}
]
},
{
"node_id" : "g2bPDi1XRcSuxYY5XAQwDQ",
"node_name" : "cem-physical5",
"transport_address" : "10.199.90.85:9300",
"node_attributes" : {
"ml.machine_memory" : "67387641856",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "replica_after_primary_active",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
},
{
"decider" : "throttling",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
}
]
},
{
"node_id" : "p89vICIvS1CASt6429YkjA",
"node_name" : "tela21prahvl",
"transport_address" : "10.42.167.151:9300",
"node_attributes" : {
"ml.machine_memory" : "67386851328",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "replica_after_primary_active",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
},
{
"decider" : "throttling",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
}
]
},
{
"node_id" : "scNdonFlSpKJXKVn29e3Zg",
"node_name" : "cem-physical2",
"transport_address" : "10.41.134.197:9300",
"node_attributes" : {
"ml.machine_memory" : "67387637760",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "replica_after_primary_active",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
},
{
"decider" : "throttling",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
}
]
},
{
"node_id" : "tK-9GkOqQ5WlVWiSet3TNA",
"node_name" : "tela14prahvl",
"transport_address" : "10.42.167.144:9300",
"node_attributes" : {
"ml.machine_memory" : "67386851328",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "replica_after_primary_active",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
},
{
"decider" : "throttling",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
}
]
},
{
"node_id" : "tYMZgsZwQ-e4pFMQxrWWVA",
"node_name" : "tela15prahvl",
"transport_address" : "10.42.167.145:9300",
"node_attributes" : {
"ml.machine_memory" : "67386851328",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "replica_after_primary_active",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
},
{
"decider" : "throttling",
"decision" : "NO",
"explanation" : "primary shard for this replica is not yet active"
}
]
}
]
}
