Hi, Today we had a big crash in our Netapp cabinet and our elastic cluster went down
After starting up all our indices appeared to be ok with the exception of this internal's elasticsearch index .async-search
which is in red state.
I have not problems running normal searches but I'm no able to use discovery/visualizations/etc (I suposse kibana uses async_search)
(below the cluster allocation explain)
{
"index" : ".async-search",
"shard" : 0,
"primary" : true,
"current_state" : "unassigned",
"unassigned_info" : {
"reason" : "ALLOCATION_FAILED",
"at" : "2021-06-10T18:30:42.366Z",
"failed_allocation_attempts" : 5,
"details" : "failed shard on node [FjP6WEThRfi6q4ef5FYW1Q]: failed recovery, failure RecoveryFailedException[[.async-search][0]: Recovery failed on {srvelk-log-warm}{FjP6WEThRfi6q4ef5FYW1Q}{LmJvuosgQN6Okm9b-dFAgQ}{192.168.1.86}{192.168.1.86:9300}{cdfhilmrstw}{ml.machine_memory=16825327616, xpack.installed=true, data=warm, transform.node=true, ml.max_open_jobs=20, ml.max_jvm_size=4294967296}]; nested: IndexShardRecoveryException[failed recovery]; nested: ElasticsearchException[java.io.IOException: failed to read /hist_log/elasticsearch/nodes/0/indices/EaB1cZowSsmjB845URxQLQ/0/_state/retention-leases-18581.st]; nested: IOException[failed to read /hist_log/elasticsearch/nodes/0/indices/EaB1cZowSsmjB845URxQLQ/0/_state/retention-leases-18581.st]; nested: IllegalArgumentException[[-1:5] [retention_leases] unknown field [.async-search]]; ",
"last_allocation_status" : "no"
},
"can_allocate" : "yes",
"allocate_explanation" : "can allocate the shard",
"target_node" : {
"id" : "FjP6WEThRfi6q4ef5FYW1Q",
"name" : "srvelk-log-warm",
"transport_address" : "192.168.1.86:9300",
"attributes" : {
"ml.machine_memory" : "16825335808",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true",
"ml.max_jvm_size" : "4294967296",
"data" : "warm",
"transform.node" : "true"
}
},
"allocation_id" : "TUzZ_86BTbqWYr6vGXtOsw",
"node_allocation_decisions" : [
{
"node_id" : "FjP6WEThRfi6q4ef5FYW1Q",
"node_name" : "srvelk-log-warm",
"transport_address" : "192.168.1.86:9300",
"node_attributes" : {
"ml.machine_memory" : "16825335808",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true",
"ml.max_jvm_size" : "4294967296",
"data" : "warm",
"transform.node" : "true"
},
"node_decision" : "yes",
"store" : {
"in_sync" : true,
"allocation_id" : "TUzZ_86BTbqWYr6vGXtOsw"
}
},
{
"node_id" : "kHmAnzalS_W2qRMqg94k-Q",
"node_name" : "srvlek-log-hot",
"transport_address" : "192.168.1.93:9300",
"node_attributes" : {
"ml.machine_memory" : "25279803392",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true",
"ml.max_jvm_size" : "12884901888",
"data" : "hot",
"transform.node" : "true"
},
"node_decision" : "yes",
"store" : {
"in_sync" : true,
"allocation_id" : "OWCqGQJAR_-_4_Ngb4AVtQ"
}
}
]
}
Any help will be appreciated
Thank you