Hello there,
We are facing a RED cluster.
GET _cluster/health
{
"cluster_name": "my_cluster",
"status": "red",
"timed_out": false,
"number_of_nodes": 20,
"number_of_data_nodes": 13,
"active_primary_shards": 385,
"active_shards": 1206,
"relocating_shards": 0,
"initializing_shards": 0,
"unassigned_shards": 30,
"delayed_unassigned_shards": 0,
"number_of_pending_tasks": 0,
"number_of_in_flight_fetch": 0,
"task_max_waiting_in_queue_millis": 0,
"active_shards_percent_as_number": 97.57281553398059
}
GET _cluster/allocation/explain
{
"note": "No shard was specified in the explain API request, so this response explains a randomly chosen unassigned shard. There may be other unassigned shards in this cluster which cannot be assigned for different reasons. It may not be possible to assign this shard until one of the other shards is assigned correctly. To explain the allocation of other shards (whether assigned or unassigned) you must specify the target shard in the request to this API.",
"index": "my-index-000002",
"shard": 3,
"primary": true,
"current_state": "unassigned",
"unassigned_info": {
"reason": "CLUSTER_RECOVERED",
"at": "2023-07-19T12:23:21.537Z",
"last_allocation_status": "no_valid_shard_copy"
},
"can_allocate": "no_valid_shard_copy",
"allocate_explanation": "cannot allocate because all found copies of the shard are either stale or corrupt",
"node_allocation_decisions": [
{
"node_id": "7u2i1dDCSum3XahhQwDSiw",
"node_name": "my-node-data-0",
"transport_address": "x.x.x.x:9300",
"node_attributes": {
"xpack.installed": "true",
"transform.node": "false"
},
"node_decision": "no",
"store": {
"in_sync": true,
"allocation_id": "JAGX8zqjSteAnE49RW-zhQ",
"store_exception": {
"type": "file_not_found_exception",
"reason": "no segments* file found in NIOFSDirectory@/usr/share/elasticsearch/data/nodes/0/indices/JxYK7aWFRaKrETf3SFitKg/3/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@e35d91b: files: []"
}
}
},
{
"node_id": "9_bBm86CQbidiFPzIzZfkw",
"node_name": "my-node-data-12",
"transport_address": "x.x.x.x:9300",
"node_attributes": {
"xpack.installed": "true",
"transform.node": "false"
},
"node_decision": "no",
"store": {
"found": false
}
},
{
"node_id": "Hf2Iw_EHSxOwePDkbiVl5g",
"node_name": "my-node-data-4",
"transport_address": "x.x.x.x:9300",
"node_attributes": {
"xpack.installed": "true",
"transform.node": "false"
},
"node_decision": "no",
"store": {
"found": false
}
},
{
"node_id": "II3jqdMVQ4mZ1--QUhKDsA",
"node_name": "my-node-data-10",
"transport_address": "x.x.x.x:9300",
"node_attributes": {
"xpack.installed": "true",
"transform.node": "false"
},
"node_decision": "no",
"store": {
"found": false
}
},
{
"node_id": "MMEiLZ7pRt6L65EKM_fRJA",
"node_name": "my-node-data-9",
"transport_address": "x.x.x.x:9300",
"node_attributes": {
"xpack.installed": "true",
"transform.node": "false"
},
"node_decision": "no",
"store": {
"found": false
}
},
{
"node_id": "Pkze3hQpSCyPbRrrPxaL4A",
"node_name": "my-node-data-7",
"transport_address": "x.x.x.x:9300",
"node_attributes": {
"xpack.installed": "true",
"transform.node": "false"
},
"node_decision": "no",
"store": {
"found": false
}
},
{
"node_id": "Qv33-UHPRq6-HeRzxsN1uw",
"node_name": "my-node-data-3",
"transport_address": "x.x.x.x:9300",
"node_attributes": {
"xpack.installed": "true",
"transform.node": "false"
},
"node_decision": "no",
"store": {
"found": false
}
},
{
"node_id": "TBp-UkfGQOaCsIQHHpkwWw",
"node_name": "my-node-data-8",
"transport_address": "x.x.x.x:9300",
"node_attributes": {
"xpack.installed": "true",
"transform.node": "false"
},
"node_decision": "no",
"store": {
"found": false
}
},
{
"node_id": "Yn4cQvWlQmuDVSBjPGjQnw",
"node_name": "my-node-data-2",
"transport_address": "x.x.x.x:9300",
"node_attributes": {
"xpack.installed": "true",
"transform.node": "false"
},
"node_decision": "no",
"store": {
"in_sync": false,
"allocation_id": "GTO1bElkTs2O5qay11r5kQ",
"store_exception": {
"type": "file_not_found_exception",
"reason": "no segments* file found in NIOFSDirectory@/usr/share/elasticsearch/data/nodes/0/indices/JxYK7aWFRaKrETf3SFitKg/3/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@1fffd548: files: []"
}
}
},
{
"node_id": "cbNnZ8HrRd-nak9rnDLx7w",
"node_name": "my-node-data-5",
"transport_address": "x.x.x.x:9300",
"node_attributes": {
"xpack.installed": "true",
"transform.node": "false"
},
"node_decision": "no",
"store": {
"found": false
}
},
{
"node_id": "kEgf75neRjmrCZrJOGOenQ",
"node_name": "my-node-data-1",
"transport_address": "x.x.x.x:9300",
"node_attributes": {
"xpack.installed": "true",
"transform.node": "false"
},
"node_decision": "no",
"store": {
"found": false
}
},
{
"node_id": "qfJkhKC2T9usePMAk6iZ7A",
"node_name": "my-node-data-11",
"transport_address": "x.x.x.x:9300",
"node_attributes": {
"xpack.installed": "true",
"transform.node": "false"
},
"node_decision": "no",
"store": {
"found": false
}
},
{
"node_id": "skvo0HFrRIepSEr1xZ7pRA",
"node_name": "my-node-data-6",
"transport_address": "x.x.x.x:9300",
"node_attributes": {
"xpack.installed": "true",
"transform.node": "false"
},
"node_decision": "no",
"store": {
"found": false
}
}
]
}
GET my-index/_settings
{
"my-index-000002": {
"settings": {
"index": {
"lifecycle": {
"name": "rollover-primary_shard_10gb",
"rollover_alias": "my-index"
},
"routing": {
"allocation": {
"include": {
"_tier_preference": "data_content"
}
}
},
"number_of_shards": "5",
"provided_name": "my-index-000002",
"creation_date": "1645099648821",
"history": {
"uuid": "DQyMPk-RRxaQmu-DO25Q_Q"
},
"number_of_replicas": "2",
"uuid": "JxYK7aWFRaKrETf3SFitKg",
"version": {
"created": "7160299"
}
}
}
}
}
What do you think the cause can be?
Thank you for the help in advance!
Best regards,
Laszlo