Hi,
I've got many error messages like that:
{
"index" : "logstash-prod_operations_clear-001098",
"shard" : 0,
"primary" : false,
"current_state" : "unassigned",
"unassigned_info" : {
"reason" : "ALLOCATION_FAILED",
"at" : "2023-03-07T09:23:48.390Z",
"failed_allocation_attempts" : 5,
"details" : "failed shard on node [8zemZfddQOm3iNFio-GgsA]: failed recovery, failure RecoveryFailedException[[logstash-prod_operations_clear-001098][0]: Recovery failed from {elastic-cold-com-15-rz1}{UTpMrVw1SW6PAJlgjzEbAg}{ng25sO6yTmyxVSoJighbZQ}{10.1.6.87}{10.1.6.87:9300}{cdfhrstw}{rz=rz1, xpack.installed=true, storage=hdd, transform.node=true} into {elastic-cold-com-16-rz2}{8zemZfddQOm3iNFio-GgsA}{7RT9r9OeQbKZ0qz7E7eECw}{10.2.6.87}{10.2.6.87:9300}{cdfhrstw}{xpack.installed=true, transform.node=true, rz=rz2, storage=hdd}]; nested: RemoteTransportException[[elastic-cold-com-15-rz1][10.1.6.87:9300][internal:index/shard/recovery/start_recovery]]; nested: CircuitBreakingException[[parent] Data too large, data for [internal:index/shard/recovery/start_recovery] would be [6363867618/5.9gb], which is larger than the limit of [6120328396/5.6gb], real usage: [6363866232/5.9gb], new bytes reserved: [1386/1.3kb], usages [request=8736/8.5kb, fielddata=144290/140.9kb, in_flight_requests=1386/1.3kb, model_inference=0/0b, accounting=578797672/551.9mb]]; ",
"last_allocation_status" : "no_attempt"
},
"can_allocate" : "awaiting_info",
"allocate_explanation" : "cannot allocate because information about existing shard data is still being retrieved from some of the nodes",
"node_allocation_decisions" : [
{
"node_id" : "-LZu73W2TgOa87dU87Nx0A",
"node_name" : "elastic-cold-com-22-rz2",
"transport_address" : "10.2.6.91:9300",
"node_attributes" : {
"rz" : "rz2",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "yes"
},
{
"node_id" : "3jgTzoQdQJSpQaLaQhlMPg",
"node_name" : "elastic-cold-com-4-rz2",
"transport_address" : "10.2.6.78:9300",
"node_attributes" : {
"rz" : "rz2",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "yes"
},
{
"node_id" : "6fUpig1RQc-161P30ZG1CA",
"node_name" : "elastic-cold-com-6-rz2",
"transport_address" : "10.2.6.79:9300",
"node_attributes" : {
"rz" : "rz2",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "yes"
},
{
"node_id" : "8zemZfddQOm3iNFio-GgsA",
"node_name" : "elastic-cold-com-16-rz2",
"transport_address" : "10.2.6.87:9300",
"node_attributes" : {
"rz" : "rz2",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "yes"
},
{
"node_id" : "CE_ICelORTCTTNv8GzcICg",
"node_name" : "elastic-cold-com-8-rz2",
"transport_address" : "10.2.6.82:9300",
"node_attributes" : {
"rz" : "rz2",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "yes"
},
{
"node_id" : "P3qZHvSQTReIUUtUw-J6iA",
"node_name" : "elastic-cold-com-2-rz2",
"transport_address" : "10.2.6.77:9300",
"node_attributes" : {
"rz" : "rz2",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "yes"
},
{
"node_id" : "WTFqNa4zTh-dHsp7aryo2w",
"node_name" : "elastic-cold-com-18-rz2",
"transport_address" : "10.2.6.88:9300",
"node_attributes" : {
"rz" : "rz2",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "yes"
},
{
"node_id" : "Wv3DS25qTimNewVFmS-L_A",
"node_name" : "elastic-cold-com-10-rz2",
"transport_address" : "10.2.6.84:9300",
"node_attributes" : {
"rz" : "rz2",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "yes"
},
{
"node_id" : "diEioxR1RRSHP8GEg9Vn8g",
"node_name" : "elastic-cold-com-12-rz2",
"transport_address" : "10.2.6.85:9300",
"node_attributes" : {
"rz" : "rz2",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "yes"
},
{
"node_id" : "1d0fBM28Q2W9J3lAXg6BEA",
"node_name" : "elastic-cold-com-20-rz2",
"transport_address" : "10.2.6.89:9300",
"node_attributes" : {
"rz" : "rz2",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "throttled",
"deciders" : [
{
"decider" : "throttling",
"decision" : "THROTTLE",
"explanation" : "reached the limit of incoming shard recoveries [2], cluster setting [cluster.routing.allocation.node_concurrent_incoming_recoveries=2] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
}
]
},
{
"node_id" : "KLV3tbJvRmGC8GdgpXV0vQ",
"node_name" : "elastic-cold-com-14-rz2",
"transport_address" : "10.2.6.86:9300",
"node_attributes" : {
"rz" : "rz2",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "throttled",
"deciders" : [
{
"decider" : "throttling",
"decision" : "THROTTLE",
"explanation" : "reached the limit of incoming shard recoveries [2], cluster setting [cluster.routing.allocation.node_concurrent_incoming_recoveries=2] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
}
]
},
{
"node_id" : "4aylJLz8SJ2P64ciJVtPtg",
"node_name" : "elastic-cold-com-9-rz1",
"transport_address" : "10.1.6.84:9300",
"node_attributes" : {
"rz" : "rz1",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "awareness",
"decision" : "NO",
"explanation" : "there are too many copies of the shard allocated to nodes with attribute [rz], there are [2] total configured shard copies for this shard id and [2] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id" : "7nMPX7b-T8KDLWIkqQ8afg",
"node_name" : "elastic-cold-com-13-rz1",
"transport_address" : "10.1.6.86:9300",
"node_attributes" : {
"rz" : "rz1",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "awareness",
"decision" : "NO",
"explanation" : "there are too many copies of the shard allocated to nodes with attribute [rz], there are [2] total configured shard copies for this shard id and [2] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id" : "9pz0HbzKQhi05UW9Zpb-ng",
"node_name" : "elastic-hot-com-6-rz2",
"transport_address" : "10.2.6.74:9300",
"node_attributes" : {
"rz" : "rz2",
"xpack.installed" : "true",
"storage" : "ssd",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not match index setting [index.routing.allocation.require] filters [storage:"hdd"]"""
}
]
},
{
"node_id" : "H-HREaclQce2POdNnx7-MQ",
"node_name" : "elastic-cold-com-11-rz1",
"transport_address" : "10.1.6.85:9300",
"node_attributes" : {
"rz" : "rz1",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "awareness",
"decision" : "NO",
"explanation" : "there are too many copies of the shard allocated to nodes with attribute [rz], there are [2] total configured shard copies for this shard id and [2] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id" : "JivunSuJRfucH1OkjOsxWw",
"node_name" : "elastic-cold-com-3-rz1",
"transport_address" : "10.1.6.78:9300",
"node_attributes" : {
"rz" : "rz1",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "awareness",
"decision" : "NO",
"explanation" : "there are too many copies of the shard allocated to nodes with attribute [rz], there are [2] total configured shard copies for this shard id and [2] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id" : "NH8YRPyZS-OzJVs-Qn8wCg",
"node_name" : "elastic-cold-com-5-rz1",
"transport_address" : "10.1.6.79:9300",
"node_attributes" : {
"rz" : "rz1",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "awareness",
"decision" : "NO",
"explanation" : "there are too many copies of the shard allocated to nodes with attribute [rz], there are [2] total configured shard copies for this shard id and [2] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id" : "TggXs0TRQx2MRqYjve7YHw",
"node_name" : "elastic-cold-com-21-rz1",
"transport_address" : "10.1.6.90:9300",
"node_attributes" : {
"rz" : "rz1",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "disk_threshold",
"decision" : "NO",
"explanation" : "the node is above the low watermark cluster setting [cluster.routing.allocation.disk.watermark.low=90%], using more disk space than the maximum allowed [90.0%], actual free: [7.483555394733263%]"
},
{
"decider" : "awareness",
"decision" : "NO",
"explanation" : "there are too many copies of the shard allocated to nodes with attribute [rz], there are [2] total configured shard copies for this shard id and [2] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id" : "TndIMiQSTS-hJkapNLJHSw",
"node_name" : "elastic-hot-com-4-rz2",
"transport_address" : "10.2.6.76:9300",
"node_attributes" : {
"rz" : "rz2",
"xpack.installed" : "true",
"storage" : "ssd",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not match index setting [index.routing.allocation.require] filters [storage:"hdd"]"""
}
]
},
{
"node_id" : "Tp-57eKqTByni5z3Sy8_Aw",
"node_name" : "elastic-cold-com-17-rz1",
"transport_address" : "10.1.6.88:9300",
"node_attributes" : {
"rz" : "rz1",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "awareness",
"decision" : "NO",
"explanation" : "there are too many copies of the shard allocated to nodes with attribute [rz], there are [2] total configured shard copies for this shard id and [2] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id" : "UTpMrVw1SW6PAJlgjzEbAg",
"node_name" : "elastic-cold-com-15-rz1",
"transport_address" : "10.1.6.87:9300",
"node_attributes" : {
"rz" : "rz1",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "same_shard",
"decision" : "NO",
"explanation" : "a copy of this shard is already allocated to this node [[logstash-prod_operations_clear-001098][0], node[UTpMrVw1SW6PAJlgjzEbAg], [P], s[STARTED], a[id=ZjhFWijkTuuhoRiBUDsDAA]]"
},
{
"decider" : "throttling",
"decision" : "THROTTLE",
"explanation" : "reached the limit of incoming shard recoveries [2], cluster setting [cluster.routing.allocation.node_concurrent_incoming_recoveries=2] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
},
{
"decider" : "awareness",
"decision" : "NO",
"explanation" : "there are too many copies of the shard allocated to nodes with attribute [rz], there are [2] total configured shard copies for this shard id and [2] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id" : "bN-qQ0iXTEm7zSMXX3jVYg",
"node_name" : "elastic-cold-com-7-rz1",
"transport_address" : "10.1.6.82:9300",
"node_attributes" : {
"rz" : "rz1",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "awareness",
"decision" : "NO",
"explanation" : "there are too many copies of the shard allocated to nodes with attribute [rz], there are [2] total configured shard copies for this shard id and [2] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id" : "dpuOVN5qR2qlGHTT3RXylQ",
"node_name" : "elastic-cold-com-23-rz1",
"transport_address" : "10.1.6.91:9300",
"node_attributes" : {
"rz" : "rz1",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "throttling",
"decision" : "THROTTLE",
"explanation" : "reached the limit of incoming shard recoveries [2], cluster setting [cluster.routing.allocation.node_concurrent_incoming_recoveries=2] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
},
{
"decider" : "awareness",
"decision" : "NO",
"explanation" : "there are too many copies of the shard allocated to nodes with attribute [rz], there are [2] total configured shard copies for this shard id and [2] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id" : "fkkU0UULQtO-ETf2QcN8ww",
"node_name" : "elastic-cold-com-19-rz1",
"transport_address" : "10.1.6.89:9300",
"node_attributes" : {
"rz" : "rz1",
"xpack.installed" : "true",
"storage" : "hdd",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "awareness",
"decision" : "NO",
"explanation" : "there are too many copies of the shard allocated to nodes with attribute [rz], there are [2] total configured shard copies for this shard id and [2] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id" : "luOxP-ExTi26KigJb8J0ng",
"node_name" : "elastic-hot-com-3-rz1",
"transport_address" : "10.1.6.76:9300",
"node_attributes" : {
"rz" : "rz1",
"xpack.installed" : "true",
"storage" : "ssd",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not match index setting [index.routing.allocation.require] filters [storage:"hdd"]"""
},
{
"decider" : "awareness",
"decision" : "NO",
"explanation" : "there are too many copies of the shard allocated to nodes with attribute [rz], there are [2] total configured shard copies for this shard id and [2] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id" : "mPXUU9zNQrKic0sPxz7pVQ",
"node_name" : "elastic-hot-com-2-rz2",
"transport_address" : "10.2.6.75:9300",
"node_attributes" : {
"rz" : "rz2",
"xpack.installed" : "true",
"storage" : "ssd",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not match index setting [index.routing.allocation.require] filters [storage:"hdd"]"""
}
]
},
{
"node_id" : "oDPOAMY2RQWmdpqDqONOzA",
"node_name" : "elastic-hot-com-5-rz1",
"transport_address" : "10.1.6.77:9300",
"node_attributes" : {
"rz" : "rz1",
"xpack.installed" : "true",
"storage" : "ssd",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not match index setting [index.routing.allocation.require] filters [storage:"hdd"]"""
},
{
"decider" : "awareness",
"decision" : "NO",
"explanation" : "there are too many copies of the shard allocated to nodes with attribute [rz], there are [2] total configured shard copies for this shard id and [2] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id" : "w3GlzXS1T66aVrZJPftZ9A",
"node_name" : "elastic-hot-com-1-rz1",
"transport_address" : "10.1.6.75:9300",
"node_attributes" : {
"rz" : "rz1",
"xpack.installed" : "true",
"storage" : "ssd",
"transform.node" : "true"
},
"node_decision" : "no",
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not match index setting [index.routing.allocation.require] filters [storage:"hdd"]"""
},
{
"decider" : "awareness",
"decision" : "NO",
"explanation" : "there are too many copies of the shard allocated to nodes with attribute [rz], there are [2] total configured shard copies for this shard id and [2] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
}
]
}
Can someone give me an advice, what can I do?