X of y shards failed

Type: es_rejected_execution_exception

Reason: rejected execution of org.Elasticsearch.common.util.concurrent.TimedRunnable@5ec3e8 on QueueResizingEsThreadPoolExecutor

I had monitored the CPU and RAM and all the metrics is ok.
also tried to reroute the shard to another node.

still get the same error -
ES and Kibana - V 7.13.2

Can you share more of this error please.

What is the output from the _cluster/stats?pretty&human API?

{
"_nodes": {
"total": 32,
"successful": 32,
"failed": 0
},
"cluster_name": "NAME",
"cluster_uuid": "ID",
"timestamp": 1633232635812,
"status": "green",
"indices": {
"count": 3880,
"shards": {
"total": 8460,
"primaries": 4230,
"replication": 1.0,
"index": {
"shards": {
"min": 2,
"max": 6,
"avg": 2.1804123711340204
},
"primaries": {
"min": 1,
"max": 3,
"avg": 1.0902061855670102
},
"replication": {
"min": 1.0,
"max": 1.0,
"avg": 1.0
}
}
},
"docs": {
"count": 79624464020,
"deleted": 525733109
},
"store": {
"size": "98.1tb",
"size_in_bytes": 107966471451980,
"total_data_set_size": "98.1tb",
"total_data_set_size_in_bytes": 107966471451980,
"reserved": "0b",
"reserved_in_bytes": 0
},
"fielddata": {
"memory_size": "4.2gb",
"memory_size_in_bytes": 4575198556,
"evictions": 1152429
},
"query_cache": {
"memory_size": "40.5gb",
"memory_size_in_bytes": 43564467996,
"total_count": 204429135183,
"hit_count": 16676201921,
"miss_count": 187752933262,
"cache_size": 1222174,
"cache_count": 165961684,
"evictions": 164739510
},
"completion": {
"size": "0b",
"size_in_bytes": 0
},
"segments": {
"count": 212336,
"memory": "14.4gb",
"memory_in_bytes": 15554542606,
"terms_memory": "5.7gb",
"terms_memory_in_bytes": 6149712768,
"stored_fields_memory": "5.9gb",
"stored_fields_memory_in_bytes": 6355820488,
"term_vectors_memory": "0b",
"term_vectors_memory_in_bytes": 0,
"norms_memory": "11.5mb",
"norms_memory_in_bytes": 12161344,
"points_memory": "0b",
"points_memory_in_bytes": 0,
"doc_values_memory": "2.8gb",
"doc_values_memory_in_bytes": 3036848006,
"index_writer_memory": "15.1gb",
"index_writer_memory_in_bytes": 16307004588,
"version_map_memory": "40.4mb",
"version_map_memory_in_bytes": 42431711,
"fixed_bit_set": "101.9mb",
"fixed_bit_set_memory_in_bytes": 106883856,
"max_unsafe_auto_id_timestamp": 1633231863221,
"file_sizes": {

  }
},
"mappings": {
  "field_types": [
    {
      "name": "boolean",
      "count": 58041,
      "index_count": 3248,
      "script_count": 0
    },
    {
      "name": "date",
      "count": 31170,
      "index_count": 3865,
      "script_count": 0
    },
    {
      "name": "double",
      "count": 2269,
      "index_count": 1112,
      "script_count": 0
    },
    {
      "name": "float",
      "count": 105753,
      "index_count": 2670,
      "script_count": 0
    },
    {
      "name": "geo_point",
      "count": 2780,
      "index_count": 940,
      "script_count": 0
    },
    {
      "name": "half_float",
      "count": 56,
      "index_count": 14,
      "script_count": 0
    },
    {
      "name": "integer",
      "count": 1613,
      "index_count": 427,
      "script_count": 0
    },
    {
      "name": "ip",
      "count": 457,
      "index_count": 153,
      "script_count": 0
    },
    {
      "name": "keyword",
      "count": 651893,
      "index_count": 3867,
      "script_count": 0
    },
    {
      "name": "long",
      "count": 305951,
      "index_count": 3726,
      "script_count": 0
    },
    {
      "name": "nested",
      "count": 466,
      "index_count": 221,
      "script_count": 0
    },
    {
      "name": "object",
      "count": 156081,
      "index_count": 3154,
      "script_count": 0
    },
    {
      "name": "short",
      "count": 3,
      "index_count": 1,
      "script_count": 0
    },
    {
      "name": "text",
      "count": 4879,
      "index_count": 481,
      "script_count": 0
    }
  ],
  "runtime_field_types": [
    
  ]
},
"analysis": {
  "char_filter_types": [
    
  ],
  "tokenizer_types": [
    
  ],
  "filter_types": [
    {
      "name": "length",
      "count": 196,
      "index_count": 196
    }
  ],
  "analyzer_types": [
    {
      "name": "custom",
      "count": 426,
      "index_count": 426
    }
  ],
  "built_in_char_filters": [
    
  ],
  "built_in_tokenizers": [
    {
      "name": "lowercase",
      "count": 196,
      "index_count": 196
    },
    {
      "name": "standard",
      "count": 230,
      "index_count": 230
    }
  ],
  "built_in_filters": [
    {
      "name": "asciifolding",
      "count": 230,
      "index_count": 230
    },
    {
      "name": "lowercase",
      "count": 230,
      "index_count": 230
    }
  ],
  "built_in_analyzers": [
    {
      "name": "simple",
      "count": 60,
      "index_count": 60
    },
    {
      "name": "standard",
      "count": 760,
      "index_count": 152
    }
  ]
},
"versions": [
  {
    "version": "6.6.0",
    "index_count": 4,
    "primary_shard_count": 10,
    "total_primary_size": "101.5gb",
    "total_primary_bytes": 108997934776
  },
  {
    "version": "6.8.3",
    "index_count": 284,
    "primary_shard_count": 338,
    "total_primary_size": "2.8tb",
    "total_primary_bytes": 3104741864325
  },
  {
    "version": "7.5.2",
    "index_count": 280,
    "primary_shard_count": 308,
    "total_primary_size": "3.9tb",
    "total_primary_bytes": 4343680811140
  },
  {
    "version": "7.9.3",
    "index_count": 732,
    "primary_shard_count": 770,
    "total_primary_size": "9.5tb",
    "total_primary_bytes": 10445365690866
  },
  {
    "version": "7.13.2",
    "index_count": 2580,
    "primary_shard_count": 2804,
    "total_primary_size": "32.7tb",
    "total_primary_bytes": 35970005960525
  }
]

},
"nodes": {
"count": {
"total": 32,
"coordinating_only": 0,
"data": 27,
"data_cold": 27,
"data_content": 27,
"data_frozen": 27,
"data_hot": 27,
"data_warm": 27,
"ingest": 2,
"master": 3,
"ml": 0,
"remote_cluster_client": 32,
"transform": 27,
"voting_only": 0
},
"versions": [
"7.13.2"
],
"os": {
"available_processors": 238,
"allocated_processors": 238,
"names": [
{
"name": "Linux",
"count": 32
}
],
"pretty_names": [
{
"pretty_name": "Linux",
"count": 5
},
{
"pretty_name": "Linux",
"count": 27
}
],
"architectures": [
{
"arch": "amd64",
"count": 32
}
],
"mem": {
"total": "1.7tb",
"total_in_bytes": 1958548692992,
"free": "51.4gb",
"free_in_bytes": 55194165248,
"used": "1.7tb",
"used_in_bytes": 1903354527744,
"free_percent": 3,
"used_percent": 97
}
},
"process": {
"cpu": {
"percent": 451
},
"open_file_descriptors": {
"min": 1025,
"max": 4413,
"avg": 3813
}
},
"jvm": {
"max_uptime": "88.8d",
"max_uptime_in_millis": 7674601417,
"versions": [
{
"version": "16",
"vm_name": "OpenJDK 64-Bit Server VM",
"vm_version": "16+36",
"vm_vendor": "AdoptOpenJDK",
"bundled_jdk": true,
"using_bundled_jdk": true,
"count": 32
}
],
"mem": {
"heap_used": "488.7gb",
"heap_used_in_bytes": 524813665424,
"heap_max": "911gb",
"heap_max_in_bytes": 978178801664
},
"threads": 3704
},
"fs": {
"total": "123.2tb",
"total_in_bytes": 135478786289664,
"free": "24.7tb",
"free_in_bytes": 27170804113408,
"available": "24.7tb",
"available_in_bytes": 27170804113408
},
"plugins": [
{
"name": "discovery-ec2",
"version": "7.13.2",
"elasticsearch_version": "7.13.2",
"java_version": "1.8",
"description": "The EC2 discovery plugin allows to use AWS API for the unicast discovery mechanism.",
"classname": "org.Elasticsearch.discovery.ec2.Ec2DiscoveryPlugin",
"extended_plugins": [

    ],
    "has_native_controller": false,
    "licensed": false,
    "type": "isolated"
  },
  {
    "name": "repository-s3",
    "version": "7.13.2",
    "elasticsearch_version": "7.13.2",
    "java_version": "1.8",
    "description": "The S3 repository plugin adds S3 repositories",
    "classname": "org.elasticsearch.repositories.s3.S3RepositoryPlugin",
    "extended_plugins": [
      
    ],
    "has_native_controller": false,
    "licensed": false,
    "type": "isolated"
  }
],
"network_types": {
  "transport_types": {
    "netty4": 32
  },
  "http_types": {
    "netty4": 32
  }
},
"discovery_types": {
  "zen": 32
},
"packaging_types": [
  {
    "flavor": "default",
    "type": "rpm",
    "count": 32
  }
],
"ingest": {
  "number_of_pipelines": 8,
  "processor_stats": {
    "gsub": {
      "count": 0,
      "failed": 0,
      "current": 0,
      "time": "0s",
      "time_in_millis": 0
    },
    "remove": {
      "count": 0,
      "failed": 0,
      "current": 0,
      "time": "0s",
      "time_in_millis": 0
    },
    "rename": {
      "count": 0,
      "failed": 0,
      "current": 0,
      "time": "0s",
      "time_in_millis": 0
    },
    "script": {
      "count": 0,
      "failed": 0,
      "current": 0,
      "time": "0s",
      "time_in_millis": 0
    },
    "set": {
      "count": 0,
      "failed": 0,
      "current": 0,
      "time": "0s",
      "time_in_millis": 0
    }
  }
}

}
}

Thank you for you help

any updates ?
what could be the problem?
what do I have to check ?

Are you using Monitoring for your cluster? It looks like you are getting overloads on your node(s) and filling threadpools.

How are you indexing data into Elasticsearch?
What do your Elasticsearch logs show?

yes I have a monitoring process,
indexing - using Logstash
Logs - looks ok nothing relevant I think.