Hello,
We have problem with our Elasticsearch cluster.
Architecture :
11 physical machines (128 Go RAM - 32 CPU - Hardware HDD)
- 3 data instances by machine
- 3 physical machines carry the master's instance in addition to the 3 data instances
HEAP of data instance : 19 Go each and master instance 2 Go each
one application which generated an index of 800G per day :
"refresh_interval": "30s",
"number_of_shards": "36",
"provided_name": "BIGINDEX",
"merge": {
"scheduler": {
"max_thread_count": "1"
"number_of_replicas": "1",
Overview of cluster
JVM Heap: 47.86% (300GB / 627GB)
Documents: 5,321,404,424
Primary Shards: 3,707
Replica Shards: 3,736
Here the cluster stats API :
{
"_nodes" : {
"total" : 36,
"successful" : 36,
"failed" : 0
},
"cluster_name" : "ES_5",
"timestamp" : 1538586960513,
"status" : "green",
"indices" : {
"count" : 396,
"shards" : {
"total" : 7443,
"primaries" : 3707,
"replication" : 1.007823037496628,
"index" : {
"shards" : {
"min" : 2,
"max" : 72,
"avg" : 18.795454545454547
},
"primaries" : {
"min" : 1,
"max" : 36,
"avg" : 9.36111111111111
},
"replication" : {
"min" : 1.0,
"max" : 14.0,
"avg" : 1.0732323232323233
}
}
},
"docs" : {
"count" : 5320385349,
"deleted" : 4995852
},
"store" : {
"size" : "38.9tb",
"size_in_bytes" : 42836390979259,
"throttle_time" : "0s",
"throttle_time_in_millis" : 0
},
"fielddata" : {
"memory_size" : "1.2mb",
"memory_size_in_bytes" : 1333920,
"evictions" : 0
},
"query_cache" : {
"memory_size" : "196.8mb",
"memory_size_in_bytes" : 206415239,
"total_count" : 67884881,
"hit_count" : 24300810,
"miss_count" : 43584071,
"cache_size" : 6992,
"cache_count" : 4056221,
"evictions" : 4049229
},
"completion" : {
"size" : "0b",
"size_in_bytes" : 0
},
"segments" : {
"count" : 120238,
"memory" : "97.4gb",
"memory_in_bytes" : 104610428781,
"terms_memory" : "84.6gb",
"terms_memory_in_bytes" : 90848983399,
"stored_fields_memory" : "7.9gb",
"stored_fields_memory_in_bytes" : 8569212864,
"term_vectors_memory" : "0b",
"term_vectors_memory_in_bytes" : 0,
"norms_memory" : "910.3mb",
"norms_memory_in_bytes" : 954539776,
"points_memory" : "2.1gb",
"points_memory_in_bytes" : 2277826818,
"doc_values_memory" : "1.8gb",
"doc_values_memory_in_bytes" : 1959865924,
"index_writer_memory" : "2.5gb",
"index_writer_memory_in_bytes" : 2707509968,
"version_map_memory" : "7.1mb",
"version_map_memory_in_bytes" : 7473310,
"fixed_bit_set" : "0b",
"fixed_bit_set_memory_in_bytes" : 0,
"max_unsafe_auto_id_timestamp" : 1538545665133,
"file_sizes" : { }
}
},
"nodes" : {
"count" : {
"total" : 36,
"data" : 33,
"coordinating_only" : 0,
"master" : 3,
"ingest" : 0
},
"versions" : [
"5.5.1"
],
"os" : {
"available_processors" : 1152,
"allocated_processors" : 1152,
"names" : [
{
"name" : "Linux",
"count" : 36
}
],
"mem" : {
"total" : "4.4tb",
"total_in_bytes" : 4856688279552,
"free" : "351.1gb",
"free_in_bytes" : 377046077440,
"used" : "4tb",
"used_in_bytes" : 4479642202112,
"free_percent" : 8,
"used_percent" : 92
}
},
"process" : {
"cpu" : {
"percent" : 11
},
"open_file_descriptors" : {
"min" : 1526,
"max" : 2138,
"avg" : 1997
}
},
"jvm" : {
"max_uptime" : "7.1d",
"max_uptime_in_millis" : 617508014,
"versions" : [
{
"version" : "1.8.0_181",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "25.181-b13",
"vm_vendor" : "Oracle Corporation",
"count" : 6
},
{
"version" : "1.8.0_161",
}
],
"mem" : {
"heap_used" : "293.5gb",
"heap_used_in_bytes" : 315195193072,
"heap_max" : "626.6gb",
"heap_max_in_bytes" : 672843890688
},
"threads" : 11234
},
"fs" : {
"total" : "315.4tb",
"total_in_bytes" : 346790843449344,
"free" : "266.7tb",
"free_in_bytes" : 293311977111552,
"available" : "261tb",
"available_in_bytes" : 287033225674752,
"spins" : "true"
},
"plugins" : [
{
"name" : "search-guard-5",
"version" : "5.5.1-15",
"description" : "Provide access control related features for Elasticsearch 5",
"classname" : "com.floragunn.searchguard.SearchGuardPlugin",
"has_native_controller" : false
},
{
"name" : "x-pack",
"version" : "5.5.1",
"description" : "Elasticsearch Expanded Pack Plugin",
"classname" : "org.elasticsearch.xpack.XPackPlugin",
"has_native_controller" : true
}
],
"network_types" : {
"transport_types" : {
"com.floragunn.searchguard.ssl.http.netty.SearchGuardSSLNettyTransport" : 36
},
"http_types" : {
"com.floragunn.searchguard.http.SearchGuardHttpServerTransport" : 36
}
}
}
}
WHAT's WRONG 
Thank you for any helps