I have a four-node es cluster on four hosts, in which there are nearly 1000 indices and 2000 shards.
My physical machine has 128G memory and I assigned 60G to the elasticserach node on it.
Now I find sometimes all the four nodes use nearly 100% heap, and the cluster nearly crashed.
Here is the node stats of one node
"segments": {
"count": 22915,
"memory": "40.7gb",
"memory_in_bytes": 43800441840,
"terms_memory": "36.4gb",
"terms_memory_in_bytes": 39094305767,
"stored_fields_memory": "3.3gb",
"stored_fields_memory_in_bytes": 3643812928,
"term_vectors_memory": "0b",
"term_vectors_memory_in_bytes": 0,
"norms_memory": "9.4mb",
"norms_memory_in_bytes": 9885888,
"points_memory": "699.4mb",
"points_memory_in_bytes": 733477621,
"doc_values_memory": "304.1mb",
"doc_values_memory_in_bytes": 318959636,
"index_writer_memory": "0b",
"index_writer_memory_in_bytes": 0,
"version_map_memory": "0b",
"version_map_memory_in_bytes": 0,
"fixed_bit_set": "0b",
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": 9223372036854776000,
"file_sizes": {}
},
"timestamp": 1535621648079,
"uptime": "9d",
"uptime_in_millis": 782493238,
"mem": {
"heap_used": "54.9gb",
"heap_used_in_bytes": 59043916728,
"heap_used_percent": 91,
"heap_committed": "59.8gb",
"heap_committed_in_bytes": 64223903744,
"heap_max": "59.8gb",
"heap_max_in_bytes": 64223903744,
"non_heap_used": "152.5mb",
"non_heap_used_in_bytes": 159924504,
"non_heap_committed": "163.4mb",
"non_heap_committed_in_bytes": 171429888,
"pools": {
"young": {
"used": "898.4mb",
"used_in_bytes": 942054504,
"max": "1.4gb",
"max_in_bytes": 1605304320,
"peak_used": "1.4gb",
"peak_used_in_bytes": 1605304320,
"peak_max": "1.4gb",
"peak_max_in_bytes": 1605304320
},
"survivor": {
"used": "6.5mb",
"used_in_bytes": 6852128,
"max": "191.3mb",
"max_in_bytes": 200605696,
"peak_used": "191.3mb",
"peak_used_in_bytes": 200605696,
"peak_max": "191.3mb",
"peak_max_in_bytes": 200605696
},
"old": {
"used": "54.1gb",
"used_in_bytes": 58095010096,
"max": "58.1gb",
"max_in_bytes": 62417993728,
"peak_used": "58.1gb",
"peak_used_in_bytes": 62417993728,
"peak_max": "58.1gb",
"peak_max_in_bytes": 62417993728
}
}
},
"threads": {
"count": 339,
"peak_count": 423
},
"gc": {
"collectors": {
"young": {
"collection_count": 151963,
"collection_time": "2h",
"collection_time_in_millis": 7445345
},
"old": {
"collection_count": 16713,
"collection_time": "1h",
"collection_time_in_millis": 3723663
}
}
},
"buffer_pools": {
"direct": {
"count": 313,
"used": "1gb",
"used_in_bytes": 1087008175,
"total_capacity": "1gb",
"total_capacity_in_bytes": 1087008174
},
"mapped": {
"count": 54036,
"used": "21.5tb",
"used_in_bytes": 23726849330328,
"total_capacity": "21.5tb",
"total_capacity_in_bytes": 23726849330328
}
},
"classes": {
"current_loaded_count": 12167,
"total_loaded_count": 12683,
"total_unloaded_count": 516
}
}
Could any body give some suggestions about my problem?