Hi
Elasticsearch nodes are crashing due to java.lang.OutOfMemoryError: Java heap space.
We have 10 GB memory in each node and as per formula total memory/2 -1 we have set JVM as 4.
ava.lang.OutOfMemoryError: Java heap space
[2023-06-27T21:12:11,961][INFO ][o.e.m.j.JvmGcMonitorService] [xxelk-xxx-md2.xxxxxx.com] [gc][408948] overhead, spent [430ms] collecting in the last [1.3s]
Dumping heap to data ...
Unable to create data: File exists
Terminating due to java.lang.OutOfMemoryError: Java heap space
Please find out _cluster/stats?pretty&human
{
"_nodes" : {
"total" : 3,
"successful" : 3,
"failed" : 0
},
"cluster_name" : "xx-xxx",
"cluster_uuid" : "iT7eW-FIQbGMmlVqdKItDg",
"timestamp" : 1687928792818,
"status" : "green",
"indices" : {
"count" : 52,
"shards" : {
"total" : 183,
"primaries" : 84,
"replication" : 1.1785714285714286,
"index" : {
"shards" : {
"min" : 2,
"max" : 10,
"avg" : 3.519230769230769
},
"primaries" : {
"min" : 1,
"max" : 5,
"avg" : 1.6153846153846154
},
"replication" : {
"min" : 1.0,
"max" : 2.0,
"avg" : 1.2115384615384615
}
}
},
"docs" : {
"count" : 4596015,
"deleted" : 2118561
},
"store" : {
"size" : "32gb",
"size_in_bytes" : 34445260661,
"total_data_set_size" : "32gb",
"total_data_set_size_in_bytes" : 34445260661,
"reserved" : "0b",
"reserved_in_bytes" : 0
},
"fielddata" : {
"memory_size" : "25.5kb",
"memory_size_in_bytes" : 26168,
"evictions" : 0
},
"query_cache" : {
"memory_size" : "14.1mb",
"memory_size_in_bytes" : 14883132,
"total_count" : 1524264,
"hit_count" : 119039,
"miss_count" : 1405225,
"cache_size" : 3957,
"cache_count" : 4144,
"evictions" : 187
},
"completion" : {
"size" : "0b",
"size_in_bytes" : 0
},
"segments" : {
"count" : 663,
"memory" : "10.9mb",
"memory_in_bytes" : 11459618,
"terms_memory" : "7.3mb",
"terms_memory_in_bytes" : 7720984,
"stored_fields_memory" : "326.2kb",
"stored_fields_memory_in_bytes" : 334040,
"term_vectors_memory" : "115.2kb",
"term_vectors_memory_in_bytes" : 118024,
"norms_memory" : "1.2mb",
"norms_memory_in_bytes" : 1288128,
"points_memory" : "0b",
"points_memory_in_bytes" : 0,
"doc_values_memory" : "1.9mb",
"doc_values_memory_in_bytes" : 1998442,
"index_writer_memory" : "1.6mb",
"index_writer_memory_in_bytes" : 1781972,
"version_map_memory" : "0b",
"version_map_memory_in_bytes" : 0,
"fixed_bit_set" : "1.7mb",
"fixed_bit_set_memory_in_bytes" : 1846360,
"max_unsafe_auto_id_timestamp" : 1687926871781,
"file_sizes" : { }
},
"mappings" : {
"field_types" : [
{
"name" : "boolean",
"count" : 67,
"index_count" : 27,
"script_count" : 0
},
{
"name" : "constant_keyword",
"count" : 6,
"index_count" : 2,
"script_count" : 0
},
{
"name" : "date",
"count" : 133,
"index_count" : 33,
"script_count" : 0
},
{
"name" : "double",
"count" : 2,
"index_count" : 2,
"script_count" : 0
},
{
"name" : "float",
"count" : 79,
"index_count" : 12,
"script_count" : 0
},
{
"name" : "geo_point",
"count" : 2,
"index_count" : 2,
"script_count" : 0
},
{
"name" : "half_float",
"count" : 56,
"index_count" : 14,
"script_count" : 0
},
{
"name" : "integer",
"count" : 155,
"index_count" : 8,
"script_count" : 0
},
{
"name" : "ip",
"count" : 2,
"index_count" : 2,
"script_count" : 0
},
{
"name" : "keyword",
"count" : 913,
"index_count" : 42,
"script_count" : 0
},
{
"name" : "long",
"count" : 1360,
"index_count" : 33,
"script_count" : 0
},
{
"name" : "nested",
"count" : 29,
"index_count" : 15,
"script_count" : 0
},
{
"name" : "object",
"count" : 878,
"index_count" : 28,
"script_count" : 0
},
{
"name" : "scaled_float",
"count" : 1,
"index_count" : 1,
"script_count" : 0
},
{
"name" : "text",
"count" : 575,
"index_count" : 31,
"script_count" : 0
},
{
"name" : "version",
"count" : 4,
"index_count" : 4,
"script_count" : 0
}
],
"runtime_field_types" :
},
"analysis" : {
"char_filter_types" : ,
"tokenizer_types" : ,
"filter_types" : [
{
"name" : "stemmer",
"count" : 6,
"index_count" : 2
},
{
"name" : "stop",
"count" : 4,
"index_count" : 2
},
{
"name" : "synonym_graph",
"count" : 4,
"index_count" : 2
}
],
"analyzer_types" : [
{
"name" : "custom",
"count" : 6,
"index_count" : 2
}
],
"built_in_char_filters" : ,
"built_in_tokenizers" : [
{
"name" : "keyword",
"count" : 2,
"index_count" : 2
},
{
"name" : "standard",
"count" : 4,
"index_count" : 2
}
],
"built_in_filters" : [
{
"name" : "lowercase",
"count" : 6,
"index_count" : 2
}
],
"built_in_analyzers" : [
{
"name" : "arabic",
"count" : 5,
"index_count" : 1
},
{
"name" : "basque",
"count" : 5,
"index_count" : 1
},
{
"name" : "brazilian",
"count" : 5,
"index_count" : 1
},
{
"name" : "bulgarian",
"count" : 5,
"index_count" : 1
},
{
"name" : "catalan",
"count" : 12,
"index_count" : 2
},
{
"name" : "cjk",
"count" : 5,
"index_count" : 1
},
{
"name" : "czech",
"count" : 5,
"index_count" : 1
},
{
"name" : "danish",
"count" : 5,
"index_count" : 1
},
{
"name" : "dutch",
"count" : 10,
"index_count" : 1
},
{
"name" : "english",
"count" : 34,
"index_count" : 2
},
{
"name" : "finnish",
"count" : 5,
"index_count" : 1
},
{
"name" : "french",
"count" : 12,
"index_count" : 2
},
{
"name" : "german",
"count" : 7,
"index_count" : 2
},
{
"name" : "greek",
"count" : 5,
"index_count" : 1
},
{
"name" : "hindi",
"count" : 5,
"index_count" : 1
},
{
"name" : "hungarian",
"count" : 5,
"index_count" : 1
},
{
"name" : "italian",
"count" : 5,
"index_count" : 1
},
{
"name" : "kuromoji",
"count" : 9,
"index_count" : 2
},
{
"name" : "norwegian",
"count" : 5,
"index_count" : 1
},
{
"name" : "polish",
"count" : 5,
"index_count" : 1
},
{
"name" : "portuguese",
"count" : 5,
"index_count" : 1
},
{
"name" : "romanian",
"count" : 5,
"index_count" : 1
},
{
"name" : "russian",
"count" : 5,
"index_count" : 1
},
{
"name" : "smartcn",
"count" : 12,
"index_count" : 2
},
{
"name" : "spanish",
"count" : 7,
"index_count" : 2
},
{
"name" : "swedish",
"count" : 5,
"index_count" : 1
},
{
"name" : "thai",
"count" : 5,
"index_count" : 1
},
{
"name" : "turkish",
"count" : 5,
"index_count" : 1
}
]
},
"versions" : [
{
"version" : "6.8.7",
"index_count" : 11,
"primary_shard_count" : 39,
"total_primary_size" : "1.1mb",
"total_primary_bytes" : 1197211
},
{
"version" : "7.17.5",
"index_count" : 41,
"primary_shard_count" : 45,
"total_primary_size" : "11.4gb",
"total_primary_bytes" : 12269704622
}
]
},
"nodes" : {
"count" : {
"total" : 3,
"coordinating_only" : 0,
"data" : 3,
"data_cold" : 3,
"data_content" : 3,
"data_frozen" : 3,
"data_hot" : 3,
"data_warm" : 3,
"ingest" : 3,
"master" : 3,
"ml" : 0,
"remote_cluster_client" : 3,
"transform" : 3,
"voting_only" : 0
},
"versions" : [
"7.17.5"
],
"os" : {
"available_processors" : 12,
"allocated_processors" : 12,
"names" : [
{
"name" : "Linux",
"count" : 3
}
],
"pretty_names" : [
{
"pretty_name" : "CentOS Linux 7 (Core)",
"count" : 3
}
],
"architectures" : [
{
"arch" : "amd64",
"count" : 3
}
],
"mem" : {
"total" : "29.2gb",
"total_in_bytes" : 31451197440,
"free" : "491.6mb",
"free_in_bytes" : 515579904,
"used" : "28.8gb",
"used_in_bytes" : 30935617536,
"free_percent" : 2,
"used_percent" : 98
}
},
"process" : {
"cpu" : {
"percent" : 9
},
"open_file_descriptors" : {
"min" : 759,
"max" : 835,
"avg" : 788
}
},
"jvm" : {
"max_uptime" : "2.1d",
"max_uptime_in_millis" : 187409634,
"versions" : [
{
"version" : "18.0.1.1",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "18.0.1.1+2-6",
"vm_vendor" : "Oracle Corporation",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"count" : 3
}
],
"mem" : {
"heap_used" : "5.9gb",
"heap_used_in_bytes" : 6349027064,
"heap_max" : "12gb",
"heap_max_in_bytes" : 12884901888
},
"threads" : 244
},
"fs" : {
"total" : "294.9gb",
"total_in_bytes" : 316651905024,
"free" : "249gb",
"free_in_bytes" : 267377627136,
"available" : "233.9gb",
"available_in_bytes" : 251221807104
},
"plugins" : [
{
"name" : "analysis-phonetic",
"version" : "7.17.5",
"elasticsearch_version" : "7.17.5",
"java_version" : "1.8",
"description" : "The Phonetic Analysis plugin integrates phonetic token filter analysis with elasticsearch.",
"classname" : "org.elasticsearch.plugin.analysis.phonetic.AnalysisPhoneticPlugin",
"extended_plugins" : ,
"has_native_controller" : false,
"licensed" : false,
"type" : "isolated"
},
{
"name" : "analysis-kuromoji",
"version" : "7.17.5",
"elasticsearch_version" : "7.17.5",
"java_version" : "1.8",
"description" : "The Japanese (kuromoji) Analysis plugin integrates Lucene kuromoji analysis module into elasticsearch.",
"classname" : "org.elasticsearch.plugin.analysis.kuromoji.AnalysisKuromojiPlugin",
"extended_plugins" : ,
"has_native_controller" : false,
"licensed" : false,
"type" : "isolated"
},
{
"name" : "analysis-icu",
"version" : "7.17.5",
"elasticsearch_version" : "7.17.5",
"java_version" : "1.8",
"description" : "The ICU Analysis plugin integrates the Lucene ICU module into Elasticsearch, adding ICU-related analysis components.",
"classname" : "org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin",
"extended_plugins" : ,
"has_native_controller" : false,
"licensed" : false,
"type" : "isolated"
},
{
"name" : "analysis-ukrainian",
"version" : "7.17.5",
"elasticsearch_version" : "7.17.5",
"java_version" : "1.8",
"description" : "The Ukrainian Analysis plugin integrates the Lucene UkrainianMorfologikAnalyzer into elasticsearch.",
"classname" : "org.elasticsearch.plugin.analysis.ukrainian.AnalysisUkrainianPlugin",
"extended_plugins" : ,
"has_native_controller" : false,
"licensed" : false,
"type" : "isolated"
},
{
"name" : "analysis-smartcn",
"version" : "7.17.5",
"elasticsearch_version" : "7.17.5",
"java_version" : "1.8",
"description" : "Smart Chinese Analysis plugin integrates Lucene Smart Chinese analysis module into elasticsearch.",
"classname" : "org.elasticsearch.plugin.analysis.smartcn.AnalysisSmartChinesePlugin",
"extended_plugins" : ,
"has_native_controller" : false,
"licensed" : false,
"type" : "isolated"
},
{
"name" : "analysis-nori",
"version" : "7.17.5",
"elasticsearch_version" : "7.17.5",
"java_version" : "1.8",
"description" : "The Korean (nori) Analysis plugin integrates Lucene nori analysis module into elasticsearch.",
"classname" : "org.elasticsearch.plugin.analysis.nori.AnalysisNoriPlugin",
"extended_plugins" : ,
"has_native_controller" : false,
"licensed" : false,
"type" : "isolated"
},
{
"name" : "analysis-stempel",
"version" : "7.17.5",
"elasticsearch_version" : "7.17.5",
"java_version" : "1.8",
"description" : "The Stempel (Polish) Analysis plugin integrates Lucene stempel (polish) analysis module into elasticsearch.",
"classname" : "org.elasticsearch.plugin.analysis.stempel.AnalysisStempelPlugin",
"extended_plugins" : ,
"has_native_controller" : false,
"licensed" : false,
"type" : "isolated"
}
],
"network_types" : {
"transport_types" : {
"netty4" : 3
},
"http_types" : {
"netty4" : 3
}
},
"discovery_types" : {
"zen" : 3
},
"packaging_types" : [
{
"flavor" : "default",
"type" : "tar",
"count" : 3
}
],
"ingest" : {
"number_of_pipelines" : 3,
"processor_stats" : {
"gsub" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
},
"rename" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
},
"script" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
},
"set" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
}
}
}
}
}