Elasticsearch G1GC over CMS in resolving the GC overhead

[navaneethan (Navaneethakrishnan)]

We are using the ES 7.3 with CMS GC and we are preparing for the rolling upgrade to 7.17 which supports G1GC only

We are getting the GC overhead curently,

[2023-04-25T02:00:41,085][WARN ][o.e.m.j.JvmGcMonitorService] [es7-cf-74] [gc][434743] overhead, spent [717ms] collecting in the last [1s]
[2023-04-25T02:00:45,529][INFO ][o.e.m.j.JvmGcMonitorService] [es7-cf-74] [gc][434747] overhead, spent [438ms] collecting in the last [1.4s]

Is there a chance that G1GC handle this well and we wont get this over head in 7.17

The data node heap spikes & reduce suddenly while overhead and we are facing request failures in CMS recently

[warkolm (Mark Walkom)]

@navaneethan

Is there a chance that G1GC handle this well and we wont get this over head in 7.17

Maybe.

What is the output from the _cluster/stats?pretty&human API?

[navaneethan (Navaneethakrishnan)]

{
"_nodes": {
"total": 8,
"successful": 8,
"failed": 0
},
"cluster_name": "es7",
"cluster_uuid": "#####",
"timestamp": 1682580993561,
"status": "green",
"indices": {
"count": 40,
"shards": {
"total": 368,
"primaries": 184,
"replication": 1.0,
"index": {
"shards": {
"min": 2,
"max": 10,
"avg": 9.2
},
"primaries": {
"min": 1,
"max": 5,
"avg": 4.6
},
"replication": {
"min": 1.0,
"max": 1.0,
"avg": 1.0
}
}
},
"docs": {
"count": 55845371,
"deleted": 1115791
},
"store": {
"size": "328.9gb",
"size_in_bytes": 353241656447
},
"fielddata": {
"memory_size": "0b",
"memory_size_in_bytes": 0,
"evictions": 0
},
"query_cache": {
"memory_size": "4.2mb",
"memory_size_in_bytes": 4430350,
"total_count": 9853952,
"hit_count": 4249977,
"miss_count": 5603975,
"cache_size": 239,
"cache_count": 104683,
"evictions": 104444
},
"completion": {
"size": "0b",
"size_in_bytes": 0
},
"segments": {
"count": 2265,
"memory": "161.4mb",
"memory_in_bytes": 169322757,
"terms_memory": "31.9mb",
"terms_memory_in_bytes": 33489221,
"stored_fields_memory": "124.5mb",
"stored_fields_memory_in_bytes": 130635336,
"term_vectors_memory": "0b",
"term_vectors_memory_in_bytes": 0,
"norms_memory": "44.8kb",
"norms_memory_in_bytes": 45888,
"points_memory": "3.4mb",
"points_memory_in_bytes": 3640870,
"doc_values_memory": "1.4mb",
"doc_values_memory_in_bytes": 1511442,
"index_writer_memory": "2mb",
"index_writer_memory_in_bytes": 2154026,
"version_map_memory": "410b",
"version_map_memory_in_bytes": 410,
"fixed_bit_set": "6.4mb",
"fixed_bit_set_memory_in_bytes": 6768528,
"max_unsafe_auto_id_timestamp": 1682578685666,
"file_sizes": {
}
}
},
"nodes": {
"count": {
"total": 8,
"coordinating_only": 0,
"data": 5,
"ingest": 0,
"master": 3,
"voting_only": 0
},
"versions": [
"7.3.2"
],
"os": {
"available_processors": 106,
"allocated_processors": 106,
"names": [
{
"name": "Linux",
"count": 8
}
],
"pretty_names": [
{
"pretty_name": "Ubuntu 20.04.6 LTS",
"count": 8
}
],
"mem": {
"total": "824.2gb",
"total_in_bytes": 885082402816,
"free": "168.4gb",
"free_in_bytes": 180843520000,
"used": "655.8gb",
"used_in_bytes": 704238882816,
"free_percent": 20,
"used_percent": 80
}
},
"process": {
"cpu": {
"percent": 10
},
"open_file_descriptors": {
"min": 560,
"max": 6454,
"avg": 3351
}
},
"jvm": {
"max_uptime": "7.8d",
"max_uptime_in_millis": 682300549,
"versions": [
{
"version": "1.8.0_131",
"vm_name": "Java HotSpot(TM) 64-Bit Server VM",
"vm_version": "25.131-b11",
"vm_vendor": "Oracle Corporation",
"bundled_jdk": true,
"using_bundled_jdk": false,
"count": 8
}
],
"mem": {
"heap_used": "22.3gb",
"heap_used_in_bytes": 23950037104,
"heap_max": "56.5gb",
"heap_max_in_bytes": 60699967488
},
"threads": 963
},
"fs": {
"total": "2.8tb",
"total_in_bytes": 3081931849728,
"free": "1tb",
"free_in_bytes": 1198002884608,
"available": "1tb",
"available_in_bytes": 1197902221312
},
"plugins": [
{
"name": "repository-gcs",
"version": "7.3.2",
"elasticsearch_version": "7.3.2",
"java_version": "1.8",
"description": "The GCS repository plugin adds Google Cloud Storage support for repositories.",
"classname": "org.elasticsearch.repositories.gcs.GoogleCloudStoragePlugin",
"extended_plugins": [
],
"has_native_controller": false
}
],
"network_types": {
"transport_types": {
"security4": 8
},
"http_types": {
"security4": 8
}
},
"discovery_types": {
"zen": 8
},
"packaging_types": [
{
"flavor": "default",
"type": "tar",
"count": 8
}
]
}
}

[warkolm (Mark Walkom)]

You have 8 nodes with about 8 gig of heap each, but your hosts have 820 gig of memory, why is your heap so low? It doesn't need to be super high, but you seem to have lots of memory to use, and I would look to increase the heap size a bit to reduce the pressure.

Also upgrading Elasticsearch and your JVM will help.