Elasticsearch G1GC over CMS in resolving the GC overhead

Elastic Stack Elasticsearch
1

We are using the ES 7.3 with CMS GC and we are preparing for the rolling upgrade to 7.17 which supports G1GC only

We are getting the GC overhead curently,

[2023-04-25T02:00:41,085][WARN ][o.e.m.j.JvmGcMonitorService] [es7-cf-74] [gc][434743] overhead, spent [717ms] collecting in the last [1s]
[2023-04-25T02:00:45,529][INFO ][o.e.m.j.JvmGcMonitorService] [es7-cf-74] [gc][434747] overhead, spent [438ms] collecting in the last [1.4s]

Is there a chance that G1GC handle this well and we wont get this over head in 7.17

The data node heap spikes & reduce suddenly while overhead and we are facing request failures in CMS recently

2

Maybe.

What is the output from the _cluster/stats?pretty&human API?

3

{
"_nodes": {
"total": 8,
"successful": 8,
"failed": 0
},
"cluster_name": "es7",
"cluster_uuid": "#####",
"timestamp": 1682580993561,
"status": "green",
"indices": {
"count": 40,
"shards": {
"total": 368,
"primaries": 184,
"replication": 1.0,
"index": {
"shards": {
"min": 2,
"max": 10,
"avg": 9.2
},
"primaries": {
"min": 1,
"max": 5,
"avg": 4.6
},
"replication": {
"min": 1.0,
"max": 1.0,
"avg": 1.0
}
}
},
"docs": {
"count": 55845371,
"deleted": 1115791
},
"store": {
"size": "328.9gb",
"size_in_bytes": 353241656447
},
"fielddata": {
"memory_size": "0b",
"memory_size_in_bytes": 0,
"evictions": 0
},
"query_cache": {
"memory_size": "4.2mb",
"memory_size_in_bytes": 4430350,
"total_count": 9853952,
"hit_count": 4249977,
"miss_count": 5603975,
"cache_size": 239,
"cache_count": 104683,
"evictions": 104444
},
"completion": {
"size": "0b",
"size_in_bytes": 0
},
"segments": {
"count": 2265,
"memory": "161.4mb",
"memory_in_bytes": 169322757,
"terms_memory": "31.9mb",
"terms_memory_in_bytes": 33489221,
"stored_fields_memory": "124.5mb",
"stored_fields_memory_in_bytes": 130635336,
"term_vectors_memory": "0b",
"term_vectors_memory_in_bytes": 0,
"norms_memory": "44.8kb",
"norms_memory_in_bytes": 45888,
"points_memory": "3.4mb",
"points_memory_in_bytes": 3640870,
"doc_values_memory": "1.4mb",
"doc_values_memory_in_bytes": 1511442,
"index_writer_memory": "2mb",
"index_writer_memory_in_bytes": 2154026,
"version_map_memory": "410b",
"version_map_memory_in_bytes": 410,
"fixed_bit_set": "6.4mb",
"fixed_bit_set_memory_in_bytes": 6768528,
"max_unsafe_auto_id_timestamp": 1682578685666,
"file_sizes": {
}
}
},
"nodes": {
"count": {
"total": 8,
"coordinating_only": 0,
"data": 5,
"ingest": 0,
"master": 3,
"voting_only": 0
},
"versions": [
"7.3.2"
],
"os": {
"available_processors": 106,
"allocated_processors": 106,
"names": [
{
"name": "Linux",
"count": 8
}
],
"pretty_names": [
{
"pretty_name": "Ubuntu 20.04.6 LTS",
"count": 8
}
],
"mem": {
"total": "824.2gb",
"total_in_bytes": 885082402816,
"free": "168.4gb",
"free_in_bytes": 180843520000,
"used": "655.8gb",
"used_in_bytes": 704238882816,
"free_percent": 20,
"used_percent": 80
}
},
"process": {
"cpu": {
"percent": 10
},
"open_file_descriptors": {
"min": 560,
"max": 6454,
"avg": 3351
}
},
"jvm": {
"max_uptime": "7.8d",
"max_uptime_in_millis": 682300549,
"versions": [
{
"version": "1.8.0_131",
"vm_name": "Java HotSpot(TM) 64-Bit Server VM",
"vm_version": "25.131-b11",
"vm_vendor": "Oracle Corporation",
"bundled_jdk": true,
"using_bundled_jdk": false,
"count": 8
}
],
"mem": {
"heap_used": "22.3gb",
"heap_used_in_bytes": 23950037104,
"heap_max": "56.5gb",
"heap_max_in_bytes": 60699967488
},
"threads": 963
},
"fs": {
"total": "2.8tb",
"total_in_bytes": 3081931849728,
"free": "1tb",
"free_in_bytes": 1198002884608,
"available": "1tb",
"available_in_bytes": 1197902221312
},
"plugins": [
{
"name": "repository-gcs",
"version": "7.3.2",
"elasticsearch_version": "7.3.2",
"java_version": "1.8",
"description": "The GCS repository plugin adds Google Cloud Storage support for repositories.",
"classname": "org.elasticsearch.repositories.gcs.GoogleCloudStoragePlugin",
"extended_plugins": [
],
"has_native_controller": false
}
],
"network_types": {
"transport_types": {
"security4": 8
},
"http_types": {
"security4": 8
}
},
"discovery_types": {
"zen": 8
},
"packaging_types": [
{
"flavor": "default",
"type": "tar",
"count": 8
}
]
}
}

4

You have 8 nodes with about 8 gig of heap each, but your hosts have 820 gig of memory, why is your heap so low? It doesn't need to be super high, but you seem to have lots of memory to use, and I would look to increase the heap size a bit to reduce the pressure.

Also upgrading Elasticsearch and your JVM will help.

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.