Elasticsearch - getting GC and the node reconnect from the cluster

Elastic Stack Elasticsearch
1

we are getting GC and during that time heap usage is reducing and even sometimes the node reconnects from the cluster

below are the params we are using and the GC

-XX:+UseConcMarkSweepGC
-XX:+UseParNewGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseCMSInitiatingOccupancyOnly

[2023-03-28T20:08:48,861][WARN ][o.e.m.j.JvmGcMonitorService] [node#] [gc][353382] overhead, spent [582ms] collecting in the last [1s]

when the GC collected more that 4-5 sec the node went unresponsive

ES config
7.3.2
Java 8
CMS GC

2

What is the output from the _cluster/stats?pretty&human API?

Please note that version is EOL and no longer supported, you should be looking to upgrade as a matter of urgency.

1 Like
3

{
"_nodes": {
"total": 8,
"successful": 8,
"failed": 0
},
"cluster_name": "####",
"cluster_uuid": "####",
"timestamp": 1680616087075,
"status": "green",
"indices": {
"count": 465,
"shards": {
"total": 4626,
"primaries": 2313,
"replication": 1.0,
"index": {
"shards": {
"min": 2,
"max": 10,
"avg": 9.948387096774194
},
"primaries": {
"min": 1,
"max": 5,
"avg": 4.974193548387097
},
"replication": {
"min": 1.0,
"max": 1.0,
"avg": 1.0
}
}
},
"docs": {
"count": 375345756,
"deleted": 24576257
},
"store": {
"size": "834.5gb",
"size_in_bytes": 896067035648
},
"fielddata": {
"memory_size": "115.5kb",
"memory_size_in_bytes": 118312,
"evictions": 0
},
"query_cache": {
"memory_size": "201.7mb",
"memory_size_in_bytes": 211509652,
"total_count": 77104706,
"hit_count": 19919292,
"miss_count": 57185414,
"cache_size": 1850,
"cache_count": 2508790,
"evictions": 2506940
},
"completion": {
"size": "0b",
"size_in_bytes": 0
},
"segments": {
"count": 12168,
"memory": "449.2mb",
"memory_in_bytes": 471078152,
"terms_memory": "181.7mb",
"terms_memory_in_bytes": 190594436,
"stored_fields_memory": "239.8mb",
"stored_fields_memory_in_bytes": 251509664,
"term_vectors_memory": "0b",
"term_vectors_memory_in_bytes": 0,
"norms_memory": "1.9mb",
"norms_memory_in_bytes": 2046080,
"points_memory": "14.6mb",
"points_memory_in_bytes": 15402506,
"doc_values_memory": "10.9mb",
"doc_values_memory_in_bytes": 11525466,
"index_writer_memory": "0b",
"index_writer_memory_in_bytes": 0,
"version_map_memory": "0b",
"version_map_memory_in_bytes": 0,
"fixed_bit_set": "76.3mb",
"fixed_bit_set_memory_in_bytes": 80035008,
"max_unsafe_auto_id_timestamp": 1680528380254,
"file_sizes": {

  }
}

},
"nodes": {
"count": {
"total": 8,
"coordinating_only": 0,
"data": 5,
"ingest": 0,
"master": 3,
"voting_only": 0
},
"versions": [
"7.3.2"
],
"os": {
"available_processors": 108,
"allocated_processors": 108,
"names": [
{
"name": "Linux",
"count": 8
}
],
"pretty_names": [
{
"pretty_name": "Ubuntu 18.04.6 LTS",
"count": 8
}
],
"mem": {
"total": "848.9gb",
"total_in_bytes": 911572004864,
"free": "95.1gb",
"free_in_bytes": 102122815488,
"used": "753.8gb",
"used_in_bytes": 809449189376,
"free_percent": 11,
"used_percent": 89
}
},
"process": {
"cpu": {
"percent": 19
},
"open_file_descriptors": {
"min": 561,
"max": 47730,
"avg": 26621
}
},
"jvm": {
"max_uptime": "361.2d",
"max_uptime_in_millis": 31208114474,
"versions": [
{
"version": "1.8.0_131",
"vm_name": "Java HotSpot(TM) 64-Bit Server VM",
"vm_version": "25.131-b11",
"vm_vendor": "Oracle Corporation",
"bundled_jdk": true,
"using_bundled_jdk": false,
"count": 8
}
],
"mem": {
"heap_used": "34.8gb",
"heap_used_in_bytes": 37396632600,
"heap_max": "80.4gb",
"heap_max_in_bytes": 86433071104
},
"threads": 1193
},
"fs": {
"total": "2.9tb",
"total_in_bytes": 3219611107328,
"free": "891gb",
"free_in_bytes": 956724506624,
"available": "890.9gb",
"available_in_bytes": 956623843328
},
"plugins": [
{
"name": "repository-gcs",
"version": "7.3.2",
"elasticsearch_version": "7.3.2",
"java_version": "1.8",
"description": "The GCS repository plugin adds Google Cloud Storage support for repositories.",
"classname": "org.elasticsearch.repositories.gcs.GoogleCloudStoragePlugin",
"extended_plugins": [

    ],
    "has_native_controller": false
  }
],
"network_types": {
  "transport_types": {
    "security4": 8
  },
  "http_types": {
    "security4": 8
  }
},
"discovery_types": {
  "zen": 8
},
"packaging_types": [
  {
    "flavor": "default",
    "type": "tar",
    "count": 8
  }
]

}
}

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.