Heap usage is More.. Need to understand more details where the heap is being used and how to optimize it

Hi ,

Below are my ES cluster node Stats.. ElasticSearch version - 5.6. It is occupying more space in "old" gen space.
Why heap usage is more here ? what all are occupying more heap here. And how can i optimize here?
Thanks in advance..

	"_nodes": {
		"total": 1,
		"successful": 1,
		"failed": 0
	},
	"cluster_name": "POC_ES_CAND",
	"nodes": {
		"BGe92SBKTMql6IWtOIfXHg": {
			"timestamp": 1597160003231,
			"name": "Node_Name",
			"transport_address": "ip:9300",
			"host": "ip",
			"ip": "ip:9300",
			"roles": ["data", "ingest"],
			"attributes": {
				"ml.max_open_jobs": "10",
				"ml.enabled": "true"
			},
			"indices": {
				"docs": {
					"count": 640399599,
					"deleted": 171272600
				},
				"store": {
					"size_in_bytes": 1140644284530,
					"throttle_time_in_millis": 0
				},
				"indexing": {
					"index_total": 4038618732,
					"index_time_in_millis": 4675523387,
					"index_current": 2,
					"index_failed": 2244427,
					"delete_total": 137297704,
					"delete_time_in_millis": 3817135,
					"delete_current": 0,
					"noop_update_total": 244165988,
					"is_throttled": false,
					"throttle_time_in_millis": 3414227
				},
				"get": {
					"total": 638420519,
					"time_in_millis": 6889592526,
					"exists_total": 579815697,
					"exists_time_in_millis": 6887824851,
					"missing_total": 58604822,
					"missing_time_in_millis": 1767675,
					"current": 2
				},
				"search": {
					"open_contexts": 31,
					"query_total": 2621627373,
					"query_time_in_millis": 48073073953,
					"query_current": 8,
					"fetch_total": 122544139,
					"fetch_time_in_millis": 740764955,
					"fetch_current": 0,
					"scroll_total": 1820924,
					"scroll_time_in_millis": 483171338342,
					"scroll_current": 64,
					"suggest_total": 0,
					"suggest_time_in_millis": 0,
					"suggest_current": 0
				},
				"merges": {
					"current": 0,
					"current_docs": 0,
					"current_size_in_bytes": 0,
					"total": 17300334,
					"total_time_in_millis": 12568219513,
					"total_docs": 55892050280,
					"total_size_in_bytes": 68287952285216,
					"total_stopped_time_in_millis": 44797904,
					"total_throttled_time_in_millis": 2874082097,
					"total_auto_throttle_in_bytes": 28246818000
				},
				"refresh": {
					"total": 9494430246,
					"total_time_in_millis": 33408041189,
					"listeners": 0
				},
				"flush": {
					"total": 2269515,
					"total_time_in_millis": 138477983
				},
				"warmer": {
					"current": 1,
					"total": 126078690,
					"total_time_in_millis": 17977698078
				},
				"query_cache": {
					"memory_size_in_bytes": 628360646,
					"total_count": 55539915970,
					"hit_count": 10006789515,
					"miss_count": 45533126455,
					"cache_size": 9216,
					"cache_count": 126961459,
					"evictions": 126952243
				},
				"fielddata": {
					"memory_size_in_bytes": 5715697160,
					"evictions": 0
				},
				"completion": {
					"size_in_bytes": 0
				},
				"segments": {
					"count": 5836,
					"memory_in_bytes": 2411070245,
					"terms_memory_in_bytes": 1656934996,
					"stored_fields_memory_in_bytes": 68905880,
					"term_vectors_memory_in_bytes": 18036672,
					"norms_memory_in_bytes": 62749312,
					"points_memory_in_bytes": 12465581,
					"doc_values_memory_in_bytes": 591977804,
					"index_writer_memory_in_bytes": 270490490,
					"version_map_memory_in_bytes": 26657,
					"fixed_bit_set_memory_in_bytes": 89390376,
					"max_unsafe_auto_id_timestamp": 1596931202593,
					"file_sizes": {}
				},
				"translog": {
					"operations": 450244,
					"size_in_bytes": 4244387905
				},
				"request_cache": {
					"memory_size_in_bytes": 226932930,
					"evictions": 5959,
					"hit_count": 100240553,
					"miss_count": 1236188687
				},
				"recovery": {
					"current_as_source": 0,
					"current_as_target": 0,
					"throttle_time_in_millis": 10966734
				}
			},
			"os": {
				"timestamp": 1597160007851,
				"cpu": {
					"percent": 90,
					"load_average": {
						"1m": 13.4,
						"5m": 12.92,
						"15m": 12.5
					}
				},
				"mem": {
					"total_in_bytes": 66008440832,
					"free_in_bytes": 1290043392,
					"used_in_bytes": 64718397440,
					"free_percent": 2,
					"used_percent": 98
				},
				"swap": {
					"total_in_bytes": 0,
					"free_in_bytes": 0,
					"used_in_bytes": 0
				},
				"cgroup": {
					"cpuacct": {
						"control_group": "/",
						"usage_nanos": 180530541493425094
					},
					"cpu": {
						"control_group": "/",
						"cfs_period_micros": 100000,
						"cfs_quota_micros": -1,
						"stat": {
							"number_of_elapsed_periods": 0,
							"number_of_times_throttled": 0,
							"time_throttled_nanos": 0
						}
					}
				}
			},
			"process": {
				"timestamp": 1597160007851,
				"open_file_descriptors": 1733,
				"max_file_descriptors": 65536,
				"cpu": {
					"percent": 90,
					"total_in_millis": 175517190650
				},
				"mem": {
					"total_virtual_in_bytes": 1176113430528
				}
			},
			"jvm": {
				"timestamp": 1597160007852,
				"uptime_in_millis": 33888224284,
				"mem": {
					"heap_used_in_bytes": 23390839568,
					"heap_used_percent": 74,
					"heap_committed_in_bytes": 31343902720,
					"heap_max_in_bytes": 31343902720,
					"non_heap_used_in_bytes": 206490480,
					"non_heap_committed_in_bytes": 234209280,
					"pools": {
						"young": {
							"used_in_bytes": 205738472,
							"max_in_bytes": 907345920,
							"peak_used_in_bytes": 907345920,
							"peak_max_in_bytes": 907345920
						},
						"survivor": {
							"used_in_bytes": 104670480,
							"max_in_bytes": 113377280,
							"peak_used_in_bytes": 113377280,
							"peak_max_in_bytes": 113377280
						},
						"old": {
							"used_in_bytes": 23080430616,
							"max_in_bytes": 30323179520,
							"peak_used_in_bytes": 30319741440,
							"peak_max_in_bytes": 30323179520
						}
					}
				},
				"threads": {
					"count": 213,
					"peak_count": 260
				},
				"gc": {
					"collectors": {
						"young": {
							"collection_count": 27834813,
							"collection_time_in_millis": 1094240793
						},
						"old": {
							"collection_count": 218678,
							"collection_time_in_millis": 33703498
						}
					}
				},
				"buffer_pools": {
					"direct": {
						"count": 197,
						"used_in_bytes": 560606942,
						"total_capacity_in_bytes": 560606941
					},
					"mapped": {
						"count": 12626,
						"used_in_bytes": 1132358371410,
						"total_capacity_in_bytes": 1132358371410
					}
				},
				"classes": {
					"current_loaded_count": 16274,
					"total_loaded_count": 17752,
					"total_unloaded_count": 1478
				}
			},
			"thread_pool": {
				"bulk": {
					"threads": 16,
					"queue": 0,
					"active": 1,
					"rejected": 0,
					"largest": 16,
					"completed": 548897242
				},
				"fetch_shard_started": {
					"threads": 1,
					"queue": 0,
					"active": 0,
					"rejected": 0,
					"largest": 32,
					"completed": 129
				},
				"fetch_shard_store": {
					"threads": 1,
					"queue": 0,
					"active": 0,
					"rejected": 0,
					"largest": 32,
					"completed": 3082
				},
				"flush": {
					"threads": 5,
					"queue": 0,
					"active": 0,
					"rejected": 0,
					"largest": 5,
					"completed": 4511642
				},
				"force_merge": {
					"threads": 0,
					"queue": 0,
					"active": 0,
					"rejected": 0,
					"largest": 0,
					"completed": 0
				},
				"generic": {
					"threads": 4,
					"queue": 0,
					"active": 0,
					"rejected": 0,
					"largest": 27,
					"completed": 46579319
				},
				"get": {
					"threads": 16,
					"queue": 0,
					"active": 0,
					"rejected": 0,
					"largest": 16,
					"completed": 24
				},
				"index": {
					"threads": 16,
					"queue": 0,
					"active": 1,
					"rejected": 96,
					"largest": 16,
					"completed": 272556274
				},
				"listener": {
					"threads": 0,
					"queue": 0,
					"active": 0,
					"rejected": 0,
					"largest": 0,
					"completed": 0
				},
				"management": {
					"threads": 5,
					"queue": 0,
					"active": 2,
					"rejected": 0,
					"largest": 5,
					"completed": 50078976
				},
				"ml_autodetect": {
					"threads": 0,
					"queue": 0,
					"active": 0,
					"rejected": 0,
					"largest": 0,
					"completed": 0
				},
				"ml_datafeed": {
					"threads": 0,
					"queue": 0,
					"active": 0,
					"rejected": 0,
					"largest": 0,
					"completed": 0
				},
				"ml_utility": {
					"threads": 0,
					"queue": 0,
					"active": 0,
					"rejected": 0,
					"largest": 0,
					"completed": 0
				},
				"refresh": {
					"threads": 8,
					"queue": 0,
					"active": 0,
					"rejected": 0,
					"largest": 8,
					"completed": 14656949745
				},
				"search": {
					"threads": 25,
					"queue": 0,
					"active": 1,
					"rejected": 0,
					"largest": 25,
					"completed": 3292902912
				},
				"snapshot": {
					"threads": 0,
					"queue": 0,
					"active": 0,
					"rejected": 0,
					"largest": 0,
					"completed": 0
				},
				"warmer": {
					"threads": 5,
					"queue": 0,
					"active": 1,
					"rejected": 0,
					"largest": 5,
					"completed": 4906477343
				},
				"watcher": {
					"threads": 0,
					"queue": 0,
					"active": 0,
					"rejected": 0,
					"largest": 0,
					"completed": 0
				}
			},
			"fs": {
				"timestamp": 1597160007853,
				"total": {
					"total_in_bytes": 2147453243392,
					"free_in_bytes": 996709965824,
					"available_in_bytes": 996709965824
				},
				"data": [{
					"path": "/var/lib/elasticsearch/nodes/0",
					"mount": "/var/lib/elasticsearch (/dev/mapper/VG001-LV001)",
					"type": "xfs",
					"total_in_bytes": 2147453243392,
					"free_in_bytes": 996709965824,
					"available_in_bytes": 996709965824,
					"spins": "false"
				}],
				"io_stats": {
					"devices": [{
						"device_name": "dm-7",
						"operations": 3926200409,
						"read_operations": 574143390,
						"write_operations": 3352057019,
						"read_kilobytes": 39663481794,
						"write_kilobytes": 76246915519
					}],
					"total": {
						"operations": 3926200409,
						"read_operations": 574143390,
						"write_operations": 3352057019,
						"read_kilobytes": 39663481794,
						"write_kilobytes": 76246915519
					}
				}
			},
			"transport": {
				"server_open": 117,
				"rx_count": 23228011359,
				"rx_size_in_bytes": 28499077227102,
				"tx_count": 23228012330,
				"tx_size_in_bytes": 23672542048295
			},
			"http": {
				"current_open": 48,
				"total_opened": 218990095
			},
			"breakers": {
				"request": {
					"limit_size_in_bytes": 18806341632,
					"limit_size": "17.5gb",
					"estimated_size_in_bytes": 7089016,
					"estimated_size": "6.7mb",
					"overhead": 1.0,
					"tripped": 0
				},
				"fielddata": {
					"limit_size_in_bytes": 18806341632,
					"limit_size": "17.5gb",
					"estimated_size_in_bytes": 5715695392,
					"estimated_size": "5.3gb",
					"overhead": 1.03,
					"tripped": 0
				},
				"in_flight_requests": {
					"limit_size_in_bytes": 31343902720,
					"limit_size": "29.1gb",
					"estimated_size_in_bytes": 24006,
					"estimated_size": "23.4kb",
					"overhead": 1.0,
					"tripped": 0
				},
				"parent": {
					"limit_size_in_bytes": 21940731904,
					"limit_size": "20.4gb",
					"estimated_size_in_bytes": 5722808414,
					"estimated_size": "5.3gb",
					"overhead": 1.0,
					"tripped": 0
				}
			},
			"script": {
				"compilations": 146,
				"cache_evictions": 46
			},
			"discovery": {
				"cluster_state_queue": {
					"total": 0,
					"pending": 0,
					"committed": 0
				}
			},
			"ingest": {
				"total": {
					"count": 0,
					"time_in_millis": 0,
					"current": 0,
					"failed": 0
				},
				"pipelines": {
					"xpack_monitoring_2": {
						"count": 0,
						"time_in_millis": 0,
						"current": 0,
						"failed": 0
					},
					"xpack_monitoring_6": {
						"count": 0,
						"time_in_millis": 0,
						"current": 0,
						"failed": 0
					}
				}
			}
		}
	}
}

can you please add some context to your question? This looks fine to me 23/30gb of heap used? The old space is bigger than the new space so it is also normal that it takes up more space.

Can you be more exact what worries you here?

P.S. You can also add the human parameter to this API call, which will return human readable byte values.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.