Hi ,
Below are my ES cluster node Stats.. ElasticSearch version - 5.6. It is occupying more space in "old" gen space.
Why heap usage is more here ? what all are occupying more heap here. And how can i optimize here?
Thanks in advance..
"_nodes": {
"total": 1,
"successful": 1,
"failed": 0
},
"cluster_name": "POC_ES_CAND",
"nodes": {
"BGe92SBKTMql6IWtOIfXHg": {
"timestamp": 1597160003231,
"name": "Node_Name",
"transport_address": "ip:9300",
"host": "ip",
"ip": "ip:9300",
"roles": ["data", "ingest"],
"attributes": {
"ml.max_open_jobs": "10",
"ml.enabled": "true"
},
"indices": {
"docs": {
"count": 640399599,
"deleted": 171272600
},
"store": {
"size_in_bytes": 1140644284530,
"throttle_time_in_millis": 0
},
"indexing": {
"index_total": 4038618732,
"index_time_in_millis": 4675523387,
"index_current": 2,
"index_failed": 2244427,
"delete_total": 137297704,
"delete_time_in_millis": 3817135,
"delete_current": 0,
"noop_update_total": 244165988,
"is_throttled": false,
"throttle_time_in_millis": 3414227
},
"get": {
"total": 638420519,
"time_in_millis": 6889592526,
"exists_total": 579815697,
"exists_time_in_millis": 6887824851,
"missing_total": 58604822,
"missing_time_in_millis": 1767675,
"current": 2
},
"search": {
"open_contexts": 31,
"query_total": 2621627373,
"query_time_in_millis": 48073073953,
"query_current": 8,
"fetch_total": 122544139,
"fetch_time_in_millis": 740764955,
"fetch_current": 0,
"scroll_total": 1820924,
"scroll_time_in_millis": 483171338342,
"scroll_current": 64,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 17300334,
"total_time_in_millis": 12568219513,
"total_docs": 55892050280,
"total_size_in_bytes": 68287952285216,
"total_stopped_time_in_millis": 44797904,
"total_throttled_time_in_millis": 2874082097,
"total_auto_throttle_in_bytes": 28246818000
},
"refresh": {
"total": 9494430246,
"total_time_in_millis": 33408041189,
"listeners": 0
},
"flush": {
"total": 2269515,
"total_time_in_millis": 138477983
},
"warmer": {
"current": 1,
"total": 126078690,
"total_time_in_millis": 17977698078
},
"query_cache": {
"memory_size_in_bytes": 628360646,
"total_count": 55539915970,
"hit_count": 10006789515,
"miss_count": 45533126455,
"cache_size": 9216,
"cache_count": 126961459,
"evictions": 126952243
},
"fielddata": {
"memory_size_in_bytes": 5715697160,
"evictions": 0
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 5836,
"memory_in_bytes": 2411070245,
"terms_memory_in_bytes": 1656934996,
"stored_fields_memory_in_bytes": 68905880,
"term_vectors_memory_in_bytes": 18036672,
"norms_memory_in_bytes": 62749312,
"points_memory_in_bytes": 12465581,
"doc_values_memory_in_bytes": 591977804,
"index_writer_memory_in_bytes": 270490490,
"version_map_memory_in_bytes": 26657,
"fixed_bit_set_memory_in_bytes": 89390376,
"max_unsafe_auto_id_timestamp": 1596931202593,
"file_sizes": {}
},
"translog": {
"operations": 450244,
"size_in_bytes": 4244387905
},
"request_cache": {
"memory_size_in_bytes": 226932930,
"evictions": 5959,
"hit_count": 100240553,
"miss_count": 1236188687
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 10966734
}
},
"os": {
"timestamp": 1597160007851,
"cpu": {
"percent": 90,
"load_average": {
"1m": 13.4,
"5m": 12.92,
"15m": 12.5
}
},
"mem": {
"total_in_bytes": 66008440832,
"free_in_bytes": 1290043392,
"used_in_bytes": 64718397440,
"free_percent": 2,
"used_percent": 98
},
"swap": {
"total_in_bytes": 0,
"free_in_bytes": 0,
"used_in_bytes": 0
},
"cgroup": {
"cpuacct": {
"control_group": "/",
"usage_nanos": 180530541493425094
},
"cpu": {
"control_group": "/",
"cfs_period_micros": 100000,
"cfs_quota_micros": -1,
"stat": {
"number_of_elapsed_periods": 0,
"number_of_times_throttled": 0,
"time_throttled_nanos": 0
}
}
}
},
"process": {
"timestamp": 1597160007851,
"open_file_descriptors": 1733,
"max_file_descriptors": 65536,
"cpu": {
"percent": 90,
"total_in_millis": 175517190650
},
"mem": {
"total_virtual_in_bytes": 1176113430528
}
},
"jvm": {
"timestamp": 1597160007852,
"uptime_in_millis": 33888224284,
"mem": {
"heap_used_in_bytes": 23390839568,
"heap_used_percent": 74,
"heap_committed_in_bytes": 31343902720,
"heap_max_in_bytes": 31343902720,
"non_heap_used_in_bytes": 206490480,
"non_heap_committed_in_bytes": 234209280,
"pools": {
"young": {
"used_in_bytes": 205738472,
"max_in_bytes": 907345920,
"peak_used_in_bytes": 907345920,
"peak_max_in_bytes": 907345920
},
"survivor": {
"used_in_bytes": 104670480,
"max_in_bytes": 113377280,
"peak_used_in_bytes": 113377280,
"peak_max_in_bytes": 113377280
},
"old": {
"used_in_bytes": 23080430616,
"max_in_bytes": 30323179520,
"peak_used_in_bytes": 30319741440,
"peak_max_in_bytes": 30323179520
}
}
},
"threads": {
"count": 213,
"peak_count": 260
},
"gc": {
"collectors": {
"young": {
"collection_count": 27834813,
"collection_time_in_millis": 1094240793
},
"old": {
"collection_count": 218678,
"collection_time_in_millis": 33703498
}
}
},
"buffer_pools": {
"direct": {
"count": 197,
"used_in_bytes": 560606942,
"total_capacity_in_bytes": 560606941
},
"mapped": {
"count": 12626,
"used_in_bytes": 1132358371410,
"total_capacity_in_bytes": 1132358371410
}
},
"classes": {
"current_loaded_count": 16274,
"total_loaded_count": 17752,
"total_unloaded_count": 1478
}
},
"thread_pool": {
"bulk": {
"threads": 16,
"queue": 0,
"active": 1,
"rejected": 0,
"largest": 16,
"completed": 548897242
},
"fetch_shard_started": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 32,
"completed": 129
},
"fetch_shard_store": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 32,
"completed": 3082
},
"flush": {
"threads": 5,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 5,
"completed": 4511642
},
"force_merge": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"generic": {
"threads": 4,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 27,
"completed": 46579319
},
"get": {
"threads": 16,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 16,
"completed": 24
},
"index": {
"threads": 16,
"queue": 0,
"active": 1,
"rejected": 96,
"largest": 16,
"completed": 272556274
},
"listener": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"management": {
"threads": 5,
"queue": 0,
"active": 2,
"rejected": 0,
"largest": 5,
"completed": 50078976
},
"ml_autodetect": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"ml_datafeed": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"ml_utility": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"refresh": {
"threads": 8,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 8,
"completed": 14656949745
},
"search": {
"threads": 25,
"queue": 0,
"active": 1,
"rejected": 0,
"largest": 25,
"completed": 3292902912
},
"snapshot": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"warmer": {
"threads": 5,
"queue": 0,
"active": 1,
"rejected": 0,
"largest": 5,
"completed": 4906477343
},
"watcher": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
}
},
"fs": {
"timestamp": 1597160007853,
"total": {
"total_in_bytes": 2147453243392,
"free_in_bytes": 996709965824,
"available_in_bytes": 996709965824
},
"data": [{
"path": "/var/lib/elasticsearch/nodes/0",
"mount": "/var/lib/elasticsearch (/dev/mapper/VG001-LV001)",
"type": "xfs",
"total_in_bytes": 2147453243392,
"free_in_bytes": 996709965824,
"available_in_bytes": 996709965824,
"spins": "false"
}],
"io_stats": {
"devices": [{
"device_name": "dm-7",
"operations": 3926200409,
"read_operations": 574143390,
"write_operations": 3352057019,
"read_kilobytes": 39663481794,
"write_kilobytes": 76246915519
}],
"total": {
"operations": 3926200409,
"read_operations": 574143390,
"write_operations": 3352057019,
"read_kilobytes": 39663481794,
"write_kilobytes": 76246915519
}
}
},
"transport": {
"server_open": 117,
"rx_count": 23228011359,
"rx_size_in_bytes": 28499077227102,
"tx_count": 23228012330,
"tx_size_in_bytes": 23672542048295
},
"http": {
"current_open": 48,
"total_opened": 218990095
},
"breakers": {
"request": {
"limit_size_in_bytes": 18806341632,
"limit_size": "17.5gb",
"estimated_size_in_bytes": 7089016,
"estimated_size": "6.7mb",
"overhead": 1.0,
"tripped": 0
},
"fielddata": {
"limit_size_in_bytes": 18806341632,
"limit_size": "17.5gb",
"estimated_size_in_bytes": 5715695392,
"estimated_size": "5.3gb",
"overhead": 1.03,
"tripped": 0
},
"in_flight_requests": {
"limit_size_in_bytes": 31343902720,
"limit_size": "29.1gb",
"estimated_size_in_bytes": 24006,
"estimated_size": "23.4kb",
"overhead": 1.0,
"tripped": 0
},
"parent": {
"limit_size_in_bytes": 21940731904,
"limit_size": "20.4gb",
"estimated_size_in_bytes": 5722808414,
"estimated_size": "5.3gb",
"overhead": 1.0,
"tripped": 0
}
},
"script": {
"compilations": 146,
"cache_evictions": 46
},
"discovery": {
"cluster_state_queue": {
"total": 0,
"pending": 0,
"committed": 0
}
},
"ingest": {
"total": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
},
"pipelines": {
"xpack_monitoring_2": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
},
"xpack_monitoring_6": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
}
}
}
}