Dear Armin,
Thanks for your inputs. How can i improve the memory utilization given the below data. I am looking at optimizations other than freezing the old unused indices?
I keep running into frequent GC taking more than 30 seconds and any executing aggregation queries on many days of data times out and fails.
I have four nodes with 24 cores and 64GB RAM and 32G of JVM HEAP. Daily index and each index has about 100 million records. 90 days of data is retained in the system. Best Compression is used with 5 Shards and 1 replica.
Heap Utilization Percentage is more than 85%
Fielddata Memory is about 15 GB (per node)
Segment Memory is about 5.3 GB (per node)
Translog Size in Byttes is about 1.4 GB (per node)
Size of Compressed data consumed by all shards : 6511 GB (total size handled by 4 nodes. Sum of all values from output of "_cat/shards?")
"active_primary_shards" : 1645 (all nodes)
================================================================
Logs and output of different commands is given below
Output of : _nodes/stats/indices
"query_cache" : {
"memory_size_in_bytes" : 250483404,
"total_count" : 345506,
"hit_count" : 265900,
"miss_count" : 79606,
"cache_size" : 885,
"cache_count" : 2055,
"evictions" : 1170
},
"fielddata" : {
"memory_size_in_bytes" : 15607420544,
"evictions" : 0
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 10786,
"memory_in_bytes" : 5743504096,
"terms_memory_in_bytes" : 5435352021,
"stored_fields_memory_in_bytes" : 242611112,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 1295936,
"points_memory_in_bytes" : 61446083,
"doc_values_memory_in_bytes" : 2798944,
"index_writer_memory_in_bytes" : 17527760,
"version_map_memory_in_bytes" : 0,
"fixed_bit_set_memory_in_bytes" : 648466280,
"max_unsafe_auto_id_timestamp" : 1581278460804,
"file_sizes" : { }
},
"translog" : {
"operations" : 2831157,
"size_in_bytes" : 1549152668,
"uncommitted_operations" : 1788929,
"uncommitted_size_in_bytes" : 609003907,
"earliest_last_modified_age" : 0
},
"request_cache" : {
"memory_size_in_bytes" : 44842106,
"evictions" : 0,
"hit_count" : 309,
"miss_count" : 8221
},
Output of : _cat/nodes?v
ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
x.x.x.x 93 95 8 1.35 1.95 1.93 mdi - node-3
x.x.x.x 87 94 10 1.67 1.90 2.24 mdi - node-1
x.x.x.x 89 97 6 0.43 1.00 1.32 mdi * node-2
x.x.x.x 88 94 5 0.69 0.92 1.12 mdi - node-4
Output of : _cluster/stats?
"cluster_name" : "my-cluster",
"cluster_uuid" : "ntFmtDanQx2yTaivjvSPBw",
"timestamp" : 1581324608473,
"status" : "green",
"indices" : {
"count" : 329,
"shards" : {
"total" : 3290,
"primaries" : 1645,
"replication" : 1.0,
"index" : {
"shards" : {
"min" : 10,
"max" : 10,
"avg" : 10.0
},
"primaries" : {
"min" : 5,
"max" : 5,
"avg" : 5.0
},
"replication" : {
"min" : 1.0,
"max" : 1.0,
"avg" : 1.0
}
}
},
"docs" : {
"count" : 10358677517,
"deleted" : 0
},
"store" : {
"size_in_bytes" : 6704778880187
},
"fielddata" : {
"memory_size_in_bytes" : 62723570200,
"evictions" : 0
},
"query_cache" : {
"memory_size_in_bytes" : 1195879730,
"total_count" : 1452057,
"hit_count" : 1080694,
"miss_count" : 371363,
"cache_size" : 5393,
"cache_count" : 10727,
"evictions" : 5334
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 42867,
"memory_in_bytes" : 23026343590,
"terms_memory_in_bytes" : 21794765416,
"stored_fields_memory_in_bytes" : 970973792,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 5217344,
"points_memory_in_bytes" : 244421530,
"doc_values_memory_in_bytes" : 10965508,
"index_writer_memory_in_bytes" : 279531472,
"version_map_memory_in_bytes" : 0,
"fixed_bit_set_memory_in_bytes" : 2592254192,
"max_unsafe_auto_id_timestamp" : 1581278460804,
"file_sizes" : { }
}
},
"nodes" : {
"count" : {
"total" : 4,
"data" : 4,
"coordinating_only" : 0,
"master" : 4,
"ingest" : 4
},
"versions" : [
"6.6.1"
],
"os" : {
"available_processors" : 96,
"allocated_processors" : 96,
"names" : [
{
"name" : "Linux",
"count" : 4
}
],
"pretty_names" : [
{
"pretty_name" : "Red Hat Enterprise Linux",
"count" : 4
}
],
"mem" : {
"total_in_bytes" : 269515407360,
"free_in_bytes" : 14183038976,
"used_in_bytes" : 255332368384,
"free_percent" : 5,
"used_percent" : 95
}
},
"process" : {
"cpu" : {
"percent" : 19
},
"open_file_descriptors" : {
"min" : 11269,
"max" : 15789,
"avg" : 13896
}
},
"jvm" : {
"max_uptime_in_millis" : 564315186,
"versions" : [
{
"version" : "1.8.0_211",
"vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
"vm_version" : "25.211-b12",
"vm_vendor" : "Oracle Corporation",
"count" : 4
}
],
"mem" : {
"heap_used_in_bytes" : 119341024336,
"heap_max_in_bytes" : 137438953472
},
"threads" : 978
},
"fs" : {
"total_in_bytes" : 50306197454848,
"free_in_bytes" : 39109396488192,
"available_in_bytes" : 36553692176384
},