Hello,
In the current system we have 90 days of logs. However, most of the time, only 30 days of data is queried. So, I am thinking of following optimizations to manage old indices to reduce system memory utilization.
Option 1)
- Moving all primary shards to one node
- Shrink the data
- Forcemerge the data to a single segment (max documents is 120 million and single segment size is about 35 GB)
Will this help save heap memory significantly? How will this impact the query performance?
My current heap and ram usage is very high. Shown Below . Here ram.percent is quite high. However, sufficient RAM is free, about 20 GB of 64 GB. Why is this ram.percent very high?
curl -X GET "http://IP:PORT/_cat/nodes?v"
ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
HOST1 86 98 4 1.25 1.23 1.25 mdi - node-3
HOST2 93 97 6 1.52 1.44 1.45 mdi - node-1
HOST3 92 98 4 0.88 1.26 1.31 mdi * node-2
HOST4 95 98 6 1.39 1.74 1.67 mdi - node-4
Option 2)
Close the old indices(older than 30 days). I can provide a mechanism to Open the indices when any closed index is queried and close the same after the query execution. Will this help save the system memory resources.
Which of the above options will help optimize the system resource memory usage and what is the recommended approach?
Index Stats for one of the node is given below :
{
"_nodes" : {
"total" : 4,
"successful" : 4,
"failed" : 0
},
"cluster_name" : "xxx-cluster",
"nodes" : {
"RQ50cjGaTJyDflmJqQKj5w" : {
"timestamp" : 1581074219257,
"name" : "node-3",
"transport_address" : "",
"host" : "",
"ip" : "",
"roles" : [
"master",
"data",
"ingest"
],
"attributes" : {
"ml.machine_memory" : "67378851840",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true",
"ml.enabled" : "true"
},
"indices" : {
"docs" : {
"count" : 5088498671,
"deleted" : 0
},
"store" : {
"size_in_bytes" : 1647950022235
},
"indexing" : {
"index_total" : 44423906,
"index_time_in_millis" : 18482112,
"index_current" : 0,
"index_failed" : 0,
"delete_total" : 0,
"delete_time_in_millis" : 0,
"delete_current" : 0,
"noop_update_total" : 0,
"is_throttled" : false,
"throttle_time_in_millis" : 0
},
"get" : {
"total" : 0,
"time_in_millis" : 0,
"exists_total" : 0,
"exists_time_in_millis" : 0,
"missing_total" : 0,
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"open_contexts" : 10,
"query_total" : 14334,
"query_time_in_millis" : 530529,
"query_current" : 10,
"fetch_total" : 2864,
"fetch_time_in_millis" : 4748083,
"fetch_current" : 0,
"scroll_total" : 10,
"scroll_time_in_millis" : 47487324,
"scroll_current" : 0,
"suggest_total" : 0,
"suggest_time_in_millis" : 0,
"suggest_current" : 0
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size_in_bytes" : 0,
"total" : 4507,
"total_time_in_millis" : 12197240,
"total_docs" : 171012154,
"total_size_in_bytes" : 56604653805,
"total_stopped_time_in_millis" : 0,
"total_throttled_time_in_millis" : 5493618,
"total_auto_throttle_in_bytes" : 17097031680
},
"refresh" : {
"total" : 15005,
"total_time_in_millis" : 3205850,
"listeners" : 0
},
"flush" : {
"total" : 539,
"periodic" : 76,
"total_time_in_millis" : 305456
},
"warmer" : {
"current" : 0,
"total" : 13505,
"total_time_in_millis" : 8185595
},
"query_cache" : {
"memory_size_in_bytes" : 38048113,
"total_count" : 76546,
"hit_count" : 72281,
"miss_count" : 4265,
"cache_size" : 106,
"cache_count" : 132,
"evictions" : 26
},
"fielddata" : {
"memory_size_in_bytes" : 15439073464,
"evictions" : 0
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 10578,
"memory_in_bytes" : 5687338579,
"terms_memory_in_bytes" : 5384658391,
"stored_fields_memory_in_bytes" : 239039736,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 1299520,
"points_memory_in_bytes" : 59617508,
"doc_values_memory_in_bytes" : 2723424,
"index_writer_memory_in_bytes" : 30111104,
"version_map_memory_in_bytes" : 0,
"fixed_bit_set_memory_in_bytes" : 636700504,
"max_unsafe_auto_id_timestamp" : 1581019261329,
"file_sizes" : { }
},
"translog" : {
"operations" : 2230958,
"size_in_bytes" : 1385529357,
"uncommitted_operations" : 1316853,
"uncommitted_size_in_bytes" : 579925452,
"earliest_last_modified_age" : 0
},
"request_cache" : {
"memory_size_in_bytes" : 104000,
"evictions" : 0,
"hit_count" : 3,
"miss_count" : 25
},
"recovery" : {
"current_as_source" : 0,
"current_as_target" : 0,
"throttle_time_in_millis" : 2252275
}
}
},
