매일 0시에 다량의 문서를 추가하고 있습니다.
Old 메모리 영역이 90%를 초과하면 FullGC가 일어나는데 재시작 후 몇일이 지나면 Old 영역이 많이 비워지지 않습니다. 그러다보니 Old 영역이 90%를 초과해 FullGC 호출이 많아지고 있습니다.
Old 영역이 꾸준히 증가하는 이유가 알고 싶습니다.
시스템 구성
version : 6.2.4
master : 3
node : 3
index : 200
shard : 4
replicas : 1
node size : 3T
GC Log
Memory Analyzer
Problem Suspect 1
org.elasticsearch.index.IndexService
이것은 재시작하자마자 항상 이정도 잡혀있습니다. 크게 증가하지도 않습니다.
Problem Suspect 2
org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader
이 인스턴스의 수가 꾸준히 증가하면서 메모리 사용량이 늘어나고 있습니다.
dominator_tree 에서 org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader 검색해보면 추가된 문서의 내용이 들어가 있습니다.
Node Status
{
"timestamp":1583285014549,
"name":"node2",
"transport_address":"192.168.10.97:9300",
"host":"192.168.10.97",
"ip":"192.168.10.97:9300",
"roles":[
"data",
"ingest"
],
"indices":{
"docs":{
"count":473469050,
"deleted":30613251
},
"store":{
"size_in_bytes":3120122605868
},
"indexing":{
"index_total":7738978,
"index_time_in_millis":31520291,
"index_current":0,
"index_failed":0,
"delete_total":9880164,
"delete_time_in_millis":3249486,
"delete_current":0,
"noop_update_total":6634,
"is_throttled":false,
"throttle_time_in_millis":0
},
"get":{
"total":197214,
"time_in_millis":1088984,
"exists_total":190797,
"exists_time_in_millis":1083877,
"missing_total":6417,
"missing_time_in_millis":5107,
"current":0
},
"search":{
"open_contexts":0,
"query_total":12941307,
"query_time_in_millis":35163247,
"query_current":0,
"fetch_total":156844,
"fetch_time_in_millis":3456965,
"fetch_current":0,
"scroll_total":12656700,
"scroll_time_in_millis":92667413,
"scroll_current":0,
"suggest_total":0,
"suggest_time_in_millis":0,
"suggest_current":0
},
"merges":{
"current":0,
"current_docs":0,
"current_size_in_bytes":0,
"total":108302,
"total_time_in_millis":70229378,
"total_docs":50212065,
"total_size_in_bytes":521243115637,
"total_stopped_time_in_millis":0,
"total_throttled_time_in_millis":9023031,
"total_auto_throttle_in_bytes":9547268834
},
"refresh":{
"total":842638,
"total_time_in_millis":12606536,
"listeners":0
},
"flush":{
"total":88360,
"total_time_in_millis":2147058
},
"warmer":{
"current":0,
"total":663185,
"total_time_in_millis":15616
},
"query_cache":{
"memory_size_in_bytes":128125953,
"total_count":100847856,
"hit_count":26110536,
"miss_count":74737320,
"cache_size":5064,
"cache_count":3782702,
"evictions":3777638
},
"fielddata":{
"memory_size_in_bytes":0,
"evictions":0
},
"completion":{
"size_in_bytes":0
},
"segments":{
"count":7786,
"memory_in_bytes":9556998466,
"terms_memory_in_bytes":9403408192,
"stored_fields_memory_in_bytes":125478624,
"term_vectors_memory_in_bytes":0,
"norms_memory_in_bytes":2491520,
"points_memory_in_bytes":14737778,
"doc_values_memory_in_bytes":10882352,
"index_writer_memory_in_bytes":0,
"version_map_memory_in_bytes":0,
"fixed_bit_set_memory_in_bytes":0,
"max_unsafe_auto_id_timestamp":-1,
"file_sizes":{
}
},
"translog":{
"operations":2326722,
"size_in_bytes":11736894824,
"uncommitted_operations":0,
"uncommitted_size_in_bytes":22962
},
"request_cache":{
"memory_size_in_bytes":0,
"evictions":0,
"hit_count":0,
"miss_count":0
},
"recovery":{
"current_as_source":0,
"current_as_target":0,
"throttle_time_in_millis":0
}
},
"os":{
"timestamp":1583285014467,
"cpu":{
"percent":0,
"load_average":{
"1m":0.4,
"5m":0.12,
"15m":0.07
}
},
"mem":{
"total_in_bytes":66780020736,
"free_in_bytes":449409024,
"used_in_bytes":66330611712,
"free_percent":1,
"used_percent":99
},
"swap":{
"total_in_bytes":34359734272,
"free_in_bytes":34359734272,
"used_in_bytes":0
},
"cgroup":{
"cpuacct":{
"control_group":"/",
"usage_nanos":1701773544650625
},
"cpu":{
"control_group":"/",
"cfs_period_micros":100000,
"cfs_quota_micros":-1,
"stat":{
"number_of_elapsed_periods":0,
"number_of_times_throttled":0,
"time_throttled_nanos":0
}
},
"memory":{
"control_group":"/",
"limit_in_bytes":"9223372036854771712",
"usage_in_bytes":"63208521728"
}
}
},
"process":{
"timestamp":1583285014468,
"open_file_descriptors":13467,
"max_file_descriptors":65536,
"cpu":{
"percent":0,
"total_in_millis":165820520
},
"mem":{
"total_virtual_in_bytes":3174575865856
}
},
"jvm":{
"timestamp":1583285014474,
"uptime_in_millis":414939829,
"mem":{
"heap_used_in_bytes":25167940368,
"heap_used_percent":79,
"heap_committed_in_bytes":31568035840,
"heap_max_in_bytes":31568035840,
"non_heap_used_in_bytes":127580792,
"non_heap_committed_in_bytes":135630848,
"pools":{
"young":{
"used_in_bytes":1364504136,
"max_in_bytes":5154013184,
"peak_used_in_bytes":5154013184,
"peak_max_in_bytes":5154013184
},
"survivor":{
"used_in_bytes":128208408,
"max_in_bytes":644218880,
"peak_used_in_bytes":644218880,
"peak_max_in_bytes":644218880
},
"old":{
"used_in_bytes":23675227824,
"max_in_bytes":25769803776,
"peak_used_in_bytes":24836722312,
"peak_max_in_bytes":25769803776
}
}
},
"threads":{
"count":342,
"peak_count":390
},
"gc":{
"collectors":{
"young":{
"collection_count":7539,
"collection_time_in_millis":771140
},
"old":{
"collection_count":152,
"collection_time_in_millis":73619
}
}
},
"buffer_pools":{
"direct":{
"count":330,
"used_in_bytes":1516848701,
"total_capacity_in_bytes":1516848700
},
"mapped":{
"count":12595,
"used_in_bytes":3112309768891,
"total_capacity_in_bytes":3112309768891
}
},
"classes":{
"current_loaded_count":11345,
"total_loaded_count":11538,
"total_unloaded_count":193
}
},
"thread_pool":{
"bulk":{
"threads":48,
"queue":0,
"active":0,
"rejected":0,
"largest":48,
"completed":3648527
},
"fetch_shard_started":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":96,
"completed":731
},
"fetch_shard_store":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":96,
"completed":800
},
"flush":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":5,
"completed":176747
},
"force_merge":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"generic":{
"threads":69,
"queue":0,
"active":0,
"rejected":0,
"largest":103,
"completed":3361284
},
"get":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"index":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"listener":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"management":{
"threads":5,
"queue":0,
"active":2,
"rejected":0,
"largest":5,
"completed":3640753
},
"refresh":{
"threads":10,
"queue":0,
"active":0,
"rejected":0,
"largest":10,
"completed":2764587
},
"search":{
"threads":73,
"queue":0,
"active":0,
"rejected":0,
"largest":73,
"completed":13098151
},
"snapshot":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"warmer":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
}
},
"fs":{
"timestamp":1583285014474,
"total":{
"total_in_bytes":23394292596736,
"free_in_bytes":10204118388736,
"available_in_bytes":10204118388736
},
"data":[
{
"path":"/data/es/nodes/0",
"mount":"/data (/dev/sdb1)",
"type":"xfs",
"total_in_bytes":23394292596736,
"free_in_bytes":10204118388736,
"available_in_bytes":10204118388736
}
],
"io_stats":{
"devices":[
{
"device_name":"sdb1",
"operations":42283881,
"read_operations":14480295,
"write_operations":27803586,
"read_kilobytes":414795940,
"write_kilobytes":829011056
}
],
"total":{
"operations":42283881,
"read_operations":14480295,
"write_operations":27803586,
"read_kilobytes":414795940,
"write_kilobytes":829011056
}
}
},
"transport":{
"server_open":57,
"rx_count":35673728,
"rx_size_in_bytes":15976845128,
"tx_count":35673726,
"tx_size_in_bytes":10702175818
},
"http":{
"current_open":0,
"total_opened":0
},
"breakers":{
"request":{
"limit_size_in_bytes":18940821504,
"limit_size":"17.6gb",
"estimated_size_in_bytes":0,
"estimated_size":"0b",
"overhead":1,
"tripped":0
},
"fielddata":{
"limit_size_in_bytes":18940821504,
"limit_size":"17.6gb",
"estimated_size_in_bytes":0,
"estimated_size":"0b",
"overhead":1.03,
"tripped":0
},
"in_flight_requests":{
"limit_size_in_bytes":31568035840,
"limit_size":"29.4gb",
"estimated_size_in_bytes":16550,
"estimated_size":"16.1kb",
"overhead":1,
"tripped":0
},
"accounting":{
"limit_size_in_bytes":31568035840,
"limit_size":"29.4gb",
"estimated_size_in_bytes":9556998466,
"estimated_size":"8.9gb",
"overhead":1,
"tripped":0
},
"parent":{
"limit_size_in_bytes":25254428672,
"limit_size":"23.5gb",
"estimated_size_in_bytes":9557015016,
"estimated_size":"8.9gb",
"overhead":1,
"tripped":0
}
},
"script":{
"compilations":0,
"cache_evictions":0
},
"discovery":{
"cluster_state_queue":{
"total":0,
"pending":0,
"committed":0
},
"published_cluster_states":{
"full_states":1,
"incompatible_diffs":0,
"compatible_diffs":437
}
},
"ingest":{
"total":{
"count":0,
"time_in_millis":0,
"current":0,
"failed":0
},
"pipelines":{
}
},
"adaptive_selection":{
}
}
도와주세요!