To be honest I'm not eager to upgrade to 7.9.1 cause I've seen info about the Lucene memory leak that affected 7.9.0 version.
The size of JVM heap is 96GB per one data node. And yes, I know that in documentation you recommend to not go higher than 32GB per node. But the main reasoning you use is to avoid JVM overhead of using non-compressed object pointers. I haven't realized any official info, that there are some BUGs in ES that causes problems with larger heaps.
JVM itself works fine and G1 GC has no problem with heaps around 100GB.
I am just trying to understand what exactly is Elasticsearch doing when creating new replicas and why it allocates unlimitted amount of memory.
JVM options follow
{
"pid" : 25732,
"version" : "14.0.1",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "14.0.1+7",
"vm_vendor" : "AdoptOpenJDK",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"start_time_in_millis" : 1600069159436,
"mem" : {
"heap_init_in_bytes" : 103079215104,
"heap_max_in_bytes" : 103079215104,
"non_heap_init_in_bytes" : 7667712,
"non_heap_max_in_bytes" : 0,
"direct_max_in_bytes" : 0
},
"gc_collectors" : [
"G1 Young Generation",
"G1 Old Generation"
],
"memory_pools" : [
"CodeHeap 'non-nmethods'",
"Metaspace",
"CodeHeap 'profiled nmethods'",
"G1 Eden Space",
"G1 Old Gen",
"G1 Survivor Space",
"CodeHeap 'non-profiled nmethods'"
],
"using_compressed_ordinary_object_pointers" : "false",
"input_arguments" : [
"-Xshare:auto",
"-Des.networkaddress.cache.ttl=60",
"-Des.networkaddress.cache.negative.ttl=10",
"-XX:+AlwaysPreTouch",
"-Xss1m",
"-Djava.awt.headless=true",
"-Dfile.encoding=UTF-8",
"-Djna.nosys=true",
"-XX:-OmitStackTraceInFastThrow",
"-XX:+ShowCodeDetailsInExceptionMessages",
"-Dio.netty.noUnsafe=true",
"-Dio.netty.noKeySetOptimization=true",
"-Dio.netty.recycler.maxCapacityPerThread=0",
"-Dio.netty.allocator.numDirectArenas=0",
"-Dlog4j.shutdownHookEnabled=false",
"-Dlog4j2.disable.jmx=true",
"-Djava.locale.providers=SPI,COMPAT",
"-Xms96g",
"-Xmx96g",
"-XX:+UseG1GC",
"-XX:MaxGCPauseMillis=300",
"-XX:+ParallelRefProcEnabled",
"-XX:G1ReservePercent=50",
"-XX:InitiatingHeapOccupancyPercent=25",
"-Djava.io.tmpdir=/tmp/elasticsearch-403851464928710223",
"-XX:+HeapDumpOnOutOfMemoryError",
"-XX:HeapDumpPath=/var/lib/elasticsearch",
"-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log",
"-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m",
"-XX:MaxDirectMemorySize=51539607552",
"-Des.path.home=/usr/share/elasticsearch",
"-Des.path.conf=/etc/elasticsearch",
"-Des.distribution.flavor=default",
"-Des.distribution.type=deb",
"-Des.bundled_jdk=true"
]
}
).
