Elasticsearch heap Memory WARNING in elk docker

Elastic Stack Elasticsearch
1

Hi,
I am using elasticsearch 7.10.0. I have 3 nodes in my cluster. But in one of the node among 3, we are receiving given below warning logs:
**{"type": "server", "timestamp": "2021-06-30T04:01:53,805Z", "level": "WARN", "component": "o.e.m.j.JvmGcMonitorService", "cluster.name": "nfvi-elkstack", "node.name": "data2", "message": "[gc][young][6718527][787252] duration [1.9s], collections [1]/[2.1s], total [1.9s]/[7.2h], memory [3.5gb]->[2.9gb]/[5gb], all_pools {[young] [656mb]->[20mb]/[0b]}{[old] [2.8gb]->[2.8gb]/[5gb]}{[survivor] [28mb]->[59mb]/[0b]}", "cluster.uuid": "7yrxdBQBRPinZlUz6CwDRw", "node.id": "6JnyYl3PTgqXFthT17wPAw" }

Is there any way to resolve this because due to this CPU Utilization for this particular node is coming very high.

2

What is the output from the _cluster/stats?pretty&human API?

3

{"_nodes":{"total":3,"successful":3,"failed":0},"cluster_name":"nfvi-elkstack","cluster_uuid":"7yrxdBQBRPinZlUz6CwDRw","timestamp":1625035659072,"status":"green","indices":{"count":238,"shards":{"total":788,"primaries":394,"replication":1.0,"index":{"shards":{"min":2,"max":6,"avg":3.310924369747899},"primaries":{"min":1,"max":3,"avg":1.6554621848739495},"replication":{"min":1.0,"max":1.0,"avg":1.0}}},"docs":{"count":2097202739,"deleted":334944},"store":{"size_in_bytes":865627576770,"reserved_in_bytes":0},"fielddata":{"memory_size_in_bytes":2009272,"evictions":0},"query_cache":{"memory_size_in_bytes":1051393983,"total_count":973926615,"hit_count":93019708,"miss_count":880906907,"cache_size":11497,"cache_count":1552335,"evictions":1540838},"completion":{"size_in_bytes":0},"segments":{"count":9005,"memory_in_bytes":381411016,"terms_memory_in_bytes":130261152,"stored_fields_memory_in_bytes":98710448,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":6245824,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":146193592,"index_writer_memory_in_bytes":515950884,"version_map_memory_in_bytes":396,"fixed_bit_set_memory_in_bytes":108472,"max_unsafe_auto_id_timestamp":1625011201270,"file_sizes":{}},"mappings":{"field_types":[{"name":"alias","count":252,"index_count":84},{"name":"binary","count":12,"index_count":2},{"name":"boolean","count":3721,"index_count":140},{"name":"byte","count":87,"index_count":87},{"name":"date","count":3534,"index_count":223},{"name":"double","count":2548,"index_count":84},{"name":"flattened","count":10,"index_count":2},{"name":"float","count":5010,"index_count":114},{"name":"geo_point","count":588,"index_count":84},{"name":"geo_shape","count":1,"index_count":1},{"name":"integer","count":52,"index_count":4},{"name":"ip","count":1540,"index_count":84},{"name":"keyword","count":58431,"index_count":229},{"name":"long","count":147017,"index_count":212},{"name":"nested","count":17,"index_count":7},{"name":"object","count":132325,"index_count":209},{"name":"scaled_float","count":8933,"index_count":85},{"name":"short","count":7,"index_count":4},{"name":"text","count":4742,"index_count":165}]},"analysis":{"char_filter_types":,"tokenizer_types":,"filter_types":,"analyzer_types":,"built_in_char_filters":,"built_in_tokenizers":,"built_in_filters":,"built_in_analyzers":}},"nodes":{"count":{"total":3,"coordinating_only":0,"data":2,"data_cold":2,"data_content":2,"data_hot":2,"data_warm":2,"ingest":3,"master":1,"ml":3,"remote_cluster_client":3,"transform":2,"voting_only":0},"versions":["7.10.0"],"os":{"available_processors":36,"allocated_processors":36,"names":[{"name":"Linux","count":3}],"pretty_names":[{"pretty_name":"CentOS Linux 8 (Core)","count":3}],"mem":{"total_in_bytes":13958643712,"free_in_bytes":166309888,"used_in_bytes":13792333824,"free_percent":1,"used_percent":99}},"process":{"cpu":{"percent":35},"open_file_descriptors":{"min":588,"max":12964,"avg":8729}},"jvm":{"max_uptime_in_millis":6736124784,"versions":[{"version":"15.0.1","vm_name":"OpenJDK 64-Bit Server VM","vm_version":"15.0.1+9","vm_vendor":"AdoptOpenJDK","bundled_jdk":true,"using_bundled_jdk":true,"count":3}],"mem":{"heap_used_in_bytes":7449201120,"heap_max_in_bytes":11274289152},"threads":391},"fs":{"total_in_bytes":1571892805632,"free_in_bytes":491849428992,"available_in_bytes":491849428992},"plugins":,"network_types":{"transport_types":{"netty4":3},"http_types":{"netty4":3}},"discovery_types":{"zen":3},"packaging_types":[{"flavor":"default","type":"docker","count":3}],"ingest":{"number_of_pipelines":3,"processor_stats":{"convert":{"count":244,"failed":0,"current":0,"time_in_millis":9},"csv":{"count":244,"failed":0,"current":0,"time_in_millis":16},"gsub":{"count":0,"failed":0,"current":0,"time_in_millis":0},"remove":{"count":244,"failed":0,"current":0,"time_in_millis":0},"script":{"count":0,"failed":0,"current":0,"time_in_millis":0}}}}}

4

From cluster stats everything seems fine. But I can see this process has very high cpu utilization:
sysadm 13372 13297 99 Apr13 ? 95-04:40:22 /usr/share/elasticsearch/jdk/bin/java -Xshare:auto -Des.networkaddress.cache.ttl=60 -Des.networkaddress.cache.negative.ttl=10 -XX:+AlwaysPreTouch -Xss1m -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djna.nosys=true -XX:-OmitStackTraceInFastThrow -XX:+ShowCodeDetailsInExceptionMessages -Dio.netty.noUnsafe=true -Dio.netty.noKeySetOptimization=true -Dio.netty.recycler.maxCapacityPerThread=0 -Dio.netty.allocator.numDirectArenas=0 -Dlog4j.shutdownHookEnabled=false -Dlog4j2.disable.jmx=true -Djava.locale.providers=SPI,COMPAT -Xms1g -Xmx1g -XX:+UseG1GC -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -Djava.io.tmpdir=/tmp/elasticsearch-10070937545416279791 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=data -XX:ErrorFile=logs/hs_err_pid%p.log -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m -Des.cgroups.hierarchy.override=/ -Xms5G -Xmx5G -XX:MaxDirectMemorySize=2684354560 -Des.path.home=/usr/share/elasticsearch -Des.path.conf=/usr/share/elasticsearch/config -Des.distribution.flavor=default -Des.distribution.type=docker -Des.bundled_jdk=true -cp /usr/share/elasticsearch/lib/* org.elasticsearch.bootstrap.Elasticsearch -Ebootstrap.memory_lock=false -Expack.security.http.ssl.enabled=false -Ecluster.routing.allocation.disk.watermark.flood_stage=1gb -Enode.name=data2 -Escript.max_compilations_rate=2048/1m -Expack.security.authc.token.enabled=false -Ecluster.initial_master_nodes=master1 -Enode.store.allow_mmap=false -Epath.repo=/usr/share/elasticsearch/backups -Ecluster.routing.allocation.disk.watermark.high=5gb -Expack.security.transport.ssl.enabled=false -Ediscovery.seed_hosts=master1:9300 -Ecluster.name=nfvi-elkstack -Enode.master=false -Ecluster.routing.allocation.disk.watermark.low=10gb -Expack.security.enabled=false -Expack.security.audit.enabled=false
sysadm 14764 13372 0 Apr13 ? 00:00:00 /usr/share/elasticsearch/modules/x-pack-ml/platform/linux-x86_64/bin/controller

5

Can you check the hot threads for that node?

Please also format your code/logs/config using the </> button, or markdown style back ticks. It helps to make things easy to read which helps us help you :slight_smile:

6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.