Write bulk is stick for a long time

Elastic Stack Elasticsearch
1

elasticsearch version: 7.16.2
os: rhel 7.9

es node load very high, write task execution 1-2 hours, help me, thanks

hot_threads

100.0% [cpu=11.1%, other=88.9%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[instance-0000000011][write][T#1]'
     8/10 snapshots sharing following 26 elements
       app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
       app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
       app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:574)
       app//org.elasticsearch.index.mapper.DocumentParser.parseObject(DocumentParser.java:608)
       app//org.elasticsearch.index.mapper.DocumentParser.parseNonDynamicArray(DocumentParser.java:702)
       app//org.elasticsearch.index.mapper.DocumentParser.parseArray(DocumentParser.java:652)
       app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:504)
       app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
       app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
       app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
       app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
       app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1013)
       app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:959)
       app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:888)
       app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
       app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
       app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
       app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
       app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
       app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
       app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
       app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
       app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
       java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
       java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
       java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
     2/10 snapshots sharing following 17 elements
       app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
       app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
       app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1013)
       app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:959)
       app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:888)
       app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
       app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
       app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
       app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
       app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
       app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
       app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
       app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
       app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
       java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
       java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
       java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
2

What is the size and hardware specification of the cluster? What type of storage is being used?

What is the full output of the cluster stats API?

3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.