Our index rate ...
And management thread_pool queue:
You can see it really blew up when we cross 30k docs per second
Here's an example of hot threads
93.3% (466.6ms out of 500ms) cpu usage by thread 'elasticsearch[esdata-03ea][management][T#2]'
2/10 snapshots sharing following 16 elements
app//org.elasticsearch.index.engine.Engine.fillSegmentStats(Engine.java:830)
app//org.elasticsearch.index.engine.Engine.segmentsStats(Engine.java:812)
app//org.elasticsearch.index.shard.IndexShard.segmentStats(IndexShard.java:1028)
app//org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:210) app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:120) app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:52) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
app//org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:752) app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base@13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13/java.lang.Thread.run(Thread.java:830)
6/10 snapshots sharing following 20 elements
java.base@13/java.util.TreeMap.getEntry(TreeMap.java:357)
java.base@13/java.util.TreeMap.get(TreeMap.java:277) app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:311)
app//org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
app//org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:366)
app//org.elasticsearch.index.engine.Engine.completionStats(Engine.java:197)
app//org.elasticsearch.index.shard.IndexShard.completionStats(IndexShard.java:1048)
app//org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:207) app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:120) app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:52) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
app//org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:752) app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base@13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13/java.lang.Thread.run(Thread.java:830)
2/10 snapshots sharing following 21 elements
java.base@13/java.util.TreeMap.getEntry(TreeMap.java:357)
java.base@13/java.util.TreeMap.get(TreeMap.java:277) app//org.apache.lucene.codecs.blocktree.BlockTreeTermsReader.terms(BlockTreeTermsReader.java:304) app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:312)
app//org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
app//org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:366)
app//org.elasticsearch.index.engine.Engine.completionStats(Engine.java:197)
app//org.elasticsearch.index.shard.IndexShard.completionStats(IndexShard.java:1048)
app//org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:207) app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:120) app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:52) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
app//org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:752) app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base@13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13/java.lang.Thread.run(Thread.java:830)