Hi,
We have our production cluster full use of cpu with very little index or query. We deploy two nodes each machine，each node use more than 1000% cpu with 24 cpu cores
- JDK 1.8.0_131
- ES 6.3.1
- 2 nodes per machine ,echo node 31GB heap
- centos 7,
here is the hot_threads：
::: {node1}{SKbtlde0Tw2Lln7_o5mNpA}{MNez5no7T_GcXNcq2-zu0A}{1******}{1*****:9300}{xpack.installed=true}
Hot threads at 2020-04-17T06:52:28.124Z, interval=1s, busiestThreads=3, ignoreIdleThreads=true:
100.3% (1s out of 1s) cpu usage by thread 'elasticsearch[node1][[transport_server_worker.default]][T#32]'
4/10 snapshots sharing following 62 elements
java.util.zip.Deflater.deflateBytes(Native Method)
java.util.zip.Deflater.deflate(Deflater.java:444)
java.util.zip.Deflater.deflate(Deflater.java:366)
java.util.zip.DeflaterOutputStream.deflate(DeflaterOutputStream.java:251)
java.util.zip.DeflaterOutputStream.write(DeflaterOutputStream.java:211)
java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
java.io.BufferedOutputStream.write(BufferedOutputStream.java:95)
org.elasticsearch.common.io.stream.OutputStreamStreamOutput.writeByte(OutputStreamStreamOutput.java:35)
org.elasticsearch.common.io.stream.StreamOutput.writeBoolean(StreamOutput.java:425)
org.elasticsearch.common.io.stream.StreamOutput.writeOptionalWriteable(StreamOutput.java:806)
org.elasticsearch.cluster.routing.ShardRouting.writeToThin(ShardRouting.java:294)
org.elasticsearch.cluster.routing.IndexShardRoutingTable$Builder.writeToThin(IndexShardRoutingTable.java:834)
org.elasticsearch.cluster.routing.IndexRoutingTable.writeTo(IndexRoutingTable.java:308)
org.elasticsearch.cluster.routing.RoutingTable.writeTo(RoutingTable.java:360)
org.elasticsearch.cluster.ClusterState.writeTo(ClusterState.java:742)
org.elasticsearch.discovery.zen.PublishClusterStateAction.serializeFullClusterState(PublishClusterStateAction.java:355)
org.elasticsearch.action.admin.cluster.state.TransportClusterStateAction.masterOperation(TransportClusterStateAction.java:136)
org.elasticsearch.action.admin.cluster.state.TransportClusterStateAction.masterOperation(TransportClusterStateAction.java:44)
org.elasticsearch.action.support.master.TransportMasterNodeAction.masterOperation(TransportMasterNodeAction.java:88)
org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$2.doRun(TransportMasterNodeAction.java:174)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.EsExecutors$1.execute(EsExecutors.java:135)
org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.doStart(TransportMasterNodeAction.java:171)
org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.start(TransportMasterNodeAction.java:127)
org.elasticsearch.action.support.master.TransportMasterNodeAction.doExecute(TransportMasterNodeAction.java:105)
org.elasticsearch.action.support.master.TransportMasterNodeAction.doExecute(TransportMasterNodeAction.java:55)
org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:167)
org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:139)
org.elasticsearch.action.support.HandledTransportAction$TransportHandler.messageReceived(HandledTransportAction.java:79)
org.elasticsearch.action.support.HandledTransportAction$TransportHandler.messageReceived(HandledTransportAction.java:69)
org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1592)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.EsExecutors$1.execute(EsExecutors.java:135)
org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1550)
org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1414)
org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:64)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:545)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:499)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
java.lang.Thread.run(Thread.java:748)
thx for any help