Elasticsearch version : 6.8.9
JDK : 8
RAM : 8GB
JVM arguments: -Xms4g -Xmx4g
and the remaining are default configurations provided by package.
CPU utilization is going more than 90% and also high response time taking for simple search queries also.
Captured the below logs during the search load, unable to understand how to resolve the issue.
37.2% (186.1ms out of 500ms) cpu usage by thread 'elasticsearch[uyHufGe][http_server_worker][T#3]'
2/10 snapshots sharing following 36 elements
java.math.BigInteger.oddModPow(BigInteger.java:2941)
java.math.BigInteger.modPow(BigInteger.java:2603)
sun.security.rsa.RSACore.crtCrypt(RSACore.java:185)
sun.security.rsa.RSACore.rsa(RSACore.java:122)
sun.security.rsa.RSASignature.engineSign(RSASignature.java:192)
java.security.Signature$Delegate.engineSign(Signature.java:1382)
java.security.Signature.sign(Signature.java:698)
sun.security.ssl.HandshakeMessage$ECDH_ServerKeyExchange.(HandshakeMessage.java:1041)
sun.security.ssl.ServerHandshaker.clientHello(ServerHandshaker.java:1053)
sun.security.ssl.ServerHandshaker.processMessage(ServerHandshaker.java:222)
sun.security.ssl.Handshaker.processLoop(Handshaker.java:1082)
sun.security.ssl.Handshaker$1.run(Handshaker.java:1015)
sun.security.ssl.Handshaker$1.run(Handshaker.java:1012)
java.security.AccessController.doPrivileged(Native Method)
sun.security.ssl.Handshaker$DelegatedTask.run(Handshaker.java:1520)
io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1464)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1369)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1203)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1247)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:502)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:441)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:278)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909)
java.lang.Thread.run(Thread.java:748)
2/10 snapshots sharing following 34 elements
sun.security.ec.ECDHKeyAgreement.deriveKeyImpl(ECDHKeyAgreement.java:239)
sun.security.ec.ECDHKeyAgreement.engineGenerateSecret(ECDHKeyAgreement.java:169)
sun.security.ec.ECDHKeyAgreement.engineGenerateSecret(ECDHKeyAgreement.java:202)
javax.crypto.KeyAgreement.generateSecret(KeyAgreement.java:648)
sun.security.ssl.ECDHCrypt.getAgreedSecret(ECDHCrypt.java:102)
sun.security.ssl.ECDHCrypt.getAgreedSecret(ECDHCrypt.java:120)
sun.security.ssl.ServerHandshaker.clientKeyExchange(ServerHandshaker.java:1771)
sun.security.ssl.ServerHandshaker.processMessage(ServerHandshaker.java:287)
sun.security.ssl.Handshaker.processLoop(Handshaker.java:1082)
sun.security.ssl.Handshaker$1.run(Handshaker.java:1015)
sun.security.ssl.Handshaker$1.run(Handshaker.java:1012)
java.security.AccessController.doPrivileged(Native Method)
sun.security.ssl.Handshaker$DelegatedTask.run(Handshaker.java:1520)
io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1464)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1369)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1203)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1247)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:502)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:441)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:278)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909)
java.lang.Thread.run(Thread.java:748)
2/10 snapshots sharing following 23 elements
java.security.AccessController.doPrivileged(Native Method)
sun.security.ssl.Handshaker$DelegatedTask.run(Handshaker.java:1520)
io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1464)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1369)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1203)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1247)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:502)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:441)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:278)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909)
java.lang.Thread.run(Thread.java:748)