Slow Cluster in Elastic Cloud since updating to 7.12

Elastic Stack Elasticsearch
4

Last part:

io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:440)
               io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
               io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
               io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
               io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
               io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
               io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
               io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
               io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
               io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
               io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714)
               io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615)
               io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578)
               io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
               io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
               io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
               java.base@15.0.1/java.lang.Thread.run(Thread.java:832)
     29.9% (149.6ms out of 500ms) cpu usage by thread 'elasticsearch[instance-0000000027][transport_worker][T#1]'
             8/10 snapshots sharing following 24 elements
               io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
               io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
               io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
               io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1518)
               io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1267)
               io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1314)
               io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:501)
               io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:440)
               io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
               io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
               io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
               io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
               io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
               io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
               io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
               io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
               io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
               io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714)
               io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615)
               io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578)
               io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
               io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
               io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
               java.base@15.0.1/java.lang.Thread.run(Thread.java:832)
             2/10 snapshots sharing following 25 elements
               java.base@15.0.1/sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506)
               java.base@15.0.1/sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482)
               java.base@15.0.1/javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:637)
               io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:282)
               io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1372)
               io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1267)
               io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1314)
               io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:501)
               io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:440)
               io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
               io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
               io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
               io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
               io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
               io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
               io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
               io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
               io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
               io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714)
               io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615)
               io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578)
               io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
               io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
               io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
               java.base@15.0.1/java.lang.Thread.run(Thread.java:832)

        ::: {instance-0000000024}{ctdyx5xlTWOzgOmaq8qjsQ}{93oKMN2tS2ytEHwSMNcYHQ}{172.26.54.191}{172.26.54.191:19190}{hmrst}{logical_availability_zone=zone-0, server_name=instance-0000000024.433e4a7f73fd4ed0a7436f398beaf0bd, availability_zone=sa-east-1a, xpack.installed=true, instance_configuration=aws.data.highio.i3, transform.node=true, region=sa-east-1}
           Hot threads at 2021-03-30T23:22:13.847Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

        ::: {instance-0000000025}{aVnmVgQOS2GEO_Sb0dUNEA}{ZLIALxcCSPybk9pO_a3xYg}{172.26.70.155}{172.26.70.155:19553}{hmrst}{logical_availability_zone=zone-1, server_name=instance-0000000025.433e4a7f73fd4ed0a7436f398beaf0bd, availability_zone=sa-east-1b, xpack.installed=true, instance_configuration=aws.data.highio.i3, transform.node=true, region=sa-east-1}
           Hot threads at 2021-03-30T23:22:13.875Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
           
            4.4% (22ms out of 500ms) cpu usage by thread 'elasticsearch[instance-0000000025][transport_worker][T#1]'
             2/10 snapshots sharing following 6 elements
               io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615)
               io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578)
               io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
               io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
               io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
               java.base@15.0.1/java.lang.Thread.run(Thread.java:832)
             4/10 snapshots sharing following 9 elements
               java.base@15.0.1/sun.nio.ch.EPoll.wait(Native Method)
               java.base@15.0.1/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
               java.base@15.0.1/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
               java.base@15.0.1/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
               io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803)
               io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457)
               io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
               io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
               java.base@15.0.1/java.lang.Thread.run(Thread.java:832)
           
            1.1% (5.5ms out of 500ms) cpu usage by thread 'elasticsearch[instance-0000000025][transport_worker][T#2]'
             unique snapshot
               java.base@15.0.1/sun.nio.ch.SocketDispatcher.read0(Native Method)
               java.base@15.0.1/sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:47)
               java.base@15.0.1/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:276)
               java.base@15.0.1/sun.nio.ch.IOUtil.read(IOUtil.java:233)
               java.base@15.0.1/sun.nio.ch.IOUtil.read(IOUtil.java:223)
               java.base@15.0.1/sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:389)
               org.elasticsearch.transport.CopyBytesSocketChannel.readFromSocketChannel(CopyBytesSocketChannel.java:130)
               org.elasticsearch.transport.CopyBytesSocketChannel.doReadBytes(CopyBytesSocketChannel.java:115)
               io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:148)
               io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714)
               io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615)
               io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578)
               io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
               io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
               io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
               java.base@15.0.1/java.lang.Thread.run(Thread.java:832)