aumthanki
January 25, 2022, 4:22pm
1
Setup:
3 nodes in Azure. 2 masters and one voting_only.
Info: (Nodes)
{
"name" : "<>",
"cluster_name" : "<>",
"cluster_uuid" : "<>",
"version" : {
"number" : "7.16.1",
"build_flavor" : "default",
"build_type" : "docker",
"build_hash" : "<>",
"build_date" : "<>",
"build_snapshot" : false,
"lucene_version" : "8.10.1",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},
"tagline" : "You Know, for Search"
}
A master node in our Azure Elasticsearch cluster has high CPU usage:
I'm very new to ELK, can anyone point me in the right direction?
100.0% [cpu=85.2%, other=14.8%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[<>][management][T#2]'
3/10 snapshots sharing following 43 elements
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:394)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:394)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:394)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:394)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:394)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:394)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:394)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:394)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:394)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapSafe(AbstractXContentParser.java:304)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.mapOrdered(AbstractXContentParser.java:259)
app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:188)
app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:147)
app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:106)
app//org.elasticsearch.cluster.metadata.MappingMetadata.sourceAsMap(MappingMetadata.java:142)
app//org.elasticsearch.cluster.metadata.MappingMetadata.getSourceAsMap(MappingMetadata.java:154)
app//org.elasticsearch.action.admin.cluster.stats.MappingStats.of(MappingStats.java:63)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction$$Lambda$4805/0x0000000801a73bc8.apply(Unknown Source)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction$MetadataStatsCache.lambda$refresh$0(TransportClusterStatsAction.java:278)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction$MetadataStatsCache$$Lambda$8556/0x0000000802110668.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction$MetadataStatsCache.refresh(TransportClusterStatsAction.java:278)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction$MetadataStatsCache.refresh(TransportClusterStatsAction.java:264)
app//org.elasticsearch.common.util.CancellableSingleObjectCache$CachedItem.startRefresh(CancellableSingleObjectCache.java:280)
app//org.elasticsearch.common.util.CancellableSingleObjectCache.get(CancellableSingleObjectCache.java:158)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.newResponseAsync(TransportClusterStatsAction.java:120)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.newResponseAsync(TransportClusterStatsAction.java:54)
app//org.elasticsearch.action.support.nodes.TransportNodesAction.newResponse(TransportNodesAction.java:156)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.lambda$finishHim$0(TransportNodesAction.java:295)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction$$Lambda$7039/0x0000000801e0e230.run(Unknown Source)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
2/10 snapshots sharing following 40 elements
app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.nextToken(UTF8StreamJsonParser.java:758)
app//org.elasticsearch.xcontent.json.JsonXContentParser.nextToken(JsonXContentParser.java:73)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:320)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:394)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:394)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:394)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:394)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:394)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:394)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapSafe(AbstractXContentParser.java:304)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.mapOrdered(AbstractXContentParser.java:259)
app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:188)
app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:147)
app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:106)
app//org.elasticsearch.cluster.metadata.MappingMetadata.sourceAsMap(MappingMetadata.java:142)
app//org.elasticsearch.cluster.metadata.MappingMetadata.getSourceAsMap(MappingMetadata.java:154)
app//org.elasticsearch.action.admin.cluster.stats.MappingStats.of(MappingStats.java:63)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction$$Lambda$4805/0x0000000801a73bc8.apply(Unknown Source)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction$MetadataStatsCache.lambda$refresh$0(TransportClusterStatsAction.java:278)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction$MetadataStatsCache$$Lambda$8556/0x0000000802110668.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction$MetadataStatsCache.refresh(TransportClusterStatsAction.java:278)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction$MetadataStatsCache.refresh(TransportClusterStatsAction.java:264)
app//org.elasticsearch.common.util.CancellableSingleObjectCache$CachedItem.startRefresh(CancellableSingleObjectCache.java:280)
app//org.elasticsearch.common.util.CancellableSingleObjectCache.get(CancellableSingleObjectCache.java:158)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.newResponseAsync(TransportClusterStatsAction.java:120)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.newResponseAsync(TransportClusterStatsAction.java:54)
app//org.elasticsearch.action.support.nodes.TransportNodesAction.newResponse(TransportNodesAction.java:156)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.lambda$finishHim$0(TransportNodesAction.java:295)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction$$Lambda$7039/0x0000000801e0e230.run(Unknown Source)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
5/10 snapshots sharing following 17 elements
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction$$Lambda$4805/0x0000000801a73bc8.apply(Unknown Source)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction$MetadataStatsCache.lambda$refresh$0(TransportClusterStatsAction.java:278)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction$MetadataStatsCache$$Lambda$8556/0x0000000802110668.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction$MetadataStatsCache.refresh(TransportClusterStatsAction.java:278)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction$MetadataStatsCache.refresh(TransportClusterStatsAction.java:264)
app//org.elasticsearch.common.util.CancellableSingleObjectCache$CachedItem.startRefresh(CancellableSingleObjectCache.java:280)
app//org.elasticsearch.common.util.CancellableSingleObjectCache.get(CancellableSingleObjectCache.java:158)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.newResponseAsync(TransportClusterStatsAction.java:120)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.newResponseAsync(TransportClusterStatsAction.java:54)
app//org.elasticsearch.action.support.nodes.TransportNodesAction.newResponse(TransportNodesAction.java:156)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.lambda$finishHim$0(TransportNodesAction.java:295)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction$$Lambda$7039/0x0000000801e0e230.run(Unknown Source)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
aumthanki
January 25, 2022, 4:29pm
2
100.0% [cpu=20.9%, other=79.1%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[<>][transport_worker][T#1]'
2/10 snapshots sharing following 32 elements
java.base@17.0.1/sun.security.ssl.KAKeyDerivation.deriveKey(KAKeyDerivation.java:61)
java.base@17.0.1/sun.security.ssl.ECDHClientKeyExchange$ECDHEClientKeyExchangeConsumer.consume(ECDHClientKeyExchange.java:532)
java.base@17.0.1/sun.security.ssl.ClientKeyExchange$ClientKeyExchangeConsumer.consume(ClientKeyExchange.java:110)
java.base@17.0.1/sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396)
java.base@17.0.1/sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480)
java.base@17.0.1/sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1277)
java.base@17.0.1/sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1264)
java.base@17.0.1/java.security.AccessController.executePrivileged(AccessController.java:807)
java.base@17.0.1/java.security.AccessController.doPrivileged(AccessController.java:712)
java.base@17.0.1/sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1209)
io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1550)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1396)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1237)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1286)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
8/10 snapshots sharing following 20 elements
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1237)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1286)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
100.0% [cpu=3.7%, other=96.3%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[<>][transport_worker][T#2]'
2/10 snapshots sharing following 42 elements
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:67)
app//org.elasticsearch.transport.InboundHandler.handleRequest(InboundHandler.java:249)
app//org.elasticsearch.transport.InboundHandler.messageReceived(InboundHandler.java:106)
app//org.elasticsearch.transport.InboundHandler.inboundMessage(InboundHandler.java:88)
app//org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:743)
org.elasticsearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$6041/0x0000000801c1f210.accept(Unknown Source)
app//org.elasticsearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:147)
app//org.elasticsearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:119)
app//org.elasticsearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:84)
org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1374)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1237)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1286)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
7/10 snapshots sharing following 9 elements
java.base@17.0.1/sun.nio.ch.EPoll.wait(Native Method)
java.base@17.0.1/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
java.base@17.0.1/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
java.base@17.0.1/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:810)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
unique snapshot
java.base@17.0.1/java.util.Arrays.binarySearch0(Arrays.java:1716)
java.base@17.0.1/java.util.Arrays.binarySearch(Arrays.java:1659)
app//org.elasticsearch.common.bytes.CompositeBytesReference.getOffsetIndex(CompositeBytesReference.java:175)
app//org.elasticsearch.common.bytes.CompositeBytesReference.get(CompositeBytesReference.java:107)
app//org.elasticsearch.action.bulk.BulkRequestParser.sliceTrimmingCarriageReturn(BulkRequestParser.java:95)
app//org.elasticsearch.action.bulk.BulkRequestParser.parse(BulkRequestParser.java:377)
app//org.elasticsearch.action.bulk.BulkRequest.add(BulkRequest.java:337)
app//org.elasticsearch.rest.action.document.RestBulkAction.prepareRequest(RestBulkAction.java:94)
app//org.elasticsearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:86)
org.elasticsearch.xpack.security.rest.SecurityRestFilter.lambda$handleRequest$0(SecurityRestFilter.java:90)
org.elasticsearch.xpack.security.rest.SecurityRestFilter$$Lambda$6956/0x0000000801dedc68.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authc.support.SecondaryAuthenticator.lambda$authenticateAndAttachToContext$2(SecondaryAuthenticator.java:84)
org.elasticsearch.xpack.security.authc.support.SecondaryAuthenticator$$Lambda$6964/0x0000000801deedf8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authc.support.SecondaryAuthenticator.authenticate(SecondaryAuthenticator.java:94)
org.elasticsearch.xpack.security.authc.support.SecondaryAuthenticator.authenticateAndAttachToContext(SecondaryAuthenticator.java:78)
org.elasticsearch.xpack.security.rest.SecurityRestFilter.lambda$handleRequest$2(SecurityRestFilter.java:85)
org.elasticsearch.xpack.security.rest.SecurityRestFilter$$Lambda$6896/0x0000000801d72008.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101)
app//org.elasticsearch.action.ActionListener$RunBeforeActionListener.onResponse(ActionListener.java:389)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.writeAuthToContext(AuthenticatorChain.java:376)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.finishAuthentication(AuthenticatorChain.java:352)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.maybeLookupRunAsUser(AuthenticatorChain.java:205)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lambda$doAuthenticate$1(AuthenticatorChain.java:128)
org.elasticsearch.xpack.security.authc.AuthenticatorChain$$Lambda$6901/0x0000000801d72d10.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.core.common.IteratingActionListener.onResponse(IteratingActionListener.java:141)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lambda$getAuthenticatorConsumer$3(AuthenticatorChain.java:175)
org.elasticsearch.xpack.security.authc.AuthenticatorChain$$Lambda$6909/0x0000000801de61f8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator.lambda$consumeToken$3(RealmsAuthenticator.java:219)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator$$Lambda$6915/0x0000000801de7160.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.core.common.IteratingActionListener.onResponse(IteratingActionListener.java:141)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator.lambda$consumeToken$0(RealmsAuthenticator.java:164)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator$$Lambda$6921/0x0000000801de7a28.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm.lambda$authenticateWithCache$1(CachingUsernamePasswordRealm.java:155)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm$$Lambda$6950/0x0000000801ded380.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm.handleCachedAuthentication(CachingUsernamePasswordRealm.java:242)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm.lambda$authenticateWithCache$2(CachingUsernamePasswordRealm.java:139)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm$$Lambda$6928/0x0000000801deccf8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListenerDirectly(ListenableFuture.java:113)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:55)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm.authenticateWithCache(CachingUsernamePasswordRealm.java:134)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm.authenticate(CachingUsernamePasswordRealm.java:105)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator.lambda$consumeToken$2(RealmsAuthenticator.java:148)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator$$Lambda$6912/0x0000000801de6cf8.accept(Unknown Source)
org.elasticsearch.xpack.core.common.IteratingActionListener.run(IteratingActionListener.java:117)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator.consumeToken(RealmsAuthenticator.java:233)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator.authenticate(RealmsAuthenticator.java:84)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lambda$getAuthenticatorConsumer$5(AuthenticatorChain.java:171)
org.elasticsearch.xpack.security.authc.AuthenticatorChain$$Lambda$6904/0x0000000801d735e0.accept(Unknown Source)
org.elasticsearch.xpack.core.common.IteratingActionListener.onResponse(IteratingActionListener.java:135)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lambda$getAuthenticatorConsumer$5(AuthenticatorChain.java:165)
org.elasticsearch.xpack.security.authc.AuthenticatorChain$$Lambda$6904/0x0000000801d735e0.accept(Unknown Source)
org.elasticsearch.xpack.core.common.IteratingActionListener.onResponse(IteratingActionListener.java:135)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lambda$getAuthenticatorConsumer$5(AuthenticatorChain.java:165)
org.elasticsearch.xpack.security.authc.AuthenticatorChain$$Lambda$6904/0x0000000801d735e0.accept(Unknown Source)
org.elasticsearch.xpack.core.common.IteratingActionListener.onResponse(IteratingActionListener.java:135)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lambda$getAuthenticatorConsumer$5(AuthenticatorChain.java:165)
org.elasticsearch.xpack.security.authc.AuthenticatorChain$$Lambda$6904/0x0000000801d735e0.accept(Unknown Source)
org.elasticsearch.xpack.core.common.IteratingActionListener.run(IteratingActionListener.java:117)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.doAuthenticate(AuthenticatorChain.java:143)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:104)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:149)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:127)
org.elasticsearch.xpack.security.rest.SecurityRestFilter.handleRequest(SecurityRestFilter.java:79)
app//org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:327)
app//org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:393)
app//org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:245)
app//org.elasticsearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:382)
app//org.elasticsearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:461)
app//org.elasticsearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:357)
org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:32)
org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:18)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.elasticsearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:48)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1374)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1237)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1286)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
62.6% [cpu=16.3%, other=46.3%] (313ms out of 500ms) cpu usage by thread 'elasticsearch[<>][generic][T#24]'
2/10 snapshots sharing following 49 elements
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:394)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:394)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:394)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:318)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.readMapSafe(AbstractXContentParser.java:304)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.mapOrdered(AbstractXContentParser.java:259)
app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:210)
app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:138)
app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:106)
app//org.elasticsearch.cluster.metadata.IndexTemplateMetadata$Builder.toInnerXContent(IndexTemplateMetadata.java:460)
app//org.elasticsearch.cluster.metadata.IndexTemplateMetadata$Builder.toXContentWithTypes(IndexTemplateMetadata.java:355)
app//org.elasticsearch.cluster.metadata.Metadata$Builder.toXContent(Metadata.java:1999)
app//org.elasticsearch.cluster.metadata.Metadata.toXContent(Metadata.java:1060)
app//org.elasticsearch.gateway.PersistedClusterStateService$Writer.makeDocument(PersistedClusterStateService.java:985)
app//org.elasticsearch.gateway.PersistedClusterStateService$Writer.makeGlobalMetadataDocument(PersistedClusterStateService.java:970)
app//org.elasticsearch.gateway.PersistedClusterStateService$Writer.updateMetadata(PersistedClusterStateService.java:795)
app//org.elasticsearch.gateway.PersistedClusterStateService$Writer.writeIncrementalStateAndCommit(PersistedClusterStateService.java:745)
app//org.elasticsearch.gateway.GatewayMetaState$LucenePersistedState.setLastAcceptedState(GatewayMetaState.java:568)
app//org.elasticsearch.cluster.coordination.CoordinationState.handlePublishRequest(CoordinationState.java:413)
app//org.elasticsearch.cluster.coordination.Coordinator.handlePublishRequest(Coordinator.java:437)
app//org.elasticsearch.cluster.coordination.Coordinator$$Lambda$4571/0x00000008019b24d8.apply(Unknown Source)
app//org.elasticsearch.cluster.coordination.PublicationTransportHandler.acceptState(PublicationTransportHandler.java:242)
app//org.elasticsearch.cluster.coordination.PublicationTransportHandler.handleIncomingPublishRequest(PublicationTransportHandler.java:225)
app//org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$new$0(PublicationTransportHandler.java:111)
app//org.elasticsearch.cluster.coordination.PublicationTransportHandler$$Lambda$4574/0x00000008019b30a0.messageReceived(Unknown Source)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:341)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:404)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:394)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:620)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:250)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:136)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$6162/0x0000000801b3fd78.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:102)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:199)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:128)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:415)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:67)
app//org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:1045)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
5/10 snapshots sharing following 34 elements
app//org.elasticsearch.cluster.metadata.Metadata.toXContent(Metadata.java:1060)
app//org.elasticsearch.gateway.PersistedClusterStateService$Writer.makeDocument(PersistedClusterStateService.java:985)
app//org.elasticsearch.gateway.PersistedClusterStateService$Writer.makeGlobalMetadataDocument(PersistedClusterStateService.java:970)
app//org.elasticsearch.gateway.PersistedClusterStateService$Writer.updateMetadata(PersistedClusterStateService.java:795)
app//org.elasticsearch.gateway.PersistedClusterStateService$Writer.writeIncrementalStateAndCommit(PersistedClusterStateService.java:745)
app//org.elasticsearch.gateway.GatewayMetaState$LucenePersistedState.setLastAcceptedState(GatewayMetaState.java:568)
app//org.elasticsearch.cluster.coordination.CoordinationState.handlePublishRequest(CoordinationState.java:413)
app//org.elasticsearch.cluster.coordination.Coordinator.handlePublishRequest(Coordinator.java:437)
app//org.elasticsearch.cluster.coordination.Coordinator$$Lambda$4571/0x00000008019b24d8.apply(Unknown Source)
app//org.elasticsearch.cluster.coordination.PublicationTransportHandler.acceptState(PublicationTransportHandler.java:242)
app//org.elasticsearch.cluster.coordination.PublicationTransportHandler.handleIncomingPublishRequest(PublicationTransportHandler.java:225)
app//org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$new$0(PublicationTransportHandler.java:111)
app//org.elasticsearch.cluster.coordination.PublicationTransportHandler$$Lambda$4574/0x00000008019b30a0.messageReceived(Unknown Source)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:341)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:404)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:394)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:620)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:250)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:136)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$6162/0x0000000801b3fd78.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:102)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.j
warkolm
January 27, 2022, 3:04am
3
Why is your build hash and date not showing, that's pretty irregular, and even if you wanted to obfuscate things like your cluster name, it makes no sense on removing.
What is the output from the _cluster/stats?pretty&human API?
aumthanki
February 1, 2022, 7:44pm
4
Hi Mark,
We found the problem. We've oversharded our cluster.
Thanks for your response.
system
March 1, 2022, 7:44pm
5
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.