Hi
I'm just taking care of gatling test for my new ECK cluster.
This cluster consists with 9 nodes
3-master nodes
3-ingest nodes
3-data nodes on nvme disk
During perform gatling test with an example 30 rps (request per sec) I saw many of failure
these tests were making over REST API
but what is interesting things this cluster doesn't load balance these request and I've observer that one of data nodes are consuming 100 % of CPU (it looks like a few of search request hit to this nodes and make overboosting), index are completed with 3 shards with 1 replica
How we can tune up those things?
::: {elasticsearch-es-eck-data-0}{zenaffBuRLS0QL7_4RsXnw}{EnPvveM7TFW6J-TpFDCt2g}{10.124.222.36}{10.124.222.36:9300}{d}{k8s_node_name=aks-nvme-16105314-vmss000006, xpack.installed=true, transform.node=false}
Hot threads at 2023-03-28T12:16:47.125Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
100.0% [cpu=56.4%, other=43.6%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[elasticsearch-es-eck-data-0][search][T#2]'
2/10 snapshots sharing following 38 elements
java.base@19.0.1/sun.nio.ch.FileDispatcherImpl.pread0(Native Method)
java.base@19.0.1/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54)
java.base@19.0.1/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:337)
java.base@19.0.1/sun.nio.ch.IOUtil.read(IOUtil.java:305)
java.base@19.0.1/sun.nio.ch.IOUtil.read(IOUtil.java:282)
java.base@19.0.1/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:976)
java.base@19.0.1/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:959)
app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:170)
app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:315)
app//org.apache.lucene.store.BufferedIndexInput.readShort(BufferedIndexInput.java:267)
app//org.apache.lucene.util.packed.DirectReader$DirectPackedReader16.get(DirectReader.java:185)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$26.nextOrd(Lucene80DocValuesProducer.java:1580)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalMapping.nextOrd(GlobalOrdinalMapping.java:54)
app//org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator$3.collect(GlobalOrdinalsStringTermsAggregator.java:158)
app//org.elasticsearch.search.aggregations.LeafBucketCollector.collect(LeafBucketCollector.java:86)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:258)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:245)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:45)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:194)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:167)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:443)
app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:255)
app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:212)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:98)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$5(SearchService.java:708)
app//org.elasticsearch.search.SearchService$$Lambda$7925/0x000000080119ec70.get(Unknown Source)
app//org.elasticsearch.search.SearchService$$Lambda$6810/0x0000000801fd6450.get(Unknown Source)
app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:47)
app//org.elasticsearch.action.ActionRunnable$$Lambda$6811/0x0000000801fd6660.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:62)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@19.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@19.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@19.0.1/java.lang.Thread.run(Thread.java:1589)
3/10 snapshots sharing following 30 elements
app//org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:40)
app//org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:108)
app//org.apache.lucene.index.OrdinalMap$3.get(OrdinalMap.java:328)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalMapping.getGlobalOrd(GlobalOrdinalMapping.java:44)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalMapping.nextOrd(GlobalOrdinalMapping.java:58)
app//org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator$3.collect(GlobalOrdinalsStringTermsAggregator.java:158)
app//org.elasticsearch.search.aggregations.LeafBucketCollector.collect(LeafBucketCollector.java:86)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:258)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:245)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:45)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:194)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:167)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:443)
app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:255)
app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:212)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:98)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$5(SearchService.java:708)
app//org.elasticsearch.search.SearchService$$Lambda$7925/0x000000080119ec70.get(Unknown Source)
app//org.elasticsearch.search.SearchService$$Lambda$6810/0x0000000801fd6450.get(Unknown Source)
app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:47)
app//org.elasticsearch.action.ActionRunnable$$Lambda$6811/0x0000000801fd6660.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:62)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@19.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@19.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@19.0.1/java.lang.Thread.run(Thread.java:1589)
4/10 snapshots sharing following 24 elements
app//org.elasticsearch.search.aggregations.LeafBucketCollector.collect(LeafBucketCollector.java:86)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:258)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:245)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:45)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:194)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:167)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:443)
app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:255)
app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:212)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:98)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$5(SearchService.java:708)
app//org.elasticsearch.search.SearchService$$Lambda$7925/0x000000080119ec70.get(Unknown Source)
app//org.elasticsearch.search.SearchService$$Lambda$6810/0x0000000801fd6450.get(Unknown Source)
app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:47)
app//org.elasticsearch.action.ActionRunnable$$Lambda$6811/0x0000000801fd6660.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:62)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@19.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@19.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@19.0.1/java.lang.Thread.run(Thread.java:1589)
unique snapshot
app//org.elasticsearch.search.aggregations.InternalOrder.lambda$comparingKeys$0(InternalOrder.java:384)
app//org.elasticsearch.search.aggregations.InternalOrder$$Lambda$7271/0x00000008020977f8.compare(Unknown Source)
app//org.elasticsearch.search.aggregations.InternalOrder$SimpleOrder.lambda$partiallyBuiltBucketComparator$0(InternalOrder.java:294)
app//org.elasticsearch.search.aggregations.InternalOrder$SimpleOrder$$Lambda$7291/0x00000008020a25b8.compare(Unknown Source)
app//org.elasticsearch.search.aggregations.InternalOrder$CompoundOrder.lambda$partiallyBuiltBucketComparator$1(InternalOrder.java:193)
app//org.elasticsearch.search.aggregations.InternalOrder$CompoundOrder$$Lambda$7292/0x00000008020a2840.compare(Unknown Source)
app//org.elasticsearch.search.aggregations.bucket.terms.BucketPriorityQueue.lessThan(BucketPriorityQueue.java:25)
app//org.apache.lucene.util.PriorityQueue.upHeap(PriorityQueue.java:254)
app//org.apache.lucene.util.PriorityQueue.add(PriorityQueue.java:131)
app//org.apache.lucene.util.PriorityQueue.insertWithOverflow(PriorityQueue.java:147)
app//org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator$ResultStrategy$1.accept(GlobalOrdinalsStringTermsAggregator.java:613)
app//org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator$DenseGlobalOrds.forEach(GlobalOrdinalsStringTermsAggregator.java:480)
app//org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator$ResultStrategy.buildAggregations(GlobalOrdinalsStringTermsAggregator.java:602)
app//org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator$ResultStrategy.access$200(GlobalOrdinalsStringTermsAggregator.java:575)
app//org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator.buildAggregations(GlobalOrdinalsStringTermsAggregator.java:182)
app//org.elasticsearch.search.aggregations.Aggregator.buildTopLevel(Aggregator.java:154)
app//org.elasticsearch.search.aggregations.AggregationPhase.execute(AggregationPhase.java:67)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:104)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$5(SearchService.java:708)
app//org.elasticsearch.search.SearchService$$Lambda$7925/0x000000080119ec70.get(Unknown Source)
app//org.elasticsearch.search.SearchService$$Lambda$6810/0x0000000801fd6450.get(Unknown Source)
app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:47)
app//org.elasticsearch.action.ActionRunnable$$Lambda$6811/0x0000000801fd6660.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:62)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@19.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@19.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@19.0.1/java.lang.Thread.run(Thread.java:1589)
100.0% [cpu=55.6%, other=44.4%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[elasticsearch-es-eck-data-0][search][T#3]'
3/10 snapshots sharing following 27 elements
app//org.elasticsearch.search.aggregations.bucket.BucketsAggregator.collectExistingBucket(BucketsAggregator.java:92)
app//org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator$DenseGlobalOrds.collectGlobalOrd(GlobalOrdinalsStringTermsAggregator.java:462)
app//org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator$3.collect(GlobalOrdinalsStringTermsAggregator.java:159)
app//org.elasticsearch.search.aggregations.LeafBucketCollector.collect(LeafBucketCollector.java:86)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:258)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:245)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:45)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:194)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:167)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:443)
app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:255)
app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:212)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:98)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$5(SearchService.java:708)
app//org.elasticsearch.search.SearchService$$Lambda$7925/0x000000080119ec70.get(Unknown Source)
app//org.elasticsearch.search.SearchService$$Lambda$6810/0x0000000801fd6450.get(Unknown Source)
app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:47)
app//org.elasticsearch.action.ActionRunnable$$Lambda$6811/0x0000000801fd6660.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:62)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@19.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@19.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@19.0.1/java.lang.Thread.run(Thread.java:1589)
3/10 snapshots sharing following 30 elements
app//org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:40)
app//org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:108)
app//org.apache.lucene.index.OrdinalMap$3.get(OrdinalMap.java:328)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalMapping.getGlobalOrd(GlobalOrdinalMapping.java:44)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalMapping.nextOrd(GlobalOrdinalMapping.java:58)
app//org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator$3.collect(GlobalOrdinalsStringTermsAggregator.java:158)
app//org.elasticsearch.search.aggregations.LeafBucketCollector.collect(LeafBucketCollector.java:86)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:258)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:245)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:45)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:194)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:167)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:443)
app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:255)
app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:212)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:98)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$5(SearchService.java:708)
app//org.elasticsearch.search.SearchService$$Lambda$7925/0x000000080119ec70.get(Unknown Source)
app//org.elasticsearch.search.SearchService$$Lambda$6810/0x0000000801fd6450.get(Unknown Source)
app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:47)
app//org.elasticsearch.action.ActionRunnable$$Lambda$6811/0x0000000801fd6660.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:62)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@19.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@19.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@19.0.1/java.lang.Thread.run(Thread.java:1589)
4/10 snapshots sharing following 24 elements
app//org.elasticsearch.search.aggregations.LeafBucketCollector.collect(LeafBucketCollector.java:86)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:258)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:245)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:45)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:194)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:167)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:443)
app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:255)
app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:212)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:98)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$5(SearchService.java:708)
app//org.elasticsearch.search.SearchService$$Lambda$7925/0x000000080119ec70.get(Unknown Source)
app//org.elasticsearch.search.SearchService$$Lambda$6810/0x0000000801fd6450.get(Unknown Source)
app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:47)
app//org.elasticsearch.action.ActionRunnable$$Lambda$6811/0x0000000801fd6660.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:62)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@19.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@19.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@19.0.1/java.lang.Thread.run(Thread.java:1589)
100.0% [cpu=54.8%, other=45.2%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[elasticsearch-es-eck-data-0][search][T#1]'
3/10 snapshots sharing following 30 elements
app//org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:40)
app//org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:108)
app//org.apache.lucene.index.OrdinalMap$3.get(OrdinalMap.java:328)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalMapping.getGlobalOrd(GlobalOrdinalMapping.java:44)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalMapping.nextOrd(GlobalOrdinalMapping.java:58)
app//org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator$3.collect(GlobalOrdinalsStringTermsAggregator.java:158)
app//org.elasticsearch.search.aggregations.LeafBucketCollector.collect(LeafBucketCollector.java:86)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:258)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:245)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:45)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:194)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:167)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:443)
app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:255)
app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:212)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:98)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$5(SearchService.java:708)
app//org.elasticsearch.search.SearchService$$Lambda$7925/0x000000080119ec70.get(Unknown Source)
app//org.elasticsearch.search.SearchService$$Lambda$6810/0x0000000801fd6450.get(Unknown Source)
app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:47)
app//org.elasticsearch.action.ActionRunnable$$Lambda$6811/0x0000000801fd6660.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:62)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@19.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@19.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@19.0.1/java.lang.Thread.run(Thread.java:1589)
7/10 snapshots sharing following 22 elements
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:245)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:45)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:194)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:167)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:443)
app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:255)
app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:212)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:98)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$5(SearchService.java:708)
app//org.elasticsearch.search.SearchService$$Lambda$7925/0x000000080119ec70.get(Unknown Source)
app//org.elasticsearch.search.SearchService$$Lambda$6810/0x0000000801fd6450.get(Unknown Source)
app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:47)
app//org.elasticsearch.action.ActionRunnable$$Lambda$6811/0x0000000801fd6660.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:62)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@19.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@19.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@19.0.1/java.lang.Thread.run(Thread.java:1589)
