I am observing very high CPU usage on data nodes in my 7.9 ES cluster:
It happens during "business hours" so I am suspecting one of the applications is issuing a query (or multiple queries) that is causing such high CPU usage.
How can I investigate, which exact operation is causing this situation?
EDIT:
I queried GET /_nodes/hot_threads and this is part of the output:
55.5% (277.6ms out of 500ms) cpu usage by thread 'elasticsearch[my_node][search][T#4]'
2/10 snapshots sharing following 40 elements
app//org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getFrame(SegmentTermsEnum.java:212)
app//org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:239)
app//org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:473)
app//org.apache.lucene.index.FilterLeafReader$FilterTermsEnum.seekExact(FilterLeafReader.java:184)
app//org.apache.lucene.index.TermStates.loadTermsEnum(TermStates.java:124)
app//org.apache.lucene.index.TermStates.build(TermStates.java:109)
app//org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:447)
app//org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:38)
app//org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:429)
app//org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:429)
app//org.apache.lucene.search.BoostQuery.createWeight(BoostQuery.java:125)
app//org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:726)
app//org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:158)
app//org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:63)
app//org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:231)
app//org.apache.lucene.queries.function.FunctionScoreQuery.createWeight(FunctionScoreQuery.java:116)
app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery.createWeight(FunctionScoreQuery.java:245)
app//org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:726)
app//org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:158)
app//org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:63)
app//org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:231)
app//org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:726)
app//org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:158)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:343)
app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:298)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:150)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:362)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:435)
app//org.elasticsearch.search.SearchService.access$200(SearchService.java:136)
app//org.elasticsearch.search.SearchService$2.lambda$onResponse$0(SearchService.java:396)
app//org.elasticsearch.search.SearchService$2$$Lambda$5191/0x0000000801af4840.get(Unknown Source)
app//org.elasticsearch.search.SearchService.lambda$runAsync$0(SearchService.java:412)
app//org.elasticsearch.search.SearchService$$Lambda$5192/0x0000000801af4c40.run(Unknown Source)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:710)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@14.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.base@14.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
java.base@14.0.1/java.lang.Thread.run(Thread.java:832)
2/10 snapshots sharing following 30 elements
app//org.apache.lucene.search.BoostQuery.createWeight(BoostQuery.java:125)
app//org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:726)
app//org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:158)
app//org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:63)
app//org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:231)
app//org.apache.lucene.queries.function.FunctionScoreQuery.createWeight(FunctionScoreQuery.java:116)
app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery.createWeight(FunctionScoreQuery.java:245)
app//org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:726)
app//org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:158)
app//org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:63)
app//org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:231)
app//org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:726)
app//org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:158)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:343)
app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:298)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:150)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:362)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:435)
app//org.elasticsearch.search.SearchService.access$200(SearchService.java:136)
app//org.elasticsearch.search.SearchService$2.lambda$onResponse$0(SearchService.java:396)
app//org.elasticsearch.search.SearchService$2$$Lambda$5191/0x0000000801af4840.get(Unknown Source)
app//org.elasticsearch.search.SearchService.lambda$runAsync$0(SearchService.java:412)
app//org.elasticsearch.search.SearchService$$Lambda$5192/0x0000000801af4c40.run(Unknown Source)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:710)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@14.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.base@14.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
java.base@14.0.1/java.lang.Thread.run(Thread.java:832)
Although I cannot read anything useful from that output.