I ran the provided benchmarks and captured stack dumps and saw it spending quite a bit of its time sending can-match requests, stuff like this:
at org.elasticsearch.common.io.stream.BytesStreamOutput.writeByte(BytesStreamOutput.java:78)
at org.elasticsearch.common.io.stream.StreamOutput.write(StreamOutput.java:520)
at org.elasticsearch.transport.CompressibleBytesOutputStream.writeByte(CompressibleBytesOutputStream.java:81)
at org.elasticsearch.common.io.stream.StreamOutput.lambda$static$25(StreamOutput.java:777)
at org.elasticsearch.common.io.stream.StreamOutput$$Lambda$1308/0x000000080108bc40.write(Unknown Source)
at org.elasticsearch.common.io.stream.StreamOutput.writeGenericValue(StreamOutput.java:846)
at org.elasticsearch.common.io.stream.StreamOutput.lambda$static$12(StreamOutput.java:709)
at org.elasticsearch.common.io.stream.StreamOutput$$Lambda$1295/0x0000000801088840.write(Unknown Source)
at org.elasticsearch.common.io.stream.StreamOutput.writeGenericValue(StreamOutput.java:846)
at org.elasticsearch.index.query.TermsQueryBuilder.doWriteTo(TermsQueryBuilder.java:198)
at org.elasticsearch.index.query.AbstractQueryBuilder.writeTo(AbstractQueryBuilder.java:79)
at org.elasticsearch.common.io.stream.StreamOutput.writeNamedWriteable(StreamOutput.java:1115)
at org.elasticsearch.index.query.AbstractQueryBuilder.writeQueries(AbstractQueryBuilder.java:256)
at org.elasticsearch.index.query.BoolQueryBuilder.doWriteTo(BoolQueryBuilder.java:103)
at org.elasticsearch.index.query.AbstractQueryBuilder.writeTo(AbstractQueryBuilder.java:79)
at org.elasticsearch.common.io.stream.StreamOutput.writeNamedWriteable(StreamOutput.java:1115)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalNamedWriteable(StreamOutput.java:1126)
at org.elasticsearch.search.builder.SearchSourceBuilder.writeTo(SearchSourceBuilder.java:301)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalWriteable(StreamOutput.java:976)
at org.elasticsearch.search.internal.ShardSearchRequest.innerWriteTo(ShardSearchRequest.java:266)
at org.elasticsearch.search.internal.ShardSearchRequest.writeTo(ShardSearchRequest.java:255)
at org.elasticsearch.transport.OutboundMessage.writeMessage(OutboundMessage.java:87)
at org.elasticsearch.transport.OutboundMessage.serialize(OutboundMessage.java:64)
at org.elasticsearch.transport.OutboundHandler$MessageSerializer.get(OutboundHandler.java:163)
at org.elasticsearch.transport.OutboundHandler$MessageSerializer.get(OutboundHandler.java:149)
at org.elasticsearch.transport.OutboundHandler$SendContext.get(OutboundHandler.java:196)
at org.elasticsearch.transport.OutboundHandler.internalSend(OutboundHandler.java:130)
at org.elasticsearch.transport.OutboundHandler.sendMessage(OutboundHandler.java:125)
at org.elasticsearch.transport.OutboundHandler.sendRequest(OutboundHandler.java:89)
at org.elasticsearch.transport.TcpTransport$NodeChannels.sendRequest(TcpTransport.java:270)
at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:747)
at org.elasticsearch.transport.TransportService$$Lambda$3621/0x000000080164f040.sendRequest(Unknown Source)
at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor.sendWithUser(SecurityServerTransportInterceptor.java:162)
at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor.access$300(SecurityServerTransportInterceptor.java:53)
at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$1.sendRequest(SecurityServerTransportInterceptor.java:129)
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:661)
at org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:712)
at org.elasticsearch.action.search.SearchTransportService.sendCanMatch(SearchTransportService.java:116)
at org.elasticsearch.action.search.CanMatchPreFilterSearchPhase.executePhaseOnShard(CanMatchPreFilterSearchPhase.java:88)
at org.elasticsearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:244)
at org.elasticsearch.action.search.AbstractSearchAsyncAction$$Lambda$5672/0x0000000801b04440.run(Unknown Source)
at org.elasticsearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:279)
at org.elasticsearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:215)
at org.elasticsearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:365)
at org.elasticsearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:182)
at org.elasticsearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:685)
at org.elasticsearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:511)
at org.elasticsearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:303)
at org.elasticsearch.action.search.TransportSearchAction$$Lambda$5629/0x0000000801af8c40.accept(Unknown Source)
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
at org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:114)
at org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:87)
at org.elasticsearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:346)
at org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:229)
at org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:105)
at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:179)
There was also a warning logged about spending too much time on the transport thread:
[2021-01-05T18:34:03,672][WARN ][o.e.t.InboundHandler ] [node-0] handling inbound transport message [InboundMessage{Header{56}{7.10.1}{3369}{false}{false}{false}{false}{NO_ACTION_NAME_FOR_RESPONSES}}] took [5024ms] which is above the warn threshold of [5000ms]
I think it's worth reporting as a bug, thanks.