Hi,
there are many topics about CPU load but I can't find an answer.
Our standalone Elasticsearch 7.12.0 node runs on a 16 GB RAM Linux server. Every 10 minutes or so elasticsearch consumes 100% CPU and queries are very slow..
I managed to get the return of the hot_threads command but I don't know how to analyze it, could you tell me what is causing this massive CPU usage and how I can solve this problem? The problem occurs whether or not there are people connected to the database.
GET _cat/nodes?v=true&s=cpu:desc
ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
XX.XX.XX.XX 55 97 17 2.90 4.47 5.80 cdfhilmrstw * hostname.domain
GET _cat/health?v=true
epoch timestamp cluster stat
us node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent
1695033705 10:41:45 elasticsearch green 1 1 2234 2234 0 0 0 0 - 100.0%
GET _cluster/stats?pretty&human
{
"_nodes" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"cluster_name" : "elasticsearch",
"cluster_uuid" : "3K6hfBlWQXi-0uk-08urTw",
"timestamp" : 1695033736725,
"status" : "green",
"indices" : {
"count" : 2234,
"shards" : {
"total" : 2234,
"primaries" : 2234,
"replication" : 0.0,
"index" : {
"shards" : {
"min" : 1,
"max" : 1,
"avg" : 1.0
},
"primaries" : {
"min" : 1,
"max" : 1,
"avg" : 1.0
},
"replication" : {
"min" : 0.0,
"max" : 0.0,
"avg" : 0.0
}
}
},
"docs" : {
"count" : 126881864,
"deleted" : 507968
},
"store" : {
"size" : "68.2gb",
"size_in_bytes" : 73271657370,
"reserved" : "0b",
"reserved_in_bytes" : 0
},
"fielddata" : {
"memory_size" : "0b",
"memory_size_in_bytes" : 0,
"evictions" : 0
},
"query_cache" : {
"memory_size" : "31.3mb",
"memory_size_in_bytes" : 32873065,
"total_count" : 390719,
"hit_count" : 51632,
"miss_count" : 339087,
"cache_size" : 9507,
"cache_count" : 11190,
"evictions" : 1683
},
"completion" : {
"size" : "0b",
"size_in_bytes" : 0
},
"segments" : {
"count" : 14260,
"memory" : "123.7mb",
"memory_in_bytes" : 129811504,
"terms_memory" : "95.8mb",
"terms_memory_in_bytes" : 100469888,
"stored_fields_memory" : "6.7mb",
"stored_fields_memory_in_bytes" : 7104672,
"term_vectors_memory" : "0b",
"term_vectors_memory_in_bytes" : 0,
"norms_memory" : "11.6mb",
"norms_memory_in_bytes" : 12171904,
"points_memory" : "0b",
"points_memory_in_bytes" : 0,
"doc_values_memory" : "9.5mb",
"doc_values_memory_in_bytes" : 10065040,
"index_writer_memory" : "18.5mb",
"index_writer_memory_in_bytes" : 19427824,
"version_map_memory" : "140.5kb",
"version_map_memory_in_bytes" : 143914,
"fixed_bit_set" : "353.9kb",
"fixed_bit_set_memory_in_bytes" : 362416,
"max_unsafe_auto_id_timestamp" : 1695027241008,
"file_sizes" : { }
},
"mappings" : {
"field_types" : [
{
"name" : "binary",
"count" : 1,
"index_count" : 1
},
{
"name" : "boolean",
"count" : 7618,
"index_count" : 2057
},
{
"name" : "byte",
"count" : 1,
"index_count" : 1
},
{
"name" : "date",
"count" : 4536,
"index_count" : 2226
},
{
"name" : "date_nanos",
"count" : 1,
"index_count" : 1
},
{
"name" : "date_range",
"count" : 1,
"index_count" : 1
},
{
"name" : "double",
"count" : 1,
"index_count" : 1
},
{
"name" : "double_range",
"count" : 1,
"index_count" : 1
},
{
"name" : "float",
"count" : 5662,
"index_count" : 1608
},
{
"name" : "float_range",
"count" : 1,
"index_count" : 1
},
{
"name" : "geo_point",
"count" : 1,
"index_count" : 1
},
{
"name" : "geo_shape",
"count" : 1,
"index_count" : 1
},
{
"name" : "half_float",
"count" : 9,
"index_count" : 3
},
{
"name" : "integer",
"count" : 23,
"index_count" : 2
},
{
"name" : "integer_range",
"count" : 1,
"index_count" : 1
},
{
"name" : "ip",
"count" : 1,
"index_count" : 1
},
{
"name" : "ip_range",
"count" : 1,
"index_count" : 1
},
{
"name" : "keyword",
"count" : 32512,
"index_count" : 2226
},
{
"name" : "long",
"count" : 27700,
"index_count" : 2124
},
{
"name" : "long_range",
"count" : 1,
"index_count" : 1
},
{
"name" : "nested",
"count" : 7,
"index_count" : 5
},
{
"name" : "object",
"count" : 3002,
"index_count" : 2206
},
{
"name" : "shape",
"count" : 1,
"index_count" : 1
},
{
"name" : "short",
"count" : 1,
"index_count" : 1
},
{
"name" : "text",
"count" : 32372,
"index_count" : 2224
}
]
},
"analysis" : {
"char_filter_types" : [ ],
"tokenizer_types" : [ ],
"filter_types" : [ ],
"analyzer_types" : [ ],
"built_in_char_filters" : [ ],
"built_in_tokenizers" : [ ],
"built_in_filters" : [ ],
"built_in_analyzers" : [ ]
},
"versions" : [
{
"version" : "7.12.0",
"index_count" : 2234,
"primary_shard_count" : 2234,
"total_primary_size" : "68.2gb",
"total_primary_bytes" : 73271657370
}
]
},
"nodes" : {
"count" : {
"total" : 1,
"coordinating_only" : 0,
"data" : 1,
"data_cold" : 1,
"data_content" : 1,
"data_frozen" : 1,
"data_hot" : 1,
"data_warm" : 1,
"ingest" : 1,
"master" : 1,
"ml" : 1,
"remote_cluster_client" : 1,
"transform" : 1,
"voting_only" : 0
},
"versions" : [
"7.12.0"
],
"os" : {
"available_processors" : 4,
"allocated_processors" : 4,
"names" : [
{
"name" : "Linux",
"count" : 1
}
],
"pretty_names" : [
{
"pretty_name" : "CentOS Linux 7 (Core)",
"count" : 1
}
],
"architectures" : [
{
"arch" : "amd64",
"count" : 1
}
],
"mem" : {
"total" : "15.4gb",
"total_in_bytes" : 16619679744,
"free" : "355.2mb",
"free_in_bytes" : 372465664,
"used" : "15.1gb",
"used_in_bytes" : 16247214080,
"free_percent" : 2,
"used_percent" : 98
}
},
"process" : {
"cpu" : {
"percent" : 40
},
"open_file_descriptors" : {
"min" : 19977,
"max" : 19977,
"avg" : 19977
}
},
"jvm" : {
"max_uptime" : "1.8h",
"max_uptime_in_millis" : 6559319,
"versions" : [
{
"version" : "15.0.1",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "15.0.1+9",
"vm_vendor" : "AdoptOpenJDK",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"count" : 1
}
],
"mem" : {
"heap_used" : "3.7gb",
"heap_used_in_bytes" : 4008591864,
"heap_max" : "8gb",
"heap_max_in_bytes" : 8589934592
},
"threads" : 186
},
"fs" : {
"total" : "89.9gb",
"total_in_bytes" : 96610549760,
"free" : "20.7gb",
"free_in_bytes" : 22257397760,
"available" : "20.7gb",
"available_in_bytes" : 22257397760
},
"plugins" : [ ],
"network_types" : {
"transport_types" : {
"security4" : 1
},
"http_types" : {
"security4" : 1
}
},
"discovery_types" : {
"single-node" : 1
},
"packaging_types" : [
{
"flavor" : "default",
"type" : "tar",
"count" : 1
}
],
"ingest" : {
"number_of_pipelines" : 1,
"processor_stats" : {
"gsub" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
},
"script" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
}
}
}
}
}
GET _nodes/hot_threads?pretty&human
::: {hostname.domain}{zlApshC6TROjZLKxlE3FAw}{30yJimmZSzaM-RvSwm3jJA}{XX.XX.XX.XX}{XX.XX.XX.XX:9300}{cdfhilmrstw}{ml.machine_memory=16619679744, xpack.installed=true, transform.node=true, ml.max_open_jobs=20, ml.max_jvm_size=8589934592}
Hot threads at 2023-09-18T09:24:48.941Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
100.7% (503.2ms out of 500ms) cpu usage by thread 'elasticsearch[hostname.domain][transport_worker][T#1]'
7/10 snapshots sharing following 362 elements
org.elasticsearch.xpack.core.security.authz.permission.Role.authorize(Role.java:174)
org.elasticsearch.xpack.security.authz.RBACEngine.buildIndicesAccessControl(RBACEngine.java:555)
org.elasticsearch.xpack.security.authz.RBACEngine.lambda$authorizeIndexAction$4(RBACEngine.java:336)
org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$5863/0x000000080186f568.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117)
org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.lambda$getAsync$0(AuthorizationService.java:722)
org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier$$Lambda$5865/0x000000080186f9c8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117)
org.elasticsearch.xpack.security.authz.AuthorizationService.resolveIndexNames(AuthorizationService.java:599)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$6(AuthorizationService.java:290)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5867/0x0000000801970040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117)
org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.lambda$getAsync$0(AuthorizationService.java:722)
org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier$$Lambda$5865/0x000000080186f9c8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117)
org.elasticsearch.xpack.security.authz.RBACEngine.loadAuthorizedIndices(RBACEngine.java:367)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$5(AuthorizationService.java:286)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5857/0x000000080186e5f0.getAsync(Unknown Source)
org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.getAsync(AuthorizationService.java:720)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$8(AuthorizationService.java:289)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5858/0x000000080186e818.getAsync(Unknown Source)
org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.getAsync(AuthorizationService.java:720)
org.elasticsearch.xpack.security.authz.RBACEngine.lambda$authorizeIndexAction$5(RBACEngine.java:328)
org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$5861/0x000000080186f108.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117)
org.elasticsearch.xpack.security.authz.RBACEngine.authorizeIndexActionName(RBACEngine.java:352)
org.elasticsearch.xpack.security.authz.RBACEngine.authorizeIndexAction(RBACEngine.java:325)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:300)
org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:265)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$1(AuthorizationService.java:229)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5851/0x000000080186d8b0.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:32)
org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$1(RBACEngine.java:127)
org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$5853/0x000000080186dd10.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.roles(CompositeRolesStore.java:161)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:278)
org.elasticsearch.xpack.security.authz.RBACEngine.getRoles(RBACEngine.java:133)
org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:121)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:231)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:130)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$5319/0x0000000801715bb8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117)
org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:330)
org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$5321/0x0000000801716b98.accept(Unknown Source)
org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:391)
org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$5322/0x0000000801716dd0.run(Unknown Source)
org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:402)
org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:327)
org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:268)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:178)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:121)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:322)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:61)
app//org.elasticsearch.transport.TransportService.sendLocalRequest(TransportService.java:908)
app//org.elasticsearch.transport.TransportService.access$100(TransportService.java:68)
app//org.elasticsearch.transport.TransportService$3.sendRequest(TransportService.java:140)
app//org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:852)
app//org.elasticsearch.transport.TransportService$$Lambda$3918/0x0000000801575668.sendRequest(Unknown Source)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor.sendWithUser(SecurityServerTransportInterceptor.java:164)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor.access$300(SecurityServerTransportInterceptor.java:55)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$1.sendRequest(SecurityServerTransportInterceptor.java:131)
app//org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:766)
app//org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:817)
app//org.elasticsearch.action.search.SearchTransportService.sendCanMatch(SearchTransportService.java:113)
app//org.elasticsearch.action.search.CanMatchPreFilterSearchPhase.executePhaseOnShard(CanMatchPreFilterSearchPhase.java:81)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$4(AbstractSearchAsyncAction.java:273)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction$$Lambda$6011/0x000000080198aab8.run(Unknown Source)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.elasticsearch.action.search.CanMatchPreFilterSearchPhase.performPhaseOnShard(CanMatchPreFilterSearchPhase.java:132)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:226)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:397)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:184)
app//org.elasticsearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:668)
app//org.elasticsearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:489)
app//org.elasticsearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:283)
app//org.elasticsearch.action.search.TransportSearchAction$$Lambda$5970/0x0000000801982168.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117)
app//org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:103)
app//org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:76)
app//org.elasticsearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:324)
app//org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:212)
app//org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:91)
app//org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:173)
app//org.elasticsearch.action.support.ActionFilter$Simple.apply(ActionFilter.java:42)
app//org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:171)
org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$applyInternal$3(SecurityActionFilter.java:161)
org.elasticsearch.xpack.security.action.filter.SecurityActionFilter$$Lambda$5452/0x0000000801717690.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:167)
app//org.elasticsearch.common.util.concurrent.ListenableFuture$1.doRun(ListenableFuture.java:101)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:95)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.lambda$done$0(ListenableFuture.java:87)
app//org.elasticsearch.common.util.concurrent.ListenableFuture$$Lambda$5433/0x0000000801805358.accept(Unknown Source)
java.base@15.0.1/java.util.ArrayList.forEach(ArrayList.java:1511)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.done(ListenableFuture.java:87)
app//org.elasticsearch.common.util.concurrent.BaseFuture.set(BaseFuture.java:133)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.onResponse(ListenableFuture.java:116)
app//org.elasticsearch.action.StepListener.innerOnResponse(StepListener.java:53)
app//org.elasticsearch.action.NotifyOnceListener.onResponse(NotifyOnceListener.java:29)
org.elasticsearch.xpack.security.authz.interceptor.IndicesAliasesRequestInterceptor.intercept(IndicesAliasesRequestInterceptor.java:107)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$runRequestInterceptors$14(AuthorizationService.java:379)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5873/0x00000008019704a0.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117)
app//org.elasticsearch.common.util.concurrent.ListenableFuture$1.doRun(ListenableFuture.java:101)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:95)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.lambda$done$0(ListenableFuture.java:87)
app//org.elasticsearch.common.util.concurrent.ListenableFuture$$Lambda$5433/0x0000000801805358.accept(Unknown Source)
java.base@15.0.1/java.util.ArrayList.forEach(ArrayList.java:1511)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.done(ListenableFuture.java:87)
app//org.elasticsearch.common.util.concurrent.BaseFuture.set(BaseFuture.java:133)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.onResponse(ListenableFuture.java:116)
app//org.elasticsearch.action.StepListener.innerOnResponse(StepListener.java:53)
app//org.elasticsearch.action.NotifyOnceListener.onResponse(NotifyOnceListener.java:29)
org.elasticsearch.xpack.security.authz.interceptor.ResizeRequestInterceptor.intercept(ResizeRequestInterceptor.java:87)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$runRequestInterceptors$14(AuthorizationService.java:379)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5873/0x00000008019704a0.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117)
app//org.elasticsearch.common.util.concurrent.ListenableFuture$1.doRun(ListenableFuture.java:101)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:95)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.lambda$done$0(ListenableFuture.java:87)
app//org.elasticsearch.common.util.concurrent.ListenableFuture$$Lambda$5433/0x0000000801805358.accept(Unknown Source)
java.base@15.0.1/java.util.ArrayList.forEach(ArrayList.java:1511)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.done(ListenableFuture.java:87)
app//org.elasticsearch.common.util.concurrent.BaseFuture.set(BaseFuture.java:133)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.onResponse(ListenableFuture.java:116)
app//org.elasticsearch.action.StepListener.innerOnResponse(StepListener.java:53)
app//org.elasticsearch.action.NotifyOnceListener.onResponse(NotifyOnceListener.java:29)
org.elasticsearch.xpack.security.authz.interceptor.FieldAndDocumentLevelSecurityRequestInterceptor.intercept(FieldAndDocumentLevelSecurityRequestInterceptor.java:77)
org.elasticsearch.xpack.security.authz.interceptor.UpdateRequestInterceptor.intercept(UpdateRequestInterceptor.java:24)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$runRequestInterceptors$14(AuthorizationService.java:379)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5873/0x00000008019704a0.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117)
app//org.elasticsearch.common.util.concurrent.ListenableFuture$1.doRun(ListenableFuture.java:101)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:95)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.lambda$done$0(ListenableFuture.java:87)
app//org.elasticsearch.common.util.concurrent.ListenableFuture$$Lambda$5433/0x0000000801805358.accept(Unknown Source)
java.base@15.0.1/java.util.ArrayList.forEach(ArrayList.java:1511)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.done(ListenableFuture.java:87)
app//org.elasticsearch.common.util.concurrent.BaseFuture.set(BaseFuture.java:133)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.onResponse(ListenableFuture.java:116)
app//org.elasticsearch.action.StepListener.innerOnResponse(StepListener.java:53)
app//org.elasticsearch.action.NotifyOnceListener.onResponse(NotifyOnceListener.java:29)
org.elasticsearch.xpack.security.authz.interceptor.BulkShardRequestInterceptor.intercept(BulkShardRequestInterceptor.java:77)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$runRequestInterceptors$14(AuthorizationService.java:379)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5873/0x00000008019704a0.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117)
app//org.elasticsearch.common.util.concurrent.ListenableFuture$1.doRun(ListenableFuture.java:101)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:95)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.lambda$done$0(ListenableFuture.java:87)
app//org.elasticsearch.common.util.concurrent.ListenableFuture$$Lambda$5433/0x0000000801805358.accept(Unknown Source)
java.base@15.0.1/java.util.ArrayList.forEach(ArrayList.java:1511)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.done(ListenableFuture.java:87)
app//org.elasticsearch.common.util.concurrent.BaseFuture.set(BaseFuture.java:133)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.onResponse(ListenableFuture.java:116)
app//org.elasticsearch.action.StepListener.innerOnResponse(StepListener.java:53)
app//org.elasticsearch.action.NotifyOnceListener.onResponse(NotifyOnceListener.java:29)
org.elasticsearch.xpack.security.authz.interceptor.FieldAndDocumentLevelSecurityRequestInterceptor.intercept(FieldAndDocumentLevelSecurityRequestInterceptor.java:77)
org.elasticsearch.xpack.security.authz.interceptor.SearchRequestInterceptor.intercept(SearchRequestInterceptor.java:22)
org.elasticsearch.xpack.security.authz.AuthorizationService.runRequestInterceptors(AuthorizationService.java:385)
org.elasticsearch.xpack.security.authz.AuthorizationService.handleIndexActionAuthorizationResult(AuthorizationService.java:362)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$9(AuthorizationService.java:302)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5859/0x000000080186ec98.accept(Unknown Source)
org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:684)
org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:659)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:32)
org.elasticsearch.xpack.security.authz.RBACEngine.buildIndicesAccessControl(RBACEngine.java:556)
org.elasticsearch.xpack.security.authz.RBACEngine.lambda$authorizeIndexAction$3(RBACEngine.java:320)
org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$5962/0x0000000801979ac8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117)
org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.lambda$getAsync$0(AuthorizationService.java:722)
org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier$$Lambda$5865/0x000000080186f9c8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117)
hot_threads output is truncated due to message character limit
