I am running a single node ELK stack which is ingesting logs from an Application Load Balancer on AWS.
It is a decent sized instance with 8 VCPUs and 32GB RAM.
I am seeing a consistently high CPU usage on the instance (in excess of 95%). Stack monitoring in Kibana seems to indicate that the CPU usage is because of elasticsearch processes (Logstash monitoring is showing very low CPU consumption). My JVM heap size is set at 8 GB and the total IO rate being reported averages at around 25/sec.
I am wondering if this excessively high CPU usage is due to any misconfiguration on my end.
Can anyone give me pointers to what I should be looking at to reduce the load? Thanks.
Check or share node stats & hot thread
GET _cluster/stats
GET _nodes/stats
GET _nodes/hot_threads
3 Likes
sorry for the delayed response. But here they are
cluster/stats
"_nodes" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"cluster_name" : "elasticsearch",
"cluster_uuid" : "0dOmVRWlR4K8hNifmZOnbw",
"timestamp" : 1612837887456,
"status" : "green",
"indices" : {
"count" : 279,
"shards" : {
"total" : 279,
"primaries" : 279,
"replication" : 0.0,
"index" : {
"shards" : {
"min" : 1,
"max" : 1,
"avg" : 1.0
},
"primaries" : {
"min" : 1,
"max" : 1,
"avg" : 1.0
},
"replication" : {
"min" : 0.0,
"max" : 0.0,
"avg" : 0.0
}
}
},
"docs" : {
"count" : 71147808,
"deleted" : 326
},
"store" : {
"size_in_bytes" : 42586385814,
"reserved_in_bytes" : 0
},
"fielddata" : {
"memory_size_in_bytes" : 0,
"evictions" : 0
},
"query_cache" : {
"memory_size_in_bytes" : 0,
"total_count" : 65657836,
"hit_count" : 6,
"miss_count" : 65657830,
"cache_size" : 0,
"cache_count" : 2,
"evictions" : 2
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 3054,
"memory_in_bytes" : 79335992,
"terms_memory_in_bytes" : 60115544,
"stored_fields_memory_in_bytes" : 1556608,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 4633856,
"points_memory_in_bytes" : 0,
"doc_values_memory_in_bytes" : 13029984,
"index_writer_memory_in_bytes" : 75407832,
"version_map_memory_in_bytes" : 4319132,
"fixed_bit_set_memory_in_bytes" : 6455512,
"max_unsafe_auto_id_timestamp" : 1612832291672,
"file_sizes" : { }
},
"mappings" : {
"field_types" : [
{
"name" : "alias",
"count" : 6,
"index_count" : 2
},
{
"name" : "binary",
"count" : 15,
"index_count" : 4
},
{
"name" : "boolean",
"count" : 262,
"index_count" : 35
},
{
"name" : "byte",
"count" : 2,
"index_count" : 2
},
{
"name" : "constant_keyword",
"count" : 3,
"index_count" : 1
},
{
"name" : "date",
"count" : 941,
"index_count" : 277
},
{
"name" : "date_range",
"count" : 2,
"index_count" : 2
},
{
"name" : "double",
"count" : 258,
"index_count" : 17
},
{
"name" : "flattened",
"count" : 9,
"index_count" : 1
},
{
"name" : "float",
"count" : 1119,
"index_count" : 241
},
{
"name" : "geo_point",
"count" : 256,
"index_count" : 232
},
{
"name" : "geo_shape",
"count" : 1,
"index_count" : 1
},
{
"name" : "half_float",
"count" : 383,
"index_count" : 174
},
{
"name" : "integer",
"count" : 211,
"index_count" : 19
},
{
"name" : "ip",
"count" : 833,
"index_count" : 233
},
{
"name" : "keyword",
"count" : 19326,
"index_count" : 278
},
{
"name" : "long",
"count" : 8736,
"index_count" : 272
},
{
"name" : "nested",
"count" : 62,
"index_count" : 20
},
{
"name" : "object",
"count" : 7754,
"index_count" : 278
},
{
"name" : "scaled_float",
"count" : 242,
"index_count" : 2
},
{
"name" : "short",
"count" : 21,
"index_count" : 7
},
{
"name" : "text",
"count" : 15419,
"index_count" : 255
}
]
},
"analysis" : {
"char_filter_types" : [ ],
"tokenizer_types" : [ ],
"filter_types" : [
{
"name" : "pattern_capture",
"count" : 1,
"index_count" : 1
}
],
"analyzer_types" : [
{
"name" : "custom",
"count" : 1,
"index_count" : 1
}
],
"built_in_char_filters" : [ ],
"built_in_tokenizers" : [
{
"name" : "uax_url_email",
"count" : 1,
"index_count" : 1
}
],
"built_in_filters" : [
{
"name" : "lowercase",
"count" : 1,
"index_count" : 1
},
{
"name" : "unique",
"count" : 1,
"index_count" : 1
}
],
"built_in_analyzers" : [
{
"name" : "simple",
"count" : 4,
"index_count" : 2
}
]
}
},
"nodes" : {
"count" : {
"total" : 1,
"coordinating_only" : 0,
"data" : 1,
"data_cold" : 1,
"data_content" : 1,
"data_hot" : 1,
"data_warm" : 1,
"ingest" : 1,
"master" : 1,
"ml" : 1,
"remote_cluster_client" : 1,
"transform" : 1,
"voting_only" : 0
},
"versions" : [
"7.10.0"
],
"os" : {
"available_processors" : 8,
"allocated_processors" : 8,
"names" : [
{
"name" : "Linux",
"count" : 1
}
],
"pretty_names" : [
{
"pretty_name" : "Ubuntu 18.04.3 LTS",
"count" : 1
}
],
"mem" : {
"total_in_bytes" : 33016766464,
"free_in_bytes" : 18470899712,
"used_in_bytes" : 14545866752,
"free_percent" : 56,
"used_percent" : 44
}
},
"process" : {
"cpu" : {
"percent" : 83
},
"open_file_descriptors" : {
"min" : 3470,
"max" : 3470,
"avg" : 3470
}
},
"jvm" : {
"max_uptime_in_millis" : 5630651,
"versions" : [
{
"version" : "15.0.1",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "15.0.1+9",
"vm_vendor" : "AdoptOpenJDK",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"count" : 1
}
],
"mem" : {
"heap_used_in_bytes" : 5201697320,
"heap_max_in_bytes" : 8589934592
},
"threads" : 111
},
"fs" : {
"total_in_bytes" : 208111882240,
"free_in_bytes" : 139090227200,
"available_in_bytes" : 139073449984
},
"plugins" : [ ],
"network_types" : {
"transport_types" : {
"security4" : 1
},
"http_types" : {
"security4" : 1
}
},
"discovery_types" : {
"zen" : 1
},
"packaging_types" : [
{
"flavor" : "default",
"type" : "deb",
"count" : 1
}
],
"ingest" : {
"number_of_pipelines" : 13,
"processor_stats" : {
"conditional" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
},
"geoip" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
},
"grok" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
},
"gsub" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
},
"remove" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
},
"rename" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
},
"script" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
},
"set" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
}
}
}
}
}```nodes/stats
"_nodes" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"cluster_name" : "elasticsearch",
"nodes" : {
"UQaBOuaLS-i4C8XpvIOOoA" : {
"timestamp" : 1612838086445,
"name" : "bold-mountain-kibana",
"transport_address" : "127.0.0.1:9300",
"host" : "127.0.0.1",
"ip" : "127.0.0.1:9300",
"roles" : [
"data",
"data_cold",
"data_content",
"data_hot",
"data_warm",
"ingest",
"master",
"ml",
"remote_cluster_client",
"transform"
],
"attributes" : {
"ml.machine_memory" : "33016766464",
"xpack.installed" : "true",
"transform.node" : "true",
"ml.max_open_jobs" : "20"
},
"indices" : {
"docs" : {
"count" : 71148847,
"deleted" : 350
},
"store" : {
"size_in_bytes" : 42587950828,
"reserved_in_bytes" : 0
},
"indexing" : {
"index_total" : 199340,
"index_time_in_millis" : 95192,
"index_current" : 0,
"index_failed" : 161193,
"delete_total" : 22,
"delete_time_in_millis" : 80,
"delete_current" : 0,
"noop_update_total" : 0,
"is_throttled" : false,
"throttle_time_in_millis" : 0
},
"get" : {
"total" : 1293,
"time_in_millis" : 777,
"exists_total" : 1189,
"exists_time_in_millis" : 589,
"missing_total" : 104,
"missing_time_in_millis" : 188,
"current" : 0
},
"search" : {
"open_contexts" : 2,
"query_total" : 19587894,
"query_time_in_millis" : 6005057,
"query_current" : 0,
"fetch_total" : 6432,
"fetch_time_in_millis" : 1744,
"fetch_current" : 0,
"scroll_total" : 2516,
"scroll_time_in_millis" : 10768,
"scroll_current" : 0,
"suggest_total" : 0,
"suggest_time_in_millis" : 0,
"suggest_current" : 0
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size_in_bytes" : 0,
"total" : 224,
"total_time_in_millis" : 100165,
"total_docs" : 1019780,
"total_size_in_bytes" : 568673952,
"total_stopped_time_in_millis" : 0,
"total_throttled_time_in_millis" : 0,
"total_auto_throttle_in_bytes" : 5934940160
},
"refresh" : {
"total" : 2783,
"total_time_in_millis" : 49759,
"external_total" : 2771,
"external_total_time_in_millis" : 52825,
"listeners" : 0
},
"flush" : {
"total" : 274,
"periodic" : 1,
"total_time_in_millis" : 8859
},
"warmer" : {
"current" : 0,
"total" : 2483,
"total_time_in_millis" : 1678
},
"query_cache" : {
"memory_size_in_bytes" : 0,
"total_count" : 68111762,
"hit_count" : 6,
"miss_count" : 68111756,
"cache_size" : 0,
"cache_count" : 2,
"evictions" : 2
},
"fielddata" : {
"memory_size_in_bytes" : 0,
"evictions" : 0
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 3037,
"memory_in_bytes" : 78987052,
"terms_memory_in_bytes" : 59842296,
"stored_fields_memory_in_bytes" : 1548312,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 4609792,
"points_memory_in_bytes" : 0,
"doc_values_memory_in_bytes" : 12986652,
"index_writer_memory_in_bytes" : 80322896,
"version_map_memory_in_bytes" : 5023462,
"fixed_bit_set_memory_in_bytes" : 6455608,
"max_unsafe_auto_id_timestamp" : 1612832291672,
"file_sizes" : { }
},
"translog" : {
"operations" : 71176,
"size_in_bytes" : 293204805,
"uncommitted_operations" : 71176,
"uncommitted_size_in_bytes" : 293204805,
"earliest_last_modified_age" : 0
},
"request_cache" : {
"memory_size_in_bytes" : 72867,
"evictions" : 0,
"hit_count" : 2901,
"miss_count" : 28
},
"recovery" : {
"current_as_source" : 0,
"current_as_target" : 0,
"throttle_time_in_millis" : 0
}
},
"os" : {
"timestamp" : 1612838086658,
"cpu" : {
"percent" : 96,
"load_average" : {
"1m" : 13.34,
"5m" : 12.81,
"15m" : 10.95
}
},
"mem" : {
"total_in_bytes" : 33016766464,
"free_in_bytes" : 18429169664,
"used_in_bytes" : 14587596800,
"free_percent" : 56,
"used_percent" : 44
},
"swap" : {
"total_in_bytes" : 0,
"free_in_bytes" : 0,
"used_in_bytes" : 0
},
"cgroup" : {
"cpuacct" : {
"control_group" : "/system.slice/elasticsearch.service",
"usage_nanos" : 39471096693327
},
"cpu" : {
"control_group" : "/system.slice/elasticsearch.service",
"cfs_period_micros" : 100000,
"cfs_quota_micros" : -1,
"stat" : {
"number_of_elapsed_periods" : 0,
"number_of_times_throttled" : 0,
"time_throttled_nanos" : 0
}
},
"memory" : {
"control_group" : "/system.slice/elasticsearch.service",
"limit_in_bytes" : "9223372036854771712",
"usage_in_bytes" : "10934870016"
}
}
},
"process" : {
"timestamp" : 1612838086658,
"open_file_descriptors" : 3472,
"max_file_descriptors" : 65535,
"cpu" : {
"percent" : 95,
"total_in_millis" : 39467740
},
"mem" : {
"total_virtual_in_bytes" : 40882126848
}
},
"jvm" : {
"timestamp" : 1612838086660,
"uptime_in_millis" : 5829934,
"mem" : {
"heap_used_in_bytes" : 2465541776,
"heap_used_percent" : 28,
"heap_committed_in_bytes" : 8589934592,
"heap_max_in_bytes" : 8589934592,
"non_heap_used_in_bytes" : 216321656,
"non_heap_committed_in_bytes" : 224821248,
"pools" : {
"young" : {
"used_in_bytes" : 297795584,
"max_in_bytes" : 0,
"peak_used_in_bytes" : 5171576832,
"peak_max_in_bytes" : 0
},
"old" : {
"used_in_bytes" : 2137960960,
"max_in_bytes" : 8589934592,
"peak_used_in_bytes" : 2137960960,
"peak_max_in_bytes" : 8589934592
},
"survivor" : {
"used_in_bytes" : 29785232,
"max_in_bytes" : 0,
"peak_used_in_bytes" : 199229440,
"peak_max_in_bytes" : 0
}
}
},
"threads" : {
"count" : 111,
"peak_count" : 130
},
"gc" : {
"collectors" : {
"young" : {
"collection_count" : 3039,
"collection_time_in_millis" : 60848
},
"old" : {
"collection_count" : 0,
"collection_time_in_millis" : 0
}
}
},
"buffer_pools" : {
"mapped" : {
"count" : 7012,
"used_in_bytes" : 25825509501,
"total_capacity_in_bytes" : 25825509501
},
"direct" : {
"count" : 93,
"used_in_bytes" : 10030048,
"total_capacity_in_bytes" : 10030047
},
"mapped - 'non-volatile memory'" : {
"count" : 0,
"used_in_bytes" : 0,
"total_capacity_in_bytes" : 0
}
},
"classes" : {
"current_loaded_count" : 23559,
"total_loaded_count" : 23559,
"total_unloaded_count" : 0
}
},
"thread_pool" : {
"analyze" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"ccr" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"fetch_shard_started" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 16,
"completed" : 283
},
"fetch_shard_store" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"flush" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 4,
"completed" : 283
},
"force_merge" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"generic" : {
"threads" : 35,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 35,
"completed" : 122604
},
"get" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"listener" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"management" : {
"threads" : 5,
"queue" : 0,
"active" : 1,
"rejected" : 0,
"largest" : 5,
"completed" : 148331
},
"ml_datafeed" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"ml_job_comms" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"ml_utility" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 1,
"completed" : 5801
},
"refresh" : {
"threads" : 4,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 4,
"completed" : 535601
},
"rollup_indexing" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"search" : {
"threads" : 13,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 13,
"completed" : 20213419
},
"search_throttled" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"searchable_snapshots_cache_fetch_async" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"searchable_snapshots_cache_prewarming" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"security-crypto" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
nodes/hot_threads
::: {bold-mountain-kibana}{UQaBOuaLS-i4C8XpvIOOoA}{vC4-i3ScQVSzKnO97pRcng}{127.0.0.1}{127.0.0.1:9300}{cdhilmrstw}{ml.machine_memory=33016766464, xpack.installed=true, transform.node=true, ml.max_open_jobs=20}
Hot threads at 2021-02-09T02:38:28.581Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
96.5% (482.6ms out of 500ms) cpu usage by thread 'elasticsearch[bold-mountain-kibana][transport_worker][T#5]'
2/10 snapshots sharing following 559 elements
java.base@15.0.1/java.util.HashMap.put(HashMap.java:612)
org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission.authorize(IndicesPermission.java:312)
org.elasticsearch.xpack.core.security.authz.permission.Role.authorize(Role.java:173)
org.elasticsearch.xpack.security.authz.RBACEngine.buildIndicesAccessControl(RBACEngine.java:554)
org.elasticsearch.xpack.security.authz.RBACEngine.lambda$authorizeIndexAction$4(RBACEngine.java:335)
org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$5512/0x00000008018a08f0.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.lambda$getAsync$0(AuthorizationService.java:678)
org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier$$Lambda$5463/0x00000008017fe978.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
org.elasticsearch.xpack.security.authz.AuthorizationService.resolveIndexNames(AuthorizationService.java:579)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$6(AuthorizationService.java:273)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5465/0x00000008017fedd8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.lambda$getAsync$0(AuthorizationService.java:678)
org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier$$Lambda$5463/0x00000008017fe978.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
org.elasticsearch.xpack.security.authz.RBACEngine.loadAuthorizedIndices(RBACEngine.java:366)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$5(AuthorizationService.java:269)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5457/0x00000008017fda10.getAsync(Unknown Source)
org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.getAsync(AuthorizationService.java:676)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$8(AuthorizationService.java:272)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5458/0x00000008017fdc38.getAsync(Unknown Source)
org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.getAsync(AuthorizationService.java:676)
org.elasticsearch.xpack.security.authz.RBACEngine.lambda$authorizeIndexAction$5(RBACEngine.java:327)
org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$5509/0x00000008018a0490.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
org.elasticsearch.xpack.security.authz.RBACEngine.authorizeIndexActionName(RBACEngine.java:351)
org.elasticsearch.xpack.security.authz.RBACEngine.authorizeIndexAction(RBACEngine.java:324)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:283)
org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:248)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$1(AuthorizationService.java:212)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5453/0x00000008017fccd0.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:43)
org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$1(RBACEngine.java:126)
org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$5455/0x00000008017fd130.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.roles(CompositeRolesStore.java:159)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:276)
org.elasticsearch.xpack.security.authz.RBACEngine.getRoles(RBACEngine.java:132)
org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:120)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:214)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4735/0x0000000801654ec0.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:323)
org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4737/0x0000000801655ea0.accept(Unknown Source)
org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:384)
org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4738/0x00000008016560d8.run(Unknown Source)
org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:395)
org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:320)
org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:261)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:173)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:72)
app//org.elasticsearch.transport.TransportService.sendLocalRequest(TransportService.java:803)
app//org.elasticsearch.transport.TransportService.access$100(TransportService.java:76)
app//org.elasticsearch.transport.TransportService$3.sendRequest(TransportService.java:130)
app//org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:747)
app//org.elasticsearch.transport.TransportService$$Lambda$3643/0x00000008014f3428.sendRequest(Unknown Source)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor.sendWithUser(SecurityServerTransportInterceptor.java:162)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor.access$300(SecurityServerTransportInterceptor.java:53)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$1.sendRequest(SecurityServerTransportInterceptor.java:129)
app//org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:661)
app//org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:712)
app//org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:704)
app//org.elasticsearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:139)
app//org.elasticsearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:83)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:244)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction$$Lambda$5504/0x000000080188f058.run(Unknown Source)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:690)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:277)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:215)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:365)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:182)
app//org.elasticsearch.action.search.TransportSearchAction$5.run(TransportSearchAction.java:799)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:365)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:359)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:603)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction.successfulShardExecution(AbstractSearchAsyncAction.java:503)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardResultConsumed(AbstractSearchAsyncAction.java:491)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction.lambda$onShardResult$8(AbstractSearchAsyncAction.java:474)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction$$Lambda$5583/0x00000008018cc638.run(Unknown Source)
app//org.elasticsearch.action.search.CanMatchPreFilterSearchPhase$CanMatchSearchPhaseResults.consumeResult(CanMatchPreFilterSearchPhase.java:171)
app//org.elasticsearch.action.search.CanMatchPreFilterSearchPhase$CanMatchSearchPhaseResults.consumeResult(CanMatchPreFilterSearchPhase.java:155)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardResult(AbstractSearchAsyncAction.java:474)
app//org.elasticsearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:249)
app//org.elasticsearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:45)
app//org.elasticsearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:29)
app//org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54)
app//org.elasticsearch.transport.TransportService$6.handleResponse(TransportService.java:634)
app//org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171)
app//org.elasticsearch.transport.TransportService$DirectResponseChannel.processResponse(TransportService.java:1249)```This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.