Elasticsearch CPU usage High

I am running a single node ELK stack which is ingesting logs from an Application Load Balancer on AWS.
It is a decent sized instance with 8 VCPUs and 32GB RAM.
I am seeing a consistently high CPU usage on the instance (in excess of 95%). Stack monitoring in Kibana seems to indicate that the CPU usage is because of elasticsearch processes (Logstash monitoring is showing very low CPU consumption). My JVM heap size is set at 8 GB and the total IO rate being reported averages at around 25/sec.
I am wondering if this excessively high CPU usage is due to any misconfiguration on my end.
Can anyone give me pointers to what I should be looking at to reduce the load? Thanks.

Check or share node stats & hot thread

GET _cluster/stats
GET _nodes/stats
GET _nodes/hot_threads
3 Likes

sorry for the delayed response. But here they are
cluster/stats

  "_nodes" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "0dOmVRWlR4K8hNifmZOnbw",
  "timestamp" : 1612837887456,
  "status" : "green",
  "indices" : {
    "count" : 279,
    "shards" : {
      "total" : 279,
      "primaries" : 279,
      "replication" : 0.0,
      "index" : {
        "shards" : {
          "min" : 1,
          "max" : 1,
          "avg" : 1.0
        },
        "primaries" : {
          "min" : 1,
          "max" : 1,
          "avg" : 1.0
        },
        "replication" : {
          "min" : 0.0,
          "max" : 0.0,
          "avg" : 0.0
        }
      }
    },
    "docs" : {
      "count" : 71147808,
      "deleted" : 326
    },
    "store" : {
      "size_in_bytes" : 42586385814,
      "reserved_in_bytes" : 0
    },
    "fielddata" : {
      "memory_size_in_bytes" : 0,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size_in_bytes" : 0,
      "total_count" : 65657836,
      "hit_count" : 6,
      "miss_count" : 65657830,
      "cache_size" : 0,
      "cache_count" : 2,
      "evictions" : 2
    },
    "completion" : {
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 3054,
      "memory_in_bytes" : 79335992,
      "terms_memory_in_bytes" : 60115544,
      "stored_fields_memory_in_bytes" : 1556608,
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory_in_bytes" : 4633856,
      "points_memory_in_bytes" : 0,
      "doc_values_memory_in_bytes" : 13029984,
      "index_writer_memory_in_bytes" : 75407832,
      "version_map_memory_in_bytes" : 4319132,
      "fixed_bit_set_memory_in_bytes" : 6455512,
      "max_unsafe_auto_id_timestamp" : 1612832291672,
      "file_sizes" : { }
    },
    "mappings" : {
      "field_types" : [
        {
          "name" : "alias",
          "count" : 6,
          "index_count" : 2
        },
        {
          "name" : "binary",
          "count" : 15,
          "index_count" : 4
        },
        {
          "name" : "boolean",
          "count" : 262,
          "index_count" : 35
        },
        {
          "name" : "byte",
          "count" : 2,
          "index_count" : 2
        },
        {
          "name" : "constant_keyword",
          "count" : 3,
          "index_count" : 1
        },
        {
          "name" : "date",
          "count" : 941,
          "index_count" : 277
        },
        {
          "name" : "date_range",
          "count" : 2,
          "index_count" : 2
        },
        {
          "name" : "double",
          "count" : 258,
          "index_count" : 17
        },
        {
          "name" : "flattened",
          "count" : 9,
          "index_count" : 1
        },
        {
          "name" : "float",
          "count" : 1119,
          "index_count" : 241
        },
        {
          "name" : "geo_point",
          "count" : 256,
          "index_count" : 232
        },
        {
          "name" : "geo_shape",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "half_float",
          "count" : 383,
          "index_count" : 174
        },
        {
          "name" : "integer",
          "count" : 211,
          "index_count" : 19
        },
        {
          "name" : "ip",
          "count" : 833,
          "index_count" : 233
        },
        {
          "name" : "keyword",
          "count" : 19326,
          "index_count" : 278
        },
        {
          "name" : "long",
          "count" : 8736,
          "index_count" : 272
        },
        {
          "name" : "nested",
          "count" : 62,
          "index_count" : 20
        },
        {
          "name" : "object",
          "count" : 7754,
          "index_count" : 278
        },
        {
          "name" : "scaled_float",
          "count" : 242,
          "index_count" : 2
        },
        {
          "name" : "short",
          "count" : 21,
          "index_count" : 7
        },
        {
          "name" : "text",
          "count" : 15419,
          "index_count" : 255
        }
      ]
    },
    "analysis" : {
      "char_filter_types" : [ ],
      "tokenizer_types" : [ ],
      "filter_types" : [
        {
          "name" : "pattern_capture",
          "count" : 1,
          "index_count" : 1
        }
      ],
      "analyzer_types" : [
        {
          "name" : "custom",
          "count" : 1,
          "index_count" : 1
        }
      ],
      "built_in_char_filters" : [ ],
      "built_in_tokenizers" : [
        {
          "name" : "uax_url_email",
          "count" : 1,
          "index_count" : 1
        }
      ],
      "built_in_filters" : [
        {
          "name" : "lowercase",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "unique",
          "count" : 1,
          "index_count" : 1
        }
      ],
      "built_in_analyzers" : [
        {
          "name" : "simple",
          "count" : 4,
          "index_count" : 2
        }
      ]
    }
  },
  "nodes" : {
    "count" : {
      "total" : 1,
      "coordinating_only" : 0,
      "data" : 1,
      "data_cold" : 1,
      "data_content" : 1,
      "data_hot" : 1,
      "data_warm" : 1,
      "ingest" : 1,
      "master" : 1,
      "ml" : 1,
      "remote_cluster_client" : 1,
      "transform" : 1,
      "voting_only" : 0
    },
    "versions" : [
      "7.10.0"
    ],
    "os" : {
      "available_processors" : 8,
      "allocated_processors" : 8,
      "names" : [
        {
          "name" : "Linux",
          "count" : 1
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "Ubuntu 18.04.3 LTS",
          "count" : 1
        }
      ],
      "mem" : {
        "total_in_bytes" : 33016766464,
        "free_in_bytes" : 18470899712,
        "used_in_bytes" : 14545866752,
        "free_percent" : 56,
        "used_percent" : 44
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 83
      },
      "open_file_descriptors" : {
        "min" : 3470,
        "max" : 3470,
        "avg" : 3470
      }
    },
    "jvm" : {
      "max_uptime_in_millis" : 5630651,
      "versions" : [
        {
          "version" : "15.0.1",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "15.0.1+9",
          "vm_vendor" : "AdoptOpenJDK",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 1
        }
      ],
      "mem" : {
        "heap_used_in_bytes" : 5201697320,
        "heap_max_in_bytes" : 8589934592
      },
      "threads" : 111
    },
    "fs" : {
      "total_in_bytes" : 208111882240,
      "free_in_bytes" : 139090227200,
      "available_in_bytes" : 139073449984
    },
    "plugins" : [ ],
    "network_types" : {
      "transport_types" : {
        "security4" : 1
      },
      "http_types" : {
        "security4" : 1
      }
    },
    "discovery_types" : {
      "zen" : 1
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "deb",
        "count" : 1
      }
    ],
    "ingest" : {
      "number_of_pipelines" : 13,
      "processor_stats" : {
        "conditional" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        },
        "geoip" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        },
        "grok" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        },
        "gsub" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        },
        "remove" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        },
        "rename" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        },
        "script" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        },
        "set" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        }
      }
    }
  }
}```

nodes/stats

  "_nodes" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "cluster_name" : "elasticsearch",
  "nodes" : {
    "UQaBOuaLS-i4C8XpvIOOoA" : {
      "timestamp" : 1612838086445,
      "name" : "bold-mountain-kibana",
      "transport_address" : "127.0.0.1:9300",
      "host" : "127.0.0.1",
      "ip" : "127.0.0.1:9300",
      "roles" : [
        "data",
        "data_cold",
        "data_content",
        "data_hot",
        "data_warm",
        "ingest",
        "master",
        "ml",
        "remote_cluster_client",
        "transform"
      ],
      "attributes" : {
        "ml.machine_memory" : "33016766464",
        "xpack.installed" : "true",
        "transform.node" : "true",
        "ml.max_open_jobs" : "20"
      },
      "indices" : {
        "docs" : {
          "count" : 71148847,
          "deleted" : 350
        },
        "store" : {
          "size_in_bytes" : 42587950828,
          "reserved_in_bytes" : 0
        },
        "indexing" : {
          "index_total" : 199340,
          "index_time_in_millis" : 95192,
          "index_current" : 0,
          "index_failed" : 161193,
          "delete_total" : 22,
          "delete_time_in_millis" : 80,
          "delete_current" : 0,
          "noop_update_total" : 0,
          "is_throttled" : false,
          "throttle_time_in_millis" : 0
        },
        "get" : {
          "total" : 1293,
          "time_in_millis" : 777,
          "exists_total" : 1189,
          "exists_time_in_millis" : 589,
          "missing_total" : 104,
          "missing_time_in_millis" : 188,
          "current" : 0
        },
        "search" : {
          "open_contexts" : 2,
          "query_total" : 19587894,
          "query_time_in_millis" : 6005057,
          "query_current" : 0,
          "fetch_total" : 6432,
          "fetch_time_in_millis" : 1744,
          "fetch_current" : 0,
          "scroll_total" : 2516,
          "scroll_time_in_millis" : 10768,
          "scroll_current" : 0,
          "suggest_total" : 0,
          "suggest_time_in_millis" : 0,
          "suggest_current" : 0
        },
        "merges" : {
          "current" : 0,
          "current_docs" : 0,
          "current_size_in_bytes" : 0,
          "total" : 224,
          "total_time_in_millis" : 100165,
          "total_docs" : 1019780,
          "total_size_in_bytes" : 568673952,
          "total_stopped_time_in_millis" : 0,
          "total_throttled_time_in_millis" : 0,
          "total_auto_throttle_in_bytes" : 5934940160
        },
        "refresh" : {
          "total" : 2783,
          "total_time_in_millis" : 49759,
          "external_total" : 2771,
          "external_total_time_in_millis" : 52825,
          "listeners" : 0
        },
        "flush" : {
          "total" : 274,
          "periodic" : 1,
          "total_time_in_millis" : 8859
        },
        "warmer" : {
          "current" : 0,
          "total" : 2483,
          "total_time_in_millis" : 1678
        },
        "query_cache" : {
          "memory_size_in_bytes" : 0,
          "total_count" : 68111762,
          "hit_count" : 6,
          "miss_count" : 68111756,
          "cache_size" : 0,
          "cache_count" : 2,
          "evictions" : 2
        },
        "fielddata" : {
          "memory_size_in_bytes" : 0,
          "evictions" : 0
        },
        "completion" : {
          "size_in_bytes" : 0
        },
        "segments" : {
          "count" : 3037,
          "memory_in_bytes" : 78987052,
          "terms_memory_in_bytes" : 59842296,
          "stored_fields_memory_in_bytes" : 1548312,
          "term_vectors_memory_in_bytes" : 0,
          "norms_memory_in_bytes" : 4609792,
          "points_memory_in_bytes" : 0,
          "doc_values_memory_in_bytes" : 12986652,
          "index_writer_memory_in_bytes" : 80322896,
          "version_map_memory_in_bytes" : 5023462,
          "fixed_bit_set_memory_in_bytes" : 6455608,
          "max_unsafe_auto_id_timestamp" : 1612832291672,
          "file_sizes" : { }
        },
        "translog" : {
          "operations" : 71176,
          "size_in_bytes" : 293204805,
          "uncommitted_operations" : 71176,
          "uncommitted_size_in_bytes" : 293204805,
          "earliest_last_modified_age" : 0
        },
        "request_cache" : {
          "memory_size_in_bytes" : 72867,
          "evictions" : 0,
          "hit_count" : 2901,
          "miss_count" : 28
        },
        "recovery" : {
          "current_as_source" : 0,
          "current_as_target" : 0,
          "throttle_time_in_millis" : 0
        }
      },
      "os" : {
        "timestamp" : 1612838086658,
        "cpu" : {
          "percent" : 96,
          "load_average" : {
            "1m" : 13.34,
            "5m" : 12.81,
            "15m" : 10.95
          }
        },
        "mem" : {
          "total_in_bytes" : 33016766464,
          "free_in_bytes" : 18429169664,
          "used_in_bytes" : 14587596800,
          "free_percent" : 56,
          "used_percent" : 44
        },
        "swap" : {
          "total_in_bytes" : 0,
          "free_in_bytes" : 0,
          "used_in_bytes" : 0
        },
        "cgroup" : {
          "cpuacct" : {
            "control_group" : "/system.slice/elasticsearch.service",
            "usage_nanos" : 39471096693327
          },
          "cpu" : {
            "control_group" : "/system.slice/elasticsearch.service",
            "cfs_period_micros" : 100000,
            "cfs_quota_micros" : -1,
            "stat" : {
              "number_of_elapsed_periods" : 0,
              "number_of_times_throttled" : 0,
              "time_throttled_nanos" : 0
            }
          },
          "memory" : {
            "control_group" : "/system.slice/elasticsearch.service",
            "limit_in_bytes" : "9223372036854771712",
            "usage_in_bytes" : "10934870016"
          }
        }
      },
      "process" : {
        "timestamp" : 1612838086658,
        "open_file_descriptors" : 3472,
        "max_file_descriptors" : 65535,
        "cpu" : {
          "percent" : 95,
          "total_in_millis" : 39467740
        },
        "mem" : {
          "total_virtual_in_bytes" : 40882126848
        }
      },
      "jvm" : {
        "timestamp" : 1612838086660,
        "uptime_in_millis" : 5829934,
        "mem" : {
          "heap_used_in_bytes" : 2465541776,
          "heap_used_percent" : 28,
          "heap_committed_in_bytes" : 8589934592,
          "heap_max_in_bytes" : 8589934592,
          "non_heap_used_in_bytes" : 216321656,
          "non_heap_committed_in_bytes" : 224821248,
          "pools" : {
            "young" : {
              "used_in_bytes" : 297795584,
              "max_in_bytes" : 0,
              "peak_used_in_bytes" : 5171576832,
              "peak_max_in_bytes" : 0
            },
            "old" : {
              "used_in_bytes" : 2137960960,
              "max_in_bytes" : 8589934592,
              "peak_used_in_bytes" : 2137960960,
              "peak_max_in_bytes" : 8589934592
            },
            "survivor" : {
              "used_in_bytes" : 29785232,
              "max_in_bytes" : 0,
              "peak_used_in_bytes" : 199229440,
              "peak_max_in_bytes" : 0
            }
          }
        },
        "threads" : {
          "count" : 111,
          "peak_count" : 130
        },
        "gc" : {
          "collectors" : {
            "young" : {
              "collection_count" : 3039,
              "collection_time_in_millis" : 60848
            },
            "old" : {
              "collection_count" : 0,
              "collection_time_in_millis" : 0
            }
          }
        },
        "buffer_pools" : {
          "mapped" : {
            "count" : 7012,
            "used_in_bytes" : 25825509501,
            "total_capacity_in_bytes" : 25825509501
          },
          "direct" : {
            "count" : 93,
            "used_in_bytes" : 10030048,
            "total_capacity_in_bytes" : 10030047
          },
          "mapped - 'non-volatile memory'" : {
            "count" : 0,
            "used_in_bytes" : 0,
            "total_capacity_in_bytes" : 0
          }
        },
        "classes" : {
          "current_loaded_count" : 23559,
          "total_loaded_count" : 23559,
          "total_unloaded_count" : 0
        }
      },
      "thread_pool" : {
        "analyze" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 0,
          "completed" : 0
        },
        "ccr" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 0,
          "completed" : 0
        },
        "fetch_shard_started" : {
          "threads" : 1,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 16,
          "completed" : 283
        },
        "fetch_shard_store" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 0,
          "completed" : 0
        },
        "flush" : {
          "threads" : 1,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 4,
          "completed" : 283
        },
        "force_merge" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 0,
          "completed" : 0
        },
        "generic" : {
          "threads" : 35,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 35,
          "completed" : 122604
        },
        "get" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 0,
          "completed" : 0
        },
        "listener" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 0,
          "completed" : 0
        },
        "management" : {
          "threads" : 5,
          "queue" : 0,
          "active" : 1,
          "rejected" : 0,
          "largest" : 5,
          "completed" : 148331
        },
        "ml_datafeed" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 0,
          "completed" : 0
        },
        "ml_job_comms" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 0,
          "completed" : 0
        },
        "ml_utility" : {
          "threads" : 1,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 1,
          "completed" : 5801
        },
        "refresh" : {
          "threads" : 4,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 4,
          "completed" : 535601
        },
        "rollup_indexing" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 0,
          "completed" : 0
        },
        "search" : {
          "threads" : 13,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 13,
          "completed" : 20213419
        },
        "search_throttled" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 0,
          "completed" : 0
        },
        "searchable_snapshots_cache_fetch_async" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 0,
          "completed" : 0
        },
        "searchable_snapshots_cache_prewarming" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 0,
          "completed" : 0
        },
        "security-crypto" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,

nodes/hot_threads

::: {bold-mountain-kibana}{UQaBOuaLS-i4C8XpvIOOoA}{vC4-i3ScQVSzKnO97pRcng}{127.0.0.1}{127.0.0.1:9300}{cdhilmrstw}{ml.machine_memory=33016766464, xpack.installed=true, transform.node=true, ml.max_open_jobs=20}
   Hot threads at 2021-02-09T02:38:28.581Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
   
   96.5% (482.6ms out of 500ms) cpu usage by thread 'elasticsearch[bold-mountain-kibana][transport_worker][T#5]'
     2/10 snapshots sharing following 559 elements
       java.base@15.0.1/java.util.HashMap.put(HashMap.java:612)
       org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission.authorize(IndicesPermission.java:312)
       org.elasticsearch.xpack.core.security.authz.permission.Role.authorize(Role.java:173)
       org.elasticsearch.xpack.security.authz.RBACEngine.buildIndicesAccessControl(RBACEngine.java:554)
       org.elasticsearch.xpack.security.authz.RBACEngine.lambda$authorizeIndexAction$4(RBACEngine.java:335)
       org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$5512/0x00000008018a08f0.accept(Unknown Source)
       app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
       org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.lambda$getAsync$0(AuthorizationService.java:678)
       org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier$$Lambda$5463/0x00000008017fe978.accept(Unknown Source)
       app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
       org.elasticsearch.xpack.security.authz.AuthorizationService.resolveIndexNames(AuthorizationService.java:579)
       org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$6(AuthorizationService.java:273)
       org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5465/0x00000008017fedd8.accept(Unknown Source)
       app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
       org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.lambda$getAsync$0(AuthorizationService.java:678)
       org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier$$Lambda$5463/0x00000008017fe978.accept(Unknown Source)
       app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
       org.elasticsearch.xpack.security.authz.RBACEngine.loadAuthorizedIndices(RBACEngine.java:366)
       org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$5(AuthorizationService.java:269)
       org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5457/0x00000008017fda10.getAsync(Unknown Source)
       org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.getAsync(AuthorizationService.java:676)
       org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$8(AuthorizationService.java:272)
       org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5458/0x00000008017fdc38.getAsync(Unknown Source)
       org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.getAsync(AuthorizationService.java:676)
       org.elasticsearch.xpack.security.authz.RBACEngine.lambda$authorizeIndexAction$5(RBACEngine.java:327)
       org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$5509/0x00000008018a0490.accept(Unknown Source)
       app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
       org.elasticsearch.xpack.security.authz.RBACEngine.authorizeIndexActionName(RBACEngine.java:351)
       org.elasticsearch.xpack.security.authz.RBACEngine.authorizeIndexAction(RBACEngine.java:324)
       org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:283)
       org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:248)
       org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$1(AuthorizationService.java:212)
       org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5453/0x00000008017fccd0.accept(Unknown Source)
       app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
       app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:43)
       org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$1(RBACEngine.java:126)
       org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$5455/0x00000008017fd130.accept(Unknown Source)
       app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
       org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.roles(CompositeRolesStore.java:159)
       org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:276)
       org.elasticsearch.xpack.security.authz.RBACEngine.getRoles(RBACEngine.java:132)
       org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:120)
       org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:214)
       org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129)
       org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4735/0x0000000801654ec0.accept(Unknown Source)
       app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:323)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4737/0x0000000801655ea0.accept(Unknown Source)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:384)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4738/0x00000008016560d8.run(Unknown Source)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:395)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:320)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:261)
       org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:173)
       org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120)
       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313)
       app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:72)
       app//org.elasticsearch.transport.TransportService.sendLocalRequest(TransportService.java:803)
       app//org.elasticsearch.transport.TransportService.access$100(TransportService.java:76)
       app//org.elasticsearch.transport.TransportService$3.sendRequest(TransportService.java:130)
       app//org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:747)
       app//org.elasticsearch.transport.TransportService$$Lambda$3643/0x00000008014f3428.sendRequest(Unknown Source)
       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor.sendWithUser(SecurityServerTransportInterceptor.java:162)
       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor.access$300(SecurityServerTransportInterceptor.java:53)
       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$1.sendRequest(SecurityServerTransportInterceptor.java:129)
       app//org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:661)
       app//org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:712)
       app//org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:704)
       app//org.elasticsearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:139)
       app//org.elasticsearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:83)
       app//org.elasticsearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:244)
       app//org.elasticsearch.action.search.AbstractSearchAsyncAction$$Lambda$5504/0x000000080188f058.run(Unknown Source)
       app//org.elasticsearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:690)
       app//org.elasticsearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:277)
       app//org.elasticsearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:215)
       app//org.elasticsearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:365)
       app//org.elasticsearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:182)
       app//org.elasticsearch.action.search.TransportSearchAction$5.run(TransportSearchAction.java:799)
       app//org.elasticsearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:365)
       app//org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:359)
       app//org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:603)
       app//org.elasticsearch.action.search.AbstractSearchAsyncAction.successfulShardExecution(AbstractSearchAsyncAction.java:503)
       app//org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardResultConsumed(AbstractSearchAsyncAction.java:491)
       app//org.elasticsearch.action.search.AbstractSearchAsyncAction.lambda$onShardResult$8(AbstractSearchAsyncAction.java:474)
       app//org.elasticsearch.action.search.AbstractSearchAsyncAction$$Lambda$5583/0x00000008018cc638.run(Unknown Source)
       app//org.elasticsearch.action.search.CanMatchPreFilterSearchPhase$CanMatchSearchPhaseResults.consumeResult(CanMatchPreFilterSearchPhase.java:171)
       app//org.elasticsearch.action.search.CanMatchPreFilterSearchPhase$CanMatchSearchPhaseResults.consumeResult(CanMatchPreFilterSearchPhase.java:155)
       app//org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardResult(AbstractSearchAsyncAction.java:474)
       app//org.elasticsearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:249)
       app//org.elasticsearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:45)
       app//org.elasticsearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:29)
       app//org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54)
       app//org.elasticsearch.transport.TransportService$6.handleResponse(TransportService.java:634)
       app//org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171)
       app//org.elasticsearch.transport.TransportService$DirectResponseChannel.processResponse(TransportService.java:1249)```