Es_rejected_execution_exception

In Elasticsearch v7.9.3, I'm getting the below shown error while sending Kubernetes Apiserver Audit logs to Elasticsearch. Looks like Elasticsearch is not able to keep up with the number of requests being sent. Can you please let me know on how I can resolve this issue.

{\"error\":{\"root_cause\":[{\"type\":\"es_rejected_execution_exception\",\"reason\":\"rejected execution of coordinating operation [coordinating_and_primary_bytes=268419661, replica_bytes=0, all_bytes=268419661, coordinating_operation_bytes=21647, max_coordinating_and_primary_bytes=268435456]\"}],\"type\":\"es_rejected_execution_exception\",\"reason\":\"rejected execution of coordinating operation [coordinating_and_primary_bytes=268419661, replica_bytes=0, all_bytes=268419661, coordinating_operation_bytes=21647, max_coordinating_and_primary_bytes=268435456]\"},\"status\":429}

What is the output from the _cluster/stats?pretty&human API?

@warkolm , below is the output of _cluster/stats?pretty&human API

{
  "_nodes" : {
    "total" : 8,
    "successful" : 8,
    "failed" : 0
  },
  "cluster_name" : "elastic",
  "cluster_uuid" : "JO0bbAhJTfixMU47DEkM_P",
  "timestamp" : 1624245620728,
  "status" : "green",
  "indices" : {
    "count" : 35,
    "shards" : {
      "total" : 350,
      "primaries" : 127,
      "replication" : 1.7559055118110236,
      "index" : {
        "shards" : {
          "min" : 2,
          "max" : 15,
          "avg" : 10.0
        },
        "primaries" : {
          "min" : 1,
          "max" : 5,
          "avg" : 3.6285714285714286
        },
        "replication" : {
          "min" : 1.0,
          "max" : 2.0,
          "avg" : 1.5714285714285714
        }
      }
    },
    "docs" : {
      "count" : 96637333,
      "deleted" : 3
    },
    "store" : {
      "size" : "96.1gb",
      "size_in_bytes" : 103238321106,
      "reserved" : "0b",
      "reserved_in_bytes" : 0
    },
    "fielddata" : {
      "memory_size" : "0b",
      "memory_size_in_bytes" : 0,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size" : "0b",
      "memory_size_in_bytes" : 0,
      "total_count" : 1470,
      "hit_count" : 0,
      "miss_count" : 1470,
      "cache_size" : 0,
      "cache_count" : 0,
      "evictions" : 0
    },
    "completion" : {
      "size" : "0b",
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 3170,
      "memory" : "66.6mb",
      "memory_in_bytes" : 69917592,
      "terms_memory" : "52.3mb",
      "terms_memory_in_bytes" : 54869856,
      "stored_fields_memory" : "2.6mb",
      "stored_fields_memory_in_bytes" : 2773440,
      "term_vectors_memory" : "0b",
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory" : "7.3mb",
      "norms_memory_in_bytes" : 7728768,
      "points_memory" : "0b",
      "points_memory_in_bytes" : 0,
      "doc_values_memory" : "4.3mb",
      "doc_values_memory_in_bytes" : 4545528,
      "index_writer_memory" : "1.5gb",
      "index_writer_memory_in_bytes" : 1635060688,
      "version_map_memory" : "0b",
      "version_map_memory_in_bytes" : 0,
      "fixed_bit_set" : "1.8mb",
      "fixed_bit_set_memory_in_bytes" : 1908768,
      "max_unsafe_auto_id_timestamp" : 1624237454742,
      "file_sizes" : { }
    },
    "mappings" : {
      "field_types" : [
        {
          "name" : "boolean",
          "count" : 67,
          "index_count" : 7
        },
        {
          "name" : "date",
          "count" : 57,
          "index_count" : 27
        },
        {
          "name" : "float",
          "count" : 2,
          "index_count" : 1
        },
        {
          "name" : "integer",
          "count" : 16,
          "index_count" : 3
        },
        {
          "name" : "keyword",
          "count" : 1665,
          "index_count" : 30
        },
        {
          "name" : "long",
          "count" : 361,
          "index_count" : 18
        },
        {
          "name" : "nested",
          "count" : 11,
          "index_count" : 3
        },
        {
          "name" : "object",
          "count" : 220,
          "index_count" : 24
        },
        {
          "name" : "text",
          "count" : 1622,
          "index_count" : 24
        }
      ]
    },
    "analysis" : {
      "char_filter_types" : [ ],
      "tokenizer_types" : [ ],
      "filter_types" : [ ],
      "analyzer_types" : [ ],
      "built_in_char_filters" : [ ],
      "built_in_tokenizers" : [ ],
      "built_in_filters" : [ ],
      "built_in_analyzers" : [
        {
          "name" : "keyword",
          "count" : 2,
          "index_count" : 2
        }
      ]
    }
  },
  "nodes" : {
    "count" : {
      "total" : 8,
      "coordinating_only" : 0,
      "data" : 3,
      "ingest" : 2,
      "master" : 3,
      "remote_cluster_client" : 8
    },
    "versions" : [
      "7.9.3"
    ],
    "os" : {
      "available_processors" : 27,
      "allocated_processors" : 27,
      "names" : [
        {
          "name" : "Linux",
          "count" : 8
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "CentOS Linux 7 (Core)",
          "count" : 8
        }
      ],
      "mem" : {
        "total" : "135gb",
        "total_in_bytes" : 144955146240,
        "free" : "62.1gb",
        "free_in_bytes" : 66754519040,
        "used" : "72.8gb",
        "used_in_bytes" : 78200627200,
        "free_percent" : 46,
        "used_percent" : 54
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 80
      },
      "open_file_descriptors" : {
        "min" : 399,
        "max" : 4712,
        "avg" : 1760
      }
    },
    "jvm" : {
      "max_uptime" : "11h",
      "max_uptime_in_millis" : 39643127,
      "versions" : [
        {
          "version" : "15",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "15+36-1562",
          "vm_vendor" : "Oracle Corporation",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 8
        }
      ],
      "mem" : {
        "heap_used" : "10.7gb",
        "heap_used_in_bytes" : 11561351416,
        "heap_max" : "27.5gb",
        "heap_max_in_bytes" : 29527900160
      },
      "threads" : 447
    },
    "fs" : {
      "total" : "478.9gb",
      "total_in_bytes" : 514319720448,
      "free" : "368gb",
      "free_in_bytes" : 395172507648,
      "available" : "367.9gb",
      "available_in_bytes" : 395038285824
    },
    "plugins" : [
      {
        "name" : "search-guard-7",
        "version" : "7.9.3-47.1.0",
        "elasticsearch_version" : "7.9.3",
        "java_version" : "1.8",
        "description" : "Provide access control related features for Elasticsearch 7",
        "classname" : "com.floragunn.searchguard.SearchGuardPlugin",
        "extended_plugins" : [
          "lang-painless"
        ],
        "has_native_controller" : false
      }
    ],
    "network_types" : {
      "transport_types" : {
        "com.floragunn.searchguard.ssl.http.netty.SearchGuardSSLNettyTransport" : 8
      },
      "http_types" : {
        "com.floragunn.searchguard.http.SearchGuardHttpServerTransport" : 8
      }
    },
    "discovery_types" : {
      "zen" : 8
    },
    "packaging_types" : [
      {
        "flavor" : "oss",
        "type" : "docker",
        "count" : 8
      }
    ],
    "ingest" : {
      "number_of_pipelines" : 0,
      "processor_stats" : { }
    }
  }
}

Can you upgrade, 7.9 is relatively old.

@warkolm, At present we won't be able to upgrade the elasticsearch. We have the plan of upgrading it in the coming days. Is there any workaround or resolution for this with v7.9.3?

What is the output from GET /_cat/thread_pool?v?

@warkolm, Please find the output

node_name            name                active queue rejected
fed-elastic-data-1   analyze                  0     0        0
fed-elastic-data-1   fetch_shard_started      0     0        0
fed-elastic-data-1   fetch_shard_store        0     0        0
fed-elastic-data-1   flush                    0     0        0
fed-elastic-data-1   force_merge              0     0        0
fed-elastic-data-1   generic                  0     0        0
fed-elastic-data-1   get                      0     0        0
fed-elastic-data-1   listener                 0     0        0
fed-elastic-data-1   management               1     0        0
fed-elastic-data-1   refresh                  0     0        0
fed-elastic-data-1   search                   0     0        0
fed-elastic-data-1   search_throttled         0     0        0
fed-elastic-data-1   snapshot                 0     0        0
fed-elastic-data-1   warmer                   0     0        0
fed-elastic-data-1   write                    4  3361        0
fed-elastic-data-0   analyze                  0     0        0
fed-elastic-data-0   fetch_shard_started      0     0        0
fed-elastic-data-0   fetch_shard_store        0     0        0
fed-elastic-data-0   flush                    0     0        0
fed-elastic-data-0   force_merge              0     0        0
fed-elastic-data-0   generic                  0     0        0
fed-elastic-data-0   get                      0     0        0
fed-elastic-data-0   listener                 0     0        0
fed-elastic-data-0   management               1     0        0
fed-elastic-data-0   refresh                  0     0        0
fed-elastic-data-0   search                   0     0        0
fed-elastic-data-0   search_throttled         0     0        0
fed-elastic-data-0   snapshot                 0     0        0
fed-elastic-data-0   warmer                   0     0        0
fed-elastic-data-0   write                    4     7        0
fed-elastic-client-0 analyze                  0     0        0
fed-elastic-client-0 fetch_shard_started      0     0        0
fed-elastic-client-0 fetch_shard_store        0     0        0
fed-elastic-client-0 flush                    0     0        0
fed-elastic-client-0 force_merge              0     0        0
fed-elastic-client-0 generic                  0     0        0
fed-elastic-client-0 get                      0     0        0
fed-elastic-client-0 listener                 0     0        0
fed-elastic-client-0 management               1     0        0
fed-elastic-client-0 refresh                  0     0        0
fed-elastic-client-0 search                   0     0        0
fed-elastic-client-0 search_throttled         0     0        0
fed-elastic-client-0 snapshot                 0     0        0
fed-elastic-client-0 warmer                   0     0        0
fed-elastic-client-0 write                    0     0        0
fed-elastic-data-2   analyze                  0     0        0
fed-elastic-data-2   fetch_shard_started      0     0        0
fed-elastic-data-2   fetch_shard_store        0     0        0
fed-elastic-data-2   flush                    0     0        0
fed-elastic-data-2   force_merge              0     0        0
fed-elastic-data-2   generic                  0     0        0
fed-elastic-data-2   get                      0     0        0
fed-elastic-data-2   listener                 0     0        0
fed-elastic-data-2   management               1     0        0
fed-elastic-data-2   refresh                  0     0        0
fed-elastic-data-2   search                   0     0        0
fed-elastic-data-2   search_throttled         0     0        0
fed-elastic-data-2   snapshot                 0     0        0
fed-elastic-data-2   warmer                   0     0        0
fed-elastic-data-2   write                    4    10        0
fed-elastic-master-2 analyze                  0     0        0
fed-elastic-master-2 fetch_shard_started      0     0        0
fed-elastic-master-2 fetch_shard_store        0     0        0
fed-elastic-master-2 flush                    0     0        0
fed-elastic-master-2 force_merge              0     0        0
fed-elastic-master-2 generic                  0     0        0
fed-elastic-master-2 get                      0     0        0
fed-elastic-master-2 listener                 0     0        0
fed-elastic-master-2 management               1     0        0
fed-elastic-master-2 refresh                  0     0        0
fed-elastic-master-2 search                   0     0        0
fed-elastic-master-2 search_throttled         0     0        0
fed-elastic-master-2 snapshot                 0     0        0
fed-elastic-master-2 warmer                   0     0        0
fed-elastic-master-2 write                    0     0        0
fed-elastic-master-0 analyze                  0     0        0
fed-elastic-master-0 fetch_shard_started      0     0        0
fed-elastic-master-0 fetch_shard_store        0     0        0
fed-elastic-master-0 flush                    0     0        0
fed-elastic-master-0 force_merge              0     0        0
fed-elastic-master-0 generic                  0     0        0
fed-elastic-master-0 get                      0     0        0
fed-elastic-master-0 listener                 0     0        0
fed-elastic-master-0 management               1     0        0
fed-elastic-master-0 refresh                  0     0        0
fed-elastic-master-0 search                   0     0        0
fed-elastic-master-0 search_throttled         0     0        0
fed-elastic-master-0 snapshot                 0     0        0
fed-elastic-master-0 warmer                   0     0        0
fed-elastic-master-0 write                    0     0        0
fed-elastic-master-1 analyze                  0     0        0
fed-elastic-master-1 fetch_shard_started      0     0        0
fed-elastic-master-1 fetch_shard_store        0     0        0
fed-elastic-master-1 flush                    0     0        0
fed-elastic-master-1 force_merge              0     0        0
fed-elastic-master-1 generic                  0     0        0
fed-elastic-master-1 get                      0     0        0
fed-elastic-master-1 listener                 0     0        0
fed-elastic-master-1 management               1     0        0
fed-elastic-master-1 refresh                  0     0        0
fed-elastic-master-1 search                   0     0        0
fed-elastic-master-1 search_throttled         0     0        0
fed-elastic-master-1 snapshot                 0     0        0
fed-elastic-master-1 warmer                   0     0        0
fed-elastic-master-1 write                    0     0        0
fed-elastic-client-1 analyze                  0     0        0
fed-elastic-client-1 fetch_shard_started      0     0        0
fed-elastic-client-1 fetch_shard_store        0     0        0
fed-elastic-client-1 flush                    0     0        0
fed-elastic-client-1 force_merge              0     0        0
fed-elastic-client-1 generic                  0     0        0
fed-elastic-client-1 get                      0     0        0
fed-elastic-client-1 listener                 0     0        0
fed-elastic-client-1 management               1     0        0
fed-elastic-client-1 refresh                  0     0        0
fed-elastic-client-1 search                   0     0        0
fed-elastic-client-1 search_throttled         0     0        0
fed-elastic-client-1 snapshot                 0     0        0
fed-elastic-client-1 warmer                   0     0        0
fed-elastic-client-1 write                    0     0        0

It seems like you are sending all writes to a single node?
What is sending the data to Elasticsearch?

I'm using Fluentd to send data to Elasticsearch.

Does it support load balancing?

@warkolm, Yes, It does support load balancing.

Then you will want to configure it to send to more than one client, as that is what it looks like is happening now.

fluent-plugin-elasticsearch is responsible for sending data to elasticsearch. I have already raised issue in their discussion forum as well. Sure, if I find any resolution to my issue I will post it here as well. If anyone ends up in this issue, it will be helpful for them. GitHub - uken/fluent-plugin-elasticsearch

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.