Circuit break exception

Elastic Stack Elasticsearch
1

Hi all,

I have been using Elasticsearch with multiple node setup from last 2 years, but currently all of sudden , got below exception of circuit_breaking_exception .
<
{"error":{"root_cause":[{"type":"circuit_breaking_exception","reason":"[parent] Data too large, data for [<http_request>] would be [4101763462/3.8gb], which is larger than the limit of [4080218931/3.7gb], real usage: [4099648288/3.8gb], new bytes reserved: [2115174/2mb], usages [request=0/0b, fielddata=0/0b, in_flight_requests=2115174/2mb, accounting=25208/24.6kb]","bytes_wanted":4101763462,"bytes_limit":4080218931,"durability":"TRANSIENT"}
/>
Can anyone guide me what should i look into?

Regards,
Rajesh

2

Welcome to our community! :smiley:

What is the output from the _cluster/stats?pretty&human API?

3

Hello,

Below is the response ,

{
  "_nodes" : {
    "total" : 3,
    "successful" : 3,
    "failed" : 0
  },
  "cluster_name" : "es-prod-cluster",
  "cluster_uuid" : "V-wgu8A_SBeR8S_cobyNvQ",
  "timestamp" : 1656910658528,
  "status" : "green",
  "indices" : {
    "count" : 11,
    "shards" : {
      "total" : 22,
      "primaries" : 11,
      "replication" : 1.0,
      "index" : {
        "shards" : {
          "min" : 2,
          "max" : 2,
          "avg" : 2.0
        },
        "primaries" : {
          "min" : 1,
          "max" : 1,
          "avg" : 1.0
        },
        "replication" : {
          "min" : 1.0,
          "max" : 1.0,
          "avg" : 1.0
        }
      }
    },
    "docs" : {
      "count" : 120064,
      "deleted" : 41796
    },
    "store" : {
      "size" : "1.6gb",
      "size_in_bytes" : 1724034581
    },
    "fielddata" : {
      "memory_size" : "0b",
      "memory_size_in_bytes" : 0,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size" : "255kb",
      "memory_size_in_bytes" : 261152,
      "total_count" : 1073794,
      "hit_count" : 53114,
      "miss_count" : 1020680,
      "cache_size" : 24,
      "cache_count" : 1848,
      "evictions" : 1824
    },
    "completion" : {
      "size" : "0b",
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 78,
      "memory" : "1.8mb",
      "memory_in_bytes" : 1888838,
      "terms_memory" : "1mb",
      "terms_memory_in_bytes" : 1108752,
      "stored_fields_memory" : "52.2kb",
      "stored_fields_memory_in_bytes" : 53456,
      "term_vectors_memory" : "0b",
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory" : "139.6kb",
      "norms_memory_in_bytes" : 143040,
      "points_memory" : "0b",
      "points_memory_in_bytes" : 0,
      "doc_values_memory" : "569.9kb",
      "doc_values_memory_in_bytes" : 583590,
      "index_writer_memory" : "0b",
      "index_writer_memory_in_bytes" : 0,
      "version_map_memory" : "0b",
      "version_map_memory_in_bytes" : 0,
      "fixed_bit_set" : "0b",
      "fixed_bit_set_memory_in_bytes" : 0,
      "max_unsafe_auto_id_timestamp" : -1,
      "file_sizes" : { }
    },
    "mappings" : {
      "field_types" : [
        {
          "name" : "boolean",
          "count" : 16,
          "index_count" : 2
        },
        {
          "name" : "date",
          "count" : 30,
          "index_count" : 2
        },
        {
          "name" : "float",
          "count" : 11,
          "index_count" : 1
        },
        {
          "name" : "keyword",
          "count" : 106,
          "index_count" : 4
        },
        {
          "name" : "long",
          "count" : 78,
          "index_count" : 4
        },
        {
          "name" : "object",
          "count" : 43,
          "index_count" : 2
        },
        {
          "name" : "search_as_you_type",
          "count" : 3,
          "index_count" : 3
        },
        {
          "name" : "text",
          "count" : 106,
          "index_count" : 4
        }
      ]
    },
    "analysis" : {
      "char_filter_types" : [ ],
      "tokenizer_types" : [ ],
      "filter_types" : [ ],
      "analyzer_types" : [ ],
      "built_in_char_filters" : [ ],
      "built_in_tokenizers" : [ ],
      "built_in_filters" : [ ],
      "built_in_analyzers" : [ ]
    }
  },
  "nodes" : {
    "count" : {
      "total" : 3,
      "coordinating_only" : 0,
      "data" : 3,
      "ingest" : 3,
      "master" : 3,
      "ml" : 3,
      "remote_cluster_client" : 3,
      "transform" : 3,
      "voting_only" : 0
    },
    "versions" : [
      "7.7.1"
    ],
    "os" : {
      "available_processors" : 6,
      "allocated_processors" : 6,
      "names" : [
        {
          "name" : "Linux",
          "count" : 3
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "Amazon Linux 2",
          "count" : 3
        }
      ],
      "mem" : {
        "total" : "23.1gb",
        "total_in_bytes" : 24840437760,
        "free" : "533.2mb",
        "free_in_bytes" : 559173632,
        "used" : "22.6gb",
        "used_in_bytes" : 24281264128,
        "free_percent" : 2,
        "used_percent" : 98
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 2
      },
      "open_file_descriptors" : {
        "min" : 339,
        "max" : 350,
        "avg" : 345
      }
    },
    "jvm" : {
      "max_uptime" : "52.8d",
      "max_uptime_in_millis" : 4562175180,
      "versions" : [
        {
          "version" : "14.0.1",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "14.0.1+7",
          "vm_vendor" : "AdoptOpenJDK",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 3
        }
      ],
      "mem" : {
        "heap_used" : "2.5gb",
        "heap_used_in_bytes" : 2762084184,
        "heap_max" : "12gb",
        "heap_max_in_bytes" : 12884901888
      },
      "threads" : 138
    },
    "fs" : {
      "total" : "149.9gb",
      "total_in_bytes" : 160982630400,
      "free" : "148gb",
      "free_in_bytes" : 158994112512,
      "available" : "148gb",
      "available_in_bytes" : 158994112512
    },
    "plugins" : [ ],
    "network_types" : {
      "transport_types" : {
        "security4" : 3
      },
      "http_types" : {
        "security4" : 3
      }
    },
    "discovery_types" : {
      "zen" : 3
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "rpm",
        "count" : 3
      }
    ],
    "ingest" : {
      "number_of_pipelines" : 2,
      "processor_stats" : {
        "gsub" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time" : "0s",
          "time_in_millis" : 0
        },
        "script" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time" : "0s",
          "time_in_millis" : 0
        }
      }
    }
  }
}

Regards,
Rajesh

4

7.7 is EOL and you should really upgrade ASAP. That would be the first thing to do.

1 Like
5

Sure will do that.
But can help me out what causes this exception.

6

Hello @Rajesh_Sahu

The data that you are trying to ingest is 4GB but your node can handle only upto 3.7GB
Hence please increase the resource to fix this issue.

Keep Posted!!! Thanks !!!

7

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.