ES 7.8 often giving errors "Data too large for http_request", but some evidence makes no sense

summary of error message:

[parent] Data too large, data for [<http_request>] would be [31095660714/28.9gb], which is larger than the limit of [30601641984/28.5gb], real usage: [31095659536/28.9gb], new bytes reserved: [1178/1.1kb], usages [request=384401408/366.5mb, fielddata=3335271/3.1mb, in_flight_requests=1178/1.1kb, accounting=1239148848/1.1gb]

Main points:

  • I am not making any request that large and there is not object that large in db
  • In yaml config I have set "http.max_content_length". It does not seems to respected it
  • It seems to be stacking up the use of JVM Heap as image below. You can see it goes down whenever I restart ES.
  • I tried cleaning all caches of requests ( I could recall all commands used here if needed) when JVM Heap gets in high load

Screen Shot 2021-09-23 at 12.32.17

My feeling is that something (maybe the requests responses) is getting cached in JVM Heap and not being cleaned right after, but I can't find what exactly.

The server has 60GB of RAM, so I have set 30GB to JVM Heap

What is the full output of the cluster stats API?

Adding to Christian's question that 7.2 is super old. You should upgrade to 7.14.2.

correction, the version is 7.8.
I changed in title

here it is


 "cluster_name" : "es_atlas",
  "nodes" : {
    "iT6Lz5-EQgO8ViY44Q8_Ag" : {
      "timestamp" : 1632419313888,
      "name" : "ax-es-cluster-node1",
      "transport_address" : "xxxx",
      "host" : "xxxx",
      "ip" : "xxxxx",
      "roles" : [
        "data",
        "ingest",
        "master",
        "ml",
        "remote_cluster_client",
        "transform"
      ],
      "attributes" : {
        "ml.machine_memory" : "67543736320",
        "xpack.installed" : "true",
        "transform.node" : "true",
        "ml.max_open_jobs" : "20"
      },
      "os" : {
        "timestamp" : 1632419313889,
        "cpu" : {
          "percent" : 36,
          "load_average" : {
            "1m" : 6.84,
            "5m" : 7.68,
            "15m" : 7.72
          }
        },
        "mem" : {
          "total" : "62.9gb",
          "total_in_bytes" : 67543736320,
          "free" : "596.9mb",
          "free_in_bytes" : 625938432,
          "used" : "62.3gb",
          "used_in_bytes" : 66917797888,
          "free_percent" : 1,
          "used_percent" : 99
        },
        "swap" : {
          "total" : "31.9gb",
          "total_in_bytes" : 34325131264,
          "free" : "26.9gb",
          "free_in_bytes" : 28884344832,
          "used" : "5gb",
          "used_in_bytes" : 5440786432
        },
        "cgroup" : {
          "cpuacct" : {
            "control_group" : "/system.slice",
            "usage_nanos" : 79313071466713723
          },
          "cpu" : {
            "control_group" : "/system.slice",
            "cfs_period_micros" : 100000,
            "cfs_quota_micros" : -1,
            "stat" : {
              "number_of_elapsed_periods" : 0,
              "number_of_times_throttled" : 0,
              "time_throttled_nanos" : 0
            }
          },
          "memory" : {
            "control_group" : "/",
            "limit_in_bytes" : "9223372036854771712",
            "usage_in_bytes" : "65493512192"
          }
        }
      }
    }
  }
}

That does not seem to be the full output or possibly the correct API.

My answer is still accurate :wink:

haha right I will follow your advice, I will update this weekend

Let me see if this is the right one:

{
  "_nodes" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "cluster_name" : "xxxx",
  "cluster_uuid" : "oxZ15HsjTPqA3E_YxKVX8A",
  "timestamp" : 1632430226070,
  "status" : "green",
  "indices" : {
    "count" : 33,
    "shards" : {
      "total" : 44,
      "primaries" : 44,
      "replication" : 0.0,
      "index" : {
        "shards" : {
          "min" : 1,
          "max" : 12,
          "avg" : 1.3333333333333333
        },
        "primaries" : {
          "min" : 1,
          "max" : 12,
          "avg" : 1.3333333333333333
        },
        "replication" : {
          "min" : 0.0,
          "max" : 0.0,
          "avg" : 0.0
        }
      }
    },
    "docs" : {
      "count" : 50856827,
      "deleted" : 17587160
    },
    "store" : {
      "size_in_bytes" : 423580151880
    },
    "fielddata" : {
      "memory_size_in_bytes" : 16118000,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size_in_bytes" : 27663480,
      "total_count" : 7318557,
      "hit_count" : 12439,
      "miss_count" : 7306118,
      "cache_size" : 693,
      "cache_count" : 693,
      "evictions" : 0
    },
    "completion" : {
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 586,
      "memory_in_bytes" : 380022278,
      "terms_memory_in_bytes" : 275573552,
      "stored_fields_memory_in_bytes" : 2545584,
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory_in_bytes" : 41257344,
      "points_memory_in_bytes" : 0,
      "doc_values_memory_in_bytes" : 60645798,
      "index_writer_memory_in_bytes" : 17153264,
      "version_map_memory_in_bytes" : 648329,
      "fixed_bit_set_memory_in_bytes" : 379120,
      "max_unsafe_auto_id_timestamp" : 1632397952114,
      "file_sizes" : { }
    },
    "mappings" : {
      "field_types" : [
        {
          "name" : "binary",
          "count" : 3,
          "index_count" : 1
        },
        {
          "name" : "boolean",
          "count" : 118,
          "index_count" : 27
        },
        {
          "name" : "date",
          "count" : 279,
          "index_count" : 33
        },
        {
          "name" : "flattened",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "float",
          "count" : 722,
          "index_count" : 10
        },
        {
          "name" : "geo_point",
          "count" : 2,
          "index_count" : 1
        },
        {
          "name" : "geo_shape",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "half_float",
          "count" : 64,
          "index_count" : 16
        },
        {
          "name" : "integer",
          "count" : 203,
          "index_count" : 10
        },
        {
          "name" : "keyword",
          "count" : 17341,
          "index_count" : 33
        },
        {
          "name" : "long",
          "count" : 2129,
          "index_count" : 26
        },
        {
          "name" : "nested",
          "count" : 36,
          "index_count" : 14
        },
        {
          "name" : "object",
          "count" : 1682,
          "index_count" : 33
        },
        {
          "name" : "text",
          "count" : 16635,
          "index_count" : 24
        }
      ]
    },
    "analysis" : {
      "char_filter_types" : [ ],
      "tokenizer_types" : [ ],
      "filter_types" : [ ],
      "analyzer_types" : [ ],
      "built_in_char_filters" : [ ],
      "built_in_tokenizers" : [ ],
      "built_in_filters" : [ ],
      "built_in_analyzers" : [ ]
    }
  },
  "nodes" : {
    "count" : {
      "total" : 1,
      "coordinating_only" : 0,
      "data" : 1,
      "ingest" : 1,
      "master" : 1,
      "ml" : 1,
      "remote_cluster_client" : 1,
      "transform" : 1,
      "voting_only" : 0
    },
    "versions" : [
      "7.8.0"
    ],
    "os" : {
      "available_processors" : 16,
      "allocated_processors" : 16,
      "names" : [
        {
          "name" : "Linux",
          "count" : 1
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "Ubuntu 18.04.4 LTS",
          "count" : 1
        }
      ],
      "mem" : {
        "total_in_bytes" : 67543736320,
        "free_in_bytes" : 849154048,
        "used_in_bytes" : 66694582272,
        "free_percent" : 1,
        "used_percent" : 99
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 4
      },
      "open_file_descriptors" : {
        "min" : 943,
        "max" : 943,
        "avg" : 943
      }
    },
    "jvm" : {
      "max_uptime_in_millis" : 3501176,
      "versions" : [
        {
          "version" : "14.0.1",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "14.0.1+7",
          "vm_vendor" : "AdoptOpenJDK",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 1
        }
      ],
      "mem" : {
        "heap_used_in_bytes" : 16672165888,
        "heap_max_in_bytes" : 32212254720
      },
      "threads" : 151
    },
    "fs" : {
      "total_in_bytes" : 972576313344,
      "free_in_bytes" : 344868917248,
      "available_in_bytes" : 295393366016
    },
    "plugins" : [
      {
        "name" : "repository-azure",
        "version" : "7.8.0",
        "elasticsearch_version" : "7.8.0",
        "java_version" : "1.8",
        "description" : "The Azure Repository plugin adds support for Azure storage repositories.",
        "classname" : "org.elasticsearch.repositories.azure.AzureRepositoryPlugin",
        "extended_plugins" : [ ],
        "has_native_controller" : false
      },
      {
        "name" : "repository-s3",
        "version" : "7.8.0",
        "elasticsearch_version" : "7.8.0",
        "java_version" : "1.8",
        "description" : "The S3 repository plugin adds S3 repositories",
        "classname" : "org.elasticsearch.repositories.s3.S3RepositoryPlugin",
        "extended_plugins" : [ ],
        "has_native_controller" : false
      }
    ],
    "network_types" : {
      "transport_types" : {
        "security4" : 1
      },
      "http_types" : {
        "security4" : 1
      }
    },
    "discovery_types" : {
      "zen" : 1
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "deb",
        "count" : 1
      }
    ],
    "ingest" : {
      "number_of_pipelines" : 1,
      "processor_stats" : {
        "gsub" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        },
        "script" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        }
      }
    }
  }
}