[fielddata] New used memory 13315258923 [12.4gb] for data of [_id] would be larger than configured breaker: 13314398617 [12.3gb], breaking
I'm getting the above warning and wondering why.
Does it mean my "_id" field (the build in one) is using up too much RAM?
We do configure custom document ID in some index, but the value is not like paragraphs. It could be string like UUID, etc
I'm baffled how it could take up GB of RAM.
Or am I reading this warning incorrectly?
Thanks.
Which version of Elasticsearch are you using?
Can you please share the full log entry?
What is the full output of the cluster stats API?
version 7.15
I know it's EOL and we are going to upgrade soon. But I still wish to know what this means and how I can work around it in the future.
Thanks.
[2023-03-06T22:39:55,560][WARN ][o.e.i.b.fielddata ] [es715-data-140-production.xxx.pvt] [fielddata] New used memory 13318952016 [12.4gb] for data of [_id] would be larger than configured breaker: 13314398617 [12.3gb], breaking
[2023-03-06T22:40:01,204][WARN ][o.e.i.b.fielddata ] [es715-data-140-production.xxx.pvt] [fielddata] New used memory 13319678808 [12.4gb] for data of [_id] would be larger than configured breaker: 13314398617 [12.3gb], breaking
[2023-03-06T22:40:07,662][WARN ][o.e.i.b.fielddata ] [es715-data-140-production.xxx.pvt] [fielddata] New used memory 13316246695 [12.4gb] for data of [_id] would be larger than configured breaker: 13314398617 [12.3gb], breaking
[2023-03-06T22:40:13,697][WARN ][o.e.i.b.fielddata ] [es715-data-140-production.xxx.pvt] [fielddata] New used memory 13316832284 [12.4gb] for data of [_id] would be larger than configured breaker: 13314398617 [12.3gb], breaking
[2023-03-06T22:40:19,814][WARN ][o.e.i.b.fielddata ] [es715-data-140-production.xxx.pvt] [fielddata] New used memory 13317065941 [12.4gb] for data of [_id] would be larger than configured breaker: 13314398617 [12.3gb], breaking
[2023-03-06T22:40:25,537][WARN ][o.e.i.b.fielddata ] [es715-data-140-production.xxx.pvt] [fielddata] New used memory 13315116658 [12.4gb] for data of [_id] would be larger than configured breaker: 13314398617 [12.3gb], breaking
[2023-03-06T22:40:31,545][WARN ][o.e.i.b.fielddata ] [es715-data-140-production.xxx.pvt] [fielddata] New used memory 13315116658 [12.4gb] for data of [_id] would be larger than configured breaker: 13314398617 [12.3gb], breaking
[2023-03-06T22:40:37,935][WARN ][o.e.i.b.fielddata ] [es715-data-140-production.xxx.pvt] [fielddata] New used memory 13315113486 [12.4gb] for data of [_id] would be larger than configured breaker: 13314398617 [12.3gb], breaking
[2023-03-06T22:40:44,634][WARN ][o.e.i.b.fielddata ] [es715-data-140-production.xxx.pvt] [fielddata] New used memory 13315113486 [12.4gb] for data of [_id] would be larger than configured breaker: 13314398617 [12.3gb], breaking
[2023-03-06T22:40:51,878][WARN ][o.e.i.b.fielddata ] [es715-data-140-production.xxx.pvt] [fielddata] New used memory 13318004561 [12.4gb] for data of [_id] would be larger than configured breaker: 13314398617 [12.3gb], breaking
[2023-03-06T22:40:57,660][WARN ][o.e.i.b.fielddata ] [es715-data-140-production.xxx.pvt] [fielddata] New used memory 13317997722 [12.4gb] for data of [_id] would be larger than configured breaker: 13314398617 [12.3gb], breaking
[2023-03-06T22:41:03,412][WARN ][o.e.i.b.fielddata ] [es715-data-140-production.xxx.pvt] [fielddata] New used memory 13314977059 [12.4gb] for data of [_id] would be larger than configured breaker: 13314398617 [12.3gb], breaking
[2023-03-06T22:41:09,480][WARN ][o.e.i.b.fielddata ] [es715-data-140-production.xxx.pvt] [fielddata] New used memory 13319363004 [12.4gb] for data of [_id] would be larger than configured breaker: 13314398617 [12.3gb], breaking
[2023-03-06T22:41:15,506][WARN ][o.e.i.b.fielddata ] [es715-data-140-production.xxx.pvt] [fielddata] New used memory 13314414688 [12.4gb] for data of [_id] would be larger than configured breaker: 13314398617 [12.3gb], breaking
{
"_nodes" : {
"total" : 51,
"successful" : 51,
"failed" : 0
},
"cluster_name" : "Elasticsearch715_Cluster",
"cluster_uuid" : "zGrKBe8NRNSPmDiPjCwXmQ",
"timestamp" : 1678213683206,
"status" : "green",
"indices" : {
"count" : 472,
"shards" : {
"total" : 6688,
"primaries" : 3324,
"replication" : 1.0120336943441637,
"index" : {
"shards" : {
"min" : 2,
"max" : 60,
"avg" : 14.169491525423728
},
"primaries" : {
"min" : 1,
"max" : 30,
"avg" : 7.0423728813559325
},
"replication" : {
"min" : 1.0,
"max" : 2.0,
"avg" : 1.0593220338983051
}
}
},
"docs" : {
"count" : 148492011427,
"deleted" : 626358181
},
"store" : {
"size_in_bytes" : 105622713287866,
"total_data_set_size_in_bytes" : 105622713287866,
"reserved_in_bytes" : 0
},
"fielddata" : {
"memory_size_in_bytes" : 460217324300,
"evictions" : 0
},
"query_cache" : {
"memory_size_in_bytes" : 68975214403,
"total_count" : 180676597695,
"hit_count" : 12352600696,
"miss_count" : 168323996999,
"cache_size" : 1405762,
"cache_count" : 97893617,
"evictions" : 96487855
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 171646,
"memory_in_bytes" : 2671218670,
"terms_memory_in_bytes" : 1814765192,
"stored_fields_memory_in_bytes" : 113505536,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 127706432,
"points_memory_in_bytes" : 0,
"doc_values_memory_in_bytes" : 615241510,
"index_writer_memory_in_bytes" : 16087203652,
"version_map_memory_in_bytes" : 1000538607,
"fixed_bit_set_memory_in_bytes" : 48000,
"max_unsafe_auto_id_timestamp" : 1678212000522,
"file_sizes" : { }
},
"mappings" : {
"field_types" : [
{
"name" : "boolean",
"count" : 1287,
"index_count" : 244,
"script_count" : 0
},
{
"name" : "byte",
"count" : 20,
"index_count" : 10,
"script_count" : 0
},
{
"name" : "date",
"count" : 1345,
"index_count" : 445,
"script_count" : 0
},
{
"name" : "dense_vector",
"count" : 9,
"index_count" : 9,
"script_count" : 0
},
{
"name" : "double",
"count" : 6,
"index_count" : 2,
"script_count" : 0
},
{
"name" : "flattened",
"count" : 3,
"index_count" : 3,
"script_count" : 0
},
{
"name" : "float",
"count" : 3429,
"index_count" : 167,
"script_count" : 0
},
{
"name" : "integer",
"count" : 157,
"index_count" : 61,
"script_count" : 0
},
{
"name" : "ip",
"count" : 11,
"index_count" : 7,
"script_count" : 0
},
{
"name" : "keyword",
"count" : 16289,
"index_count" : 449,
"script_count" : 0
},
{
"name" : "long",
"count" : 2509,
"index_count" : 301,
"script_count" : 0
},
{
"name" : "nested",
"count" : 4,
"index_count" : 4,
"script_count" : 0
},
{
"name" : "object",
"count" : 1617,
"index_count" : 197,
"script_count" : 0
},
{
"name" : "short",
"count" : 116,
"index_count" : 47,
"script_count" : 0
},
{
"name" : "text",
"count" : 10914,
"index_count" : 412,
"script_count" : 0
},
{
"name" : "version",
"count" : 4,
"index_count" : 4,
"script_count" : 0
}
],
"runtime_field_types" : [ ]
},
"analysis" : {
"char_filter_types" : [ ],
"tokenizer_types" : [ ],
"filter_types" : [ ],
"analyzer_types" : [ ],
"built_in_char_filters" : [ ],
"built_in_tokenizers" : [ ],
"built_in_filters" : [ ],
"built_in_analyzers" : [
{
"name" : "english",
"count" : 14,
"index_count" : 7
}
]
},
"versions" : [
{
"version" : "7.15.0",
"index_count" : 472,
"primary_shard_count" : 3324,
"total_primary_bytes" : 52760195446575
}
]
},
"nodes" : {
"count" : {
"total" : 51,
"coordinating_only" : 0,
"data" : 48,
"data_cold" : 0,
"data_content" : 0,
"data_frozen" : 0,
"data_hot" : 0,
"data_warm" : 0,
"ingest" : 3,
"master" : 3,
"ml" : 0,
"remote_cluster_client" : 0,
"transform" : 0,
"voting_only" : 0
},
"versions" : [
"7.15.0"
],
"os" : {
"available_processors" : 792,
"allocated_processors" : 792,
"names" : [
{
"name" : "Linux",
"count" : 51
}
],
"pretty_names" : [
{
"pretty_name" : "Ubuntu 20.04.2 LTS",
"count" : 51
}
],
"architectures" : [
{
"arch" : "amd64",
"count" : 51
}
],
"mem" : {
"total_in_bytes" : 6481442992128,
"free_in_bytes" : 203170045952,
"used_in_bytes" : 6278272946176,
"free_percent" : 3,
"used_percent" : 97
}
},
"process" : {
"cpu" : {
"percent" : 561
},
"open_file_descriptors" : {
"min" : 2938,
"max" : 4217,
"avg" : 4002
}
},
"jvm" : {
"max_uptime_in_millis" : 38657749039,
"versions" : [
{
"version" : "16.0.2",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "16.0.2+7",
"vm_vendor" : "Eclipse Foundation",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"count" : 51
}
],
"mem" : {
"heap_used_in_bytes" : 1105075036848,
"heap_max_in_bytes" : 1647354839040
},
"threads" : 8144
},
"fs" : {
"total_in_bytes" : 177438503313408,
"free_in_bytes" : 71585487626240,
"available_in_bytes" : 62568525897728
},
"plugins" : [
{
"name" : "repository-s3",
"version" : "7.15.0",
"elasticsearch_version" : "7.15.0",
"java_version" : "1.8",
"description" : "The S3 repository plugin adds S3 repositories",
"classname" : "org.elasticsearch.repositories.s3.S3RepositoryPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"licensed" : false,
"type" : "isolated"
}
],
"network_types" : {
"transport_types" : {
"netty4" : 51
},
"http_types" : {
"netty4" : 51
}
},
"discovery_types" : {
"zen" : 51
},
"packaging_types" : [
{
"flavor" : "default",
"type" : "deb",
"count" : 51
}
],
"ingest" : {
"number_of_pipelines" : 2,
"processor_stats" : {
"gsub" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
},
"script" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
}
}
}
}
}
And this is the stats for that particular node.
{
"_nodes" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"cluster_name" : "Elasticsearch715_Cluster",
"cluster_uuid" : "zGrKBe8NRNSPmDiPjCwXmQ",
"timestamp" : 1678214962396,
"indices" : {
"count" : 126,
"shards" : {
"total" : 139,
"primaries" : 81,
"replication" : 0.7160493827160493,
"index" : {
"shards" : {
"min" : 1,
"max" : 2,
"avg" : 1.1031746031746033
},
"primaries" : {
"min" : 0,
"max" : 2,
"avg" : 0.6428571428571429
},
"replication" : {
"min" : 0.0,
"max" : 1.0,
"avg" : 0.0873015873015873
}
}
},
"docs" : {
"count" : 3318568641,
"deleted" : 12658533
},
"store" : {
"size" : "1.9tb",
"size_in_bytes" : 2126011624031,
"total_data_set_size" : "1.9tb",
"total_data_set_size_in_bytes" : 2126011624031,
"reserved" : "0b",
"reserved_in_bytes" : 0
},
"fielddata" : {
"memory_size" : "11.1gb",
"memory_size_in_bytes" : 12020336580,
"evictions" : 0
},
"query_cache" : {
"memory_size" : "974.2mb",
"memory_size_in_bytes" : 1021545666,
"total_count" : 9565523694,
"hit_count" : 318703367,
"miss_count" : 9246820327,
"cache_size" : 28833,
"cache_count" : 3794973,
"evictions" : 3766140
},
"completion" : {
"size" : "0b",
"size_in_bytes" : 0
},
"segments" : {
"count" : 3499,
"memory" : "53mb",
"memory_in_bytes" : 55612208,
"terms_memory" : "35.8mb",
"terms_memory_in_bytes" : 37624048,
"stored_fields_memory" : "2.1mb",
"stored_fields_memory_in_bytes" : 2299192,
"term_vectors_memory" : "0b",
"term_vectors_memory_in_bytes" : 0,
"norms_memory" : "2.6mb",
"norms_memory_in_bytes" : 2806080,
"points_memory" : "0b",
"points_memory_in_bytes" : 0,
"doc_values_memory" : "12.2mb",
"doc_values_memory_in_bytes" : 12882888,
"index_writer_memory" : "288.3mb",
"index_writer_memory_in_bytes" : 302334376,
"version_map_memory" : "28.7mb",
"version_map_memory_in_bytes" : 30166658,
"fixed_bit_set" : "0b",
"fixed_bit_set_memory_in_bytes" : 0,
"max_unsafe_auto_id_timestamp" : 1678147200960,
"file_sizes" : { }
},
"mappings" : {
"field_types" : [
{
"name" : "boolean",
"count" : 1287,
"index_count" : 244,
"script_count" : 0
},
{
"name" : "byte",
"count" : 20,
"index_count" : 10,
"script_count" : 0
},
{
"name" : "date",
"count" : 1345,
"index_count" : 445,
"script_count" : 0
},
{
"name" : "dense_vector",
"count" : 9,
"index_count" : 9,
"script_count" : 0
},
{
"name" : "double",
"count" : 6,
"index_count" : 2,
"script_count" : 0
},
{
"name" : "flattened",
"count" : 3,
"index_count" : 3,
"script_count" : 0
},
{
"name" : "float",
"count" : 3429,
"index_count" : 167,
"script_count" : 0
},
{
"name" : "integer",
"count" : 157,
"index_count" : 61,
"script_count" : 0
},
{
"name" : "ip",
"count" : 11,
"index_count" : 7,
"script_count" : 0
},
{
"name" : "keyword",
"count" : 16289,
"index_count" : 449,
"script_count" : 0
},
{
"name" : "long",
"count" : 2509,
"index_count" : 301,
"script_count" : 0
},
{
"name" : "nested",
"count" : 4,
"index_count" : 4,
"script_count" : 0
},
{
"name" : "object",
"count" : 1617,
"index_count" : 197,
"script_count" : 0
},
{
"name" : "short",
"count" : 116,
"index_count" : 47,
"script_count" : 0
},
{
"name" : "text",
"count" : 10914,
"index_count" : 412,
"script_count" : 0
},
{
"name" : "version",
"count" : 4,
"index_count" : 4,
"script_count" : 0
}
],
"runtime_field_types" : [ ]
},
"analysis" : {
"char_filter_types" : [ ],
"tokenizer_types" : [ ],
"filter_types" : [ ],
"analyzer_types" : [ ],
"built_in_char_filters" : [ ],
"built_in_tokenizers" : [ ],
"built_in_filters" : [ ],
"built_in_analyzers" : [
{
"name" : "english",
"count" : 14,
"index_count" : 7
}
]
},
"versions" : [
{
"version" : "7.15.0",
"index_count" : 472,
"primary_shard_count" : 3324,
"total_primary_size" : "1.1tb",
"total_primary_bytes" : 1257779512225
}
]
},
"nodes" : {
"count" : {
"total" : 1,
"coordinating_only" : 0,
"data" : 1,
"data_cold" : 0,
"data_content" : 0,
"data_frozen" : 0,
"data_hot" : 0,
"data_warm" : 0,
"ingest" : 0,
"master" : 0,
"ml" : 0,
"remote_cluster_client" : 0,
"transform" : 0,
"voting_only" : 0
},
"versions" : [
"7.15.0"
],
"os" : {
"available_processors" : 16,
"allocated_processors" : 16,
"names" : [
{
"name" : "Linux",
"count" : 1
}
],
"pretty_names" : [
{
"pretty_name" : "Ubuntu 20.04.2 LTS",
"count" : 1
}
],
"architectures" : [
{
"arch" : "amd64",
"count" : 1
}
],
"mem" : {
"total" : "123.8gb",
"total_in_bytes" : 132962983936,
"free" : "3.8gb",
"free_in_bytes" : 4155506688,
"used" : "119.9gb",
"used_in_bytes" : 128807477248,
"free_percent" : 3,
"used_percent" : 97
}
},
"process" : {
"cpu" : {
"percent" : 18
},
"open_file_descriptors" : {
"min" : 3999,
"max" : 3999,
"avg" : 3999
}
},
"jvm" : {
"max_uptime" : "202.4d",
"max_uptime_in_millis" : 17490757862,
"versions" : [
{
"version" : "16.0.2",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "16.0.2+7",
"vm_vendor" : "Eclipse Foundation",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"count" : 1
}
],
"mem" : {
"heap_used" : "20.7gb",
"heap_used_in_bytes" : 22331315704,
"heap_max" : "31gb",
"heap_max_in_bytes" : 33285996544
},
"threads" : 161
},
"fs" : {
"total" : "3.3tb",
"total_in_bytes" : 3690063474688,
"free" : "1.4tb",
"free_in_bytes" : 1559041359872,
"available" : "1.2tb",
"available_in_bytes" : 1371524583424
},
"plugins" : [
{
"name" : "repository-s3",
"version" : "7.15.0",
"elasticsearch_version" : "7.15.0",
"java_version" : "1.8",
"description" : "The S3 repository plugin adds S3 repositories",
"classname" : "org.elasticsearch.repositories.s3.S3RepositoryPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"licensed" : false,
"type" : "isolated"
}
],
"network_types" : {
"transport_types" : {
"netty4" : 1
},
"http_types" : {
"netty4" : 1
}
},
"discovery_types" : {
"zen" : 1
},
"packaging_types" : [
{
"flavor" : "default",
"type" : "deb",
"count" : 1
}
],
"ingest" : {
"number_of_pipelines" : 2,
"processor_stats" : {
"gsub" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
},
"script" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
}
}
}
}
}
does anybody have some hints as to what might've been the cause?
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.
© 2020. All Rights Reserved - Elasticsearch
Apache, Apache Lucene, Apache Hadoop, Hadoop, HDFS and the yellow elephant logo are trademarks of the Apache Software Foundation in the United States and/or other countries.