I am shipping logs using fluentd in k8s cluster i see a bunch of the following messages and logs stop flowing to ES
warn]: [elasticsearch] failed to write data into buffer by buffer overflow action=:block
Any thoughts or solution
Sounds like there might be a problem with the Elasticsearch cluster. Is there anything in the Elasticsearch logs?
I see a bunch of the following
[2019-07-15T05:20:32,226][INFO ][o.e.m.j.JvmGcMonitorService] [elasticsearch-logging-0] [gc][39] overhead, spent [403ms] collecting in the last [1s]
just another data point meteric beat continues to work in the same setup
ES 6.2.5
{
"_nodes" : {
"total" : 2,
"successful" : 2,
"failed" : 0
},
"cluster_name" : "kubernetes-logging",
"timestamp" : 1563168679582,
"status" : "yellow",
"indices" : {
"count" : 97,
"shards" : {
"total" : 481,
"primaries" : 481,
"replication" : 0.0,
"index" : {
"shards" : {
"min" : 1,
"max" : 5,
"avg" : 4.958762886597938
},
"primaries" : {
"min" : 1,
"max" : 5,
"avg" : 4.958762886597938
},
"replication" : {
"min" : 0.0,
"max" : 0.0,
"avg" : 0.0
}
}
},
"docs" : {
"count" : 58508426,
"deleted" : 6
},
"store" : {
"size_in_bytes" : 17602171642
},
"fielddata" : {
"memory_size_in_bytes" : 0,
"evictions" : 0
},
"query_cache" : {
"memory_size_in_bytes" : 0,
"total_count" : 0,
"hit_count" : 0,
"miss_count" : 0,
"cache_size" : 0,
"cache_count" : 0,
"evictions" : 0
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 3060,
"memory_in_bytes" : 82584599,
"terms_memory_in_bytes" : 49103002,
"stored_fields_memory_in_bytes" : 13437920,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 3027392,
"points_memory_in_bytes" : 1778397,
"doc_values_memory_in_bytes" : 15237888,
"index_writer_memory_in_bytes" : 0,
"version_map_memory_in_bytes" : 0,
"fixed_bit_set_memory_in_bytes" : 0,
"max_unsafe_auto_id_timestamp" : 1563168028913,
"file_sizes" : { }
}
},
"nodes" : {
"count" : {
"total" : 2,
"data" : 2,
"coordinating_only" : 0,
"master" : 2,
"ingest" : 2
},
"versions" : [
"6.2.4"
],
"os" : {
"available_processors" : 16,
"allocated_processors" : 16,
"names" : [
{
"name" : "Linux",
"count" : 2
}
],
"mem" : {
"total_in_bytes" : 32434601984,
"free_in_bytes" : 8051560448,
"used_in_bytes" : 24383041536,
"free_percent" : 25,
"used_percent" : 75
}
},
"process" : {
"cpu" : {
"percent" : 1
},
"open_file_descriptors" : {
"min" : 852,
"max" : 1236,
"avg" : 1044
}
},
"jvm" : {
"max_uptime_in_millis" : 715637,
"versions" : [
{
"version" : "1.8.0_161",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "25.161-b14",
"vm_vendor" : "Oracle Corporation",
"count" : 2
}
],
"mem" : {
"heap_used_in_bytes" : 1055361600,
"heap_max_in_bytes" : 2075918336
},
"threads" : 179
},
"fs" : {
"total_in_bytes" : 44120965120,
"free_in_bytes" : 6204526592,
"available_in_bytes" : 3916115968
},
"plugins" : [
{
"name" : "ingest-user-agent",
"version" : "6.2.4",
"description" : "Ingest processor that extracts information from a user agent",
"classname" : "org.elasticsearch.ingest.useragent.IngestUserAgentPlugin",
"extended_plugins" : ,
"has_native_controller" : false,
"requires_keystore" : false
},
{
"name" : "ingest-geoip",
"version" : "6.2.4",
"description" : "Ingest processor that uses looksup geo data based on ip adresses using the Maxmind geo database",
"classname" : "org.elasticsearch.ingest.geoip.IngestGeoIpPlugin",
"extended_plugins" : ,
"has_native_controller" : false,
"requires_keystore" : false
}
],
"network_types" : {
"transport_types" : {
"netty4" : 2
},
"http_types" : {
"netty4" : 2
}
}
}
}
You seem to have a good amount of RAM but a default 1GB heap. I would recommend you increase the heap size of the nodes to e.g. 4GB to see if that helps.
Hi i see file to change in a native rpm install but if i am getting it from a docker image
k8s.gcr.io/elasticsearch:v6.2.5
where would one make such a change.
I have not used that image so am not sure. Would however recommend using the official docker images.
just to confirm it will be the following to a say 2048m to start with for example
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
That is 512 MB heap, which is smaller than what you have now.
Hi i see that my heap used is 1GB less than max so how does this help? Anything else to try?
thanks
"mem" : {
"heap_used_in_bytes" : 1055361600,
"heap_max_in_bytes" : 2075918336
},
"threads" : 179
That is across 2 nodes, so each node only have 1GB heap configured. Update your jvm.options files and increase the heap size to 2GB or 4GB.
added the another node and updated the mem - output below but still same error
"mem" : {
"heap_used" : "1.9gb",
"heap_used_in_bytes" : 2073279480,
"heap_max" : "5.8gb",
"heap_max_in_bytes" : 6233260032
},
"threads" : 307
Anything else as the last idea did not work.
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.