I have this happen every few minutes or so. As I am using Kibana, I get timeouts and bad gateway errors. Eventually elasticsearch goes on the red on Kibana (or its plugin sees it that it does under status). Then elasticsearch comes back up and most of the queries run fine for a while but the problem happens again. This is visible if I try to query last 15 minute records or 30 day records.
I read somewhere that there may be an imbalance between shards and free memory but I am not sure which lines on the stats output I need to look at to fix the configuration. Any help with this would be much appreciated
{
"_nodes" : {
"total" : 3,
"successful" : 3,
"failed" : 0
},
"cluster_name" : "xxx",
"timestamp" : 1544034563257,
"status" : "red",
"indices" : {
"count" : 184,
"shards" : {
"total" : 1830,
"primaries" : 915,
"replication" : 1.0,
"index" : {
"shards" : {
"min" : 2,
"max" : 10,
"avg" : 9.945652173913043
},
"primaries" : {
"min" : 1,
"max" : 5,
"avg" : 4.9728260869565215
},
"replication" : {
"min" : 1.0,
"max" : 1.0,
"avg" : 1.0
}
}
},
"docs" : {
"count" : 4415317094,
"deleted" : 3
},
"store" : {
"size" : "4.5tb",
"size_in_bytes" : 4977641999266,
"throttle_time" : "0s",
"throttle_time_in_millis" : 0
},
"fielddata" : {
"memory_size" : "155mb",
"memory_size_in_bytes" : 162621008,
"evictions" : 0
},
"query_cache" : {
"memory_size" : "4.2gb",
"memory_size_in_bytes" : 4550589609,
"total_count" : 1102065,
"hit_count" : 365166,
"miss_count" : 736899,
"cache_size" : 76407,
"cache_count" : 76476,
"evictions" : 69
},
"completion" : {
"size" : "0b",
"size_in_bytes" : 0
},
"segments" : {
"count" : 33937,
"memory" : "8gb",
"memory_in_bytes" : 8674290123,
"terms_memory" : "5.2gb",
"terms_memory_in_bytes" : 5606092484,
"stored_fields_memory" : "1.5gb",
"stored_fields_memory_in_bytes" : 1664997152,
"term_vectors_memory" : "0b",
"term_vectors_memory_in_bytes" : 0,
"norms_memory" : "5.7kb",
"norms_memory_in_bytes" : 5888,
"points_memory" : "592.3mb",
"points_memory_in_bytes" : 621152111,
"doc_values_memory" : "745.8mb",
"doc_values_memory_in_bytes" : 782042488,
"index_writer_memory" : "85mb",
"index_writer_memory_in_bytes" : 89174360,
"version_map_memory" : "251.1kb",
"version_map_memory_in_bytes" : 257168,
"fixed_bit_set" : "0b",
"fixed_bit_set_memory_in_bytes" : 0,
"max_unsafe_auto_id_timestamp" : 1543968000622,
"file_sizes" : { }
}
},
"nodes" : {
"count" : {
"total" : 3,
"data" : 3,
"coordinating_only" : 0,
"master" : 3,
"ingest" : 3
},
"versions" : [
"5.5.1"
],
"os" : {
"available_processors" : 72,
"allocated_processors" : 72,
"names" : [
{
"name" : "Linux",
"count" : 3
}
],
"mem" : {
"total" : "188.7gb",
"total_in_bytes" : 202631995392,
"free" : "1.6gb",
"free_in_bytes" : 1735139328,
"used" : "187gb",
"used_in_bytes" : 200896856064,
"free_percent" : 1,
"used_percent" : 99
}
},
"process" : {
"cpu" : {
"percent" : 3
},
"open_file_descriptors" : {
"min" : 2016,
"max" : 2149,
"avg" : 2083
}
},
"jvm" : {
"max_uptime" : "37d",
"max_uptime_in_millis" : 3200280527,
"versions" : [
{
"version" : "1.8.0_181",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "25.181-b13",
"vm_vendor" : "Oracle Corporation",
"count" : 1
},
{
"version" : "1.8.0_161",
"vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
"vm_version" : "25.161-b12",
"vm_vendor" : "Oracle Corporation",
"count" : 2
}
],
"mem" : {
"heap_used" : "50.6gb",
"heap_used_in_bytes" : 54421706512,
"heap_max" : "95.5gb",
"heap_max_in_bytes" : 102608142336
},
"threads" : 565
},
"fs" : {
"total" : "11.8tb",
"total_in_bytes" : 12982740295680,
"free" : "7.2tb",
"free_in_bytes" : 8001277722624,
"available" : "6.6tb",
"available_in_bytes" : 7341721743360,
"spins" : "true"
},
"plugins" : [ ],
"network_types" : {
"transport_types" : {
"netty4" : 3
},
"http_types" : {
"netty4" : 3
}
}
}
}