after print some debug info from kibana api, I found the ".monitoring-es-*" index missing the docs with the type of "cluster_stats" which kibana need, only "node_stats" type exists.
I have check the cluster:
{
"_nodes" : {
"total" : 3,
"successful" : 3,
"failed" : 0
},
"cluster_name" : "es-nqms",
"timestamp" : 1502356448842,
"status" : "green",
"indices" : {
"count" : 1355,
"shards" : {
"total" : 8263,
"primaries" : 4129,
"replication" : 1.001210946960523,
"index" : {
"shards" : {
"min" : 2,
"max" : 15,
"avg" : 6.098154981549816
},
"primaries" : {
"min" : 1,
"max" : 5,
"avg" : 3.047232472324723
},
"replication" : {
"min" : 1.0,
"max" : 2.0,
"avg" : 1.0007380073800738
}
}
},
"docs" : {
"count" : 408129101,
"deleted" : 12759428
},
"store" : {
"size_in_bytes" : 544052242420,
"throttle_time_in_millis" : 0
},
"fielddata" : {
"memory_size_in_bytes" : 268168,
"evictions" : 0
},
"query_cache" : {
"memory_size_in_bytes" : 1082465136,
"total_count" : 695830968,
"hit_count" : 50998442,
"miss_count" : 644832526,
"cache_size" : 70110,
"cache_count" : 114486,
"evictions" : 44376
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 42004,
"memory_in_bytes" : 1178028250,
"terms_memory_in_bytes" : 825021469,
"stored_fields_memory_in_bytes" : 174053208,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 31181824,
"points_memory_in_bytes" : 71276961,
"doc_values_memory_in_bytes" : 76494788,
"index_writer_memory_in_bytes" : 49851276,
"version_map_memory_in_bytes" : 93441,
"fixed_bit_set_memory_in_bytes" : 0,
"max_unsafe_auto_id_timestamp" : 9223372036854775807,
"file_sizes" : { }
}
},
"nodes" : {
"count" : {
"total" : 3,
"data" : 3,
"coordinating_only" : 0,
"master" : 3,
"ingest" : 3
},
"versions" : [
"5.2.2"
],
"os" : {
"available_processors" : 72,
"allocated_processors" : 72,
"names" : [
{
"name" : "Linux",
"count" : 3
}
],
"mem" : {
"total_in_bytes" : 151668895744,
"free_in_bytes" : 1748615168,
"used_in_bytes" : 149920280576,
"free_percent" : 1,
"used_percent" : 99
}
},
"process" : {
"cpu" : {
"percent" : 12
},
"open_file_descriptors" : {
"min" : 6134,
"max" : 7361,
"avg" : 6768
}
},
"jvm" : {
"max_uptime_in_millis" : 2828901155,
"versions" : [
{
"version" : "1.8.0_131",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "25.131-b11",
"vm_vendor" : "Oracle Corporation",
"count" : 1
},
{
"version" : "1.8.0_91",
"vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
"vm_version" : "25.91-b14",
"vm_vendor" : "Oracle Corporation",
"count" : 2
}
],
"mem" : {
"heap_used_in_bytes" : 21279546416,
"heap_max_in_bytes" : 31741181952
},
"threads" : 901
},
"fs" : {
"total_in_bytes" : 1234125553664,
"free_in_bytes" : 620783722496,
"available_in_bytes" : 558080036864
},
"plugins" : [
{
"name" : "x-pack",
"version" : "5.2.2",
"description" : "Elasticsearch Expanded Pack Plugin",
"classname" : "org.elasticsearch.xpack.XPackPlugin"
},
{
"name" : "analysis-ik",
"version" : "5.2.2",
"description" : "IK Analyzer for Elasticsearch",
"classname" : "org.elasticsearch.plugin.analysis.ik.AnalysisIkPlugin"
}
],
"network_types" : {
"transport_types" : {
"netty4" : 3
},
"http_types" : {
"netty4" : 3
}
}
}
}
I found the only strange point from above info is "os.mem". And not sure if there is something wrong with other info...