Kibana monitor index exists, but dashboard says "No Monitoring Data Found"

[abbshr (ran)]

There did exist elasticsearch monitor indices for a week, but Kibana Monitoring can not show the metrics.

Until yesterday, everything is ok. And I didn't find any related error or warning logs outputted by elasticsearch and kibana recently.

So, is there any way to detect what's wrong?

elastic stack version: v5.2.2

[abbshr (ran)]

after print some debug info from kibana api, I found the ".monitoring-es-*" index missing the docs with the type of "cluster_stats" which kibana need, only "node_stats" type exists.

I have check the cluster:

{
  "_nodes" : {
    "total" : 3,
    "successful" : 3,
    "failed" : 0
  },
  "cluster_name" : "es-nqms",
  "timestamp" : 1502356448842,
  "status" : "green",
  "indices" : {
    "count" : 1355,
    "shards" : {
      "total" : 8263,
      "primaries" : 4129,
      "replication" : 1.001210946960523,
      "index" : {
        "shards" : {
          "min" : 2,
          "max" : 15,
          "avg" : 6.098154981549816
        },
        "primaries" : {
          "min" : 1,
          "max" : 5,
          "avg" : 3.047232472324723
        },
        "replication" : {
          "min" : 1.0,
          "max" : 2.0,
          "avg" : 1.0007380073800738
        }
      }
    },
    "docs" : {
      "count" : 408129101,
      "deleted" : 12759428
    },
    "store" : {
      "size_in_bytes" : 544052242420,
      "throttle_time_in_millis" : 0
    },
    "fielddata" : {
      "memory_size_in_bytes" : 268168,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size_in_bytes" : 1082465136,
      "total_count" : 695830968,
      "hit_count" : 50998442,
      "miss_count" : 644832526,
      "cache_size" : 70110,
      "cache_count" : 114486,
      "evictions" : 44376
    },
    "completion" : {
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 42004,
      "memory_in_bytes" : 1178028250,
      "terms_memory_in_bytes" : 825021469,
      "stored_fields_memory_in_bytes" : 174053208,
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory_in_bytes" : 31181824,
      "points_memory_in_bytes" : 71276961,
      "doc_values_memory_in_bytes" : 76494788,
      "index_writer_memory_in_bytes" : 49851276,
      "version_map_memory_in_bytes" : 93441,
      "fixed_bit_set_memory_in_bytes" : 0,
      "max_unsafe_auto_id_timestamp" : 9223372036854775807,
      "file_sizes" : { }
    }
  },
  "nodes" : {
    "count" : {
      "total" : 3,
      "data" : 3,
      "coordinating_only" : 0,
      "master" : 3,
      "ingest" : 3
    },
    "versions" : [
      "5.2.2"
    ],
    "os" : {
      "available_processors" : 72,
      "allocated_processors" : 72,
      "names" : [
        {
          "name" : "Linux",
          "count" : 3
        }
      ],
      "mem" : {
        "total_in_bytes" : 151668895744,
        "free_in_bytes" : 1748615168,
        "used_in_bytes" : 149920280576,
        "free_percent" : 1,
        "used_percent" : 99
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 12
      },
      "open_file_descriptors" : {
        "min" : 6134,
        "max" : 7361,
        "avg" : 6768
      }
    },
    "jvm" : {
      "max_uptime_in_millis" : 2828901155,
      "versions" : [
        {
          "version" : "1.8.0_131",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "25.131-b11",
          "vm_vendor" : "Oracle Corporation",
          "count" : 1
        },
        {
          "version" : "1.8.0_91",
          "vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
          "vm_version" : "25.91-b14",
          "vm_vendor" : "Oracle Corporation",
          "count" : 2
        }
      ],
      "mem" : {
        "heap_used_in_bytes" : 21279546416,
        "heap_max_in_bytes" : 31741181952
      },
      "threads" : 901
    },
    "fs" : {
      "total_in_bytes" : 1234125553664,
      "free_in_bytes" : 620783722496,
      "available_in_bytes" : 558080036864
    },
    "plugins" : [
      {
        "name" : "x-pack",
        "version" : "5.2.2",
        "description" : "Elasticsearch Expanded Pack Plugin",
        "classname" : "org.elasticsearch.xpack.XPackPlugin"
      },
      {
        "name" : "analysis-ik",
        "version" : "5.2.2",
        "description" : "IK Analyzer for Elasticsearch",
        "classname" : "org.elasticsearch.plugin.analysis.ik.AnalysisIkPlugin"
      }
    ],
    "network_types" : {
      "transport_types" : {
        "netty4" : 3
      },
      "http_types" : {
        "netty4" : 3
      }
    }
  }
}

I found the only strange point from above info is "os.mem". And not sure if there is something wrong with other info...

[abbshr (ran)]

Fixed~

perhaps it's the issue of max jvm heap size settings, I just lift the -Xms.