So it appears that the data exists in the monitoring index, just not displayed in Kibana.
If I try to zoom on the blank timeframe in Kibana I get "Monitoring Request Failed. Unable to find the cluster in the selected time range. UUID: vd63AAWCTc6rZ1RiIRSK4g . HTTP 404"
Here's a screenshot of the timeframe:
And here's the query and response
Query:
{
"query": {
"bool": {
"filter": [
{
"term": {
"cluster_uuid": "vd63AAWCTc6rZ1RiIRSK4g"
}
},
{
"range": {
"timestamp": {
"gte": "2019-12-17T03:48:00.080Z",
"lte": "2019-12-17T03:49:00.080Z"
}
}
}
]
}
},
"aggs": {
"check": {
"date_histogram": {
"field": "timestamp",
"fixed_interval": "30s"
},
"aggs": {
"metric": {
"max": {
"field": "indices_stats._all.total.search.query_total"
}
},
"metric_deriv": {
"derivative": {
"buckets_path": "metric",
"gap_policy": "skip",
"unit": "1s"
}
}
}
}
}
}
Response (truncated):
{
"took": 10,
"timed_out": false,
"_shards": {
"total": 7,
"successful": 7,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 222,
"relation": "eq"
},
"max_score": 0.0,
"hits": [
{
"_index": ".monitoring-es-7-2019.12.17",
"_type": "_doc",
"_id": "FcP4EW8Byx4LD79xQ0Os",
"_score": 0.0,
"_source": {
"cluster_uuid": "vd63AAWCTc6rZ1RiIRSK4g",
"timestamp": "2019-12-17T03:48:00.543Z",
"interval_ms": 10000,
"type": "node_stats",
"source_node": {
"uuid": "5SmrUGH4SRuLDYLdrOt77g",
"host": "172.30.1.242",
"transport_address": "172.30.1.242:9300",
"ip": "172.30.1.242",
"name": "ip-172-30-1-242.ec2.internal",
"timestamp": "2019-12-17T03:48:00.543Z"
},
"node_stats": {
"node_id": "5SmrUGH4SRuLDYLdrOt77g",
"node_master": false,
"mlockall": false,
"indices": {
"docs": {
"count": 26609220
},
"store": {
"size_in_bytes": 49286464235
},
"indexing": {
"index_total": 3303155,
"index_time_in_millis": 1646136,
"throttle_time_in_millis": 0
},
"search": {
"query_total": 142097245,
"query_time_in_millis": 855398284
},
"query_cache": {
"memory_size_in_bytes": 32790152,
"hit_count": 227876976,
"miss_count": 249589745,
"evictions": 20350234
},
"fielddata": {
"memory_size_in_bytes": 0,
"evictions": 0
},
"segments": {
"count": 88,
"memory_in_bytes": 11952968186,
"terms_memory_in_bytes": 11943687306,
"stored_fields_memory_in_bytes": 7562824,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 54656,
"points_memory_in_bytes": 978040,
"doc_values_memory_in_bytes": 685360,
"index_writer_memory_in_bytes": 849632,
"version_map_memory_in_bytes": 748,
"fixed_bit_set_memory_in_bytes": 132592
},
"request_cache": {
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 30,
"miss_count": 11390
}
},
"os": {
"cpu": {
"load_average": {
"1m": 0.2,
"5m": 0.2,
"15m": 1.11
}
},
"cgroup": {
"cpuacct": {
"control_group": "/",
"usage_nanos": 1407906507480565
},
"cpu": {
"control_group": "/",
"cfs_period_micros": 100000,
"cfs_quota_micros": -1,
"stat": {
"number_of_elapsed_periods": 0,
"number_of_times_throttled": 0,
"time_throttled_nanos": 0
}
},
"memory": {
"control_group": "/",
"limit_in_bytes": "9223372036854771712",
"usage_in_bytes": "98930262016"
}
}
},
"process": {
"open_file_descriptors": 898,
"max_file_descriptors": 65535,
"cpu": {
"percent": 1
}
},
"jvm": {
"mem": {
"heap_used_in_bytes": 19752021000,
"heap_used_percent": 57,
"heap_max_in_bytes": 34246361088
},
"gc": {
"collectors": {
"young": {
"collection_count": 1582085,
"collection_time_in_millis": 36134761
},
"old": {
"collection_count": 33,
"collection_time_in_millis": 6668
}
}
}
},
"thread_pool": {
"generic": {
"threads": 20,
"queue": 0,
"rejected": 0
},
"get": {
"threads": 1,
"queue": 0,
"rejected": 0
},
"management": {
"threads": 5,
"queue": 0,
"rejected": 0
},
"search": {
"threads": 25,
"queue": 0,
"rejected": 0
},
"watcher": {
"threads": 0,
"queue": 0,
"rejected": 0
},
"write": {
"threads": 16,
"queue": 0,
"rejected": 0
}
},
"fs": {
"total": {
"total_in_bytes": 1056750854144,
"free_in_bytes": 995028893696,
"available_in_bytes": 951978315776
},
"io_stats": {
"total": {
"operations": 11420617,
"read_operations": 444892,
"write_operations": 10975725,
"read_kilobytes": 16952040,
"write_kilobytes": 290891716
}
}
}
}
}
},
{
"_index": ".monitoring-es-7-2019.12.17",
"_type": "_doc",
"_id": "8x34EW8BQZsPnUc7RfgG",
"_score": 0.0,
"_source": {
"cluster_uuid": "vd63AAWCTc6rZ1RiIRSK4g",
"timestamp": "2019-12-17T03:48:00.875Z",
.
.
.
"aggregations": {
"check": {
"buckets": [
{
"key_as_string": "2019-12-17T03:48:00.000Z",
"key": 1576554480000,
"doc_count": 111,
"metric": {
"value": 3.96591825E8
}
},
{
"key_as_string": "2019-12-17T03:48:30.000Z",
"key": 1576554510000,
"doc_count": 111,
"metric": {
"value": 3.966284E8
},
"metric_deriv": {
"value": 36575.0,
"normalized_value": 1219.1666666666667
}
}
]
}
}
}
