Hello, everyone.
We are trying to make our Micronaut app push metrics to our APM server without using a Java agent. For that we are using this Elasticsearch Registry as the Micrometer metrics reporter.
This lib first creates a legacy index template, if it doesn't already exists, and then starts pushing bulk metrics to Elasticsearch through POST /micrometer-metrics-2023-05/_bulk with the following body:
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"jvm_buffer_memory_used","type":"gauge","id":"direct","value":21265.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"jvm_threads_states","type":"gauge","state":"blocked","value":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"executor_pool_max","type":"gauge","name_tag":"scheduled","value":2.147483647E9}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"hikaricp_connections_max","type":"gauge","pool":"HikariPool-1","value":10.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"executor_completed","type":"counter","name_tag":"scheduled","count":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"process_uptime","type":"gauge","value":107256.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"jvm_threads_states","type":"gauge","state":"runnable","value":8.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"jvm_memory_used","type":"gauge","area":"nonheap","id":"Compressed Class Space","value":9886992.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"system_load_average_1m","type":"gauge","value":1.1650390625}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"executor","type":"timer","name_tag":"scheduled","count":0,"sum":0.0,"mean":0.0,"max":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"hikaricp_connections_usage","type":"timer","pool":"HikariPool-1","count":0,"sum":0.0,"mean":0.0,"max":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"executor","type":"timer","name_tag":"io","count":0,"sum":0.0,"mean":0.0,"max":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"jvm_memory_used","type":"gauge","area":"nonheap","id":"Metaspace","value":5.5698392E7}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"executor_queued","type":"gauge","name_tag":"scheduled","value":1.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"logback_events","type":"counter","level":"trace","count":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"jvm_buffer_count","type":"gauge","id":"mapped - 'non-volatile memory'","value":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"jvm_memory_committed","type":"gauge","area":"heap","id":"G1 Old Gen","value":6.5011712E7}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"executor_queued","type":"gauge","name_tag":"io","value":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"jvm_memory_used","type":"gauge","area":"nonheap","id":"CodeHeap 'non-profiled nmethods'","value":2413312.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"executor_active","type":"gauge","name_tag":"io","value":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"jvm_buffer_total_capacity","type":"gauge","id":"direct","value":21264.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"jvm_memory_committed","type":"gauge","area":"heap","id":"G1 Eden Space","value":9.2274688E7}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"jvm_gc_pause","type":"timer","action":"end of minor GC","cause":"Metadata GC Threshold","count":0,"sum":0.0,"mean":0.0,"max":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.153Z","name":"jvm_memory_max","type":"gauge","area":"heap","id":"G1 Eden Space","value":-1.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"system_cpu_usage","type":"gauge","value":0.20670391061452514}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_threads_live","type":"gauge","value":74.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_classes_unloaded","type":"counter","count":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_memory_committed","type":"gauge","area":"nonheap","id":"Compressed Class Space","value":1.0027008E7}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_classes_loaded","type":"gauge","value":11552.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_threads_peak","type":"gauge","value":74.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_memory_max","type":"gauge","area":"nonheap","id":"CodeHeap 'profiled nmethods'","value":1.22908672E8}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_memory_committed","type":"gauge","area":"nonheap","id":"CodeHeap 'profiled nmethods'","value":8454144.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"hikaricp_connections_acquire","type":"timer","pool":"HikariPool-1","count":0,"sum":0.0,"mean":0.0,"max":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"logback_events","type":"counter","level":"warn","count":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"hikaricp_connections_min","type":"gauge","pool":"HikariPool-1","value":10.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"logback_events","type":"counter","level":"error","count":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"executor_pool_core","type":"gauge","name_tag":"scheduled","value":24.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_gc_memory_promoted","type":"counter","count":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"hikaricp_connections","type":"gauge","pool":"HikariPool-1","value":10.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_memory_max","type":"gauge","area":"nonheap","id":"Compressed Class Space","value":1.073741824E9}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_threads_states","type":"gauge","state":"waiting","value":12.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_buffer_count","type":"gauge","id":"mapped","value":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_memory_used","type":"gauge","area":"heap","id":"G1 Eden Space","value":5.0331648E7}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"executor_pool_core","type":"gauge","name_tag":"io","value":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"hikaricp_connections_timeout","type":"counter","pool":"HikariPool-1","count":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"hikaricp_connections_pending","type":"gauge","pool":"HikariPool-1","value":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_gc_memory_allocated","type":"counter","count":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"executor_pool_size","type":"gauge","name_tag":"io","value":2.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"logback_events","type":"counter","level":"info","count":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_memory_max","type":"gauge","area":"nonheap","id":"CodeHeap 'non-profiled nmethods'","value":1.22908672E8}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_memory_used","type":"gauge","area":"heap","id":"G1 Survivor Space","value":2336912.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_memory_used","type":"gauge","area":"nonheap","id":"CodeHeap 'non-nmethods'","value":1345536.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_gc_pause","type":"timer","action":"end of minor GC","cause":"G1 Evacuation Pause","count":0,"sum":0.0,"mean":0.0,"max":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_buffer_total_capacity","type":"gauge","id":"mapped","value":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_gc_max_data_size","type":"gauge","value":4.160749568E9}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"executor_active","type":"gauge","name_tag":"scheduled","value":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_buffer_memory_used","type":"gauge","id":"mapped - 'non-volatile memory'","value":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_memory_used","type":"gauge","area":"nonheap","id":"CodeHeap 'profiled nmethods'","value":8445312.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_memory_max","type":"gauge","area":"heap","id":"G1 Old Gen","value":4.160749568E9}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"system_cpu_count","type":"gauge","value":12.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"hikaricp_connections_active","type":"gauge","pool":"HikariPool-1","value":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_memory_max","type":"gauge","area":"nonheap","id":"CodeHeap 'non-nmethods'","value":5840896.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"hikaricp_connections_idle","type":"gauge","pool":"HikariPool-1","value":10.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"process_start_time","type":"gauge","value":1.684776042908E12}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_buffer_memory_used","type":"gauge","id":"mapped","value":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_buffer_count","type":"gauge","id":"direct","value":5.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"executor_completed","type":"counter","name_tag":"io","count":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"process_files_max","type":"gauge","value":1048576.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_threads_daemon","type":"gauge","value":20.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"executor_pool_max","type":"gauge","name_tag":"io","value":2.147483647E9}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_threads_states","type":"gauge","state":"new","value":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_buffer_total_capacity","type":"gauge","id":"mapped - 'non-volatile memory'","value":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_memory_committed","type":"gauge","area":"nonheap","id":"CodeHeap 'non-nmethods'","value":2555904.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_memory_committed","type":"gauge","area":"heap","id":"G1 Survivor Space","value":4194304.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_threads_states","type":"gauge","state":"timed-waiting","value":54.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"executor_queue_remaining","type":"gauge","name_tag":"scheduled","value":2.147483647E9}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_memory_used","type":"gauge","area":"heap","id":"G1 Old Gen","value":1.6160256E7}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.154Z","name":"jvm_memory_committed","type":"gauge","area":"nonheap","id":"Metaspace","value":5.603328E7}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.155Z","name":"jvm_memory_max","type":"gauge","area":"nonheap","id":"Metaspace","value":-1.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.155Z","name":"jvm_memory_max","type":"gauge","area":"heap","id":"G1 Survivor Space","value":-1.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.155Z","name":"jvm_threads_states","type":"gauge","state":"terminated","value":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.155Z","name":"hikaricp_connections_creation","type":"timer","pool":"HikariPool-1","count":0,"sum":0.0,"mean":0.0,"max":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.155Z","name":"executor_pool_size","type":"gauge","name_tag":"scheduled","value":2.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.155Z","name":"process_files_open","type":"gauge","value":236.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.155Z","name":"jvm_memory_committed","type":"gauge","area":"nonheap","id":"CodeHeap 'non-profiled nmethods'","value":2555904.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.155Z","name":"jvm_gc_live_data_size","type":"gauge","value":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.155Z","name":"logback_events","type":"counter","level":"debug","count":1.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.155Z","name":"executor_queue_remaining","type":"gauge","name_tag":"io","value":0.0}
{ "create" : {} }
{"@timestamp":"2023-05-22T17:22:30.155Z","name":"process_cpu_usage","type":"gauge","value":0.03428571428571429}
I can visualize the newly created index template and found the documents on Kibana by creating a data view with the index pattern micrometer-*, but all of them are shown containing only 3 fields: _id, _index, _version and _score. Not even the @timestamp field is shown. Example:
{
"_index": "micrometer-metrics-2023-05",
"_id": "eOMRRYgBzmpw9hGxlYXM",
"_version": 1,
"_score": 0
}
I've tried adding each field manually on the data view, but didn't work.
Our final goal is to make these metrics shown in the APM UI, but we are struggling to make them appear even on Kibana.
Could you guys point me some direction? Is there any configuration I must do to work with legacy index template on Elastic version 8.5.1?