hello, i have an MTR running between nodes around the worlds and they are sending JSON logs of the MTR report to elastic.
in maps i use Point to Point with the intention of drawing lines between them and on each line will be shown the avg loss between the two nodes that the line is drawn from.
The only metrics i can use with average Aggregation are @timestamp, event.Ingested and log.offset , with the following sanitized document for an example why i can use Loss as a metric?
{
"_version": 1,
"_score": 0,
"_source": {
"source_location": "22.3193,114.1694",
"hub.19.loss": 0,
"hub.13.loss": 0,
"hub.6.count": 6,
"hub.21.loss": 100,
"hub.22.count": 22,
"hub.12.count": 12,
"source": "as-xxxxxx",
"hub.5.loss": 0,
"hub.4.loss": 100,
"hub.12.loss": 0,
"hub.25.count": 25,
"hub.15.count": 15,
"hub.27.loss": 0,
"hub.7.count": 7,
"hub.9.count": 9,
"hub.13.count": 13,
"hub.25.loss": 100,
"hub.28.loss": 100,
"hub.11.loss": 0,
"hub.16.count": 16,
"hub.21.count": 21,
"hub.17.loss": 0,
"hub.3.count": 3,
"hub.22.loss": 100,
"hub.14.loss": 0,
"hub.19.count": 19,
"hub.1.count": 1,
"hub.14.count": 14,
"hub.16.loss": 0,
"hub.2.loss": 0,
"hub.24.count": 24,
"offset": 0,
"hub.8.count": 8,
"hub.24.loss": 100,
"destination": "il-xxxx",
"hub.8.loss": 0,
"hub.27.count": 27,
"hub.10.loss": 0,
"hub.4.count": 4,
"hub.7.loss": 0,
"hub.18.count": 18,
"hub.28.count": 28,
"hub.10.count": 10,
"hub.20.count": 20,
"hub.1.loss": 0,
"hub.23.count": 23,
"hub.2.count": 2,
"hub.3.loss": 0,
"hub.20.loss": 100,
"hub.15.loss": 0,
"hub.18.loss": 0,
"hub.23.loss": 100,
"hub.26.loss": 0,
"hub.5.count": 5,
"hub.17.count": 17,
"hub.11.count": 11,
"@timestamp": "2024-06-22T20:24:28.541Z",
"hub.26.count": 26,
"destination_location": "32.1058,34.8556",
"hub.6.loss": 0,
"hub.9.loss": 0
},
"fields": {
"hub.19.loss": [
0
"hub.21.loss": [
100
],
"hub.7.count": [
7
],
"hub.13.count": [
13
],
"hub.16.count": [
16
],
"hub.21.count": [
21
],
"hub.17.loss": [
0
],
"hub.22.loss": [
100
],
"hub.19.count": [
19
],
"hub.1.count": [
1
],
"hub.14.count": [
"14"
],
"hub.16.loss": [
0
],
"hub.8.count": [
8
],
"hub.27.count": [
27
],
"hub.10.loss": [
0
],
"hub.20.count": [
20
],
"hub.2.count": [
2
],
"hub.20.loss": [
100
],
"hub.3.loss": [
0
],
"hub.23.loss": [
100
],
"hub.26.loss": [
0
],
"event.ingested": [
"2024-06-22T20:24:43.000Z"
],
"@timestamp": [
"2024-06-22T20:24:28.541Z"
],
"hub.26.count": [
26
"destination_location": [
{
"coordinates": [
34.8556,
32.1058
],
"type": "Point"
}
],
"hub.6.loss": [
0
],
"hub.9.loss": [
0
],
"source_location": [
{
"coordinates": [
114.1694,
22.3193
],
"type": "Point"
}
],
"hub.13.loss": [
0
],
"hub.6.count": [
6
],
"hub.22.count": [
22
],
"hub.12.count": [
12
],
"source": [
"as-xxxx"
],
"hub.5.loss": [
0
],
"hub.4.loss": [
100
],
"hub.12.loss": [
0
],
"hub.25.count": [
25
],
"hub.15.count": [
"15"
],
"hub.27.loss": [
0
],
"hub.9.count": [
9
],
"log.offset": [
0
],
],
"hub.25.loss": [
100
],
"hub.28.loss": [
100
],
"hub.11.loss": [
0
],
],
"hub.3.count": [
3
],
"hub.14.loss": [
0
],
"hub.2.loss": [
0
],
"hub.24.count": [
24
],
"hub.24.loss": [
100
],
"destination": [
"il-xxxxx"
],
"hub.8.loss": [
0
],
"hub.4.count": [
4
],
"hub.18.count": [
18
],
"hub.7.loss": [
0
],
"hub.28.count": [
28
],
"hub.10.count": [
10
],
"hub.1.loss": [
0
],
"hub.23.count": [
23
],
"hub.15.loss": [
0
],
"hub.18.loss": [
0
],
],
"hub.5.count": [
5
],
"hub.17.count": [
17
],
"hub.11.count": [
11
],
}
}
while pasting this i noticed that the fields are kind of duplicated on each log. but on elastic i only see one set of fields and the same in the data view of the index. i do use a pipeline here its config.
and of course i use template to set the source_location and destination_location to geo_points. do i need to format the loss and count fields as well? to what?
PUT _ingest/pipeline/logs-mtr.monitor@custom
{
"description": "Extract src and dst fields, convert to geo_points and add them as source and destination",
"processors": [
{
"json": {
"field": "message",
"target_field": "parsed_message"
}
},
{
"script": {
"lang": "painless",
"source": """
// Mapping from hostnames to geographical coordinates
Map hostToGeo = [
'us-ny2-xxxxxxx': '40.7128,-74.0060',
'eu-lo-xxxxxxx': '51.5074,-0.1278',
// Add more mappings as needed
];
// Assign source and destination coordinates
ctx.source_location = hostToGeo.get(ctx.parsed_message.report.mtr.src);
ctx.destination_location = hostToGeo.get(ctx.parsed_message.report.mtr.dst);
if (ctx.parsed_message.report.hubs != null) {
for (hub in ctx.parsed_message.report.hubs) {
ctx['hub_' + hub.count + '_count'] = hub.count;
ctx['hub_' + hub.count + '_loss'] = hub['Loss%'];
}
}
"""
}
},
{
"set": {
"field": "source",
"value": "{{parsed_message.report.mtr.src}}"
}
},
{
"set": {
"field": "destination",
"value": "{{parsed_message.report.mtr.dst}}"
}
},
{
"remove": {
"field": "parsed_message"
}
}
]
}
thank you kindly!