Hi there,
I'm somehow battling with geoip plugin in logstash in 7.1.
I'm sending data to logstash and I do expect logstash will perform geoip lookup.
The fileds with ip data (ie. destination.ip) are present in the source json as the structure is pretty common packetbeat:
{
"_index": "packetbeat-20190617",
"_type": "_doc",
"_id": "WgLzZWsBmVD51cfr8dou",
"_version": 1,
"_score": null,
"_source": {
"tags": [
"beats_input_raw_event",
"_geoip_lookup_failure"
],
"network": {
"community_id": "1:UfJ4pWHrirjs8GSsmdcAxKedSbE=",
"bytes": 56,
"transport": "icmp",
"type": "ipv4",
"direction": "outbound"
},
"error": {
"message": "Response was received without an associated request."
},
"host": {
"hostname": "server.xx.xx.xx",
"os": {
"version": "7 (Core)",
"family": "redhat",
"kernel": "3.10.0-957.12.1.el7.x86_64",
"name": "CentOS Linux",
"codename": "Core",
"platform": "centos"
},
"id": "085cc053c2d249d5b3dce3e3bff02c29",
"name": "d1clcsec1.dc.vzp.cz",
"containerized": false,
"architecture": "x86_64"
},
"icmp": {
"response": {
"code": 0,
"type": 0,
"message": "EchoReply(0)"
},
"version": 4
},
"ecs": {
"version": "1.0.0"
},
"@timestamp": "2019-06-17T15:00:12.638Z",
"source": {
"ip": "10.128.2.18"
},
"path": "104.20.38.240",
"agent": {
"ephemeral_id": "a80a131b-907a-45ef-a87a-2f38a924af1b",
"hostname": "server.xx.xx.xx",
"id": "8f26e80c-b595-44c3-b46d-a1bfa7321b46",
"type": "packetbeat",
"version": "7.1.1"
},
"type": "icmp",
"status": "Error",
"event": {
"category": "network_traffic",
"dataset": "icmp",
"end": "2019-06-17T15:00:12.638Z",
"kind": "event"
},
"server": {
"ip": "104.20.38.240",
"bytes": 56
},
"client": {
"ip": "10.128.2.18"
},
"destination": {
"ip": "104.20.38.240",
"bytes": 56
},
"@version": "1"
},
"fields": {
"event.end": [
"2019-06-17T15:00:12.638Z"
],
"@timestamp": [
"2019-06-17T15:00:12.638Z"
]
},
"highlight": {
"path": [
"@kibana-highlighted-field@104.20.38.240@/kibana-highlighted-field@"
],
"destination.ip.keyword": [
"@kibana-highlighted-field@104.20.38.240@/kibana-highlighted-field@"
],
"server.ip.keyword": [
"@kibana-highlighted-field@104.20.38.240@/kibana-highlighted-field@"
],
"server.ip": [
"@kibana-highlighted-field@104.20.38.240@/kibana-highlighted-field@"
],
"destination.ip": [
"@kibana-highlighted-field@104.20.38.240@/kibana-highlighted-field@"
],
"path.keyword": [
"@kibana-highlighted-field@104.20.38.240@/kibana-highlighted-field@"
]
},
"sort": [
1560783612638
]
}
As I've read the geoip is bundled by default with 7.1 so I just put in the logstash config the snippet below (tried different variants), but all ends up with :
_geoip_lookup_failure
The logstash filter part is there:
filter {
#geoip {
# source => "client.ip"
# target => "client.geoip"
# add_field => [ "[client.geoip][coordinates]", "%{[client.geoip][longitude]}" ]
# add_field => [ "[client.geoip][coordinates]", "%{[client.geoip][latitude]}" ]
#}
#geoip {
# source => "source.ip"
# target => "source.geoip"
# add_field => [ "[source.geoip][coordinates]", "%{[source.geoip][longitude]}" ]
# add_field => [ "source.geoip[][coordinates]", "%{[source.geoip][latitude]}" ]
#}
geoip {
source => "destination.ip"
target => "destination.geo"
add_field => [ "[destination.geo][coordinates]", "%{[destination.geo][longitude]}" ]
add_field => [ "[destination.geo][coordinates]", "%{[destination.geo][latitude]}" ]
}
mutate {
convert => [ "[destination.geo][coordinates]", "float"]
}
#geoip {
# source => "server.ip"
# target => "server.geoip"
# add_field => [ "[server.geoip][coordinates]", "%{[server.geoip][longitude]}" ]
# add_field => [ "[server.geoip][coordinates]", "%{[server.geoip][latitude]}" ]
#}
#geoip {
# source => "host.ip"
# target => "host.geoip"
# add_field => [ "[host.geoip][coordinates]", "%{[host.geoip][longitude]}" ]
# add_field => [ "[host.geoip][coordinates]", "%{[host.geoip][latitude]}" ]
#}
}
while start logstash only says:
[2019-06-17T17:23:20,619][INFO ][logstash.outputs.elasticsearch] Attempting to install template {:manage_template=>{"index_patterns"=>"logstash-*", "version"=>60001, "settings"=>{"index.refresh_interval"=>"5s", "number_of_shards"=>1}, "mappings"=>{"dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"keyword"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}
Thanks for hints...