Logstash 7.1 and _geoip_lookup_failure

[alchy]

Hi there,

I'm somehow battling with geoip plugin in logstash in 7.1.
I'm sending data to logstash and I do expect logstash will perform geoip lookup.

The fileds with ip data (ie. destination.ip) are present in the source json as the structure is pretty common packetbeat:

{
  "_index": "packetbeat-20190617",
  "_type": "_doc",
  "_id": "WgLzZWsBmVD51cfr8dou",
  "_version": 1,
  "_score": null,
  "_source": {
    "tags": [
      "beats_input_raw_event",
      "_geoip_lookup_failure"
    ],
    "network": {
      "community_id": "1:UfJ4pWHrirjs8GSsmdcAxKedSbE=",
      "bytes": 56,
      "transport": "icmp",
      "type": "ipv4",
      "direction": "outbound"
    },
    "error": {
      "message": "Response was received without an associated request."
    },
    "host": {
      "hostname": "server.xx.xx.xx",
      "os": {
        "version": "7 (Core)",
        "family": "redhat",
        "kernel": "3.10.0-957.12.1.el7.x86_64",
        "name": "CentOS Linux",
        "codename": "Core",
        "platform": "centos"
      },
      "id": "085cc053c2d249d5b3dce3e3bff02c29",
      "name": "d1clcsec1.dc.vzp.cz",
      "containerized": false,
      "architecture": "x86_64"
    },
    "icmp": {
      "response": {
        "code": 0,
        "type": 0,
        "message": "EchoReply(0)"
      },
      "version": 4
    },
    "ecs": {
      "version": "1.0.0"
    },
    "@timestamp": "2019-06-17T15:00:12.638Z",
    "source": {
      "ip": "10.128.2.18"
    },
    "path": "104.20.38.240",
    "agent": {
      "ephemeral_id": "a80a131b-907a-45ef-a87a-2f38a924af1b",
      "hostname": "server.xx.xx.xx",
      "id": "8f26e80c-b595-44c3-b46d-a1bfa7321b46",
      "type": "packetbeat",
      "version": "7.1.1"
    },
    "type": "icmp",
    "status": "Error",
    "event": {
      "category": "network_traffic",
      "dataset": "icmp",
      "end": "2019-06-17T15:00:12.638Z",
      "kind": "event"
    },
    "server": {
      "ip": "104.20.38.240",
      "bytes": 56
    },
    "client": {
      "ip": "10.128.2.18"
    },
    "destination": {
      "ip": "104.20.38.240",
      "bytes": 56
    },
    "@version": "1"
  },
  "fields": {
    "event.end": [
      "2019-06-17T15:00:12.638Z"
    ],
    "@timestamp": [
      "2019-06-17T15:00:12.638Z"
    ]
  },
  "highlight": {
    "path": [
      "@kibana-highlighted-field@104.20.38.240@/kibana-highlighted-field@"
    ],
    "destination.ip.keyword": [
      "@kibana-highlighted-field@104.20.38.240@/kibana-highlighted-field@"
    ],
    "server.ip.keyword": [
      "@kibana-highlighted-field@104.20.38.240@/kibana-highlighted-field@"
    ],
    "server.ip": [
      "@kibana-highlighted-field@104.20.38.240@/kibana-highlighted-field@"
    ],
    "destination.ip": [
      "@kibana-highlighted-field@104.20.38.240@/kibana-highlighted-field@"
    ],
    "path.keyword": [
      "@kibana-highlighted-field@104.20.38.240@/kibana-highlighted-field@"
    ]
  },
  "sort": [
    1560783612638
  ]
}

As I've read the geoip is bundled by default with 7.1 so I just put in the logstash config the snippet below (tried different variants), but all ends up with :

_geoip_lookup_failure

The logstash filter part is there:

filter {
  #geoip {
  #  source => "client.ip"
  #  target => "client.geoip"
  #  add_field => [ "[client.geoip][coordinates]", "%{[client.geoip][longitude]}" ]
  #  add_field => [ "[client.geoip][coordinates]", "%{[client.geoip][latitude]}"  ]
  #}
  #geoip {
  #  source => "source.ip"
  #  target => "source.geoip"
  #  add_field => [ "[source.geoip][coordinates]", "%{[source.geoip][longitude]}" ]
  #  add_field => [ "source.geoip[][coordinates]", "%{[source.geoip][latitude]}"  ]
  #}


  geoip {
    source => "destination.ip"
    target => "destination.geo"
    add_field => [ "[destination.geo][coordinates]", "%{[destination.geo][longitude]}" ]
    add_field => [ "[destination.geo][coordinates]", "%{[destination.geo][latitude]}"  ]
  }
  mutate {
      convert => [ "[destination.geo][coordinates]", "float"]
  }


  #geoip {
  #  source => "server.ip"
  #  target => "server.geoip"
  #  add_field => [ "[server.geoip][coordinates]", "%{[server.geoip][longitude]}" ]
  #  add_field => [ "[server.geoip][coordinates]", "%{[server.geoip][latitude]}"  ]
  #}
  #geoip {
  #  source => "host.ip"
  #  target => "host.geoip"
  #  add_field => [ "[host.geoip][coordinates]", "%{[host.geoip][longitude]}" ]
  #  add_field => [ "[host.geoip][coordinates]", "%{[host.geoip][latitude]}"  ]
  #}
}

while start logstash only says:

[2019-06-17T17:23:20,619][INFO ][logstash.outputs.elasticsearch] Attempting to install template {:manage_template=>{"index_patterns"=>"logstash-*", "version"=>60001, "settings"=>{"index.refresh_interval"=>"5s", "number_of_shards"=>1}, "mappings"=>{"dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"keyword"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}

Thanks for hints...

[Badger]

[destination.ip] refers to a field with a dot in its name. That's not what you have. [destination][ip] refers to the ip field in the destination object. That is what you have.

Note that geoip will not be able to do a lookup of anything in the 10/8 network since that is reserved and has different locations in different environments. You could build a custom DB if you need to geo-locate reserved addresses.

[alchy]

that's it! great, thanks!