Problem with GeoIP using filebeat->logstash->ES->Kibana (No geo_point type)

Elastic Stack Logstash
1

This is my config:

filter {
  if [type] == "apache-access" {
    grok {
      match => [ "message", "%{IP:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:apache_timestamp}\] \"%{WORD:method} /%{NOTSPACE:request_page} HTTP/%{NUMBER:http_version}\" %{NUMBER:server_response} " ]
      add_tag => [ "web", "apache" ]
    }
    geoip {
      source => "clientip"
      target => "geoip"
      database => "/etc/logstash/GeoLite2-City.mmdb"
      add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
      add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
    }
    mutate {
      convert => [ "[geoip][coordinates]", "float"]
    }

  }
}



filter {
  if [type] == "apache-error" {
    grok {
      match => { "message" => "%{COMBINEDAPACHELOG}" }
    }
  }
}

filebeat.yml

{
  "mappings": {
    "_default_": {
      "_all": {
        "norms": false
      },
      "_meta": {
        "version": "5.2.0"
      },
      "dynamic_templates": [
        { 
          "strings_as_keyword": {
            "mapping": {
              "ignore_above": 1024,
              "type": "keyword"
            },
            "match_mapping_type": "string"
          }
        }
      ],
      "properties": {
        "@timestamp": {
          "type": "date"
        },
        "beat": {
          "properties": {
            "hostname": {
              "ignore_above": 1024,
              "type": "keyword"
            },
            "name": {
              "ignore_above": 1024,
              "type": "keyword"
            },
            "version": {
              "ignore_above": 1024,
              "type": "keyword"
            }
          }
        },
        "input_type": {
          "ignore_above": 1024,
          "type": "keyword"
        },
        "message": {
          "norms": false,
          "type": "text"
        },
        "meta": {
          "properties": {
            "cloud": {
              "properties": {
                "availability_zone": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "instance_id": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "machine_type": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "project_id": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "provider": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "region": {
                  "ignore_above": 1024,
                  "type": "keyword"
                }
              }
            }
          }
        },
        "offset": {
          "type": "long"
        },
        "source": {
          "ignore_above": 1024,
          "type": "keyword"
        },
        "tags": {
          "ignore_above": 1024,
          "type": "keyword"
        },
        "type": {
          "ignore_above": 1024,
          "type": "keyword"
        }
      }
    }
  },
  "order": 0,
  "settings": {
    "index.mapping.total_fields.limit": 10000,
    "index.refresh_interval": "5s"
  },
  "template": "filebeat-*"
}

I had refresh de index fields... I readed ALL the threads. I dont know what to do.

Thanks in advance!

2

This is the map in the index...

"geoip" : {
            "properties" : {
              "city_name" : {
                "type" : "keyword",
                "ignore_above" : 1024
              },
              "continent_code" : {
                "type" : "keyword",
                "ignore_above" : 1024
              },
              "coordinates" : {
                "type" : "float"
              },
              "country_code2" : {
                "type" : "keyword",
                "ignore_above" : 1024
              },
              "country_code3" : {
                "type" : "keyword",
                "ignore_above" : 1024
              },
              "country_name" : {
                "type" : "keyword",
                "ignore_above" : 1024
              },
              "dma_code" : {
                "type" : "long"
              },
              "ip" : {
                "type" : "keyword",
                "ignore_above" : 1024
              },
              "latitude" : {
                "type" : "float"
              },
              "location" : {
                "type" : "float"
              },
              "longitude" : {
                "type" : "float"
              },
              "postal_code" : {
                "type" : "keyword",
                "ignore_above" : 1024
              },
              "region_code" : {
                "type" : "keyword",
                "ignore_above" : 1024
              },
              "region_name" : {
                "type" : "keyword",
                "ignore_above" : 1024
              },
              "timezone" : {
                "type" : "keyword",
                "ignore_above" : 1024
              }
            }
          },
3

Don't bother with this, the plugin creates a geoip.location field that should map correctly.

1 Like
4

Thanks so much! Now its working. I just deleted all the old indexes for incompatibiliy types and now work fine.

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.