Trying to figure out geo_point mapping

[mw_jko]

Hi there,

I'm using the latest ELK stack (official 5.3.0 docker images) and trying to figure out a setup for geoip mapping. Without much success and I don't really know why:

logstash.conf

input {
    file {
        path => "/usr/share/logstash/sampledata/sample.log"
        type => "nginxaccess"
        start_position => "beginning"
        sincedb_path => "/dev/null"
        ignore_older => 0
    }
}
filter {
  if [type] == "nginxaccess" {
    grok {
      patterns_dir => ["/usr/share/logstash/patterns"]
      match => { "message" => "%{NGINXACCESS}" }
    }
    geoip {
      source => "clientip"
      target => "geoip"
      database => "/usr/share/logstash/plugins/data/GeoLite2-City.mmdb"
    }
  }
}

output {
    if [type] == "nginxaccess" {
        stdout { codec => rubydebug }
        elasticsearch {
            hosts => "elasticsearch:9200"
            index => "nginxaccess-%{+yyyy-MM-dd}"
            document_type => "nginxaccess"
            manage_template => false
            template => "/usr/share/logstash/templates/elasticsearch-template-filebeat.json"
            template_name => "nginxaccess"
        }
    }
}

My custom template looks like this atm (alot of "nginxaccess" because I could not figure out yet what is the responsible part):

{
    "template" : "nginxaccess*",
    "settings" : {
      "index" : {
        "refresh_interval" : "5s"
      }
    },
    "mappings" : {
      "nginxaccess" : {
        "dynamic_templates" : [
          {
            "message_field" : {
              "path_match" : "message",
              "mapping" : {
                "norms" : false,
                "type" : "text"
              },
              "match_mapping_type" : "string"
            }
          },
          {
            "string_fields" : {
              "mapping" : {
                "norms" : false,
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "type" : "keyword"
                  }
                }
              },
              "match_mapping_type" : "string",
              "match" : "*"
            }
          }
        ],
        "_all" : {
          "norms" : false,
          "enabled" : false
        },
        "properties" : {
          "@timestamp" : {
            "include_in_all" : false,
            "type" : "date"
          },
          "geoip" : {
            "dynamic" : true,
            "properties" : {
              "ip" : {
                "type" : "ip"
              },
              "latitude" : {
                "type" : "half_float"
              },
              "location" : {
                "type" : "geo_point"
              }
              "longitude" : {
                "type" : "half_float"
              }
            }
          },
          "@version" : {
            "include_in_all" : false,
            "type" : "keyword"
          }
        }
      }
    }
}

It does not work though:

data

          "geoip" : {
            "ip" : "SUPERSECRET",
            "latitude" : 51.2993,
            "country_code2" : "DE",
            "country_name" : "Germany",
            "coordinates" : "9.491, 51.2993",
            "continent_code" : "EU",
            "country_code3" : "DE",
            "location" : [
              9.491,
              51.2993
            ],
            "longitude" : 9.491
          },

mapping (http://localhost:9200/nginxaccess-2017-04-10/_mapping/nginxaccess/?pretty=true)

{
  "nginxaccess-2017-04-10" : {
    "mappings" : {
      "nginxaccess" : {
        "properties" : {
          "@timestamp" : {
            "type" : "date"
          },
          "@version" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
# exluded some fields to keep it small
          "geoip" : {
            "properties" : {
              "continent_code" : {
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "type" : "keyword",
                    "ignore_above" : 256
                  }
                }
              },
              "country_code2" : {
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "type" : "keyword",
                    "ignore_above" : 256
                  }
                }
              },
              "country_code3" : {
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "type" : "keyword",
                    "ignore_above" : 256
                  }
                }
              },
              "country_name" : {
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "type" : "keyword",
                    "ignore_above" : 256
                  }
                }
              },
              "ip" : {
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "type" : "keyword",
                    "ignore_above" : 256
                  }
                }
              },
              "latitude" : {
                "type" : "float"
              },
              "location" : {
                "type" : "float"
              },
              "longitude" : {
                "type" : "float"
              }
            }
          }
# exluded some fields to keep it small
        }
      }
    }
  }
}

Why is location still "float"? Thank you in advance!

[mw_jko]

Is the mapping done on the logstash side or the elasticsearch side? I just created the template in the logstash container and added it to the elasticsearch output in the logstash.conf. Do I need to push it to elasticsearch?

[bhatch (Brandon Hatch)]

Logstash can do a bit of mapping, but the real mapping happens on the Elasticsearch side.
Remember that template changes to the mapping only take effect upon index creation. So any index created prior to your template won't have your changes.

[theuntergeek (Aaron Mildenstein)]

Try following the example in this blog post, as it covers the geoip mapping, specifically.

[mw_jko]

Hi Brandon,

thanks for the reply, this was something I already knew. In my test setup I always deleted the index before pushing triggering sample.log.

[mw_jko]

Hi Aaron,

thank you for this link. I will do this from scratch and try to port it to my setup. Will report back then