Unable to use geo_point

Elastic Stack Kibana
1

Hi everybody,

I try to parse my JSON in the logstash.conf to retrieve GPS coordinates and do data viewing with Coordinate Map.

Here is my logstash.conf :


input {
  kafka {
    bootstrap_servers => "localhost:9092"
    topics => ["villorep"]
  }
}

filter {
  json {
    source => "[message]"
    target => "doc"
    add_field => [ "available_bikes", "%{[doc][fields][available_bikes]}" ]
    add_field => [ "available_bike_stands", "%{[doc][fields][available_bike_stands]}" ]
    add_field => [ "bike_stands", "%{[doc][fields][bike_stands]}" ]
    add_field => [ "name", "%{[doc][fields][name]}" ]
    add_field => [ "address", "%{[doc][fields][address]}" ]
    add_field => [ "[geoip][location]", "%{[doc][fields][position][0]}"]
    add_field => [ "[geoip][location]", "%{[doc][fields][position][1]}"]
  }

  mutate {
    convert => { "available_bikes" => "integer" }
    convert => { "available_bike_stands" => "integer" }
    convert => { "bike_stands" => "integer" }
    remove_field => "fields"
    remove_field => "doc"
    remove_field => "message"
    convert => { "[geoip][location]" => "float" }
  }
}

output {
  elasticsearch {
    hosts => ["localhost:9200"]
    index => "villorep"
    workers => 1
  }
}

Here is my mapping :

{
  "villorep": {
    "aliases": {},
    "mappings": {
      "doc": {
        "properties": {
          "@timestamp": {
            "type": "date"
          },
          "@version": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "address": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "available_bike_stands": {
            "type": "long"
          },
          "available_bikes": {
            "type": "long"
          },
          "bike_stands": {
            "type": "long"
          },
          "geoip": {
            "properties": {
              "location": {
                "type": "float"
              }
            }
          },
          "name": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          }
        }
      }
    },
    "settings": {
      "index": {
        "creation_date": "1540222008168",
        "number_of_shards": "5",
        "number_of_replicas": "1",
        "uuid": "7RJvlyisSb2kUivf9vpkgw",
        "version": {
          "created": "6040299"
        },
        "provided_name": "villorep"
      }
    }
  }
}

The log is the following :

Capture%20du%202018-10-22%2017-41-31
Capture du 2018-10-22 17-41-31.png412×518 24.3 KB

How should I map my location with de geo_point type ? What is going wrong in my configuration file ?Thank you in advance for paying attention to my issue.

2

You'll need to use an index template to tell Elasticsearch that the villorep index has a geoip field that is a geo point. You can see how the default index template created by the Elasticsearch output for Logstash does that here: https://github.com/logstash-plugins/logstash-output-elasticsearch/blob/master/lib/logstash/outputs/elasticsearch/elasticsearch-template-es7x.json#L34

3

Finally got it ! Thank you for your help. Please find my solution below. :grinning:

logstash.conf :

filter {
json {
source => "[message]"
target => "doc"
add_field => [ "available_bikes", "%{[doc][fields][available_bikes]}" ]
add_field => [ "available_bike_stands", "%{[doc][fields][available_bike_stands]}" ]
add_field => [ "bike_stands", "%{[doc][fields][bike_stands]}" ]
add_field => [ "name", "%{[doc][fields][name]}" ]
add_field => [ "address", "%{[doc][fields][address]}" ]
add_field => [ "[location]", "%{[doc][fields][position][1]}"]
add_field => [ "[location]", "%{[doc][fields][position][0]}"]
}

mutate {
  convert => { "available_bikes" => "integer" }
  convert => { "available_bike_stands" => "integer" }
  convert => { "bike_stands" => "integer" }
  remove_field => "fields"
  remove_field => "doc"
  remove_field => "message"
  convert => { "[geoip][location]" => "float" }
}

I switched [lat, lng] for [lng, lat] as the template shows.

Then I modified my mapping as :

"location": {
     "type": "geo_point"
}
4

Glad to hear!

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.