Aha! It works now. Your mapping puts the geo_point underneath the mapping _type of event. If you do not send the event through Logstash with the correct document_type, then the mapping template will not apply, as the default document_type in Logstash is logs.
With this configuration:
input { stdin {} }
filter {
mutate {
add_field => { "[attrs][source]" => "8.8.8.8" }
}
geoip { source => "[attrs][source]" }
}
output {
stdout { codec => rubydebug }
elasticsearch {
manage_template => false
# This prevents the default Logstash template from interfering with your mapping
# template, which I manually applied
}
}
I get this output:
This will fail
{
"@timestamp" => 2017-04-07T15:47:51.879Z,
"geoip" => {
"timezone" => "America/Los_Angeles",
"ip" => "8.8.8.8",
"latitude" => 37.386,
"continent_code" => "NA",
"city_name" => "Mountain View",
"country_code2" => "US",
"country_name" => "United States",
"dma_code" => 807,
"country_code3" => "US",
"region_name" => "California",
"location" => [
[0] -122.0838,
[1] 37.386
],
"postal_code" => "94035",
"longitude" => -122.0838,
"region_code" => "CA"
},
"@version" => "1",
"host" => "thunderbolt-display.untergeek.net",
"message" => "This will fail",
"attrs" => {
"source" => "8.8.8.8"
}
}
[2017-04-07T09:47:52,031][WARN ][logstash.outputs.elasticsearch] Failed action. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"logstash-2017.04.07", :_type=>"logs", :_routing=>nil}, 2017-04-07T15:47:51.879Z thunderbolt-display.untergeek.net This will fail], :response=>{"index"=>{"_index"=>"logstash-2017.04.07", "_type"=>"logs", "_id"=>"AVtJGbBDIxmgySnf2h_6", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse", "caused_by"=>{"type"=>"parse_exception", "reason"=>"geo_point expected"}}}}}
This what you've been seeing: The geo_point expected error.
Now I change the configuration to add this:
input {
stdin { type => "event" }
}
filter {
mutate {
add_field => { "[attrs][source]" => "8.8.8.8" }
}
geoip { source => "[attrs][source]" }
}
output {
stdout { codec => rubydebug }
elasticsearch {
manage_template => false
}
}
Not only do I get no error:
This will work
{
"@timestamp" => 2017-04-07T15:51:12.303Z,
"geoip" => {
"timezone" => "America/Los_Angeles",
"ip" => "8.8.8.8",
"latitude" => 37.386,
"continent_code" => "NA",
"city_name" => "Mountain View",
"country_code2" => "US",
"country_name" => "United States",
"dma_code" => 807,
"country_code3" => "US",
"region_name" => "California",
"location" => [
[0] -122.0838,
[1] 37.386
],
"postal_code" => "94035",
"longitude" => -122.0838,
"region_code" => "CA"
},
"@version" => "1",
"host" => "REDACTED",
"message" => "This will work",
"type" => "event",
"attrs" => {
"source" => "8.8.8.8"
}
}
^C[2017-04-07T09:51:14,675][WARN ][logstash.runner ] SIGINT received. Shutting down the agent.
[2017-04-07T09:51:14,683][WARN ][logstash.agent ] stopping pipeline {:id=>"main"}
I also can query that document:
curl -XGET localhost:9200/_search?pretty -d '{"query":{ "match_all" : {} } }'
{
"took" : 31,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"failed" : 0
},
"hits" : {
"total" : 1,
"max_score" : 1.0,
"hits" : [
{
"_index" : "logstash-2017.04.07",
"_type" : "event",
"_id" : "AVtJHL_tqkbjadN9d_gn",
"_score" : 1.0,
"_source" : {
"@timestamp" : "2017-04-07T15:51:12.303Z",
"geoip" : {
"timezone" : "America/Los_Angeles",
"ip" : "8.8.8.8",
"latitude" : 37.386,
"continent_code" : "NA",
"city_name" : "Mountain View",
"country_code2" : "US",
"country_name" : "United States",
"dma_code" : 807,
"country_code3" : "US",
"region_name" : "California",
"location" : [
-122.0838,
37.386
],
"postal_code" : "94035",
"longitude" : -122.0838,
"region_code" : "CA"
},
"@version" : "1",
"host" : "REDACTED",
"message" : "This will work",
"type" : "event",
"attrs" : {
"source" : "8.8.8.8"
}
}
}
]
}
}
Note that _type is "event". This is why your mapping is failing to do GeoIP properly.
