Filebeat to logstash geo_point missing

[rajsolanki (rajesh)]

Ok Please pardon me for asking same question regarding geo_point. I have done tons of reading in this forum and have tried everything i can think of after reading but i just cant find a solution to my issue.

So here it is.

I am using file beat to ingest IBM HTTP server to Logstash. Logstash config has following filter

filter {
grok {
match => { "message" => '%{IPORHOST:clientip} %{USER:ident} %{USER:auth} [%{HTTPDATE:timestamp}] %{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion} %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{NUMBER:responsetime} "%{DATA:referrer}" "%{DATA:agent}"' }
remove_field => [ "message" ]
}

geoip {
source => "clientip"
target => "geoip"
add_tag => [ "webserver-geoip" ]
}

if "_grokparsefailure" in [tags] {
drop { }
}
date {
match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
remove_field => [ "timestamp" ]
}
mutate {
convert => {
"bytes" => "integer"
"response" => "integer"
"responsetime" => "float"
}
}
}

output {
elasticsearch {
hosts => [ "http://xx.xx.xx.xx:xxxxx" ]
template => "/mytemplate/template.json"
template_overwrite => true
index => "mywebservers-%{+YYYY.MM.dd}"
}
}

Here is my template.json

{
"template": "logstash-",
"settings": {"index.refresh_interval": "-1"},
"mappings": {
"default": {
"_all": {"enabled": false},
"date_detection": false,
"dynamic_templates": [
{"string_fields": {
"match": "",
"match_mapping_type": "string",
"mapping": {"type": "keyword"}
}}
],
"properties": {
"@timestamp": {"type": "date", "format": "dateOptionalTime"},
"agent": {"type": "text", "fields": {"raw": {"type": "keyword"}}},
"referrer": {"type": "text", "fields": {"raw": {"type": "keyword"}}},
"request": {"type": "text", "fields": {"raw": {"type": "keyword"}}},
"host": {"type": "keyword"},
"httpversion": {"type": "keyword"},
"user": {"type": "keyword"},
"operation": {"type": "keyword"},
"bytes": {"type": "long"},
"response": {"type": "short"},
"responsetime":{"type":"long" },
"clientip": {"type": "ip"},
"geoip" : {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "float" },
"longitude" : { "type" : "float" }
}
}
}
}
}
}

I am able to see all documents which has geo ip related details populated. Enitre json record has all geo related values populated. But there is no geo_point. My template.json is taking care of it but in visualization when i try to create coordinated map and select geo hash it gives me this error.

No Compatible Fields: The "mywebservers*" index pattern does not contain any of the following field types: geo_point

have i missed something ? is it very simple solution where in template.json "template": "logstash-", should be mywebservers ?

thanks

Raj

[warkolm (Mark Walkom)]

What does the mapping for the index look like?

[rajsolanki (rajesh)]

"other-webservers-ppd-2018.10.17": {
"mappings": {
"doc": {
"properties": {
"@timestamp": {
"type": "date"
},
"@version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"agent": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"auth": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"beat": {
"properties": {
"hostname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"bytes": {
"type": "long"
},
"clientip": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"geoip": {
"properties": {
"city_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"continent_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country_code2": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country_code3": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"dma_code": {
"type": "long"
},
"ip": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"latitude": {
"type": "float"
},
"location": {
"properties": {
"lat": {
"type": "float"
},
"lon": {
"type": "float"
}
}
},
"longitude": {
"type": "float"
},
"postal_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"region_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"region_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"timezone": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"host": {
"properties": {
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"httpversion": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ident": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"input": {
"properties": {
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"offset": {
"type": "long"
},
"prospector": {
"properties": {
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"referrer": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},

[rajsolanki (rajesh)]

"request": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"response": {
"type": "long"
},
"responsetime": {
"type": "float"
},
"source": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"tags": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"verb": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}

[warkolm (Mark Walkom)]

It'd be easier to read if you format things using the </> button, or put markdown code backticks around the code.

[rajsolanki (rajesh)]

apologies for not using </>. I think i have narrowed it down. If i dont use template at all logstash uses default one. so when i start logstash with no template i get this output.

Attempting to install template {:manage_template=>{"template"=>"logstash-", "version"=>60001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"default"=>{"dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"keyword"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}

where as if i use my template i see this output.

:manage_template=>{"template"=>"logstash-", "settings"=>{"index.refresh_interval"=>"-1"}, "mappings"=>{"default"=>{"_all"=>{"enabled"=>false}, "date_detection"=>false, "dynamic_templates"=>[{"string_fields"=>{"match"=>"", "match_mapping_type"=>"string", "mapping"=>{"type"=>"keyword"}}}], "properties"=>{"@timestamp"=>{"type"=>"date", "format"=>"dateOptionalTime"}, "agent"=>{"type"=>"text", "fields"=>{"raw"=>{"type"=>"keyword"}}}, "referrer"=>{"type"=>"text", "fields"=>{"raw"=>{"type"=>"keyword"}}}, "request"=>{"type"=>"text", "fields"=>{"raw"=>{"type"=>"keyword"}}}, "host"=>{"type"=>"keyword"}, "httpversion"=>{"type"=>"keyword"}, "dealer"=>{"type"=>"keyword"}, "operation"=>{"type"=>"keyword"}, "bytes"=>{"type"=>"long"}, "response"=>{"type"=>"short"}, "responsetime"=>{"type"=>"long"}, "clientip"=>{"type"=>"ip"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"float"}, "longitude"=>{"type"=>"float"}}}}}}}}

I have verified my format its valid json. only diff i see is longitude and latitude are half float in default where as float in mine. can that be it ?