Hi,
I am trying to fetch data from a csv file and store it in elastic search. Finally i want to visualize it on Tile Map.
logstash conf file
input {
file {
path => "/airports_test.csv"
codec => "plain"
start_position => "beginning"
}
}
filter {
csv {
separator => ","
columns => ["airport_id", "name", "city", "country", "iato", "icao", "latitude", "longitude", "altitude", "timezone", "dst", "tz", "type", "source"]
}
mutate {
remove_field => ["message","host"]
}
mutate {
convert => {"latitude" => "float"}
convert => {"longitude" => "float"}
}
mutate {
rename => {
"latitude" => "[location][lat]"
"longitude" => "[location][lon]"
}
}
}
output {
stdout {
codec => dots
}
elasticsearch {
hosts => "localhost"
index => "flights_dataset"
template => "/flights_location.json"
template_name => "flights_location"
}
}
template file
{
"template" : "flights_location",
"settings" : {
"index.refresh_interval" : "5s"
},
"mappings" : {
"_default_" : {
"_all" : {"enabled" : true, "omit_norms" : true},
"dynamic_templates" : [ {
"message_field" : {
"match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "string", "index" : "analyzed", "omit_norms" : true
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "string", "index" : "analyzed", "omit_norms" : true,
"fields" : {
"raw" : {"type": "string", "index" : "not_analyzed", "ignore_above" : 256}
}
}
}
} ],
"properties" : {
"@version": { "type": "string", "index": "not_analyzed" },
"coordinates" : {
"type" : "object",
"dynamic" : true,
"properties" : {
"coordinates" : { "type" : "geo_point" }
}
},
"geoip" : {
"type" : "object",
"dynamic": true,
"properties" : {
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "float" },
"longitude" : { "type" : "float" }
}
},
"location": {
"type": "geo_point"
}
}
}
}
}
Sample csv file
1,"Goroka Airport","Goroka","Papua New Guinea","GKA","AYGA",-6.081689834590001,145.391998291,5282,10,"U","Pacific/Port_Moresby","airport","OurAirports"
2,"Madang Airport","Madang","Papua New Guinea","MAG","AYMD",-5.20707988739,145.789001465,20,10,"U","Pacific/Port_Moresby","airport","OurAirports"
3,"Mount Hagen Kagamuga Airport","Mount Hagen","Papua New Guinea","HGU","AYMH",-5.826789855957031,144.29600524902344,5388,10,"U","Pacific/Port_Moresby","airport","OurAirports"
4,"Nadzab Airport","Nadzab","Papua New Guinea","LAE","AYNZ",-6.569803,146.725977,239,10,"U","Pacific/Port_Moresby","airport","OurAirports"
5,"Port Moresby Jacksons International Airport","Port Moresby","Papua New Guinea","POM","AYPY",-9.443380355834961,147.22000122070312,146,10,"U","Pacific/Port_Moresby","airport","OurAirports"
where fields are in order as specified above in logstash conf file.
Finally after starting logstash i get mapping in elastic search as
{
"_index": "flights_dataset",
"_type": "airport",
"_id": "AVwOMstowd_eWoVNFBrH",
"_score": 1,
"_source": {
"country": "Papua New Guinea",
"altitude": "239",
"iato": "LAE",
"airport_id": "4",
"dst": "U",
"city": "Nadzab",
"timezone": "10",
"tz": "Pacific/Port_Moresby",
"source": "OurAirports",
"type": "airport",
"path": "/airports_test.csv",
"@timestamp": "2017-05-15T22:20:28.473Z",
"@version": "1",
"name": "Nadzab Airport",
"icao": "AYNZ",
"location": {
"lon": 146.725977,
"lat": -6.569803
}
},
"fields": {
"@timestamp": [
1494886828473
]
}
}
But then when i try to create a tile map i get this error
No Compatible Fields: The "flights_dataset" index pattern does not contain any of the following field types: geo_point
Please can anyone suggest what i am doing wrong?
Thanks in advance.
