I create a giop filed in my logstash config file (location).
In my data source the latitude and the longitude fields can be an empty string.
so whene i load data into elasticserach i lose all the documents wich have this empty fields.
[2020-09-17T12:56:30,368][WARN ][logstash.outputs.elasticsearch][main][42d2611b98bc06a090ed5c6a07b113b86623d] Could not index event to Elasticsearch. {:status=>400, :action=>["update", {:_id=>"17164", :_index=>"hotels", :routing=>nil, :_type=>"_doc", :retry_on_conflict=>1}, #LogStash::Event:0x151670a5], :response=>{"update"=>{"_index"=>"hotels", "_type"=>"_doc", "_id"=>"17164", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse field [long] of type [float] in document with id '17164'. Preview of field's value: '10.5618093,17'", "caused_by"=>{"type"=>"number_format_exception", "reason"=>"For input string: "10.5618093,17""}}}}}
[2020-09-17T12:56:30,372][WARN ][logstash.outputs.elasticsearch][main][42d2611b98bc06a090ed5c6a07b113b86623d79c5] Could not index event to Elasticsearch. {:status=>400, :action=>["update", {:_id=>"17283", :_index=>"hotels", :routing=>nil, :_type=>"_doc", :retry_on_conflict=>1}, #LogStash::Event:0xf5abdd5], :response=>{"update"=>{"_index"=>"hotels", "_type"=>"_doc", "_id"=>"17283", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse field [lat] of type [float] in document with id '17283'. Preview of field's value: '33.877462,7'", "caused_by"=>{"type"=>"number_format_exception", "reason"=>"For input string: "33.877462,7""}}}}}
My config file
input {
http_poller {
urls => {
url => "XXXXXX"
}
request_timeout => 60
schedule => { every => "60s" }
codec => "json"
}
}
filter {
split { field => "[hotels]" }
mutate {
rename => {
"[hotels][hotelId]" => "hotelId"
"[hotels][name]" => "name"
"[hotels][rating]" => "rating"
"[hotels][marketingtext]" => "marketingtext"
"[hotels][lat]" => "lat"
"[hotels][long]" => "long"
"[hotels][score]" => "score"
"[hotels][address]" => "address"
"[hotels][addressSuite]" => "addressSuite"
"[hotels][postCode]" => "postCode"
"[hotels][cityId]" => "cityId"
"[hotels][countryId]" => "countryId"
"[hotels][zoneId]" => "zoneId"
"[hotels][hotelChainId]" => "hotelChainId"
"[hotels][distanceCity]" => "distanceCity"
"[hotels][distanceAirport]" => "distanceAirport"
"[hotels][photos]" => "photos"
"[hotels][themes]" => "themes"
"[hotels][facilities]" => "facilities"
"[hotels][descriptions]" => "descriptions"
}
add_field => {"location" => "%{lat},%{long}"}
remove_field => ["marketingtext", "addressSuite", "postCode", "photos", "themes", "facilities", "descriptions"]
}
}
output {
elasticsearch {
user => XXXXX
password => XXXX
hosts => "localhost:9200"
index => "hotels"
action => "update"
doc_as_upsert => "true"
document_id => "%{hotelId}"
}
stdout { codec => rubydebug }
}