I have data that contains src_latitude, src_longitude, dst_latitiude, and dst_longitude. Some of the data is zero which seems to throw off elastic.
[2017-07-31T15:17:54,787][DEBUG][o.e.a.b.TransportShardBulkAction] [UzWEM4s] [netskope-31071411][0] failed to execute bulk item (index) BulkShardRequest [[netskope-31071411][0]] containing [988] requests
org.elasticsearch.index.mapper.MapperParsingException: failed to parse.
Caused by: java.lang.NumberFormatException: For input string: "%{src_latitude}"
at sun.misc.FloatingDecimal.readJavaFormatString(FloatingDecimal.java:2043) ~[?:?]
at sun.misc.FloatingDecimal.parseDouble(FloatingDecimal.java:110) ~[?:?]
at java.lang.Double.parseDouble(Double.java:538) ~[?:1.8.0_91]
at org.elasticsearch.common.geo.GeoPoint.resetFromString(GeoPoint.java:84) ~[elasticsearch-5.5.0.jar:5.5.0]
I am mapping the coordinate data to a float in the logstash config and in map template. I think this is working as the raw data is 0 and the data output by rubydebug is like this:"src_latitude" => 0.0,
Does anyine know what is the cause of the elastic errors I am seeing?
logstash config
input {
file {
path => [ "/Users/schroew/Documents/Scripts/NetskopeAPI/allNetSkopeEvents.json" ]
type => "netSkopeAPI"
tags => "netSkopeAPI"
codec => "json"
}
} #close input
filter {
if [path] == "/Users/schroew/Documents/Scripts/NetskopeAPI/allNetSkopeEvents.json" {
split {
field => "data"
}
mutate {
rename => { "[data][dsthost]" => "dst_host" }
rename => { "[data][dst_region]" => "dst_region" }
rename => { "[data][srcip]" => "srcip" }
rename => { "[data][fromlogs]" => "fromlogs" }
rename => { "[data][dstport]" => "dstport" }
rename => { "[data][src_zipcode]" => "src_zipcode" }
rename => { "[data][dstip]" => "dstip" }
rename => { "[data][suppression_end_time]" => "suppression_end_time" }
rename => { "[data][server_bytes]" => "server_bytes" }
rename => { "[data][src_country]" => "src_country" }
rename => { "[data][dst_zipcode]" => "dst_zipcode" }
rename => { "[data][dst_longitude]" => "dst_longitude" }
rename => { "[data][_id]" => "id" }
rename => { "[data][numbytes]" => "numbytes" }
rename => { "[data][device]" => "device" }
rename => { "[data][_insertion_epoch_timestamp]" => "insertion_epoch_timestamp" }
rename => { "[data][app_session_id]" => "app_session_id" }
rename => { "[data][access_method]" => "access_method" }
rename => { "[data][traffic_type]" => "traffic_type" }
rename => { "[data][src_longitude]" => "src_longitude" }
rename => { "[data][alert_type]" => "alert_type" }
rename => { "[data][resp_cnt]" => "resp_cnt" }
rename => { "[data][alert]" => "alert" }
rename => { "[data][browser]" => "browser" }
rename => { "[data][policy]" => "policy" }
rename => { "[data][dst_latitude]" => "dst_latitude" }
rename => { "[data][timestamp]" => "timestamp" }
rename => { "[data][dst_country]" => "dst_country" }
rename => { "[data][os]" => "os" }
rename => { "[data][org]" => "org" }
rename => { "[data][client_bytes]" => "client_bytes" }
rename => { "[data][src_region]" => "src_region" }
rename => { "[data][acked]" => "acked" }
rename => { "[data][src_location]" => "src_location" }
rename => { "[data][site]" => "site" }
rename => { "[data][dst_location]" => "dst_location" }
rename => { "[data][site]" => "site" }
rename => { "[data][dst_location]" => "dst_location" }
rename => { "[data][serial]" => "serial" }
rename => { "[data][src_latitude]" => "src_latitude" }
rename => { "[data][category]" => "category" }
rename => { "[data][user]" => "user" }
rename => { "[data][alert_name]" => "alert_name" }
rename => { "[data][activity]" => "activity" }
rename => { "[data][app_activity]" => "app_activity" }
rename => { "[data][dlp_file]" => "dlp_file" }
rename => { "[data][dlp_incident_id]" => "dlp_incident_id" }
rename => { "[data][dlp_parent_id]" => "dlp_parent_id" }
rename => { "[data][dlp_profile]" => "dlp_profile" }
rename => { "[data][dlp_rule]" => "dlp_rule" }
rename => { "[data][dlp_rule_count]" => "dlp_rule_count" }
rename => { "[data][dlp_rule_severity]" => "dlp_rule_severity" }
rename => { "[data][dst_timezone]" => "dst_timezone" }
rename => { "[data][file_password_protected]" => "file_password_protected" }
rename => { "[data][file_lang]" => "file_lang" }
rename => { "[data][file_path]" => "file_path" }
rename => { "[data][file_size]" => "file_size" }
rename => { "[data][file_type]" => "file_type" }
rename => { "[data][instance_id]" => "instance_id" }
rename => { "[data][managed_app]" => "managed_app" }
rename => { "[data][md5]" => "md5" }
rename => { "[data][mime_type]" => "mime_type" }
rename => { "[data][modified]" => "modified" }
rename => { "[data][object]" => "object" }
rename => { "[data][object_id]" => "object_id" }
rename => { "[data][object_type]" => "object_type" }
rename => { "[data][os_version]" => "os_version" }
rename => { "[data][owner]" => "owner" }
rename => { "[data][page_id]" => "page_id" }
rename => { "[data][scan_type]" => "scan_type" }
rename => { "[data][shared]" => "shared" }
rename => { "[data][src_timezone]" => "src_timezone" }
rename => { "[data][sv]" => "sv" }
rename => { "[data][transaction_id]" => "transaction_id" }
rename => { "[data][url]" => "url" }
rename => { "[data][userkey]" => "userkey" }
convert => {"insertion_epoch_timestamp" => "integer"}
convert => {"suppression_end_time" => "integer"}
convert => {"src_latitude" => "float"}
convert => {"dst_latitude" => "float"}
convert => {"src_longitude" => "float"}
convert => {"dst_longitude" => "float"}
} #Close mutate
mutate {
add_field => { "src_geo_location" => "%{src_latitude},%{src_longitude}" }
add_field => { "dst_geo_location" => "%{dst_latitude},%{dst_longitude}" }
remove_field => ["%{data}"]
} #Close mutate
date { match => [ "%{insertion_epoch_timestamp}", "UNIX" ] }
date { match => [ "%{suppression_end_time}", "UNIX" ] }
} #close if
} # close filter
output {
stdout {codec => rubydebug}
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "netskope-31071411"
}
} #close output
My field map template
PUT _template/netskopetemplate1
{
"template": "netskope-*",
"mappings": {
"eventGeoLocations": {
"properties": {
"src_geo_location": {
"type": "geo_point"
},
"dst_geo_location": {
"type": "geo_point"
},
"src_latitude": {
"type": "float"
},
"src_longitude": {
"type": "float"
},
"dst_latitude": {
"type": "float"
},
"dst_longitude": {
"type": "float"
},
"insertion_epoch_timestamp": {
"type": "date"
}
}
}
}
}