I got this to work based on a different posting from 2016 found here:
I think some additional documentation around JSON would be helpful. I am open to help if someone from Elastic can assist as I am pretty dangerously new to this.
My config now looks like this:
input {
file {
path => [ "/Users/schroew/Documents/Scripts/NetskopeAPI/allNetSkopeEvents.json" ]
type => "netSkopeAPI"
tags => "netSkopeAPI"
codec => "json"
}
} #close input
filter {
if [path] == "/Users/schroew/Documents/Scripts/NetskopeAPI/allNetSkopeEvents.json" {
split {
field => "data"
}
mutate {
rename => { "[data][dst_region]" => "dst_region" }
rename => { "[data][srcip]" => "srcip" }
rename => { "[data][fromlogs]" => "fromlogs" }
rename => { "[data][dstport]" => "dstport" }
rename => { "[data][src_zipcode]" => "src_zipcode" }
rename => { "[data][cci]" => "cci" }
rename => { "[data][suppression_start_time]" => "suppression_start_time" }
rename => { "[data][req_cnt]" => "req_cnt" }
rename => { "[data][type]" => "type" }
rename => { "[data][page_duration]" => "page_duration" }
rename => { "[data][ccl]" => "ccl" }
rename => { "[data][_session_begin]" => "session_begin" }
rename => { "[data][action]" => "action" }
rename => { "[data][dstip]" => "dstip" }
rename => { "[data][aggregated_user]" => "aggregated_user" }
rename => { "[data][sfwder]" => "sfwder" }
rename => { "[data][app]" => "app" }
rename => { "[data][appcategory]" => "appcategory" }
rename => { "[data][count]" => "count" }
rename => { "[data][log_file_name]" => "log_file_name" }
rename => { "[data][organization_unit]" => "organization_unit" }
rename => { "[data][suppression_end_time]" => "suppression_end_time" }
rename => { "[data][server_bytes]" => "server_bytes" }
rename => { "[data][src_country]" => "src_country" }
rename => { "[data][dst_zipcode]" => "dst_zipcode" }
rename => { "[data][dst_longitude]" => "dst_longitude" }
rename => { "[data][_id]" => "id" }
rename => { "[data][numbytes]" => "numbytes" }
rename => { "[data][device]" => "device" }
rename => { "[data][_insertion_epoch_timestamp]" => "insertion_epoch_timestamp" }
rename => { "[data][app_session_id]" => "app_session_id" }
rename => { "[data][access_method]" => "access_method" }
rename => { "[data][traffic_type]" => "traffic_type" }
rename => { "[data][src_longitude]" => "src_longitude" }
rename => { "[data][alert_type]" => "alert_type" }
rename => { "[data][resp_cnt]" => "resp_cnt" }
rename => { "[data][alert]" => "alert" }
rename => { "[data][browser]" => "browser" }
rename => { "[data][policy]" => "policy" }
rename => { "[data][dst_latitude]" => "dst_latitude" }
rename => { "[data][timestamp]" => "timestamp" }
rename => { "[data][dst_country]" => "dst_country" }
rename => { "[data][os]" => "os" }
rename => { "[data][org]" => "org" }
rename => { "[data][client_bytes]" => "client_bytes" }
rename => { "[data][src_region]" => "src_region" }
rename => { "[data][acked]" => "acked" }
rename => { "[data][src_location]" => "src_location" }
rename => { "[data][site]" => "site" }
rename => { "[data][dst_location]" => "dst_location" }
rename => { "[data][serial]" => "serial" }
rename => { "[data][src_latitude]" => "src_latitude" }
rename => { "[data][category]" => "category" }
rename => { "[data][user]" => "user" }
rename => { "[data][alert_name]" => "alert_name" }
convert => {"src_latitude" => "float"}
convert => {"dst_latitude" => "float"}
convert => {"src_longitude" => "float"}
convert => {"dst_longitude" => "float"}
}
date {
match => [ "insertion_epoch_timestamp", "UNIX" ]
}
} #close if
} # close filter
output {
stdout {codec => rubydebug}
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "netskope-28071138"
}
} #close output