Hi,
Big problem here. I upgraded from 5.4.0 to 5.4.1 and now I'm having big problems with netflow.
On 5.4.0 I had two netflow v9 devices inputting data, this appeared to work fine though I did not reboot the server after adding the second device.
After upgrading I'm now getting a lot of errors. Both IP's still log some of the netflow packets but the packets containing the actual data (host, source, dst etc) are not there anymore. I tried connecting just one device but no change.
error
2017-06-09T14:52:31,382][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"netflow-2017.06.09", :_type=>"netflow", :_routing=>nil}, 2017-06-09T05:52:30.000Z 2.2.2.2 %{message}], :response=>{"index"=>{"_index"=>"netflow-2017.06.09", "_type"=>"netflow", "_id"=>"AVyLaUaI6EcdITz9UjqF", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse [netflow.application_id]", "caused_by"=>{"type"=>"number_format_exception", "reason"=>"For input string: \"0:0\""}}}}}
[2017-06-09T14:52:41,286][INFO ][logstash.filters.translate] refreshing dictionary file
[2017-06-09T14:52:41,586][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"netflow-2017.06.09", :_type=>"netflow", :_routing=>nil}, 2017-06-09T05:49:32.000Z 1.1.1.1 %{message}], :response=>{"index"=>{"_index"=>"netflow-2017.06.09", "_type"=>"netflow", "_id"=>"AVyLaW5V6EcdITz9UjqG", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse [netflow.application_id]", "caused_by"=>{"type"=>"number_format_exception", "reason"=>"For input string: \"0:0\""}}}}}
[2017-06-09T14:52:41,634][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"netflow-2017.06.09", :_type=>"netflow", :_routing=>nil}, 2017-06-09T05:49:32.000Z 1.1.1.1 %{message}], :response=>{"index"=>{"_index"=>"netflow-2017.06.09", "_type"=>"netflow", "_id"=>"AVyLaW6a6EcdITz9UjqH", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse [netflow.application_id]", "caused_by"=>{"type"=>"number_format_exception", "reason"=>"For input string: \"0:0\""}}}}}
[2017-06-09T14:53:04,996][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"netflow-2017.06.09", :_type=>"netflow", :_routing=>nil}, 2017-06-09T05:49:56.000Z 1.1.1.1 %{message}], :response=>{"index"=>{"_index"=>"netflow-2017.06.09", "_type"=>"netflow", "_id"=>"AVyLacnU6EcdITz9UjqJ", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse [netflow.application_id]", "caused_by"=>{"type"=>"number_format_exception", "reason"=>"For input string: \"0:0\""}}}}}
[2017-06-09T14:53:05,158][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"netflow-2017.06.09", :_type=>"netflow", :_routing=>nil}, 2017-06-09T05:49:56.000Z 1.1.1.1 %{message}], :response=>{"index"=>{"_index"=>"netflow-2017.06.09", "_type"=>"netflow", "_id"=>"AVyLacp46EcdITz9UjqK", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse [netflow.application_id]", "caused_by"=>{"type"=>"number_format_exception", "reason"=>"For input string: \"0:0\""}}}}}
[2017-06-09T14:53:26,890][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"netflow-2017.06.09", :_type=>"netflow", :_routing=>nil}, 2017-06-09T05:50:17.000Z 1.1.1.1 %{message}], :response=>{"index"=>{"_index"=>"netflow-2017.06.09", "_type"=>"netflow", "_id"=>"AVyLah9f6EcdITz9Ujqk", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse [netflow.application_id]", "caused_by"=>{"type"=>"number_format_exception", "reason"=>"For input string: \"0:0\""}}}}}
[2017-06-09T14:53:26,918][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"netflow-2017.06.09", :_type=>"netflow", :_routing=>nil}, 2017-06-09T05:50:17.000Z 1.1.1.1 %{message}], :response=>{"index"=>{"_index"=>"netflow-2017.06.09", "_type"=>"netflow", "_id"=>"AVyLah966EcdITz9Ujql", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse [netflow.application_id]", "caused_by"=>{"type"=>"number_format_exception", "reason"=>"For input string: \"0:0\""}}}}}
[2017-06-09T14:53:41,366][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"netflow-2017.06.09", :_type=>"netflow", :_routing=>nil}, 2017-06-09T05:53:40.000Z 2.2.2.2 %{message}], :response=>{"index"=>{"_index"=>"netflow-2017.06.09", "_type"=>"netflow", "_id"=>"AVyLalft6EcdITz9Ujqm", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse [netflow.application_id]", "caused_by"=>{"type"=>"number_format_exception", "reason"=>"For input string: \"0:0\""}}}}}config (worked fine with 5.4.0)
[code]input {
udp {
port => 9995
type => "netflow"
codec => netflow {
versions => [9]
}
}
}
filter {
mutate {
add_field => {
"[netflow][ipv4_dst_host]" => "%{[netflow][ipv4_dst_addr]}"
"[netflow][ipv4_src_host]" => "%{[netflow][ipv4_src_addr]}"
}
}
if ([netflow][l4_dst_port]) {
mutate {
add_field => {
"[netflow][l4_dst_port_translation]" => "%{[netflow][l4_dst_port]}"
}
}
}
translate {
dictionary_path => '/etc/logstash/port_translation.yaml'
field => "[netflow][l4_dst_port_translation]"
override => true
destination => "[netflow][l4_dst_port_translation]"
}
dns {
action => 'replace'
reverse => "[netflow][ipv4_dst_host]"
}
dns {
action => 'replace'
reverse => "[netflow][ipv4_src_host]"
}
}
output {
if [type] == "netflow" {
elasticsearch {
hosts => localhost
index => "netflow-%{+YYYY.MM.dd}"
}
}
}
[/code]
Please help. For my project its essential I have netflow from multiple locations (all v9) working.