Netflow codec UDP input worker silently crashes

Elastic Stack Logstash
1

On...

Red Hat Enterprise Linux Server 7.4 (Maipo)
logstash 5.6.4-1
logstash-codec-netflow-3.8.1
openjdk-1.8.0_151-b12

lscpu

Architecture:          x86_64
CPU op-mode(s):        32-bit, 64-bit
Byte Order:            Little Endian
CPU(s):                8
On-line CPU(s) list:   0-7
Thread(s) per core:    1
Core(s) per socket:    8
Socket(s):             1
NUMA node(s):          1
Vendor ID:             GenuineIntel
CPU family:            6
Model:                 45
Model name:            Intel(R) Xeon(R) CPU E5-2698 v3 @ 2.30GHz
Stepping:              2
CPU MHz:               2297.339
BogoMIPS:              4594.67
Hypervisor vendor:     VMware
Virtualization type:   full
L1d cache:             32K
L1i cache:             32K
L2 cache:              256K
L3 cache:              40960K
NUMA node0 CPU(s):     0-7
Flags:                 fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts mmx fxsr sse sse2 ss ht syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts nopl xtopology tsc_reliable nonstop_tsc aperfmperf pni pclmulqdq ssse3 cx16 pcid sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx hypervisor lahf_lm epb tsc_adjust dtherm ida arat pln pts

Config

input {
  udp {
    host => "0.0.0.0"
    port => 2055
    codec => netflow { versions => [9] netflow_definitions => "/etc/logstash/netflow.yaml" }
    type => "netflow"
    receive_buffer_bytes => 16777216
    workers => 6
  }
}

filter {

        if [netflow][in_bytes] == "0" {
                drop { }
        }

        mutate {
                rename => ["[netflow][ipv4_dst_addr]", "ipv4_dst_addr"]
        }

        mutate {
                rename => ["[netflow][ipv4_src_addr]", "ipv4_src_addr"]
        }

        mutate {
                rename => ["[netflow][in_bytes]", "in_bytes"]
        }

        mutate {
                rename => ["[netflow][app_id]", "app_id"]
        }

        mutate {
                remove_field => ["netflow"]
        }

        prune {
                whitelist_names => ["ipv4_dst_addr", "ipv4_src_addr", "app_id", "@timestamp", "in_bytes", "type"]
        }

        metrics {
                meter => "events"
                add_tag => "metrics"
        }

}

output {

if "metrics" in [tags] {
        elasticsearch {
                hosts => ["itddb-es1.nd.gov"]
                index => "metrics-%{+YYYY.MM.dd}"
         }
}

else {
        elasticsearch {
                hosts => ["itddb-es1.nd.gov"]
                index => "%{type}-%{+YYYY.MM.dd}"
         }
}

}

Input worker 0 randomly crashed after about six days...

Any advice would be appreciated. @jorritfolmer?

2 
# cat /var/log/logstash/logstash-plain-2017-11-29.log
[2017-11-29T12:22:11,001][ERROR][logstash.inputs.udp      ] block_0$RUBY$__file__)", "org.jruby.runtime.CompiledBlock19.yield(CompiledBlock19.java:135)", "org.jruby.runtime.Block.yield(Block.java:142)", "org.jruby.RubyArray.eachCommon(RubyArray.java:1606)", "org.jruby.RubyArray.each(RubyArray.java:1613)", "org.jruby.RubyArray$INVOKER$i$0$0$each.call(RubyArray$INVOKER$i$0$0$each.gen)", "org.jruby.runtime.callsite.CachingCallSite.callBlock(CachingCallSite.java:143)", "org.jruby.runtime.callsite.CachingCallSite.callIter(CachingCallSite.java:154)", "rubyjit.BinData::Struct$$do_read_1c4fd213e42dce2f793f696ded3811836a783d0a1975012498.__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/bindata-2.4.1/lib/bindata/struct.rb:139)", "rubyjit.BinData::Struct$$do_read_1c4fd213e42dce2f793f696ded3811836a783d0a1975012498.__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/bindata-2.4.1/lib/bindata/struct.rb)", "org.jruby.internal.runtime.methods.JittedMethod.call(JittedMethod.java:181)", "org.jruby.runtime.callsite.CachingCallSite.cacheAndCall(CachingCallSite.java:326)", "org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:170)", "rubyjit.BinData::ReadUntilEOFPlugin$$do_read_11c3a092d1caf48b046694b4f8bb5a4be54ac0451975012498.chained_0_rescue_1$RUBY$SYNTHETIC__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/bindata-2.4.1/lib/bindata/array.rb:310)", "rubyjit.BinData::ReadUntilEOFPlugin$$do_read_11c3a092d1caf48b046694b4f8bb5a4be54ac0451975012498.block_0$RUBY$__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/bindata-2.4.1/lib/bindata/array.rb:309)", "rubyjit$BinData::ReadUntilEOFPlugin$$do_read_11c3a092d1caf48b046694b4f8bb5a4be54ac0451975012498$block_0$RUBY$__file__.call(rubyjit$BinData::ReadUntilEOFPlugin$$do_read_11c3a092d1caf48b046694b4f8bb5a4be54ac0451975012498$block_0$RUBY$__file__)", "org.jruby.runtime.CompiledBlock19.yieldSpecificInternal(CompiledBlock19.java:117)", "org.jruby.runtime.CompiledBlock19.yieldSpecific(CompiledBlock19.java:92)", "org.jruby.runtime.Block.yieldSpecific(Block.java:111)", "org.jruby.RubyKernel.loop(RubyKernel.java:1479)", "org.jruby.RubyKernel$INVOKER$s$0$0$loop.call(RubyKernel$INVOKER$s$0$0$loop.gen)", "org.jruby.runtime.callsite.CachingCallSite.cacheAndCall(CachingCallSite.java:316)", "org.jruby.runtime.callsite.CachingCallSite.callBlock(CachingCallSite.java:145)", "org.jruby.runtime.callsite.CachingCallSite.callIter(CachingCallSite.java:154)", "rubyjit.BinData::ReadUntilEOFPlugin$$do_read_11c3a092d1caf48b046694b4f8bb5a4be54ac0451975012498.__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/bindata-2.4.1/lib/bindata/array.rb:307)", "rubyjit.BinData::ReadUntilEOFPlugin$$do_read_11c3a092d1caf48b046694b4f8bb5a4be54ac0451975012498.__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/bindata-2.4.1/lib/bindata/array.rb)", "org.jruby.internal.runtime.methods.JittedMethod.call(JittedMethod.java:181)", "org.jruby.runtime.callsite.CachingCallSite.cacheAndCall(CachingCallSite.java:326)", "org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:170)", "rubyjit.BinData::Struct$$do_read_1c4fd213e42dce2f793f696ded3811836a783d0a1975012498.block_0$RUBY$__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/bindata-2.4.1/lib/bindata/struct.rb:139)", "rubyjit$BinData::Struct$$do_read_1c4fd213e42dce2f793f696ded3811836a783d0a1975012498$block_0$RUBY$__file__.call(rubyjit$BinData::Struct$$do_read_1c4fd213e42dce2f793f696ded3811836a783d0a1975012498$block_0$RUBY$__file__)", "org.jruby.runtime.CompiledBlock19.yield(CompiledBlock19.java:135)", "org.jruby.runtime.Block.yield(Block.java:142)", "org.jruby.RubyArray.eachCommon(RubyArray.java:1606)", "org.jruby.RubyArray.each(RubyArray.java:1613)", "org.jruby.RubyArray$INVOKER$i$0$0$each.call(RubyArray$INVOKER$i$0$0$each.gen)", "org.jruby.runtime.callsite.CachingCallSite.callBlock(CachingCallSite.java:143)", "org.jruby.runtime.callsite.CachingCallSite.callIter(CachingCallSite.java:154)", "rubyjit.BinData::Struct$$do_read_1c4fd213e42dce2f793f696ded3811836a783d0a1975012498.__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/bindata-2.4.1/lib/bindata/struct.rb:139)", "rubyjit.BinData::Struct$$do_read_1c4fd213e42dce2f793f696ded3811836a783d0a1975012498.__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/bindata-2.4.1/lib/bindata/struct.rb)", "org.jruby.internal.runtime.methods.JittedMethod.call(JittedMethod.java:181)", "org.jruby.runtime.callsite.CachingCallSite.cacheAndCall(CachingCallSite.java:326)", "org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:170)", "rubyjit.BinData::Base$$read_f3243693220f293515d4f1b4961720520f9fb7051975012498.block_0$RUBY$__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/bindata-2.4.1/lib/bindata/base.rb:147)", "rubyjit$BinData::Base$$read_f3243693220f293515d4f1b4961720520f9fb7051975012498$block_0$RUBY$__file__.call(rubyjit$BinData::Base$$read_f3243693220f293515d4f1b4961720520f9fb7051975012498$block_0$RUBY$__file__)", "org.jruby.runtime.CompiledBlock19.yieldSpecificInternal(CompiledBlock19.java:117)", "org.jruby.runtime.CompiledBlock19.yieldSpecific(CompiledBlock19.java:92)", "org.jruby.runtime.Block.yieldSpecific(Block.java:111)", "rubyjit.BinData::Base$$start_read_79c44a88ee47828c87af1769119f08be241597cf1975012498.chained_0_ensure_1$RUBY$__ensure__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/bindata-2.4.1/lib/bindata/base.rb:254)",
3

"rubyjit.BinData::Base$$start_read_79c44a88ee47828c87af1769119f08be241597cf1975012498.__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/bindata-2.4.1/lib/bindata/base.rb)", "rubyjit.BinData::Base$$start_read_79c44a88ee47828c87af1769119f08be241597cf1975012498.__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/bindata-2.4.1/lib/bindata/base.rb)", "org.jruby.internal.runtime.methods.JittedMethod.call(JittedMethod.java:161)", "org.jruby.runtime.callsite.CachingCallSite.cacheAndCall(CachingCallSite.java:316)", "org.jruby.runtime.callsite.CachingCallSite.callBlock(CachingCallSite.java:145)", "org.jruby.runtime.callsite.CachingCallSite.callIter(CachingCallSite.java:154)", "rubyjit.BinData::Base$$read_f3243693220f293515d4f1b4961720520f9fb7051975012498.__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/bindata-2.4.1/lib/bindata/base.rb:145)", "rubyjit.BinData::Base$$read_f3243693220f293515d4f1b4961720520f9fb7051975012498.__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/bindata-2.4.1/lib/bindata/base.rb)", "org.jruby.internal.runtime.methods.JittedMethod.call(JittedMethod.java:201)", "org.jruby.runtime.callsite.CachingCallSite.cacheAndCall(CachingCallSite.java:336)", "org.jruby.runtime.callsite.CachingCallSite.callBlock(CachingCallSite.java:179)", "org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:183)", "rubyjit.BinData::Base$$read_e841921dc5de78f7b7c79511bb1bbc6653960c8c1975012498.__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/bindata-2.4.1/lib/bindata/base.rb:21)", "rubyjit.BinData::Base$$read_e841921dc5de78f7b7c79511bb1bbc6653960c8c1975012498.__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/bindata-2.4.1/lib/bindata/base.rb)", "org.jruby.ast.executable.AbstractScript.__file__(AbstractScript.java:42)", "org.jruby.internal.runtime.methods.JittedMethod.call(JittedMethod.java:181)", "org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:168)", "rubyjit.LogStash::Codecs::Netflow$$decode_daae12b98c64e545e3ea7edb0b826a8eda6af61e1975012498.chained_0_rescue_1$RUBY$SYNTHETIC__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-codec-netflow-3.8.1/lib/logstash/codecs/netflow.rb:114)", "rubyjit.LogStash::Codecs::Netflow$$decode_daae12b98c64e545e3ea7edb0b826a8eda6af61e1975012498.__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-codec-netflow-3.8.1/lib/logstash/codecs/netflow.rb)", "rubyjit.LogStash::Codecs::Netflow$$decode_daae12b98c64e545e3ea7edb0b826a8eda6af61e1975012498.__file__(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-codec-netflow-3.8.1/lib/logstash/codecs/netflow.rb)", "org.jruby.ast.executable.AbstractScript.__file__(AbstractScript.java:42)", "org.jruby.internal.runtime.methods.JittedMethod.call(JittedMethod.java:201)", "org.jruby.runtime.callsite.CachingCallSite.callBlock(CachingCallSite.java:177)", "org.jruby.runtime.callsite.CachingCallSite.callIter(CachingCallSite.java:188)", "org.jruby.ast.CallOneArgBlockNode.interpret(CallOneArgBlockNode.java:60)", "org.jruby.ast.NewlineNode.interpret(NewlineNode.java:105)", "org.jruby.ast.BlockNode.interpret(BlockNode.java:71)", "org.jruby.ast.WhileNode.interpret(WhileNode.java:131)", "org.jruby.ast.NewlineNode.interpret(NewlineNode.java:105)", "org.jruby.ast.RescueNode.executeBody(RescueNode.java:221)", "org.jruby.ast.RescueNode.interpret(RescueNode.java:116)", "org.jruby.ast.BeginNode.interpret(BeginNode.java:83)", "org.jruby.ast.NewlineNode.interpret(NewlineNode.java:105)", "org.jruby.ast.BlockNode.interpret(BlockNode.java:71)", "org.jruby.evaluator.ASTInterpreter.INTERPRET_METHOD(ASTInterpreter.java:74)", "org.jruby.internal.runtime.methods.InterpretedMethod.call(InterpretedMethod.java:182)", "org.jruby.internal.runtime.methods.DefaultMethod.call(DefaultMethod.java:203)", "org.jruby.runtime.callsite.CachingCallSite.cacheAndCall(CachingCallSite.java:326)", "org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:170)", "org.jruby.ast.FCallOneArgNode.interpret(FCallOneArgNode.java:36)", "org.jruby.ast.NewlineNode.interpret(NewlineNode.java:105)", "org.jruby.evaluator.ASTInterpreter.INTERPRET_BLOCK(ASTInterpreter.java:112)", "org.jruby.runtime.Interpreted19Block.evalBlockBody(Interpreted19Block.java:206)", "org.jruby.runtime.Interpreted19Block.yield(Interpreted19Block.java:194)", "org.jruby.runtime.Interpreted19Block.call(Interpreted19Block.java:125)", "org.jruby.runtime.Block.call(Block.java:101)", "org.jruby.RubyProc.call(RubyProc.java:300)", "org.jruby.RubyProc.call(RubyProc.java:230)", "org.jruby.internal.runtime.RubyRunnable.run(RubyRunnable.java:103)", "java.lang.Thread.run(Thread.java:748)"]}

4

I've now suffered a complete crash of all UDP input workers this morning. Logstash itself did not crash, but all netflow processing obviously ceased. Logs do not show anything for this event.

I've checked for plugin updates again and I see 3.8.3 was released for the netflow plugin. Updated to 3.8.3 and am testing again.

5

Another silent crash happened. Upgraded to 3.9.0 and testing again.

6

Two more UDP input worker crashes over the weekend on 3.9.0.