Here is background
My logstash version : logstash_oss 681
My logstash is a dockerversion (install into k8s) and my logstash docker file like this
FROM docker.elastic.co/logstash/logstash-oss:6.8.1
RUN logstash-plugin install logstash-output-kinesis && logstash-plugin install logstash-filter-prune
ENTRYPOINT ["/usr/local/bin/docker-entrypoint"]
I have a logstash running in my design , logstash sometime will terminate itself
I checked the cpu and memory usage which it does not seems related to reaching their limit
When it terminate a log show like this
warning: thread "[main]>worker0" terminated with exception (report_on_exception is true):
java.lang.IllegalStateException: java.lang.InterruptedException
at org.logstash.config.ir.compiler.OutputDelegatorExt.doOutput(OutputDelegatorExt.java:80)
at org.logstash.config.ir.compiler.AbstractOutputDelegatorExt.multiReceive(AbstractOutputDelegatorExt.java:101)
at usr.share.logstash.logstash_minus_core.lib.logstash.pipeline.RUBY$block$output_batch$3(/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:390)
at org.jruby.runtime.CompiledIRBlockBody.yieldDirect(CompiledIRBlockBody.java:146)
at org.jruby.runtime.BlockBody.yield(BlockBody.java:114)
at org.jruby.runtime.Block.yield(Block.java:165)
at org.jruby.RubyHash$12.visit(RubyHash.java:1438)
at org.jruby.RubyHash$12.visit(RubyHash.java:1435)
at org.jruby.RubyHash.visitLimited(RubyHash.java:690)
at org.jruby.RubyHash.visitAll(RubyHash.java:675)
at org.jruby.RubyHash.iteratorVisitAll(RubyHash.java:1395)
at org.jruby.RubyHash.each_pairCommon(RubyHash.java:1430)
at org.jruby.RubyHash.each(RubyHash.java:1419)
at usr.share.logstash.logstash_minus_core.lib.logstash.pipeline.RUBY$method$output_batch$0(/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:389)
at org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:130)
at org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:189)
at org.jruby.internal.runtime.methods.DynamicMethod.call(DynamicMethod.java:208)
at org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:199)
at org.jruby.ir.interpreter.InterpreterEngine.processCall(InterpreterEngine.java:326)
at org.jruby.ir.interpreter.StartupInterpreterEngine.interpret(StartupInterpreterEngine.java:72)
at org.jruby.ir.interpreter.InterpreterEngine.interpret(InterpreterEngine.java:92)
at org.jruby.internal.runtime.methods.MixedModeIRMethod.INTERPRET_METHOD(MixedModeIRMethod.java:204)
at org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:191)
at org.jruby.internal.runtime.methods.DynamicMethod.call(DynamicMethod.java:208)
at usr.share.logstash.logstash_minus_core.lib.logstash.pipeline.RUBY$block$start_workers$2(/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:304)
at org.jruby.runtime.CompiledIRBlockBody.callDirect(CompiledIRBlockBody.java:136)
at org.jruby.runtime.IRBlockBody.call(IRBlockBody.java:77)
at org.jruby.runtime.Block.call(Block.java:124)
at org.jruby.RubyProc.call(RubyProc.java:295)
at org.jruby.RubyProc.call(RubyProc.java:274)
at org.jruby.RubyProc.call(RubyProc.java:270)
at org.jruby.internal.runtime.RubyRunnable.run(RubyRunnable.java:105)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.InterruptedException
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220)
at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335)
at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:400)
at org.logstash.config.ir.compiler.OutputStrategyExt$LegacyOutputStrategyExt.output(OutputStrategyExt.java:192)
at org.logstash.config.ir.compiler.OutputStrategyExt$AbstractOutputStrategyExt.multiReceive(OutputStrategyExt.java:118)
at org.logstash.config.ir.compiler.OutputDelegatorExt.doOutput(OutputDelegatorExt.java:78)
... 32 more
[2021-08-19T06:07:30,000][FATAL][logstash.runner ] An unexpected error occurred! {:error=>java.lang.IllegalStateException: java.lang.InterruptedException, :backtrace=>["org.logstash.config.ir.compiler.OutputDelegatorExt.doOutput(OutputDelegatorExt.java:80)", "org.logstash.config.ir.compiler.AbstractOutputDelegatorExt.multiReceive(AbstractOutputDelegatorExt.java:101)", "usr.share.logstash.logstash_minus_core.lib.logstash.pipeline.RUBY$block$output_batch$3(/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:390)", "org.jruby.runtime.CompiledIRBlockBody.yieldDirect(CompiledIRBlockBody.java:146)", "org.jruby.runtime.BlockBody.yield(BlockBody.java:114)", "org.jruby.runtime.Block.yield(Block.java:165)", "org.jruby.RubyHash$12.visit(RubyHash.java:1438)", "org.jruby.RubyHash$12.visit(RubyHash.java:1435)", "org.jruby.RubyHash.visitLimited(RubyHash.java:690)", "org.jruby.RubyHash.visitAll(RubyHash.java:675)", "org.jruby.RubyHash.iteratorVisitAll(RubyHash.java:1395)", "org.jruby.RubyHash.each_pairCommon(RubyHash.java:1430)", "org.jruby.RubyHash.each(RubyHash.java:1419)", "usr.share.logstash.logstash_minus_core.lib.logstash.pipeline.RUBY$method$output_batch$0(/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:389)", "org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:130)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:189)", "org.jruby.internal.runtime.methods.DynamicMethod.call(DynamicMethod.java:208)", "org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:199)", "org.jruby.ir.interpreter.InterpreterEngine.processCall(InterpreterEngine.java:326)", "org.jruby.ir.interpreter.StartupInterpreterEngine.interpret(StartupInterpreterEngine.java:72)", "org.jruby.ir.interpreter.InterpreterEngine.interpret(InterpreterEngine.java:92)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.INTERPRET_METHOD(MixedModeIRMethod.java:204)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:191)", "org.jruby.internal.runtime.methods.DynamicMethod.call(DynamicMethod.java:208)", "usr.share.logstash.logstash_minus_core.lib.logstash.pipeline.RUBY$block$start_workers$2(/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:304)", "org.jruby.runtime.CompiledIRBlockBody.callDirect(CompiledIRBlockBody.java:136)", "org.jruby.runtime.IRBlockBody.call(IRBlockBody.java:77)", "org.jruby.runtime.Block.call(Block.java:124)", "org.jruby.RubyProc.call(RubyProc.java:295)", "org.jruby.RubyProc.call(RubyProc.java:274)", "org.jruby.RubyProc.call(RubyProc.java:270)", "org.jruby.internal.runtime.RubyRunnable.run(RubyRunnable.java:105)", "java.lang.Thread.run(Thread.java:748)"]}
[2021-08-19T06:07:30,127][ERROR][org.logstash.Logstash ] java.lang.IllegalStateException: Logstash stopped processing because of an error: (SystemExit) exit
any hints?