[2022-02-22T08:29:17,828][ERROR][logstash.javapipeline ][mylogs5001][e7635b2b0a223a267b10d8697a123f9520147502f99c0c0faadfac0d8bcf3960] A plugin had an unrecoverable error. Will restart this plugin.
Pipeline_id:mylogs5001
Plugin: <LogStash::Inputs::Tcp type=>"syslog", port=>514, id=>"e7635b2b0a223a267b10d8697a123f9520147502f99c0c0faadfac0d8bcf3960", enable_metric=>true, codec=><LogStash::Codecs::Line id=>"line_3bc8a176-95ad-408a-9723-d2b4cb82474e", enable_metric=>true, charset=>"UTF-8", delimiter=>"\n">, host=>"0.0.0.0", mode=>"server", proxy_protocol=>false, ssl_enable=>false, ssl_verify=>true, ssl_key_passphrase=>, tcp_keep_alive=>false, dns_reverse_lookup_enabled=>true>
Error: event executor terminated
Exception: Java::JavaUtilConcurrent::RejectedExecutionException
Stack: io.netty.util.concurrent.SingleThreadEventExecutor.reject(io/netty/util/concurrent/SingleThreadEventExecutor.java:926)
io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(io/netty/util/concurrent/SingleThreadEventExecutor.java:353)
io.netty.util.concurrent.SingleThreadEventExecutor.addTask(io/netty/util/concurrent/SingleThreadEventExecutor.java:346)
io.netty.util.concurrent.SingleThreadEventExecutor.execute(io/netty/util/concurrent/SingleThreadEventExecutor.java:828)
io.netty.util.concurrent.SingleThreadEventExecutor.execute(io/netty/util/concurrent/SingleThreadEventExecutor.java:818)
io.netty.channel.AbstractChannel$AbstractUnsafe.register(io/netty/channel/AbstractChannel.java:471)
io.netty.channel.SingleThreadEventLoop.register(io/netty/channel/SingleThreadEventLoop.java:87)
io.netty.channel.SingleThreadEventLoop.register(io/netty/channel/SingleThreadEventLoop.java:81)
io.netty.channel.MultithreadEventLoopGroup.register(io/netty/channel/MultithreadEventLoopGroup.java:86)
io.netty.bootstrap.AbstractBootstrap.initAndRegister(io/netty/bootstrap/AbstractBootstrap.java:323)
io.netty.bootstrap.AbstractBootstrap.doBind(io/netty/bootstrap/AbstractBootstrap.java:272)
io.netty.bootstrap.AbstractBootstrap.bind(io/netty/bootstrap/AbstractBootstrap.java:268)
io.netty.bootstrap.AbstractBootstrap.bind(io/netty/bootstrap/AbstractBootstrap.java:253)
org.logstash.tcp.InputLoop.run(org/logstash/tcp/InputLoop.java:87)
jdk.internal.reflect.GeneratedMethodAccessor29.invoke(jdk/internal/reflect/GeneratedMethodAccessor29)
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(jdk/internal/reflect/DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(java/lang/reflect/Method.java:566)
org.jruby.javasupport.JavaMethod.invokeDirectWithExceptionHandling(org/jruby/javasupport/JavaMethod.java:441)
org.jruby.javasupport.JavaMethod.invokeDirect(org/jruby/javasupport/JavaMethod.java:305)
local.u01.ELK.$7_dot_10_dot_0.LS.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_input_minus_tcp_minus_6_dot_0_dot_6_minus_java.lib.logstash.inputs.tcp.run(/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-input-tcp-6.0.6-java/lib/logstash/inputs/tcp.rb:163)
local.u01.ELK.$7_dot_10_dot_0.LS.logstash_minus_core.lib.logstash.java_pipeline.inputworker(/local/u01/ELK/7.10.0/LS/logstash-core/lib/logstash/java_pipeline.rb:405)
local.u01.ELK.$7_dot_10_dot_0.LS.logstash_minus_core.lib.logstash.java_pipeline.RUBY$method$inputworker$0$VARARGS(local/u01/ELK/$7_dot_10_dot_0/LS/logstash_minus_core/lib/logstash//local/u01/ELK/7.10.0/LS/logstash-core/lib/logstash/java_pipeline.rb)
local.u01.ELK.$7_dot_10_dot_0.LS.logstash_minus_core.lib.logstash.java_pipeline.start_input(/local/u01/ELK/7.10.0/LS/logstash-core/lib/logstash/java_pipeline.rb:396)
org.jruby.RubyProc.call(org/jruby/RubyProc.java:318)
java.lang.Thread.run(java/lang/Thread.java:834)
[2022-02-22T08:29:18,534][WARN ][logstash.outputs.Elasticsearch][mylogs5001] Marking url as dead. Last error: [LogStash::Outputs::Elasticsearch::HttpClient::Pool::HostUnreachableError] Elasticsearch Unreachable: [https://dnewdsinw1:xxxxxx@dsi-nw-area-youlab-tfw-i-log-es.sero.wh.rnd.internal.ericsson.com:443/][Manticore::SocketTimeout] Read timed out {:url=>https://dnewdsinw1:xxxxxx@dsi-nw-area-youlab-tfw-i-log-es.sero.wh.rnd.internal.ericsson.com:443/, :error_message=>"Elasticsearch Unreachable: [https://dnewdsinw1:xxxxxx@dsi-nw-area-youlab-tfw-i-log-es.sero.wh.rnd.internal.ericsson.com:443/][Manticore::SocketTimeout] Read timed out", :error_class=>"LogStash::Outputs::Elasticsearch::HttpClient::Pool::HostUnreachableError"}
[2022-02-22T08:29:18,542][INFO ][logstash.outputs.Elasticsearch][mylogs5001] Using a default mapping template {:es_version=>7, :ecs_compatibility=>:disabled}
[2022-02-22T08:29:18,565][ERROR][logstash.outputs.Elasticsearch][mylogs5001] Failed to install template. {:message=>"No Available connections", :class=>"LogStash::Outputs::Elasticsearch::HttpClient::Pool::NoConnectionAvailableError", :backtrace=>["/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-Elasticsearch-10.7.0-java/lib/logstash/outputs/Elasticsearch/http_client/pool.rb:413:in with_connection'", "/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.7.0-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:318:in perform_request'", "/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-Elasticsearch-10.7.0-java/lib/logstash/outputs/Elasticsearch/http_client/pool.rb:326:in block in Pool'", "/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.7.0-java/lib/logstash/outputs/elasticsearch/http_client.rb:162:in get'", "/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-Elasticsearch-10.7.0-java/lib/logstash/outputs/Elasticsearch/http_client.rb:378:in get_xpack_info'", "/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.7.0-java/lib/logstash/outputs/elasticsearch/ilm.rb:57:in ilm_ready?'", "/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-Elasticsearch-10.7.0-java/lib/logstash/outputs/Elasticsearch/ilm.rb:28:in ilm_in_use?'", "/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.7.0-java/lib/logstash/outputs/elasticsearch/template_manager.rb:15:in install_template'", "/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-Elasticsearch-10.7.0-java/lib/logstash/outputs/Elasticsearch/common.rb:218:in install_template'", "/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.7.0-java/lib/logstash/outputs/elasticsearch/common.rb:49:in block in setup_after_successful_connection'"]}
warning: thread "Ruby-0-Thread-5: :1" terminated with exception (report_on_exception is true):
LogStash::Outputs::Elasticsearch::HttpClient::Pool::NoConnectionAvailableError: No Available connections
with_connection at /local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-Elasticsearch-10.7.0-java/lib/logstash/outputs/Elasticsearch/http_client/pool.rb:413
perform_request at /local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-Elasticsearch-10.7.0-java/lib/logstash/outputs/Elasticsearch/http_client/pool.rb:318
Pool at /local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-Elasticsearch-10.7.0-java/lib/logstash/outputs/Elasticsearch/http_client/pool.rb:326
get at /local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-Elasticsearch-10.7.0-java/lib/logstash/outputs/Elasticsearch/http_client.rb:162
get_xpack_info at /local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-Elasticsearch-10.7.0-java/lib/logstash/outputs/Elasticsearch/http_client.rb:378
ilm_ready? at /local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-Elasticsearch-10.7.0-java/lib/logstash/outputs/Elasticsearch/ilm.rb:57
ilm_in_use? at /local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-Elasticsearch-10.7.0-java/lib/logstash/outputs/Elasticsearch/ilm.rb:28
setup_after_successful_connection at /local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-Elasticsearch-10.7.0-java/lib/logstash/outputs/Elasticsearch/common.rb:50
[2022-02-22T08:29:18,651][FATAL][logstash.runner ] An unexpected error occurred! {:error=>#<LogStash::Outputs::Elasticsearch::HttpClient::Pool::NoConnectionAvailableError: No Available connections>, :backtrace=>["/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-Elasticsearch-10.7.0-java/lib/logstash/outputs/Elasticsearch/http_client/pool.rb:413:in with_connection'", "/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.7.0-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:318:in perform_request'", "/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-Elasticsearch-10.7.0-java/lib/logstash/outputs/Elasticsearch/http_client/pool.rb:326:in block in Pool'", "/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.7.0-java/lib/logstash/outputs/elasticsearch/http_client.rb:162:in get'", "/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-Elasticsearch-10.7.0-java/lib/logstash/outputs/Elasticsearch/http_client.rb:378:in get_xpack_info'", "/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.7.0-java/lib/logstash/outputs/elasticsearch/ilm.rb:57:in ilm_ready?'", "/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-Elasticsearch-10.7.0-java/lib/logstash/outputs/Elasticsearch/ilm.rb:28:in ilm_in_use?'", "/local/u01/ELK/7.10.0/LS/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.7.0-java/lib/logstash/outputs/elasticsearch/common.rb:50:in block in setup_after_successful_connection'"]}
[2022-02-22T08:29:18,663][ERROR][org.logstash.Logstash ] java.lang.IllegalStateException: Logstash stopped processing because of an error: (SystemExit) exit
getting logs of above and stopped automatically , we are trying to send syslog from proxy to kibana server