I am chasing a stability problem in logstash and have turned on debug logging in the log4j config found in /etc/logstash/
Running logstash 5.6
We use only a custom tcp listener.
issues spotted:
[2018-02-27T16:19:21,353][ERROR][logstash.pipeline ] Error registering plugin {:plugin=>"#<LogStash::OutputDelegator:0x2725780d @namespaced_metric=#<LogStash::Instrument::NamespacedMetric:0x1ed32d2 @metric=#<LogStash::Instrument::Metric:0x5fb5f6db @collector=#<LogStash::Instrument::Collector:0x50b81dff
and
[2018-02-27T16:19:21,358][ERROR][logstash.agent ] Pipeline aborted due to error {:exception=>#<NameError: HttpClientBuilder not found in packages org.apache.http.client.methods, org.apache.http.client.entity, org.apache.http.client.config, org.apache.http.config,
The full log is below.
How to clean up?
[2018-02-27T16:18:27,648][ERROR][logstash.agent ] Pipeline aborted due to error {:exception=>#<NameError: HttpClientBuilder not found in packages org.apache.http.client.methods, org.apache.http.client.entity, org.apache.http.client.config, org.apache.http.config, org.apache.http.conn.socket, org.apache.http.impl, org.apache.http.impl.client, org.apache.http.impl.conn, org.apache.http.impl.auth, org.apache.http.entity, org.apache.http.message, org.apache.http.params, org.apache.http.protocol, org.apache.http.auth, java.util.concurrent, org.apache.http.client.protocol, org.apache.http.conn.ssl, java.security.cert, java.security.spec, java.security, org.apache.http.client.utils; last error: cannot load Java class org.apache.http.client.utils.HttpClientBuilder>, :backtrace=>["file:/usr/share/logstash/vendor/jruby/lib/jruby.jar!/jruby/java/core_ext/module.rb:45:in const_missing'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.6.1-java/lib/manticore/client.rb:382:in client_builder'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.6.1-java/lib/manticore/client.rb:180:in initialize'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb:26:in initialize'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/http_client.rb:271:in build_adapter'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/http_client.rb:275:in build_pool'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/http_client.rb:60:in initialize'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/http_client_builder.rb:101:in create_http_client'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/http_client_builder.rb:97:in build'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch.rb:230:in build_client'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/common.rb:24:in register'", "/usr/share/logstash/logstash-core/lib/logstash/output_delegator_strategies/shared.rb:9:in register'", "/usr/share/logstash/logstash-core/lib/logstash/output_delegator.rb:43:in register'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:290:in register_plugin'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:301:in register_plugins'", "org/jruby/RubyArray.java:1613:in each'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:301:in register_plugins'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:310:in start_workers'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:235:in run'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:408:in start_pipeline'"]}
[2018-02-27T16:18:27,689][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
[2018-02-27T16:18:30,673][WARN ][logstash.agent ] stopping pipeline {:id=>"main"}
[2018-02-27T16:18:55,454][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"netflow", :directory=>"/usr/share/logstash/modules/netflow/configuration"}
[2018-02-27T16:18:55,457][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}
[2018-02-27T16:18:55,688][ERROR][logstash.plugins.registry] Problems loading a plugin with {:type=>"input", :name=>"tcp", :path=>"logstash/inputs/tcp", :error_message=>"missing class name (org.logstash.tcp.Decoder')", :error_class=>NameError, :error_backtrace=>["org/jruby/javasupport/JavaUtilities.java:54:in get_proxy_or_package_under_package'", "file:/usr/share/logstash/vendor/jruby/lib/jruby.jar!/jruby/java/java_package_module_template.rb:14:in method_missing'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-input-tcp-4.2.4-java/lib/logstash/inputs/tcp/decoder_impl.rb:4:in (root)'", "org/jruby/RubyKernel.java:1040:in require'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/polyglot-0.3.5/lib/polyglot.rb:65:in require'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-input-tcp-4.2.4-java/lib/logstash/inputs/tcp.rb:1:in (root)'", "org/jruby/RubyKernel.java:1040:in require'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/polyglot-0.3.5/lib/polyglot.rb:65:in require'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-input-tcp-4.2.4-java/lib/logstash/inputs/tcp.rb:8:in (root)'", "/usr/share/logstash/logstash-core/lib/logstash/plugins/registry.rb:1:in (root)'", "/usr/share/logstash/logstash-core/lib/logstash/plugins/registry.rb:156:in legacy_lookup'", "/usr/share/logstash/logstash-core/lib/logstash/plugins/registry.rb:138:in lookup'", "/usr/share/logstash/logstash-core/lib/logstash/plugins/registry.rb:180:in lookup_pipeline_plugin'", "/usr/share/logstash/logstash-core/lib/logstash/plugin.rb:140:in lookup'", "org/jruby/RubyKernel.java:1079:in eval'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:103:in plugin'", "(eval):8:in initialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:75:in initialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:165:in initialize'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:296:in create_pipeline'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:95:in register_pipeline'",
... (truncated to fit max post size)