Logstash Monitor API get 500: Unknown internal error after upgrade to 5.5


#1

Hi all,
I used to run python script to monitor logstash performance through monitor API.
However, it return 500 status code when query hot_threads after I upgraded logstash from 5.4.1 to 5.5.1.
I checked 5.5 release note and didn't see any changes about monitor API.
Also, Query node/stats is working.
Any ideas? Thanks in advance.

Error message when query hot_threads by curl:

$ curl -XGET 'localhost:9600/_node/hot_threads?human=true'
{
   "status":500,
   "request_method":"GET",
   "path_info":"/_node/hot_threads",
   "query_string":"human=true",
   "http_version":"HTTP/1.1",
   "http_accept":"*/*",
   "error":"Unexpected Internal Error",
   "class":"Java::JavaLang::IllegalStateException",
   "message":"Duplicate key {cpu.time=845229211935, waited.time=-1, thread.state=runnable, waited.count=193208, thread.name=[main]<redis, blocked.count=3, thread.stacktrace=[sun.nio.ch.EPollArrayWrapper.epollWait(Native Method), sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269), sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)], blocked.time=-1}",
   "backtrace":[
      "java.util.stream.Collectors.lambda$throwingMerger$0(java/util/stream/Collectors.java:133)",
      "java.util.HashMap.merge(java/util/HashMap.java:1253)",
      "java.util.stream.Collectors.lambda$toMap$58(java/util/stream/Collectors.java:1320)",
      "java.util.stream.ReduceOps$3ReducingSink.accept(java/util/stream/ReduceOps.java:169)",
      "java.util.ArrayList$ArrayListSpliterator.forEachRemaining(java/util/ArrayList.java:1374)",
      "java.util.stream.AbstractPipeline.copyInto(java/util/stream/AbstractPipeline.java:481)",
      "java.util.stream.AbstractPipeline.wrapAndCopyInto(java/util/stream/AbstractPipeline.java:471)",
      "java.util.stream.ReduceOps$ReduceOp.evaluateSequential(java/util/stream/ReduceOps.java:708)",
      "java.util.stream.AbstractPipeline.evaluate(java/util/stream/AbstractPipeline.java:234)",
      "java.util.stream.ReferencePipeline.collect(java/util/stream/ReferencePipeline.java:499)",
      "org.logstash.instrument.reports.ThreadsReport.generate(org/logstash/instrument/reports/ThreadsReport.java:29)",
      "java.lang.reflect.Method.invoke(java/lang/reflect/Method.java:498)",
      "RUBY.initialize(/usr/share/logstash/logstash-core/lib/logstash/api/commands/hot_threads_reporter.rb:11)",
      "RUBY.hot_threads(/usr/share/logstash/logstash-core/lib/logstash/api/commands/node.rb:58)",
      "RUBY.HEAD /hot_threads(/usr/share/logstash/logstash-core/lib/logstash/api/modules/node.rb:20)",
      "org.jruby.RubyMethod.call(org/jruby/RubyMethod.java:116)",
      "RUBY.compile!(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:1611)",
      "org.jruby.RubyProc.call(org/jruby/RubyProc.java:281)",
      "RUBY.route!(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:975)",
      "RUBY.route_eval(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:994)",
      "RUBY.route!(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:975)",
      "RUBY.process_route(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:1015)",
      "org.jruby.RubyKernel.catch(org/jruby/RubyKernel.java:1242)",
      "RUBY.process_route(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:1013)",
      "RUBY.route!(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:973)",
      "org.jruby.RubyArray.each(org/jruby/RubyArray.java:1613)",
      "RUBY.route!(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:972)",
      "RUBY.dispatch!(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:1085)",
      "RUBY.invoke(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:1067)",
      "org.jruby.RubyKernel.catch(org/jruby/RubyKernel.java:1242)",
      "RUBY.invoke(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:1067)",
      "RUBY.dispatch!(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:1082)",
      "RUBY.call!(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:907)",
      "RUBY.invoke(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:1067)",
      "org.jruby.RubyKernel.catch(org/jruby/RubyKernel.java:1242)",
      "RUBY.invoke(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:1067)",
      "RUBY.call!(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:907)",
      "RUBY.call(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:895)",
      "RUBY.call(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/rack-protection-1.5.3/lib/rack/protection/xss_header.rb:18)",
      "RUBY.call(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/rack-protection-1.5.3/lib/rack/protection/path_traversal.rb:16)",
      "RUBY.call(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/rack-protection-1.5.3/lib/rack/protection/json_csrf.rb:18)",
      "RUBY.call(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/rack-protection-1.5.3/lib/rack/protection/base.rb:49)",
      "RUBY.call(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/rack-protection-1.5.3/lib/rack/protection/base.rb:49)",
      "RUBY.call(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/rack-protection-1.5.3/lib/rack/protection/frame_options.rb:31)",
      "RUBY.call(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/rack-1.6.6/lib/rack/nulllogger.rb:9)",
      "RUBY.call(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/rack-1.6.6/lib/rack/head.rb:13)",
      "RUBY.call(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:182)",
      "RUBY.call(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/sinatra-1.4.8/lib/sinatra/base.rb:2013)",
      "RUBY.call(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/rack-1.6.6/lib/rack/urlmap.rb:66)",
      "org.jruby.RubyArray.each(org/jruby/RubyArray.java:1613)",
      "RUBY.call(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/rack-1.6.6/lib/rack/urlmap.rb:50)",
      "RUBY.call(/usr/share/logstash/logstash-core/lib/logstash/api/rack_app.rb:57)",
      "RUBY.call(/usr/share/logstash/logstash-core/lib/logstash/api/rack_app.rb:31)",
      "RUBY.call(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/rack-1.6.6/lib/rack/builder.rb:153)",
      "RUBY.handle_request(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/puma-2.16.0-java/lib/puma/server.rb:557)",
      ......
   ]
}

(system) #2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.