LogStash::PipelineReporter

Hi everybody!:smiley: My logstash pipeline got stuck and giving warning logs like below

{:timestamp=>"2018-04-05T13:46:28.970000+0600", :message=>#<LogStash::PipelineReporter::Snapshot:0x46197e9a @data={:events_filtered=>7191249, :events_consumed=>7191249, :worker_count=>4, :inflight_count=>6, :worker_states=>[{:status=>"sleep", :alive=>true, :index=>0, :inflight_count=>3}, {:status=>"sleep", :alive=>true, :index=>1, :inflight_count=>2}, {:status=>"sleep", :alive=>true, :index=>2, :inflight_count=>0}, {:status=>"sleep", :alive=>true, :index=>3, :inflight_count=>1}], :output_info=>[{:type=>"redis", :config=>{"host"=>"obfuscated_host", "data_type"=>"list", "password"=>"obfuscated_password", "key"=>"logstash_%{type}", "codec"=>"json", "ALLOW_ENV"=>false}, :is_multi_worker=>false, :events_received=>7191249, :workers=><Java::JavaUtilConcurrent::CopyOnWriteArrayList:2045075981 [<LogStash::Outputs::Redis host=>["obfuscated_host"], data_type=>"list", password=>, key=>"logstash_%{type}", codec=><LogStash::Codecs::JSON charset=>"UTF-8">, workers=>1, name=>"default", shuffle_hosts=>true, port=>6379, db=>0, timeout=>5, batch=>false, batch_events=>50, batch_timeout=>5, reconnect_interval=>1, congestion_threshold=>0, congestion_interval=>1>]>, :busy_workers=>0}], :thread_info=>[{"thread_id"=>20, "name"=>"[main]<file", "plugin"=>nil, "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in `sleep'", "[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in `subscribe'", "[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/observing_tail.rb:23:in `subscribe'", "[...]/vendor/jruby/lib/ruby/1.9/forwardable.rb:201:in `subscribe'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-input-file-2.2.5/lib/logstash/inputs/file.rb:294:in `run'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:342:in `inputworker'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:336:in `start_input'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in `sleep'"}, {"thread_id"=>21, "name"=>"[main]<file", "plugin"=>nil, "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in `sleep'", "[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in `subscribe'", "[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/observing_tail.rb:23:in `subscribe'", "[...]/vendor/jruby/lib/ruby/1.9/forwardable.rb:201:in `subscribe'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-input-file-2.2.5/lib/logstash/inputs/file.rb:294:in `run'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:342:in `inputworker'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:336:in `start_input'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in `sleep'"}, {"thread_id"=>22, "name"=>"[main]<file", "plugin"=>nil, "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in `sleep'", "[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in `subscribe'", "[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/observing_tail.rb:23:in `subscribe'", "[...]/vendor/jruby/lib/ruby/1.9/forwardable.rb:201:in `subscribe'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-input-file-2.2.5/lib/logstash/inputs/file.rb:294:in `run'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:342:in
............
the rest of my log is in comment

my config looks as below:

input {
file {
path => ["obfuscated/search.log"]
type => "searchlog"
start_position => "end"
}
}
filter {
mutate {
remove_field => ["@version","path"]
}
}
output {
redis {
host => "obfuscated_host"
password => "obfuscated_password"
data_type => "list"
key => "logstash_%{type}"
codec => "json"
}
}
What can I do to make pipeline work again? please help

[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in sleep'", "[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:insubscribe'", "[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/observing_tail.rb:23:in subscribe'", "[...]/vendor/jruby/lib/ruby/1.9/forwardable.rb:201:insubscribe'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-input-file-2.2.5/lib/logstash/inputs/file.rb:294:in run'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:342:ininputworker'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:336:in start_input'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:insleep'"}, {"thread_id"=>25, "name"=>"[main]<file", "plugin"=>nil, "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in sleep'", "[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:insubscribe'", "[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/observing_tail.rb:23:in subscribe'", "[...]/vendor/jruby/lib/ruby/1.9/forwardable.rb:201:insubscribe'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-input-file-2.2.5/lib/logstash/inputs/file.rb:294:in run'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:342:ininputworker'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:336:in start_input'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:insleep'"}, {"thread_id"=>36, "name"=>"[main]>worker0", "plugin"=>["LogStash::Filters::Mutate", {"remove_field"=>["@version", "path"]}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:309:in synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:309:ininflight_batches_synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:234:in worker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:201:instart_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:309:in synchronize'"}, {"thread_id"=>37, "name"=>"[main]>worker1", "plugin"=>["LogStash::Filters::Mutate", {"remove_field"=>["@version", "path"]}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:309:insynchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:309:in inflight_batches_synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:234:inworker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:201:in start_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:309:insynchronize'"}, {"thread_id"=>38, "name"=>"[main]>worker2", "plugin"=>["LogStash::Filters::Mutate", {"remove_field"=>["@version", "path"]}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:218:in synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:218:inworker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:201:in start_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:218:insynchronize'"}, {"thread_id"=>39, "name"=>"[main]>worker3", "plugin"=>["LogStash::Filters::Mutate", {"remove_field"=>["@version", "path"]}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:309:in synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:309:ininflight_batches_synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:234:in worker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:201:instart_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:309:in synchronize'"}], :stalling_threads_info=>[{"thread_id"=>20, "name"=>"[main]<file", "plugin"=>nil, "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:insleep'"}, {"thread_id"=>21, "name"=>"[main]<file", "plugin"=>nil, "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in sleep'"}, {"thread_id"=>22, "name"=>"[main]<file", "plugin"=>nil, "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:insleep'"}, {"thread_id"=>23, "name"=>"[main]<file", "plugin"=>nil, "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in sleep'"}, {"thread_id"=>24, "name"=>"[main]<file", "plugin"=>nil, "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:insleep'"}, {"thread_id"=>25, "name"=>"[main]<file", "plugin"=>nil, "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in sleep'"}, {"thread_id"=>36, "name"=>"[main]>worker0", "plugin"=>["LogStash::Filters::Mutate", {"remove_field"=>["@version", "path"]}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:309:insynchronize'"}, {"thread_id"=>37, "name"=>"[main]>worker1", "plugin"=>["LogStash::Filters::Mutate", {"remove_field"=>["@version", "path"]}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:309:in synchronize'"}, {"thread_id"=>38, "name"=>"[main]>worker2", "plugin"=>["LogStash::Filters::Mutate", {"remove_field"=>["@version", "path"]}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:218:insynchronize'"}, {"thread_id"=>39, "name"=>"[main]>worker3", "plugin"=>["LogStash::Filters::Mutate", {"remove_field"=>["@version", "path"]}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:309:in `synchronize'"}]}>, :level=>:warn}

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.