Hi everybody!
My logstash pipeline got stuck and giving warning logs like below
{:timestamp=>"2018-04-05T13:46:28.970000+0600", :message=>#<LogStash::PipelineReporter::Snapshot:0x46197e9a @data={:events_filtered=>7191249, :events_consumed=>7191249, :worker_count=>4, :inflight_count=>6, :worker_states=>[{:status=>"sleep", :alive=>true, :index=>0, :inflight_count=>3}, {:status=>"sleep", :alive=>true, :index=>1, :inflight_count=>2}, {:status=>"sleep", :alive=>true, :index=>2, :inflight_count=>0}, {:status=>"sleep", :alive=>true, :index=>3, :inflight_count=>1}], :output_info=>[{:type=>"redis", :config=>{"host"=>"obfuscated_host", "data_type"=>"list", "password"=>"obfuscated_password", "key"=>"logstash_%{type}", "codec"=>"json", "ALLOW_ENV"=>false}, :is_multi_worker=>false, :events_received=>7191249, :workers=><Java::JavaUtilConcurrent::CopyOnWriteArrayList:2045075981 [<LogStash::Outputs::Redis host=>["obfuscated_host"], data_type=>"list", password=>, key=>"logstash_%{type}", codec=><LogStash::Codecs::JSON charset=>"UTF-8">, workers=>1, name=>"default", shuffle_hosts=>true, port=>6379, db=>0, timeout=>5, batch=>false, batch_events=>50, batch_timeout=>5, reconnect_interval=>1, congestion_threshold=>0, congestion_interval=>1>]>, :busy_workers=>0}], :thread_info=>[{"thread_id"=>20, "name"=>"[main]<file", "plugin"=>nil, "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in `sleep'", "[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in `subscribe'", "[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/observing_tail.rb:23:in `subscribe'", "[...]/vendor/jruby/lib/ruby/1.9/forwardable.rb:201:in `subscribe'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-input-file-2.2.5/lib/logstash/inputs/file.rb:294:in `run'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:342:in `inputworker'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:336:in `start_input'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in `sleep'"}, {"thread_id"=>21, "name"=>"[main]<file", "plugin"=>nil, "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in `sleep'", "[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in `subscribe'", "[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/observing_tail.rb:23:in `subscribe'", "[...]/vendor/jruby/lib/ruby/1.9/forwardable.rb:201:in `subscribe'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-input-file-2.2.5/lib/logstash/inputs/file.rb:294:in `run'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:342:in `inputworker'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:336:in `start_input'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in `sleep'"}, {"thread_id"=>22, "name"=>"[main]<file", "plugin"=>nil, "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in `sleep'", "[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/watch.rb:288:in `subscribe'", "[...]/vendor/bundle/jruby/1.9/gems/filewatch-0.9.0/lib/filewatch/observing_tail.rb:23:in `subscribe'", "[...]/vendor/jruby/lib/ruby/1.9/forwardable.rb:201:in `subscribe'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-input-file-2.2.5/lib/logstash/inputs/file.rb:294:in `run'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.4.1-java/lib/logstash/pipeline.rb:342:in
............
the rest of my log is in comment
my config looks as below:
input {
file {
path => ["obfuscated/search.log"]
type => "searchlog"
start_position => "end"
}
}
filter {
mutate {
remove_field => ["@version","path"]
}
}
output {
redis {
host => "obfuscated_host"
password => "obfuscated_password"
data_type => "list"
key => "logstash_%{type}"
codec => "json"
}
}
What can I do to make pipeline work again? please help