Thanks for the reply...The symptom has manifested after barely one hour...I'd love to troubleshoot this while in this state....
ps -ef |grep logstash |grep java
logstash 1110 1 99 09:08 ? 13:27:08 java -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -Djava.io.tmpdir=/var/lib/logstash -Xmx2g -Xss2048k -Djffi.boot.library.path=/opt/logstash/vendor/jruby/lib/jni -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -Djava.io.tmpdir=/var/lib/logstash -Xbootclasspath/a:/opt/logstash/vendor/jruby/lib/jruby.jar -classpath : -Djruby.home=/opt/logstash/vendor/jruby -Djruby.lib=/opt/logstash/vendor/jruby/lib -Djruby.script=jruby -Djruby.shell=/bin/sh org.jruby.Main --1.9 /opt/logstash/lib/bootstrap/environment.rb logstash/runner.rb agent -f /etc/logstash/conf.d -l /var/log/logstash/logstash.log -w 6
type logstash_open_logs
logstash_open_logs is a function
logstash_open_logs ()
{
lsof -p $(ps ax |grep java |grep logstash |awk '{print $1}') | LC_ALL="C" LC_ALL="C" grep --color=auto --color=auto "syslog-ng" | awk '{print $NF}'
}
logstash_open_logs |wc -l
385
top -b -n1 |head
top - 12:25:48 up 62 days, 29 min, 10 users, load average: 0.01, 0.07, 0.33
Tasks: 232 total, 1 running, 231 sleeping, 0 stopped, 0 zombie
%Cpu(s): 5.0 us, 0.8 sy, 25.7 ni, 68.3 id, 0.1 wa, 0.0 hi, 0.1 si, 0.0 st
KiB Mem : 8011104 total, 146784 free, 5940436 used, 1923884 buff/cache
KiB Swap: 1679356 total, 1346700 free, 332656 used. 1642880 avail Mem
ps -efT |grep logstash |grep java |wc -l
176