This happens frequently,and it's a show stopper. Sometimes rebooting the node fixes the problem. Sometimes not. In this case, right now, it's just repeating 'failed to flush' error.
What I"m trying to do is load web log data from 2014. I'm running this on node/host ris-webstats01
ls -lah /var/log/muo-2014/app01/domain.com-access_log-20141022
-rw-r--r-- 1 root root 180M May 28 15:24 /var/log/muo-2014/app01/domain.com-access_log-20141022
Load up screen and execute ...
cat /var/log/2014/app01/domain.com-access_log-20141022 | /opt/logstash/bin/logstash -f /root/muo/fileimport.conf
fileimport.conf
input {
stdin {
type => "apache"
codec => plain {
charset => "ISO-8859-1"
}
}
}
filter {
if [type] == "apache" {
grok {
match => { "message" => "%{COMBINEDAPACHELOG}" }
}
date {
match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
}
geoip {
source => "clientip"
target => "geoip"
database => "/etc/logstash/GeoLiteCity.dat"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float"]
}
if [clientip] in ["10.1.88.11", "10.1.88.12", "10.1.88.13", "10.1.88.14", "10.1.88.15", "10.1.88.16", "10.1.42.117", "10.1.42.118", "10.1.42.119", "10.1.88.21", "10.1.88.22", "10.1.88.23", "10.1.88.24", "10.1.88.25", "10.1.88.26", "10.1.42.127", "10.1.42.128", "10.1.42.129"] {
drop {}
}
}
}
output {
elasticsearch {
cluster => "muostats"
host => "127.0.0.1"
protocol => http
index_type => "apache"
workers => "1"
}
}
Failed to flush outgoing items {:outgoing_count=>5000, :exception=>#<Manticore::Timeout: Read timed out>, :backtrace=>["/opt/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.4.1-java/lib/manticore/response.rb:35:in `initialize'", "org/jruby/RubyProc.java:271:in `call'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.4.1-java/lib/manticore/response.rb:61:in `call'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.4.1-java/lib/manticore/response.rb:225:in `call_once'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.4.1-java/lib/manticore/response.rb:128:in `code'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.7/lib/elasticsearch/transport/transport/http/manticore.rb:50:in `perform_request'", "org/jruby/RubyProc.java:271:in `call'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.7/lib/elasticsearch/transport/transport/base.rb:187:in `perform_request'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.7/lib/elasticsearch/transport/transport/http/manticore.rb:33:in `perform_request'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.7/lib/elasticsearch/transport/client.rb:115:in `perform_request'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/elasticsearch-api-1.0.7/lib/elasticsearch/api/actions/bulk.rb:80:in `bulk'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-0.2.4-java/lib/logstash/outputs/elasticsearch/protocol.rb:100:in `bulk'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-0.2.4-java/lib/logstash/outputs/elasticsearch.rb:437:in `submit'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-0.2.4-java/lib/logstash/outputs/elasticsearch.rb:462:in `flush'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/stud-0.0.19/lib/stud/buffer.rb:219:in `buffer_flush'", "org/jruby/RubyHash.java:1341:in `each'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/stud-0.0.19/lib/stud/buffer.rb:216:in `buffer_flush'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/stud-0.0.19/lib/stud/buffer.rb:193:in `buffer_flush'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/stud-0.0.19/lib/stud/buffer.rb:159:in `buffer_receive'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-0.2.4-java/lib/logstash/outputs/elasticsearch.rb:426:in `receive'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-1.5.0-java/lib/logstash/outputs/base.rb:88:in `handle'", "(eval):159:in `output_func'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-1.5.0-java/lib/logstash/pipeline.rb:244:in `outputworker'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-1.5.0-java/lib/logstash/pipeline.rb:166:in `start_outputs'"], :level=>:warn}
top
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
1688 es 20 0 13.8g 1.4g 39m S 136.4 8.8 34:20.66 java
_
cat/health
1435761438 07:37:18 muostats green 2 2 1288 644 0 0 0 0
_cat/nodes
ris-webstats03 127.0.0.1 98 39 1.04 d * ris-webstats03
ris-webstats01 127.0.0.1 78 15 1.25 d m ris-webstats01