Failed to flush outgoing items - Help appreciated


(Brian Dunbar) #1

This happens frequently,and it's a show stopper. Sometimes rebooting the node fixes the problem. Sometimes not. In this case, right now, it's just repeating 'failed to flush' error.

What I"m trying to do is load web log data from 2014. I'm running this on node/host ris-webstats01

ls -lah /var/log/muo-2014/app01/domain.com-access_log-20141022
-rw-r--r-- 1 root root 180M May 28 15:24 /var/log/muo-2014/app01/domain.com-access_log-20141022

Load up screen and execute ...

cat /var/log/2014/app01/domain.com-access_log-20141022  |  /opt/logstash/bin/logstash -f /root/muo/fileimport.conf

fileimport.conf

input {
      stdin {
          type => "apache"
	  codec => plain {
                    charset => "ISO-8859-1"
	}
      }
}


filter {
  if [type] == "apache" {
    grok {
      match => { "message" => "%{COMBINEDAPACHELOG}" }
    }
    date {
    	 match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
    }
    geoip {
      source => "clientip"
      target => "geoip"
      database => "/etc/logstash/GeoLiteCity.dat"
      add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
      add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
    }
    mutate {
      convert => [ "[geoip][coordinates]", "float"]
    }

    if [clientip] in ["10.1.88.11", "10.1.88.12", "10.1.88.13", "10.1.88.14", "10.1.88.15", "10.1.88.16", "10.1.42.117", "10.1.42.118", "10.1.42.119", "10.1.88.21", "10.1.88.22", "10.1.88.23", "10.1.88.24", "10.1.88.25", "10.1.88.26", "10.1.42.127", "10.1.42.128", "10.1.42.129"]  {
       drop {}
    }
  }
}


output {
    elasticsearch {
        cluster => "muostats"
        host => "127.0.0.1"
        protocol => http
        index_type => "apache"
	workers => "1"
    }
}

Failed to flush outgoing items {:outgoing_count=>5000, :exception=>#<Manticore::Timeout: Read timed out>, :backtrace=>["/opt/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.4.1-java/lib/manticore/response.rb:35:in `initialize'", "org/jruby/RubyProc.java:271:in `call'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.4.1-java/lib/manticore/response.rb:61:in `call'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.4.1-java/lib/manticore/response.rb:225:in `call_once'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.4.1-java/lib/manticore/response.rb:128:in `code'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.7/lib/elasticsearch/transport/transport/http/manticore.rb:50:in `perform_request'", "org/jruby/RubyProc.java:271:in `call'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.7/lib/elasticsearch/transport/transport/base.rb:187:in `perform_request'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.7/lib/elasticsearch/transport/transport/http/manticore.rb:33:in `perform_request'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/elasticsearch-transport-1.0.7/lib/elasticsearch/transport/client.rb:115:in `perform_request'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/elasticsearch-api-1.0.7/lib/elasticsearch/api/actions/bulk.rb:80:in `bulk'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-0.2.4-java/lib/logstash/outputs/elasticsearch/protocol.rb:100:in `bulk'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-0.2.4-java/lib/logstash/outputs/elasticsearch.rb:437:in `submit'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-0.2.4-java/lib/logstash/outputs/elasticsearch.rb:462:in `flush'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/stud-0.0.19/lib/stud/buffer.rb:219:in `buffer_flush'", "org/jruby/RubyHash.java:1341:in `each'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/stud-0.0.19/lib/stud/buffer.rb:216:in `buffer_flush'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/stud-0.0.19/lib/stud/buffer.rb:193:in `buffer_flush'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/stud-0.0.19/lib/stud/buffer.rb:159:in `buffer_receive'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-0.2.4-java/lib/logstash/outputs/elasticsearch.rb:426:in `receive'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-1.5.0-java/lib/logstash/outputs/base.rb:88:in `handle'", "(eval):159:in `output_func'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-1.5.0-java/lib/logstash/pipeline.rb:244:in `outputworker'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-1.5.0-java/lib/logstash/pipeline.rb:166:in `start_outputs'"], :level=>:warn}

top

  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
 1688 es        20   0 13.8g 1.4g  39m S 136.4  8.8  34:20.66 java

_

cat/health
1435761438 07:37:18 muostats green 2 2 1288 644 0 0 0 0 

_cat/nodes
ris-webstats03 127.0.0.1 98 39 1.04 d * ris-webstats03 
ris-webstats01 127.0.0.1 78 15 1.25 d m ris-webstats01

(Ned) #2

Hi Brian, were you ever able to fix this issue? I seem to be experiencing the same thing.


(system) #3