Logstash keeps crashing with the same Fatal Error

[whicter (Viktor)]

Hello,
My logstash keeps crashing with the same reason and I couldn't find the reason. Could someone help? The log is as below:

[2018-03-06T17:44:09,393][FATAL][logstash.runner ] An unexpected error occurred! {:error=>#<SystemCallError: Unknown error (SystemCallError) - >, :backtrace=>["org/jruby/RubyIO.java:1457:in write'", "org/jruby/RubyIO.java:1428:inwrite'", "/scratch/elk/logstash-6.2.2/vendor/bundle/jruby/2.3.0/gems/logstash-output-stdout-3.1.3/lib/logstash/outputs/stdout.rb:44:in block in multi_receive_encoded'", "org/jruby/RubyArray.java:1734:ineach'", "/scratch/elk/logstash-6.2.2/vendor/bundle/jruby/2.3.0/gems/logstash-output-stdout-3.1.3/lib/logstash/outputs/stdout.rb:43:in multi_receive_encoded'", "/scratch/elk/logstash-6.2.2/logstash-core/lib/logstash/outputs/base.rb:90:inmulti_receive'", "/scratch/elk/logstash-6.2.2/logstash-core/lib/logstash/output_delegator_strategies/single.rb:15:in block in multi_receive'", "org/jruby/ext/thread/Mutex.java:148:insynchronize'", "/scratch/elk/logstash-6.2.2/logstash-core/lib/logstash/output_delegator_strategies/single.rb:14:in multi_receive'", "/scratch/elk/logstash-6.2.2/logstash-core/lib/logstash/output_delegator.rb:49:inmulti_receive'", "/scratch/elk/logstash-6.2.2/logstash-core/lib/logstash/pipeline.rb:479:in block in output_batch'", "org/jruby/RubyHash.java:1343:ineach'", "/scratch/elk/logstash-6.2.2/logstash-core/lib/logstash/pipeline.rb:478:in output_batch'", "/scratch/elk/logstash-6.2.2/logstash-core/lib/logstash/pipeline.rb:430:inworker_loop'", "/scratch/elk/logstash-6.2.2/logstash-core/lib/logstash/pipeline.rb:385:in `block in start_workers'"]}
[2018-03-06T17:44:10,038][ERROR][org.logstash.Logstash ] java.lang.IllegalStateException: org.jruby.exceptions.RaiseException: (SystemExit) exit
~

Thanks

[whicter (Viktor)]

Could someone please help?

[tag_v]

@whicter

`block in start_workers'

how your conf file looks?

[whicter (Viktor)]

Here is my logstash conf file:

input {
    beats {
        port => "5044"
    }
}
filter {

    grok {
        match => { "message" => "\A%{IP:ip} - %{PROG:unknown0} \[%{HTTPDATE:http_date}] %{PROG:ecid} %{URIPROTO:http_protocol} %{URIPATHPARAM:uri_path}%{SPACE}%{EMAILLOCALPART:http_version} %{POSINT:http_status} %{PROG:byte_size} \*\*%{SYSLOGPROG:rounded_response_time}/%{SYSLOGPROG:response_time}\*\* %{PROG:place1} %{PROG:userid} %{PROG:unknownid1} - %{PROG:locale}" }

        match => { "message" => "\A%{IP:ip} - %{PROG:unknown0} \[%{HTTPDATE:http_date}] %{PROG:ecid} %{URIPROTO:http_protocol} %{URIPATHPARAM:uri_path}%{SPACE}%{EMAILLOCALPART:http_version} %{POSINT:http_status} %{PROG:byte_size} \*\*%{SYSLOGPROG:rounded_response_time}/%{SYSLOGPROG:response_time}\*\* %{PROG:place1} %{PROG:userid} - %{PROG:unknownid1} %{PROG:locale}" }

        match => { "message" => "\A%{IP:ip} - %{PROG:unknown0} \[%{HTTPDATE:http_date}] %{PROG:ecid} %{URIPROTO:http_protocol} %{URIPATHPARAM:uri_path}%{SPACE}%{EMAILLOCALPART:http_version} %{POSINT:http_status} %{PROG:byte_size} \*\*%{SYSLOGPROG:rounded_response_time}/%{SYSLOGPROG:response_time}\*\* %{PROG:place1} %{PROG:userid} %{PROG:unknownid1} %{PROG:locale}" }

        remove_field => ["program", "prospector", "beat", "tags", "rounded_response_time"]
    }
    geoip {
        source => "ip"
    }

    date {
        match => ["http_date", "dd/MMM/yyyy:HH:mm:ss Z"]
        target => "@timestamp"
    }

    mutate {
        convert => { "response_time" => "float" }
    }

    if [userid] == "-" {
        ruby {
            code => "event.set('userid', '')"
        }
    }

    if [locale] == "-" {
        ruby {
            code => "event.set('locale', '')"
        }
    }


    if "_grokparsefailure" in [tags] {
        mutate {
            add_field => { "parsing_error" => "grokparsefailure" }
        }
    }

    else {
        ruby {
            code => "event.set('response_time', event.get('response_time')/1000000)"
        }
    }

    fingerprint {
        source => "message"
        target => "[@metadata][fingerprint]"
        method => "SHA1"
        key => "Log analytics"
        base64encode => true
    }
}


output {
    stdout { codec => rubydebug }
    elasticsearch {
        hosts => [ "rno214009.us.oracle.com:9200" ]
        user => "elastic"
        password => "elastic"
        document_id => "%{[@metadata][fingerprint]}"
    }
}