Logstash service stops after few minutes

Elastic Stack Logstash
1

Hello Greetings,

Can some one help us to fix as we start logstash service it stops with in few minutes and logs shows with the below error,

{:timestamp=>"2015-11-30T13:04:07.450000-0600", :message=>"The error reported is: \n pattern %{HOST:WebSiteName} not defined"}
~

2

What version are you on, what does your config look like?

3

Version: logstash1.5

30-lumberjack-output.conf
output {

stdout { codec => rubydebug }

if [URI] != ""{
elasticsearch {
host => "testlogs.com"
cluster => "ElasticLogs"
protocol=> "http"
index => "weblogs-%{+YYYY.MM.dd}"
manage_template => "false"
}
}

else{
elasticsearch {
host => "testlogs.com"
cluster => "ElasticLogs"
protocol=> "http"
index => "errorlog-%{+YYYY.MM.dd}"
manage_template => "false"
}
}
}

01-lumberjack-input.conf

input {
lumberjack {
port => 5043
type => "logs"
ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
}
}

4

Okay, those are the input and outputs but what about the grok filters (or whatever is producing the error message)?

5

here you go .

filter {
grok {

    match => {
            'message' => '\A%{TIMESTAMP_ISO8601:DateTime}%{SPACE}%{URIPATHPARAM:URI}%{SPACE}%{INT:Status}%{SPACE}%{QUOTEDSTRING:ComputerName}%{SPACE}%{QUOTEDSTRING:Referer}%{SPACE}%{INT:Win32Status}%{SPACE}%{NUMBER:BytesSent}%{SPACE}%{NUMBER:BytesReceived}%{SPACE}%{QUOTEDSTRING:UserAgent}%{SPACE}%{IP:ServerIP}%{SPACE}%{INT:ServerPort}%{SPACE}%{QUOTEDSTRING:Protocol}%{SPACE}%{PROG:Method}%{SPACE}%{IP:ClientIP}%{SPACE}%{NUMBER:TimeTaken}%{SPACE}%{NUMBER:RequestPerSecond}%{SPACE}%{HOST:WebSiteName}%{SPACE}%{GREEDYDATA:QRY}'

    }

}

# The timestamp may have commas instead of dots. Convert so as to store everything in the same way

    mutate {
            gsub => [
            # replace all commas with dots
            "DateTime", ",", "."
            ]
    }

    mutate {
            gsub => [
            # make the logTimestamp sortable. With a space, it is not! This does not work that well, in the end

            # but somehow apparently makes things easier for the date filter
            "DateTime", " ", ";"
            ]
    }

# User Agent

    useragent {
            add_tag => [ "UA" ]
            source => "UserAgent"
    }

    date {

            locale => "en"
            match => ["DateTime", "YYYY-MM-dd;HH:mm:ss.SSS"]
            target => "@timestamp"
    }

    if [type] == "IISlogs" {
            if [UserAgent] != "-" and [UserAgent] != "" {
                    useragent {
                    add_tag => [ "UA" ]
                    source => "UserAgent"
                    }
            }

            if "UA" in [tags] {
                    if [device] == "Other" { mutate { remove_field => "device" } }
                    if [name]   == "Other" { mutate { remove_field => "name" } }
                    if [os]     == "Other" { mutate { remove_field => "os" } }
            }
    }

    geoip {
            source => "ClientIP"
            target => "geoip"
            database =>"/etc/logstash/GeoLiteCity.dat"
	add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
            add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
    }

    mutate {
            convert => [ "[geoip][coordinates]", "float" ]
            convert => [ "BytesSent", "integer" ]
            convert => [ "BytesReceived", "integer" ]
            convert => [ "RequestsPerSecond", "integer" ]
    }

}

6

That's odd. HOST is definitely a valid pattern in Logstash 1.5. Does the problem go away if you delete that part from the grok expression? Have you double-checked that you don't have any weird non-printable characters in the expression? What's actually in your grok pattern files (have a look in /opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-*/patterns/grok-patterns)?

7

Yes, I tried {HOST:WebSiteName} changing this Grok pattern to {HOSTNAME:WebSiteName} and it works like charm.

Thanks.