Syslog + Apache log => _Grokparsefailure

Elastic Stack Logstash
1

Hi,

I'm trying to parse an Apache log that was sent by syslog but I get a _grokparsefailure error in Kibana and nothing in logstash log. Yet I tested the grok patterns on http://grokconstructor.appspot.com with several lines of log and everything was OK.

Below is a log line that I want to parse

Jun 8 15:57:51 zabbix apache-access: 192.168.0.4 - - [08/Jun/2017:15:57:42 +0200] "POST /zabbix.php?action=widget.hosts.view&sid=4c7f4ebca8593738&upd_counter=5561&pmasterid=dashboard HTTP/1.1" 200 14017 "http://zabbix.alta/zabbix.php?action=dashboard.view" "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0"

and my logstash filter is

filter {
 if [type] == "syslog" {
   grok {
     match => { "message" => "%{SYSLOGBASE} %{COMBINEDAPACHELOG}" }
   }
 }

}

How can i fix this?

Sorry for my English (Google traductor)

2

Works fine for me with both Logstash 2.3.4 and 5.4.1. Please use a stdout { codec => rubydebug } output and show what it produces.

3

I changed the config file to use the "program" field instead of the "type" field.
Below is the complete config file with rubydebug output
texte préformaté indenté par 4 espaces
input {
syslog {
type => syslog
port => 5140
}
}

filter {
if [program] == "apache-access" { grok { match => { "message" => "%{SYSLOGBASE} %{COMBINEDAPACHELOG}" } } }
else if [program] == "apache-error" {
grok {
match => { "message" => "%{HTTPD24_ERRORLOG}" }
patterns_dir => ["/etc/logstash/patterns"]
}
}

mutate {
convert => { "response" => "integer" }
convert => { "bytes" => "integer" }
}

geoip {
source => "clientip"
target => "geoip"
add_tag => ["apache-geoip"]
}

date {
match => [ "timestamp" , "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]

remove_field => [ "timestamp" ]

}

useragent {
source => "agent"
}
}

output {
stdout { codec => rubydebug }
}

4 
{
      "severity" => 6,
         "geoip" => {},
       "program" => "apache-access",
       "message" => "192.168.0.101 - - [12/Jun/2017:23:23:57 +0200] \"POST /jsrpc.php?output=json-rpc HTTP/1.1\" 200 901 \"http://zabbix.alta/zabbix.php?action=dashboard.view\" \"Mozilla/5.0 (Windows NT 6.3; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0\"",
          "type" => "syslog",
      "priority" => 134,
     "logsource" => "zabbix",
          "tags" => [
    [0] "_grokparsefailure",
    [1] "_geoip_lookup_failure"
],
    "@timestamp" => 2017-06-14T08:17:48.000Z,
      "@version" => "1",
          "host" => "192.168.0.17",
      "facility" => 16,
"severity_label" => "Informational",
     "timestamp" => "Jun 14 10:17:48",
"facility_label" => "local0"

}

5

I repost it with the right format

input {
   syslog {
  type => syslog
  port => 5140
   }
}


filter {
 if [program] == "apache-access" { grok { match => { "message" => "%{SYSLOGBASE} %{COMBINEDAPACHELOG}" } } }
 else if [program] == "apache-error" {
  grok {
match => { "message" => "%{HTTPD24_ERRORLOG}" }
patterns_dir => ["/etc/logstash/patterns"]
  }
 }

mutate {
 convert => { "response" => "integer" }
 convert => { "bytes" => "integer" }
}

geoip {
 source => "clientip"
 target => "geoip"
 add_tag => ["apache-geoip"]
}

date {
 match => [ "timestamp" , "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
#     remove_field => [ "timestamp" ]
}

useragent {
  source => "agent"
  }
}
output {
  stdout { codec => rubydebug }
}
6

Do you really need %{SYSLOGBASE} in your grok expression? I'm pretty sure the syslog input strip the syslog "header" and makes sure only the message payload itself ends up in message. The evidence you've presented suggests that such is the case.

7

Effectively logstash succeeded in recognizing that it is a syslog file and only the apache part remains to be parsed.
Thank you

8

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.