Parsing apache logs - COMBINEDAPACHELOG & COMMONAPACHELOG not working

Elastic Stack Logstash
1

I am currently trying to parse Apache access and error logs. I have used both COMBINEDAPACHELOG and COMMONAPACHELOG but no luck. For the access logs I am trying to break it down to show timestamp, source, target, useragent and message. Similar parsing for the Apache error logs. I set it to display the parsed logs on the console but nothing is displaying. Any help would be appreciated in probably parsing these logs.

apache.conf

input {
        file {
                path => [ "/logs/apache24inst0/httpd0_access.log", "/logs/apache24inst0/httpds0_access.log" ]
                type => "apache-access"
                ignore_older => 7776000
                start_position => "beginning"
                sincedb_path => "/dev/null"
        }
        file {
                path => [ "/logs/apache24inst0/httpd0_error.log", "/logs/apache24inst0/httpds0_error.log" ]
                type => "apache-error"
                ignore_older => 7776000
                start_position => "beginning"
                sincedb_path => "/dev/null"
        }	
}
filter {
        # APACHE 2
        if [type] == "apache-access" {
                # To process log data (message's content) using some regex or precompiled GROK pattern
                grok {
                        match => { "message" => [ "message", "%{COMBINEDAPACHELOG}"] }
                }
                # To extract log's time according to a date pattern
                date {
                        match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z"]
                }
                # Extraction browser information, if available.
                if [agent] != "" {
                        useragent {
                                source => "agent"
                        }
                }
                if [clientip] != "" {
                        geoip {
                                source => "clientip"
                                target => "apache_clientip"
                                add_tag => [ "geoip" ]
                        }
                }
        }
		if [type] == 'apache_error' {
			grok {
				match => [ 'message', '\[(?<timestamp>%{DAY:day} %{MONTH:month} %{MONTHDAY} %{TIME} %{YEAR})\] \[%{WORD:class}\] \[%{WORD:originator} %{IP:clientip}\] %{GREEDYDATA:errmsg}']
			}
		}
}
output {
        stdout { codec => rubydebug { metadata => true } }
}

Excerpt of the raw access logs:

[20/Feb/2017:03:42:38 -0600] : server12354-http : 169.98.93.20 : 3 : GET : 172.30.196.72 : + : "GET /favicon.ico HTTP/1.1" 404 209 "-" "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0"
[20/Feb/2017:03:42:39 -0600] : server12354-http : 169.98.93.20 : 4 : GET : 172.30.196.72 : + : "GET /favicon.ico HTTP/1.1" 404 209 "-" "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0"

Excerpt of the raw error logs:

[Mon Jan 23 08:24:20.147296 2017] [reqtimeout:info] [pid 3213:tid 140039965009664] [client 10.47.240.36:50895] AH01382: Request header read timeout
[Mon Apr 03 17:26:41.406597 2017] [mpm_worker:notice] [pid 2113:tid 140466427148096] AH00292: Apache/2.4.18 (Unix) OpenSSL/1.0.2f configured -- resuming normal operations

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.