GROK match multiple pattern drop the log

Elastic Stack Logstash
1

I used following grok pattern to extract IIS log. The match message containing multiple grok pattern due to IIS log configuration on several server.

Unfortunately one I have execute this filter, no messages were parsers. dropping all iis log without parsing.

Once I execute this code without grok messages are displaying.

Any issue on this code ?

filter {

    if ([type] == "iis-log" or [type] == "iis_log") 
    {
	if "beats_input_codec_plain_applied" in [tags] {
		mutate {
			remove_tag => ["beats_input_codec_plain_applied"]
		}
	}
					
	grok 
	{
		match => ["message", "%{TIMESTAMP_ISO8601:log_timestamp} %{WORD:iis_site} %{IPORHOST:dstip} %{WORD:method} %{URIPATH:page} %{NOTSPACE:querystring} %{NUMBER:dstport} %{NOTSPACE:username} %{IPORHOST:load_balancer_ip} %{NOTSPACE:useragent} %{NOTSPACE:request_host} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:windows_status} %{NUMBER:sent_bytes} %{NUMBER:input_bytes} %{NUMBER:time_taken} %{IPORHOST:srcip},\+%{IPORHOST:waf_ip}",
			  "message", "%{TIMESTAMP_ISO8601:log_timestamp} %{IPORHOST:dstip} %{WORD:method} %{URIPATH:page} %{NOTSPACE:querystring} %{NUMBER:dstport} %{NOTSPACE:username} %{IPORHOST:load_balancer_ip} %{NOTSPACE:useragent} %{NOTSPACE:request_host} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:windows_status} %{NUMBER:sent_bytes} %{IPORHOST:srcip},\+%{IPORHOST:waf_ip}",
			  "message", "%{TIMESTAMP_ISO8601:log_timestamp} %{WORD:iis_site} %{IPORHOST:dstip} %{WORD:method} %{URIPATH:page} %{NOTSPACE:querystring} %{NUMBER:dstport} %{NOTSPACE:username} %{IPORHOST:load_balancer_ip} %{NOTSPACE:useragent} %{NOTSPACE:request_host} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:windows_status} %{NUMBER:sent_bytes} %{NUMBER:input_bytes} %{NUMBER:time_taken}",
			  "message", "%{TIMESTAMP_ISO8601:log_timestamp} %{IPORHOST:dstip} %{WORD:method} %{URIPATH:page} %{NOTSPACE:querystring} %{NUMBER:dstport} %{NOTSPACE:username} %{IPORHOST:load_balancer_ip} %{NOTSPACE:useragent} %{NOTSPACE:request_host} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:windows_status} %{NUMBER:sent_bytes}"
		]
	}
				
	mutate 
	{
		convert => [ "dstport", "integer"]
		convert => [ "response", "integer"]
		convert => [ "sent_bytes", "integer"]
		convert => [ "input_bytes", "integer"]
		convert => [ "time_taken", "integer"]
		convert => [ "subresponse", "integer"]
		convert => [ "windows_status", "integer"]	
	}
		
	mutate {
		remove_field => ["[beat][hostname]", "[beat][name]", "[beat][version]", "[beat]", "loadbalancer_ip" ]
	}

    }
}
2

I would write that match as

match => {
    "message" => {
        "%{TIMESTAMP_ISO8601:log_timestamp} %{WORD:iis_site} %{IPORHOST:dstip} %{WORD:method} %{URIPATH:page} %{NOTSPACE:querystring} %{NUMBER:dstport} %{NOTSPACE:username} %{IPORHOST:load_balancer_ip} %{NOTSPACE:useragent} %{NOTSPACE:request_host} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:windows_status} %{NUMBER:sent_bytes} %{NUMBER:input_bytes} %{NUMBER:time_taken} %{IPORHOST:srcip},\+%{IPORHOST:waf_ip}",
        "%{TIMESTAMP_ISO8601:log_timestamp} %{IPORHOST:dstip} %{WORD:method} %{URIPATH:page} %{NOTSPACE:querystring} %{NUMBER:dstport} %{NOTSPACE:username} %{IPORHOST:load_balancer_ip} %{NOTSPACE:useragent} %{NOTSPACE:request_host} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:windows_status} %{NUMBER:sent_bytes} %{IPORHOST:srcip},\+%{IPORHOST:waf_ip}", 
        "%{TIMESTAMP_ISO8601:log_timestamp} %{WORD:iis_site} %{IPORHOST:dstip} %{WORD:method} %{URIPATH:page} %{NOTSPACE:querystring} %{NUMBER:dstport} %{NOTSPACE:username} %{IPORHOST:load_balancer_ip} %{NOTSPACE:useragent} %{NOTSPACE:request_host} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:windows_status} %{NUMBER:sent_bytes} %{NUMBER:input_bytes} %{NUMBER:time_taken}",
        "%{TIMESTAMP_ISO8601:log_timestamp} %{IPORHOST:dstip} %{WORD:method} %{URIPATH:page} %{NOTSPACE:querystring} %{NUMBER:dstport} %{NOTSPACE:username} %{IPORHOST:load_balancer_ip} %{NOTSPACE:useragent} %{NOTSPACE:request_host} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:windows_status} %{NUMBER:sent_bytes}"
    ]
}

Also, I would strongly recommend you anchor your patterns to start of line ("^%{TIMESTAMP_ISO8601:log_timestamp} ...") for reasons explained here.

3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.