Assistance with grokparsefailure

[unknown_user]

Hi All,

Very new to using logstash, so would appreciate any help.

Would would be the best way to parse this file? I'm still getting some grokparsefailures but it is picking up everything except besides the "cpu" which does seem to match on grokconstructor.

Log file:

2019-08-26 11:11:45 hostname vv admin 2 14 0.05 0
2019-08-26 11:11:51 hostname cpu 0 0 1 99
2019-08-26 11:11:56 hostname cmp 0 8020 93  3493 45 63825 852605 8 1 261 143 0 0 262 0 120 0     0 0 0 0
2019-08-26 11:11:56 hostname capvv admin 0 0 0 0 10240 10240 10240
2019-08-26 11:11:56 hostname cappd FC 24 41164800 22162432 3430400 15570944 0 1024

config:

if [type] == "3parperf" {

	grok {
			match => ["message", "%{TIMESTAMP_ISO8601} %{WORD:hostname} cappd %{WORD:tier} %{NUMBER:ndisks:float} %{NUMBER:size:float} %{NUMBER:volume:float} %{NUMBER:spare:float} %{NUMBER:free:float} %{NUMBER:unavail:float} %{NUMBER:failed:float}"]
			add_field => ["monitor", "cappd"]
	}
	grok {
			match => ["message", "%{TIMESTAMP_ISO8601} %{WORD:hostname} vv %{NOTSPACE:volume} %{NUMBER:iops:float} %{NUMBER:kbps:float} %{NUMBER:service_times:float} %{NUMBER:queue_length:float}"]
			add_field => ["monitor", "vv"]
	}
	grok {		
			match => ["message", "%{TIMESTAMP_ISO8601} %{WORD:hostname} capvv %{NOTSPACE:volume} %{NUMBER:admrsvd:float} %{NUMBER:admused:float} %{NUMBER:snaprsvd:float} %{NUMBER:snapused:float} %{NUMBER:usrrsvd:float} %{NUMBER:usrused:float} %{NUMBER:vsize:float}"]
			add_field => ["monitor", "capvv"]
	}
	grok {
			match => ["message", "%{TIMESTAMP_ISO8601} %{WORD:hostname} cpu %{NUMBER:node} %{NUMBER:user:float} %{NUMBER:sys:float} %{NUMBER:idle:float}"]
			add_field => ["monitor", "cpu"]
	}
	grok {
			match => ["message", "%{TIMESTAMP_ISO8601} %{WORD:hostname} cmp %{NUMBER:node} %{NUMBER:rdaccesses:float} %{NUMBER:rdhit:float} \s* %{NUMBER:wraccesses:float} %{NUMBER:wrhit:float} %{NUMBER:free:float} %{NUMBER:clean:float} %{NUMBER:write1:float} %{NUMBER:writen:float} %{NUMBER:wrtsched:float} %{NUMBER:writing:float} %{NUMBER:dcowpend:float} %{NUMBER:dcowproc:float} %{NUMBER:cfcdirtyt1:float} %{NUMBER:cfcdirtyt2:float} %{NUMBER:cfcdirtyt3:float} %{NUMBER:cfcdirtyt4:float} %{NUMBER:delackt1:float} %{NUMBER:delackt2:float} %{NUMBER:delackt3:float} %{NUMBER:delackt4:float}"]	
			add_field => ["monitor", "cmp"]
	}

}

[unknown_user]

I may have my own solution, i assume that the other groks don't match and tag a failure?

[Badger]

If all the lines have different numbers of fields, then you can match against an array of patterns. Also, I would use dissect to parse the fixed format prefix to the messages.

    dissect { mapping => { "message" => "%{[@metadata][timestamp]} %{+[@metadata][timestamp]} %{hostname} %{[@metadata][restOfLine]}" } }
    date { match => [ "[@metadata][timestamp]", "yyyy-MM-dd HH:mm:ss" ] }
    grok {
        match => {
            "[@metadata][restOfLine]" => [
        "^%{WORD:monitor} %{NUMBER:node} %{NUMBER:rdaccesses:float} %{NUMBER:rdhit:float} \s* %{NUMBER:wraccesses:float} %{NUMBER:wrhit:float} %{NUMBER:free:float} %{NUMBER:clean:float} %{NUMBER:write1:float} %{NUMBER:writen:float} %{NUMBER:wrtsched:float} %{NUMBER:writing:float} %{NUMBER:dcowpend:float} %{NUMBER:dcowproc:float} %{NUMBER:cfcdirtyt1:float} %{NUMBER:cfcdirtyt2:float} %{NUMBER:cfcdirtyt3:float} %{NUMBER:cfcdirtyt4:float}\s+%{NUMBER:delackt1:float} %{NUMBER:delackt2:float} %{NUMBER:delackt3:float} %{NUMBER:delackt4:float}$",
        "^%{WORD:monitor} %{NOTSPACE:volume} %{NUMBER:admrsvd:float} %{NUMBER:admused:float} %{NUMBER:snaprsvd:float} %{NUMBER:snapused:float} %{NUMBER:usrrsvd:float} %{NUMBER:usrused:float} %{NUMBER:vsize:float}$",
        "^%{WORD:monitor} %{WORD:tier} %{NUMBER:ndisks:float} %{NUMBER:size:float} %{NUMBER:volume:float} %{NUMBER:spare:float} %{NUMBER:free:float} %{NUMBER:unavail:float} %{NUMBER:failed:float}$",
        "^%{WORD:monitor} %{NOTSPACE:volume} %{NUMBER:iops:float} %{NUMBER:kbps:float} %{NUMBER:service_times:float} %{NUMBER:queue_length:float}$",
        "^%{WORD:monitor} %{NUMBER:node} %{NUMBER:user:float} %{NUMBER:sys:float} %{NUMBER:idle:float}$"
            ]
        }
    }