I have two patterns for the logs file. Grok debugger shows a match, but I see Kibana show grokparsefailure for pattern TEST2. What could be the problem?
input {
file {
type => "nginx_access"
path => "/var/log/nginx/*access*"
start_position => beginning
}
}
filter {
if [type] == "nginx_access" {
grok {
match => { "message" => "%{TEST1}" }
}
grok {
match => { "message" => "%{TEST2}" }
}
date {
match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
remove_field => [ "timestamp" ]
}
geoip {
source => "clientip"
}
}
output {
stdout { codec => rubydebug }
elasticsearch { hosts => ["localhost:9200"] }
}
TEST1 %{IP:client_1} %{WORD:method1} %{WORD:method} %{IP:client_2} - %{NOTSPACE:remote_user} \[%{HTTPDATE:timestamp}\](?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})"
TEST2 %{IP:client_2} %{WORD:method1} %{WORD:method} - - %{NOTSPACE:remote_user} [%{HTTPDATE:timestamp}](?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})"
TEST1 Oct 12 18:09:20 nginx: 1.1.1.1 forwarded for 10.10.10.10 - - [12/Oct/2016:18:09:16 +0300] "GET /images/redesign/key.png HTTP/1.0" 304 0 "https://test.com" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36
TEST2 "Oct 12 17:56:39 nginx: 1.1.1.1 forwarded for - - - [12/Oct/2016:17:56:32 +0300] "POST /process/index/gateId/1/lang/ru HTTP/1.1" 200 1150 "https://test.com" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36""