my pattern seem to not work with logstash but works fine with grokdebbuger & consturctor.
I thought its matter of order and so and and removed all other patterns except grok parser/shortcut. But that doesn't seem to help either.
Any thoughts?
[root@tt home]# cat /etc/logstash/conf.d/02-filebeat-input.conf
input {
beats {
port => 5044
type => "logs"
ssl => true
ssl_certificate => "/etc/pki/tls/certs/filebeat.crt"
ssl_key => "/etc/pki/tls/private/filebeat.key"
}
}
[root@tt home]# cat /etc/logstash/conf.d/11-nginx.conf
filter {
if [type] == "Nginx-Access-Log" {
grok {
patterns_dir => ["/etc/logstash/patterns/"]
match => { "message" => [
"%{NGINXACCESS}"
]
}
add_field => {
"received_at" => "%{@timestamp}"
"received_from" => "%{host}"
}
}
}
geoip {
source => "xff_clientip"
target => "geoip"
database => "/etc/logstash/GeoLiteCity.dat"
}
}
[root@tt home]# cat /etc/logstash/conf.d/30-elasticsearch-output.conf
output {
elasticsearch { hosts => ["localhost:9200"] }
stdout { codec => rubydebug }
}
[root@tt home]# cat /etc/logstash/patterns/nginxaccess
NGUSERNAME [a-zA-Z.@-+_%]+
NGUSER %{NGUSERNAME}
NGINXACCESS %{IPORHOST:clientip} %{NOTSPACE:ident} %{NOTSPACE:auth} [%{HTTPDATE:timestamp}] "%{NOTSPACE:host}" "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response} (?:%{NUMBER:bytes}|-) (?:"(?:%{URI:referrer}|-)"|%{QS:referrer})(?:;|) %{QS:agent} "(?<x_forwarded_for>%{IP:xff_clientip}, .*)"
log format:
192.168.1.229 - - [12/Jan/2016:09:10:20 +0000] "testsite.com" "GET /images/someimages.jpg HTTP/1.1" 200 24028 "http://testsite.com/some/url/" "Some referer" "123.12.13.118, 24.24.24.24" "1452589820.216" "0.000"
Debug Output:
"message" => "192.168.1.229 - - [12/Jan/2016:09:10:20 +0000] "testsite.com" "GET /images/someimages.jpg HTTP/1.1" 200 24028 "http://testsite.com/some/url/" "Some referer" "123.12.13.118, 24.24.24.24" "1452589820.216" "0.000"",
"@version" => "1",
"@timestamp" => "2016-01-12T16:40:46.054Z",
"beat" => {
"hostname" => "testsite.com",
"name" => "testsite.com"
},
"count" => 1,
"fields" => {
"service" => "serviceNametag",
"zone" => "eu-west-1"
},
"input_type" => "log",
"offset" => 51046638,
"source" => "/var/logs/nginx/access.log",
"type" => "Nginx_Access_log",
"host" => "testsite.com"
}