Hello,
I am new to logstash and trying to filter the haproxy logs.
Example of my haproxy logs.
<134>0 2018-05-29T03:45:29+02:00 localhost epic-webservicefrontend 18645 - [type=haproxy] [18645] [1527558316.602] 3/3/1/0/0/0/0 5/0/0/4454/12916/{|gwlprod|Rjg2MzYyRDBDQTQ0NEEzREI3OUQzRDM0QjU5QzcyRDB8ZXBpY19sdHxlcGljX2x0fHx8MHw=|5b946a6b-2663-4c03-b31b-3938833514c3}/ --NI 128.87.242.20:39120 128.87.242.31:443 128.87.242.25:8011 https-in~ RequestCookies=- | "POST /3dspace/ericsson_services/Product?WSDL HTTP/1.1" 200 | backend_pool_sit2_3dspacebatch:SIT2_3dspace_BWS_Front_1
My Grok Expression:
grok {
match => ["message", "%{NOTSPACE:string} %{TIMESTAMP_ISO8601:timestamp8601} %{IPORHOST:syslog_server} %{SYSLOGPROG:serviceend} %{INT:HaproxyPID} - %{NOTSPACE:type} %{INT:Pid} %{INT:Ts}:%{INT:ms} %{INT:ac}/%{INT:fc}/%{INT:bc}/%{INT:bq}/%{INT:sc}/%{INT:sq}%{INT:rc} %{INT:Tq}/%{INT:Tw}/%{INT:Tc}/%{INT:Tr}/%{INT:Tt}/{|%{USER:http_user}|%{DATA:Headers}/%{INT:hs} %{DATA:tsc} %{IP:client_ip}:%{INT:client_port} %{IP:frontend_ip}:%{INT:frontend_port} %{IP:server_ip}:%{INT:server_port} %{DATA:transfer_type} [RequestCookies=- | %{DATA:cookie} %{INT:http_status_code} |]%{NOTSPACE:backendserver}:%{NOTSPACE:backendservice}"]
}
Result while starting logstash.
[2018-06-01T11:02:07,611][ERROR][logstash.pipeline ] Pipeline aborted due to error {:pipeline_id=>"main", :exception=>#<RegexpError: empty range in char class: /(?NOTSPACE:string\S+) (?<TIMESTAMP_ISO8601:timestamp8601>(?:(?>\d\d){1,2})-(?:(?:0?[1-9]|1[0-2]))-(?:(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]))T :?(?:(?:[0-5][0-9]))(?::?(?:(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)))?(?:(?:Z|+-(?::?(?:(?:[0-5][0-9])))))?) (?IPORHOST:syslog_server(?:(?:(?:(?:((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?)|(?:(?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])...)(?![0-9]))))|(?:\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))(.?|\b)))) (?SYSLOGPROG:serviceend(?PROG:program[\x21-\x5A\x5C\x5E-\x7E]+)(?:[(?POSINT:pid\b(?:[1-9][0-9])\b)])?) (?INT:HaproxyPID(?:[+-]?(?:[0-9]+))) - (?NOTSPACE:type\S+) (?INT:Pid(?:[+-]?(?:[0-9]+))) (?INT:Ts(?:[+-]?(?:[0-9]+))):(?INT:ms(?:[+-]?(?:[0-9]+))) (?INT:ac(?:[+-]?(?:[0-9]+)))/(?INT:fc(?:[+-]?(?:[0-9]+)))/(?INT:bc(?:[+-]?(?:[0-9]+)))/(?INT:bq(?:[+-]?(?:[0-9]+)))/(?INT:sc(?:[+-]?(?:[0-9]+)))/(?INT:sq(?:[+-]?(?:[0-9]+)))(?INT:rc(?:[+-]?(?:[0-9]+))) (?INT:Tq(?:[+-]?(?:[0-9]+)))/(?INT:Tw(?:[+-]?(?:[0-9]+)))/(?INT:Tc(?:[+-]?(?:[0-9]+)))/(?INT:Tr(?:[+-]?(?:[0-9]+)))/(?INT:Tt(?:[+-]?(?:[0-9]+)))/{|(?USER:http_user(?:[a-zA-Z0-9._-]+))|(?<DATA:Headers>.?)/(?INT:hs(?:[+-]?(?:[0-9]+))) (?<DATA:tsc>.?) (?IP:client_ip(?:(?:((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?)|(?:(?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])...)(?![0-9])))):(?INT:client_port(?:[+-]?(?:[0-9]+))) (?IP:frontend_ip(?:(?:((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-