Greetings,
I am trying to parse log records with the following format and capture the log message date
in @timestamp:
May 24 00:00:00 localhost haproxy-https[7130]: 92.34.251.94 - - [24/May/2020:07:00:00 +0000] "GET /search/suggest/grazopr HTTP/1.1" 200 736 "" "" 64182 621 "https~" "server-backend" "10.100.30.21:80" 0 0 1 20 21 --NI 48 45 1 0 0 0 0 "" ""
May 24 00:00:00 localhost haproxy-https[7130]: 92.34.251.94 - - [24/May/2020:07:00:00 +0000] "GET /search/suggest/grazopre HTTP/1.1" 200 736 "" "" 64182 645 "https~" "server-backend" "10.100.30.21:80" 0 0 1 30 31 --NI 45 41 2 1 0 0 0 "" ""
May 24 00:00:01 localhost haproxy-https[7130]: 92.34.251.94 - - [24/May/2020:07:00:01 +0000] "GET /search/suggest/grazoprev HTTP/1.1" 200 736 "" "" 64182 927 "https~" "server-backend" "10.100.30.21:80" 0 0 1 26 27 --NI 45 41 2 1 0 0 0 "" ""
.. using the following configuration file. The grok filter successfully parses the record; however,
I am unable to use the following date filter to update the @timestamp with HTTPDATE element
from the log message. I do not get any messages of parse failure indications in the output
records that are attached at the end of this post. I continue to get the load time value
in the @timestamp field. I have patterns ignoring time zone and even patterns that don't match to try and generate some error indications without success. It is as if the plugin is not working or I am completely misunderstanding how to use the tool.
Any help would be greatly appreciated in trying to troubleshoot this issue (latest logstash and macos software).
input {
file {
path => "/Users/jwest/Desktop/ELK/east-haproxy-test.log"
start_position => "beginning"
sincedb_path => "/tmp/mydevnull"
}
}
filter {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:http_timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} %{NUMBER:client_port} %{NUMBER:date_milliseconds} \"(%{NOTSPACE:frontend_transport})\" \"(%{NOTSPACE:backend_name})\" \"(%{IP:server_ip}:%{NUMBER:server_port})\" %{NUMBER:time_recieve} %{NUMBER:Tw} %{NUMBER:Tc} %{NUMBER:time_respond} %{NUMBER:active_request_time} %{NOTSPACE:termination_state_w_cookie} %{NUMBER:active_connections} %{NUMBER:frontend_connections} %{NUMBER:backend_connections} %{NUMBER:server_connections} %{NUMBER:retries} %{NUMBER:server_queue} %{NUMBER:backend_queue}%{GREEDYDATA:message_remaining}"}
}
date {
match => [ "http_timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
target => "@timestamp"
}
}
output {
stdout { codec => rubydebug }
}
Output stream from Logstash:
Sending Logstash logs to /usr/local/Cellar/logstash-full/7.7.0/libexec/logs which is now configured via log4j2.properties
[2020-05-25T15:59:32,560][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2020-05-25T15:59:32,658][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"7.7.0"}
[2020-05-25T15:59:34,227][INFO ][org.reflections.Reflections] Reflections took 30 ms to scan 1 urls, producing 21 keys and 41 values
[2020-05-25T15:59:35,886][WARN ][org.logstash.instrument.metrics.gauge.LazyDelegatingGauge][main] A gauge metric of an unknown type (org.jruby.RubyArray) has been created for key: cluster_uuids. This may result in invalid serialization. It is recommended to log an issue to the responsible developer/development team.
[2020-05-25T15:59:35,898][INFO ][logstash.javapipeline ][main] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>12, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>1500, "pipeline.sources"=>["/Users/jwest/Desktop/ELK/logstash-haproxy-test.conf"], :thread=>"#<Thread:0x14ac2af9 run>"}
[2020-05-25T15:59:36,823][INFO ][logstash.javapipeline ][main] Pipeline started {"pipeline.id"=>"main"}
[2020-05-25T15:59:36,876][INFO ][filewatch.observingtail ][main][b45ceba3d1c9b2c05e0170e3cf960c803e6ebbadd8bd4879192c51be7ad9cb07] START, creating Discoverer, Watch with file and sincedb collections
[2020-05-25T15:59:36,879][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
[2020-05-25T15:59:37,179][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
/usr/local/Cellar/logstash-full/7.7.0/libexec/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated
{
"verb" => "POST",
"referrer" => "\"\"",
"server_ip" => "10.100.30.21",
"server_port" => "80",
"termination_state_w_cookie" => "--NI",
"host" => "rtt",
"path" => "/Users/jwest/Desktop/ELK/east-haproxy-test.log",
"syslog_timestamp" => "May 24 00:00:01",
"response" => "200",
"auth" => "-",
"date_milliseconds" => "441",
"Tw" => "0",
"Tc" => "1",
"time_recieve" => "0",
"message_remaining" => " \"\" \"\" ",
"bytes" => "977",
"@version" => "1",
"active_request_time" => "62",
"httpversion" => "1.1",
"server_queue" => "0",
"frontend_connections" => "41",
"ident" => "-",
"time_respond" => "57",
"clientip" => "92.34.251.94",
"logsource" => "localhost",
"pid" => "7130",
"program" => "haproxy-https",
"request" => "/search/data",
"@timestamp" => 2020-05-24T07:00:01.000Z,
"backend_connections" => "3",
"client_port" => "64182",
"frontend_transport" => "https~",
"retries" => "0",
"message" => "May 24 00:00:01 localhost haproxy-https[7130]: 92.34.251.94 - - [24/May/2020:07:00:01 +0000] \"POST /search/data HTTP/1.1\" 200 977 \"\" \"\" 64182 441 \"https~\" \"server-backend\" \"10.100.30.21:80\" 0 0 1 57 62 --NI 46 41 3 1 0 0 0 \"\" \"\" ",
"backend_queue" => "0",
"active_connections" => "46",
"server_connections" => "1",
"http_timestamp" => "24/May/2020:07:00:01 +0000",
"backend_name" => "server-backend",
"agent" => "\"\""
}
{
"verb" => "GET",
"referrer" => "\"\"",
"server_ip" => "10.100.30.21",
"server_port" => "80",
"termination_state_w_cookie" => "--NI",
"host" => "rtt",
"path" => "/Users/jwest/Desktop/ELK/east-haproxy-test.log",
"syslog_timestamp" => "May 24 00:00:01",
"response" => "200",
"auth" => "-",
"date_milliseconds" => "285",
"Tw" => "0",
"Tc" => "0",
"time_recieve" => "0",
"message_remaining" => " \"\" \"\" ",
"bytes" => "747",
"@version" => "1",
"active_request_time" => "19",
"httpversion" => "1.1",
"server_queue" => "0",
"frontend_connections" => "41",
"ident" => "-",
"time_respond" => "19",
"clientip" => "92.34.251.94",
"logsource" => "localhost",
"pid" => "7130",
"program" => "haproxy-https",
"request" => "/search/suggest/grazoprevir",
"@timestamp" => 2020-05-24T07:00:01.000Z,
"backend_connections" => "2",
"client_port" => "64182",
"frontend_transport" => "https~",
"retries" => "0",
"message" => "May 24 00:00:01 localhost haproxy-https[7130]: 92.34.251.94 - - [24/May/2020:07:00:01 +0000] \"GET /search/suggest/grazoprevir HTTP/1.1\" 200 747 \"\" \"\" 64182 285 \"https~\" \"server-backend\" \"10.100.30.21:80\" 0 0 0 19 19 --NI 45 41 2 1 0 0 0 \"\" \"\" ",
"backend_queue" => "0",
"active_connections" => "45",
"server_connections" => "1",
"http_timestamp" => "24/May/2020:07:00:01 +0000",
"backend_name" => "server-backend",
"agent" => "\"\""
}