stevetso
(Steve)
October 2, 2018, 11:45am
1
I have a message like this:
14/Jul/18 15:18:55 44.130.128.195 GET /url111 200 853 3 - - -
Filter:
filter {
grok {
match => { "message" => "%{MONTHDAY:day}/%{MONTH:month}/%{YEAR:year} %{TIME:time} %{IP:client_ip} %{WORD:method} %{NOTSPACE:uri_query} %{NUMBER:status} %{NUMBER:bytes} %{NUMBER:response_time}" }
}
date {
match => [ "logdate" , "dd/MMM/yy HH:mm:ss" ]
locale => "en"
timezone => "Asia/Hong_Kong"
target => "@timestamp "
}
}
In the output, the @timestamp is the parsing time instead of the timestamp in log message.
i.e.
Actual: "@timestamp " => 2018-10-02T11:37:25.708Z
Expected: "@timestamp " => 2018-07-14T15:18:55Z
Hope experts here can help.
stevetso
(Steve)
October 2, 2018, 2:30pm
2
Conf:
input {
file {
path => "/path/to/file"
start_position => "beginning"
}
}
filter {
grok {
match => { "message" => "%{MONTHDAY:day}/%{MONTH:month}/%{YEAR:year} %{TIME:time} %{IP:client_ip} %{WORD:method} %{NOTSPACE:uri_query} %{NUMBER:status} %{NUMBER:bytes} %{NUMBER:response_time}" }
}
date {
match => [ "timestamp" , "dd/MMM/yy HH:mm:ss" ]
locale => "en"
timezone => "Asia/Hong_Kong"
target => "@timestamp"
}
}
output {
stdout { codec => rubydebug }
}
Please show what your stdout output produces for an example input line.
stevetso
(Steve)
October 4, 2018, 7:28am
4
Conf:
input {
file {
path => "/var/path/to/file"
start_position => "beginning"
}
}
filter {
grok {
match => { "message" => "%{MONTHDAY:day}/%{MONTH:month}/%{YEAR:year} %{TIME:time} %{IP:client_ip} %{WORD:method} %{NOTSPACE:uri_query} %{NUMBER:status} %{NUMBER:bytes} %{NUMBER:response_time}" }
}
date {
match => [ "timestamp" , "dd/MMM/yy HH:mm:ss" ]
locale => "en"
timezone => "Asia/Hong_Kong"
}
geoip {
source => "client_ip"
}
}
output {
stdout { codec => rubydebug }
}
Output:
{
"time" => "15:18:55",
"@version" => "1",
"client_ip" => "203.198.53.198",
"path" => "/var/path/to/file",
"method" => "GET",
"day" => "14",
"status" => "200",
"geoip" => {
"latitude" => 22.291,
"ip" => "203.198.53.198",
"country_name" => "Hong Kong",
"city_name" => "Central District",
"continent_code" => "AS",
"longitude" => 114.15,
"timezone" => "Asia/Hong_Kong",
"country_code3" => "HK",
"country_code2" => "HK",
"location" => {
"lat" => 22.291,
"lon" => 114.15
}
},
"@timestamp" => 2018-10-04T07:22:06.690Z,
"month" => "Jul",
"host" => "ip-10-10-1-221",
"year" => "18",
"message" => "14/Jul/18 15:18:55 49.130.128.195 GET /xxx 200 100 3 - - -",
"uri_query" => "/xxx",
"bytes" => "100",
"response_time" => "3"
}
Your event doesn't contain a timestamp
field so the date filter obviously can't do anything. You can replace the beginning of your grok expression with
(?<timestamp>%{MONTHDAY}/%{MONTH}/%{YEAR} %{TIME})
to extract such a field that contains the whole timestamp.
stevetso
(Steve)
October 5, 2018, 9:37am
6
Thanks!!! It's work!!!!!
This technique is very useful when we deal with the non standard date time format.
Conf:
input {
file {
path => "/path/to/file"
start_position => "beginning"
}
}
filter {
grok {
match => { "message" => "(?<timestamp>%{MONTHDAY}/%{MONTH}/%{YEAR} %{TIME}) %{IP:client_ip} %{WORD:method} %{NOTSPACE:uri_query} %{NUMBER:status} %{NUMBER:bytes} %{NUMBER:response_time}" }
}
date {
match => [ "timestamp" , "dd/MMM/yy HH:mm:ss" ]
locale => "en"
timezone => "Asia/Hong_Kong"
target => "@timestamp"
}
geoip {
source => "client_ip"
}
mutate {
remove_field => [ "timestamp", "[geoip][ip]", "path", "logdate", "year", "month", "day", "time" ]
}
}
output {
stdout { codec => rubydebug }
}
Output:
{
"message" => "14/Aug/18 15:19:00 101.1.16.3 GET /url 200 853 4 - - -",
"bytes" => "853",
"host" => "ip-10-10-1-221",
"method" => "GET",
"@version" => "1",
"uri_query" => "/url",
"geoip" => {
"longitude" => 114.15,
"location" => {
"lon" => 114.15,
"lat" => 22.2833
},
"continent_code" => "AS",
"country_code3" => "HK",
"latitude" => 22.2833,
"timezone" => "Asia/Hong_Kong",
"country_name" => "Hong Kong",
"country_code2" => "HK"
},
"client_ip" => "101.1.16.3",
"@timestamp" => 2018-08-14T07:19:00.000Z,
"response_time" => "4",
"status" => "200"
}
system
(system)
Closed
November 2, 2018, 9:37am
7
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.