Hello,
I'm working on a logstash config but can't get a working date from the log line to @timestamp
the paas_apex_access lines are successfully using a date line but the paas_apex_oblog won't.
also when mutate the @ to a space the date is not working.
How do i specify a date with the following format: 2018/02/06@15:17:39.26788 ?
logline
2018/02/06@15:17:39.26788 11386 11396 INIT ERROR 0x000003B6 ../oblistrwutil.cpp:225 "Could not read file" filename^/oracle/middleware/Oracle_OAMWebGate1/webgate/ohs/config/oblog_config.xml
Using NPTL Threading Library.
Filebeat config
-
input_type: log
paths:
- /oracle/user_projects/http_instances/*/diagnostics/logs/OHS/ohs1_*/access_log*
document_type: paas_apex_access
fields:
app_id: paas_apex
-
input_type: log
paths:
- /oracle/user_projects/http_instances/*/diagnostics/logs/OHS/ohs1_*/oblog.log
document_type: paas_apex_oblog
multiline:
pattern: '^[0-9]{4}\/[0-9]{2}\/[0-9]{2}\@[0-9]{2}\:[0-9]{2}\:[0-9]{2}'
negate: true
match: after
fields:
app_id: paas_apex
logstash filter
filter {
if [type] == "paas_apex_access" or [type] == "paas_apex_oblog" {
if [type] == "paas_apex_access" {
grok {
patterns_dir => ["/etc/logstash/patterns/"]
match => { "message" => "%{IPORHOST:clientip} (?:%{USER:user}) [-] (?:%{USER:user-agent}|-) (?:%{NUMBER:reqbytes}|-) \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-)"
}
}
date {
match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
target => "@timestamp"
remove_field => [ "timestamp" ]
}
}
if [type] == "paas_apex_oblog" {
grok {
patterns_dir => ["/etc/logstash/patterns/"]
match => { "message" => "(?<apexTimeStamp>%{YEAR}\/%{MONTHNUM}\/%{MONTHDAY}\@%{TIME})%{SPACE}*%{NUMBER:process_Id}%{SPACE}*%{NUMBER:thread_Id}%{SPACE}*%{WORD:module}%{SPACE}*%{LOGLEVEL:level}%{SPACE}*(?<code>[0-9]{1}[x][0-9a-zA-Z]{8})%{SPACE}*(?<fileLine>[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*)[:]%{NUMBER:fileLineNum}%{SPACE}*[\"](?<message>([a-zA-Z .]*))[\"]%{SPACE}*(HTTPStatus\^%{NUMBER:httpStatus})?%{SPACE}*(requestedURL\^%{URIPATH:requestedURL})?%{SPACE}*(filename\^%{URIPATH:requestedURL})?%{SPACE}*(raw_code\^%{NUMBER:rawCode})?"
}
}
mutate {
gsub => [
"apexTimeStamp","@"," "
]
}
date {
#2018/02/01@11:00:04.729475 UTC
#2018/02/06@08:48:12.74629
match => [ "apexTimeStamp" , "yyyy/MM/dd HH:mm:ss.SSSSS", "ISO8601" ]
target => "@timestamp"
remove_field => [ "apexTimeStamp" ]
}
}
}
}