I'm trying to parse logs in logstash with use of grok. In error log there are two more optional fields like requestUrl and requestMethod. but when i use grok for parsing logs to logstash.these from the error log grok can't create these two fields. here is my grok pattern :
if [fields][log_type] == "finance"
{
if [level] in [ "Error", "Fatal" ]
{
grok {
match=> ["message","%{DATESTAMP:time} \[%{WORD:processId}\] %{LOGLEVEL:level} %{USERNAME:logger} %{USER:user} %{IPV4:clientIp} %{URI:requestUrl} %{USER:requestMethod} %{GREEDYDATA:message}"]
overwrite => [ "message" ]
}
mutate { gsub => ["message", "\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{4} ",""]}
mutate { gsub => ["message", "%{level}",""]}
mutate { gsub => ["message", "%{logger}",""]}
mutate { gsub => ["message", "%{clientIp}",""]}
}
else
{
grok {
match=> ["message","%{DATESTAMP:time} \[%{WORD:processId}\] %{LOGLEVEL:level} %{USERNAME:logger} %{USER:user} %{IPV4:clientIp} %{GREEDYDATA:message}" ]
overwrite => [ "message" ]
}
mutate { gsub => ["message", "\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{4} ",""]}
mutate { gsub => ["message", "%{level}",""]}
mutate { gsub => ["message", "%{logger}",""]}
mutate { gsub => ["message", "%{clientIp}",""]}
}
}
here are my log sample :
2020-10-10 12:25:07.0891 [35476] DEBUG Program bhavin 192.168.43.244 init main
2020-10-10 12:25:09.8357 [35476] WARN HttpsRedirectionMiddleware bhavin 192.168.43.244 Failed to determine the https port for redirect.
2020-10-10 12:25:10.7106 [35476] ERROR DeveloperExceptionPageMiddleware bhavin 192.168.43.244 http://bhavin/Transactions POST An unhandled exception has occurred while executing the request.System.InvalidOperationException
here is the image of kibana view :
you can see other fields are created but these two are not created