Hello,
I am trying to parse multiline logs
Log sample
06/Feb/2020:09:20 +0000 ACCESS {
"Total alerts subscribed for email only": 0,
"Total alerts subscribed for In-App only": 59579
filebeat configuration
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/test.log
multiline.pattern: '^[0-9]{1,3}'
multiline.negate: true
multiline.match: after
output.logstash:
hosts: ["172.16.0.143:5044"]
processors:
- add_cloud_metadata: ~
Logstash Configuration
input {
beats {
port => 5044
}
}
filter {
grok {
match => {
"message" => [
"%{GREEDYDATA:[app][timestamp]} %{WORD:[app][loglevel]} \{%{SPACE}\"%{GREEDYDATA}\"\: %{NUMBER:[app][totalSubscribedNumber]}\,%{SPACE}\"%{GREEDYDATA}\"\: %{NUMBER:[app][totalSubscribedNumberInApp]}"
]
}
}
mutate {
remove_field => ["agent","[log][offset]","[cloud][account]","[cloud][region]","[cloud][machine]","[cloud][image]","[cloud][availability_zone]","[cloud][provider]"]
}
}
output {
stdout {
codec => rubydebug
}
}
Output
{
"@timestamp" => 2020-02-26T12:24:58.674Z,
"tags" => [
[0] "beats_input_codec_plain_applied",
[1] "_grokparsefailure"
],
"host" => {
"name" => "beats"
},
"message" => "06/Feb/2020:09:20 +0000 ACCESS {\n \"Total alerts subscribed for email only\": 0,\n \"Total alerts subscribed for In-App only\": 59579",
"log" => {
"flags" => [
[0] "multiline"
],
"file" => {
"path" => "/var/log/test.log"
}
}
}
Note:
- Parsing multiple pattern with same logstash, can't use multiline codec on logstash
- Same pattern working perfectly fine with Kibana Grok Debugger & https://grokdebug.herokuapp.com
- Filebeat & Logstash both are running with v7.6