I have installed filebeat on a server to ingest the stderr logs for tomcat instance. When I am trying to send those logs to my logstash instance its sending as one event instead of multiple events. How can I fix that?
My filebeat config looks like:
# ============================== Filebeat inputs ===============================
filebeat.inputs:
- type: log
enabled: true
paths:
- E:\tomcat\instances\application\logs\application-stderr.*.log
exclude_lines: '^[[:space:]]*$'
multiline.type: pattern
multiline.pattern: '((([0-9]{2}\-\b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b)-[0-9]{4}%{SPACE}+[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3})|([0-9]{4}-[0-9]{2}-[0-9]{2}\s[0-9]{2}:[0-9]{2}:[0-9]{2}))'
multiline.negate: true
multiline.match: after
# ================================== Outputs ===================================
# Configure what output to use when sending the data collected by the beat.
# ------------------------------ Logstash Output -------------------------------
output.logstash:
# The Logstash hosts
hosts: ["1.2.3.4:8081"]
# ================================== Logging ===================================
# Sets log level. The default log level is info.
# Available log levels are: error, warning, info, debug
logging.level: info
My logstash config looks like:
input {
beats {
port => 8081
}
}
filter {
grok {
match => ["message", "%{DATESTAMP:timestamp}%{SPACE}%{GREEDYDATA:details}"]
match => [ "message", "\s*(?<timestamp>([0-9]{2}\-\b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b)-[0-9]{4}%{SPACE}+[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3})%{SPACE}%{LOGLEVEL:logLevel}%{SPACE}%{NOTSPACE:Method}%{SPACE}%{JAVACLASS:classname}%{SPACE}(?<details>(.|\r|\n)*)" ]
#remove_field => [ "message" ]
}
date{
match => [ "timestamp", "dd-MMM-YYYY hh:mm:ss.SSS" ]
timezone => "US/Eastern"
target => ["@timestamp"]
remove_field => ["timestamp"]
}
mutate {
add_field => { "type" => "stderr" }
remove_field => [ "log", "ecs" ]
}
}
output {
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "stderr"
user => *******
password => *********
ssl => true
ssl_certificate_verification => false
}
stdout { codec => rubydebug }
}
This is my output in kibana:
My log file looks like:
How can I separate the events?
