Hi all,
I am trying to use multiline pattern in filebeat to append multiline code in jenkins log
Below is a sample of log file:
Aug 06, 2017 12:18:19 AM hudson.UDPBroadcastThread run
INFO: Cannot listen to UDP port 33,848, skipping: java.net.SocketException: No such device
Aug 06, 2017 12:18:19 AM hudson.WebAppMain$3 run
INFO: Jenkins is fully up and running
Aug 06, 2017 12:18:19 AM javax.jmdns.impl.HostInfo newHostInfo
WARNING: Could not find any address beside the loopback.
When I used this configuration on logstash for a local log file, everything is working well:
input {
file {
path => "/var/log/jenkins/jenkins.log"
#start_position => "beginning"
#sincedb_path => "/dev/null"
codec => multiline {
pattern => "^%{MONTH}"
negate => true
what => previous
}
}
}
filter {
grok {
match => {"message" => "%{JENKINSTIMESTAMP:[jenkins][timestamp]} %{DATA:[jenkins][logger][source_class]}( %{DATA:[jenkins][logger][source_method]})?\n%{DATA:[jenkins][logger][level]}: %{GREEDYMULTILINE:[jenkins][logger][message]}"
}
pattern_definitions => {
"GREEDYMULTILINE"=> "(.|\n)*"
"JENKINSTIMESTAMP" => "%{MONTH} %{MONTHDAY}[,] %{YEAR} %{TIME} [A-P]M"
}
remove_field => "message"
}
mutate {
gsub => ["[jenkins][logger][message]", "[\r?\n|\r|\t]", " "]
}
date {
match => [ "[jenkins][timestamp]", "MMM dd, yyyy hh:mm:ss a", "MMM dd, yyyy h:mm:ss a", "MMM d, yyyy hh:mm:ss a", "MMM d, yyyy h:mm:ss a" ]
}
}
output {
file {
path => "/etc/logstash/test.json"
}
}
But when I try to get it from filebeat client with this logstash cofiguration
input {
beats {
# The port to listen on for filebeat connections.
port => 5044
# The IP address to listen for filebeat connections.
host => "0.0.0.0"
}
}
}
filter {
grok {
match => {"message" => "%{JENKINSTIMESTAMP:[jenkins][timestamp]} %{DATA:[jenkins][logger][source_class]}( %{DATA:[jenkins][logger][source_method]})?\n%{DATA:[jenkins][logger][level]}: %{GREEDYMULTILINE:[jenkins][logger][message]}"
}
pattern_definitions => {
"GREEDYMULTILINE"=> "(.|\n)*"
"JENKINSTIMESTAMP" => "%{MONTH} %{MONTHDAY}[,] %{YEAR} %{TIME} [A-P]M"
}
remove_field => "message"
}
mutate {
gsub => ["[jenkins][logger][message]", "[\r?\n|\r|\t]", " "]
}
date {
match => [ "[jenkins][timestamp]", "MMM dd, yyyy hh:mm:ss a", "MMM dd, yyyy h:mm:ss a", "MMM d, yyyy hh:mm:ss a", "MMM d, yyyy h:mm:ss a" ]
}
}
output {
file {
path => "/etc/logstash/test.json"
}
}
And filebeat configuration
filebeat.prospectors:
- input_type: log
paths:
- /var/log/jenkins/jenkins.log
exclude_files: [".gz$"]
multiline:
pattern: "^%{MONTH}"
negate: true
match: after
I get all log event concatenated in one time-stamped output, as it if filebeat doesn't detect lines that begins with {MONTH} pattern so they are concatenated to previous line.
I tried also to change {MONTH} pattern with it's defintion in logstash plugin:
filebeat.prospectors:
- input_type: log
paths:
- /var/log/jenkins/jenkins.log
exclude_files: [".gz$"]
multiline:
pattern: "^\b(?:[Jj]an(?:uary|uar)?|[Ff]eb(?:ruary|ruar)?|[Mm](?:a|ä)?r(?:ch|z)?|[Aa]pr(?:il)?|[Mm]a(?:y|i)?|[Jj]un(?:e|i)?|[Jj]ul(?:y)?|[Aa]ug(?:ust)?|[Ss]ep(?:tember)?|[Oo](?:c|k)?t(?:ober)?|[Nn]ov(?:ember)?|[Dd]e(?:c|z)(?:ember)?)\b"
negate: true
match: after
But I get the same result 