Hello. Since we've updated logstash from 5.4.2 to 6.2.4 version we have problem with high using of CPU.
As you can see on screenshots this high usage is not depend on message rate:
All config files are the same. We use your original docker images.
With INFO log level everything looks fine but log with DEBUG level contains a lot of next messages:
[2018-07-02T09:36:38,928][DEBUG][logstash.config.source.local.configpathloader] Skipping the following files while reading config since they don't match the specified glob pattern {:files=>["/usr/share/logstash/CONTRIBUTORS", "/usr/share/logstash/Gemfile", "/usr/share/logstash/Gemfile.lock", "/usr/share/logstash/LICENSE", "/usr/share/logstash/NOTICE.TXT", "/usr/share/logstash/bin", "/usr/share/logstash/config", "/usr/share/logstash/data", "/usr/share/logstash/lib", "/usr/share/logstash/logstash-core", "/usr/share/logstash/logstash-core-plugin-api", "/usr/share/logstash/modules", "/usr/share/logstash/pipeline", "/usr/share/logstash/tools", "/usr/share/logstash/vendor"]}
[2018-07-02T09:36:38,928][DEBUG][logstash.config.source.local.configpathloader] Reading config file {:config_file=>"/usr/share/logstash/pipeline/logstash.conf"}
[2018-07-02T09:36:38,930][DEBUG][logstash.agent ] Converging pipelines state {:actions_count=>0}
Less then one minute of log almost 2,5Mb. I'm not sure should I upload it fully or not.
$ cat ./logstash.conf
input {
beats {
port => 5044
}
}
filter {
if [source] =~ "services" {
grok {
match => ["source", "/%{GREEDYDATA}/%{DATA:ServiceName}\_%{GREEDYDATA}\_%{DATA:ServiceVersion}\.0.log"]
break_on_match => false
tag_on_failure => [ "failedPattern3" ]
}
grok {
match => ["message", "%{TIMESTAMP_ISO8601:timestamp}\|%{DATA:TraceID}\|%{DATA:EventID:int}\|%{DATA:Logger}\|%{LOGLEVEL:LogLevel}\|%{GREEDYDATA:Message}"]
tag_on_failure => [ "failedPattern2" ]
}
date {
timezone => "UTC"
match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSSS" ]
target => "@timestamp"
remove_field => [ "timestamp" ]
}
} else if [source] =~ "docker" {
mutate {
add_field => {
ServiceName => "docker"
ServiceVersion => "0"
LogLevel => "INFO"
}
}
grok {
match => ["message", "%{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:monthday}T%{TIME:time}Z %{DATA:Logger1} %{DATA:Logger2} %{DATA:TraceID} %{GREEDYDATA:Message}"]
tag_on_failure => [ "failedPattern4" ]
}
mutate {
add_field => {
"timestamp" => "%{year}-%{month}-%{monthday} %{time}"
}
}
date {
locale => "en"
timezone => "UTC"
match => ["timestamp", "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm:ss.SSSSSSSSS", "ISO8601"]
target => "@timestamp"
remove_field => ["timestamp", "monthday", "year", "month", "day", "time"]
}
mutate {
add_field => {
"Logger" => "%{Logger1} %{Logger2}"
}
remove_field => [ "Logger2", "Logger1" ]
}
} else {
mutate {
add_field => {
ServiceName => "rabbitmq"
ServiceVersion => "3.6.10"
}
gsub => ["message", "==\n", " "]
}
grok {
match => ["message", "=%{LOGLEVEL:LogLevel} REPORT==== %{MONTHDAY:monthday}-%{MONTH:month}-%{YEAR:year}::%{TIME:time} = %{GREEDYDATA:Message}"]
tag_on_failure => [ "failedPattern1" ]
}
mutate {
add_field => {
"timestamp" => "%{monthday}/%{month}/%{year}:%{time}"
}
}
date {
locale => "en"
timezone => "UTC"
match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss"]
target => "@timestamp"
remove_field => ["timestamp", "monthday", "year", "month", "day", "time"]
}
}
if "beats_input_codec_plain_applied" in [tags] {
# mutate {
# remove_tag => ["beats_input_codec_plain_applied"]
# }
mutate {
replace => ["@version", "%{ServiceVersion}"]
remove_field => [ "tags", "message", "host", "type", "input_type", "offset", "beat", "source", "ServiceVersion" ]
remove_field => [ '[geoip][ip]', '[geoip][location]', '[geoip][latitude]', '[geoip][longitude]' ]
}
}
}
output {
elasticsearch {
hosts => ["elasticsearch:9200"]
}
# Remove in production
# stdout {
# codec => rubydebug
# }
}