Infrastructure details ,
Running logstash 5.4.1 in a docker container.
Following is my logstash.conf
input {
http {
type => 'webhook'
}
s3 {
bucket => "abcd"
region => "xyz"
prefix => "1234"
exclude_pattern => "(?:2017030[1-9]|201703[10-31]|2017040[1-9]|201704[10-30])"
type => "known"
backup_to_bucket => "process"
delete => "true"
}
}
filter {
if [type] =~ "known" {
mutate {
gsub => ["message", "[\]", "/"]
}
grok {
patterns_dir => ["/opt/logstash/patterns", "/opt/logstash/extra_patterns"]
match => { "message" => "%{CUSTOMSTAMP:timestamp}\t(?:%{IP:client}|-)\t(?:%{WORD:method}|-)\t(?:%{URIPATHPARAM:request}|-)\t%{NUMBER:status_code:int}\t%{NUMBER:bytes:int}\t%{NUMBER:duration:int}\t(?:%{QS:referrer}|-)\t%{BLANKQUOTE}%{QS:agent}%{BLANKQUOTE}\t%{QS:cookie}" }
}
mutate {
rename => { "referrer" => "cs(Referrer)" }
rename => { "agent" => "cs(User-Agent)" }
}
useragent { source => "cs(User-Agent)" target => "useragent" }
date { match => [ "timestamp", "YYYY-MM-dd HH:mm:ss" ] remove_field => "timestamp" }
geoip { source => "client" }
}
}
output {
elasticsearch {
hosts => ["https://{{ ES_NAME }}:443"]
index => "logstash-%{type}-v1-%{+YYYY.MM.dd}"
}
}
Error message as
[ERROR][logstash.pipeline ] A plugin had an unrecoverable error. Will restart this plugin.
Plugin : LogStash::Inputs::S3
Error:
Though logstash continues processing the logs, but it is hampering the efficiency and has become really slow, due to plugin restart.
I have no clue where to start troubleshooting since the Error message is blank.