Logstash sending output to three ES nodes. When one ES node is down, Logstash doesn't forward specific outputs to other nodes, probably because the dead node was receiving the output. For example, in the Logstash conf below, when one elasticsearchhost1 node is down, I don't see traffic going to the dev-internal or test-external index but the other index receives traffic. When elasticsearchhost2 is down, I don't see traffic going to let's say test-internal index. When all nodes are up, I see all output in Kibana.
Is there a way to prevent this behavior?
input {
beats {
port => "5044"
host => "0.0.0.0"
}
}
filter
{
if "beats_input_codec_plain_applied" in [tags] {
mutate {
remove_tag => ["beats_input_codec_plain_applied"]
}
}
}
output {
if "test-int" in [fields][environment]
{
elasticsearch {
hosts => ["elasticsearchhost1:9200", "elasticsearchhost2:9200", "elasticsearchhost3:9200"]
cacert => "path/to/cert"
index => "test-internal-%{+yyyy.MM.dd}"
user => "elastic"
password => "secret"
}
}
else if "test-ext" in [fields][environment]
{
elasticsearch {
hosts => ["elasticsearchhost1:9200", "elasticsearchhost2:9200", "elasticsearchhost3:9200"]
cacert => "path/to/cert"
index => "test-external-%{+yyyy.MM.dd}"
user => "elastic"
password => "secret"
}
}
else if "dev-int" in [fields][environment]
{
elasticsearch {
hosts => ["elasticsearchhost1:9200", "elasticsearchhost2:9200", "elasticsearchhost3:9200"]
cacert => "path/to/cert"
index => "dev-external-%{+yyyy.MM.dd}"
user => "elastic"
password => "secret"
}
else if "dev-ext" in [fields][environment]
{
elasticsearch {
hosts => ["elasticsearchhost1:9200", "elasticsearchhost2:9200", "elasticsearchhost3:9200"]
cacert => "path/to/cert"
index => "dev-external-%{+yyyy.MM.dd}"
user => "elastic"
password => "secret"
}
else
{
elasticsearch {
hosts => ["elasticsearchhost1:9200", "elasticsearchhost2:9200", "elasticsearchhost3:9200"]
cacert => "path/to/cert"
index => "iis-%{+yyyy.MM.dd}"
user => "elastic"
password => "secret"
}
}
stdout { codec => rubydebug }
}