For Logstash1, 2 and 3 the input is beats, see below (to make it more readable, I skipped the pipelines.yml file)
Configuration Logstash 1 and 2
input { beats { port => 5044 } }
filter {
mutate {
add_tag => [ "my_tags" ]
add_field => { "agent_host" => "%{[beat][hostname]}" }
}
if ! [host][name] { mutate { rename => { "[host]" => "[host][name]" } } }
dns {
resolve => ["agent_host"]
action => "replace"
}
dns {
reverse => ["agent_host"]
action => "replace"
}
mutate {
replace => {"[host][name]" => "%{agent_host}"}
remove_field => ["agent_host"]
}
}
output {
lumberjack {
codec => json
hosts => ["dns_name_logstash_server_3"]
ssl_certificate => "/etc/logstash/certs/lumberjack.crt"
port => 6000
}
}
Configuration Logstash 3
input {
beats {
port => 6000
ssl => true
ssl_key => '/etc/logstash/certs/lumberjack.key'
ssl_certificate => '/etc/logstash/certs/lumberjack.crt'
codec => json
}
}
filter {
mutate {
add_tag => [ "tag_name" ]
}
uuid {
target => "uuid"
overwrite => true
}
}
output {
lumberjack {
codec => json
hosts => ["address_to_loadbalancer_logstash4_and5"]
ssl_certificate => "/etc/logstash/certs/lumberjack.crt"
port => 6000
}
}
I see the errors occur on Logstash 1 and 2, as well as on 3 (mostly at the same time):
Error Logstash2
[2020-10-01T08:20:16,117][ERROR][logstash.outputs.lumberjack] Client write error, trying connect {:e=>#<IOError: Connection reset by peer>, :backtrace=>["org/jruby/ext/openssl/SSLSocket.java:950:in syswrite'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/jls-lumberjack-0.0.26/lib/lumberjack/client.rb:107:in send_window_size'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/jls-lumberjack-0.0.26/lib/lumberjack/client.rb:127:in write_sync'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/jls-lumberjack-0.0.26/lib/lumberjack/client.rb:42:in write'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-lumberjack-3.1.7/lib/logstash/outputs/lumberjack.rb:65:in flush'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/buffer.rb:219:in block in buffer_flush'", "org/jruby/RubyHash.java:1343:in each'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/buffer.rb:216:in buffer_flush'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/buffer.rb:159:in buffer_receive'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-lumberjack-3.1.7/lib/logstash/outputs/lumberjack.rb:52:in block in register'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-codec-json-3.0.5/lib/logstash/codecs/json.rb:42:in encode'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-lumberjack-3.1.7/lib/logstash/outputs/lumberjack.rb:59:in receive'", "/usr/share/logstash/logstash-core/lib/logstash/outputs/base.rb:89:in block in multi_receive'", "org/jruby/RubyArray.java:1734:in each'", "/usr/share/logstash/logstash-core/lib/logstash/outputs/base.rb:89:in multi_receive'", "org/logstash/config/ir/compiler/OutputStrategyExt.java:114:in multi_receive'", "org/logstash/config/ir/compiler/AbstractOutputDelegatorExt.java:97:in multi_receive'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:390:in block in output_batch'", "org/jruby/RubyHash.java:1343:in each'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:389:in output_batch'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:341:in worker_loop'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:304:in block in start_workers'"]}
Error Logstash3
[2020-10-01T08:22:37,137][ERROR][logstash.outputs.lumberjack][pipeline-data] Client write error, trying connect {:e=>#<IOError: Connection reset by peer>, :backtrace=>["org/jruby/ext/openssl/SSLSocket.java:950:in syswrite'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/jls-lumberjack-0.0.26/lib/lumberjack/client.rb:107:in send_window_size'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/jls-lumberjack-0.0.26/lib/lumberjack/client.rb:127:in write_sync'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/jls-lumberjack-0.0.26/lib/lumberjack/client.rb:42:in write'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-lumberjack-3.1.7/lib/logstash/outputs/lumberjack.rb:65:in flush'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/stud-0.0.23/lib/stud/buffer.rb:219:in block in buffer_flush'", "org/jruby/RubyHash.java:1428:in each'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/stud-0.0.23/lib/stud/buffer.rb:216:in buffer_flush'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/stud-0.0.23/lib/stud/buffer.rb:159:in buffer_receive'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-lumberjack-3.1.7/lib/logstash/outputs/lumberjack.rb:52:in block in register'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-codec-json-3.0.5/lib/logstash/codecs/json.rb:42:in encode'", "/usr/share/logstash/logstash-core/lib/logstash/codecs/delegator.rb:31:in block in encode'", "org/logstash/instrument/metrics/AbstractSimpleMetricExt.java:45:in time'", "org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java:44:in time'", "/usr/share/logstash/logstash-core/lib/logstash/codecs/delegator.rb:30:in encode'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-lumberjack-3.1.7/lib/logstash/outputs/lumberjack.rb:59:in receive'", "/usr/share/logstash/logstash-core/lib/logstash/outputs/base.rb:89:in block in multi_receive'", "org/jruby/RubyArray.java:1814:in each'", "/usr/share/logstash/logstash-core/lib/logstash/outputs/base.rb:89:in multi_receive'", "org/logstash/config/ir/compiler/OutputStrategyExt.java:118:in multi_receive'", "org/logstash/config/ir/compiler/AbstractOutputDelegatorExt.java:101:in multi_receive'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:262:in block in start_workers'"]}
Ah, interesting, do you have any documentation on that? I cannot find it on the Logstash-to-logstash page