error log
[2022-11-06T10:33:45,283][ERROR][logstash.javapipeline ][main] Pipeline worker error, the pipeline will be stopped {:pipeline_id=>"main", :error=>"(IOError) Resource temporarily unavailable", :exception=>Java::OrgJrubyExceptions::IOError, :backtrace=>["org.jruby.RubyIO.sysopen(org/jruby/RubyIO.java:1234)", "org.jruby.RubyFile.initialize(org/jruby/RubyFile.java:365)", "org.jruby.RubyIO.new(org/jruby/RubyIO.java:876)", "usr.share.logstash.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_output_minus_file_minus_4_dot_3_dot_0.lib.logstash.outputs.file.open(/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-file-4.3.0/lib/logstash/outputs/file.rb:276)", "usr.share.logstash.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_output_minus_file_minus_4_dot_3_dot_0.lib.logstash.outputs.file.multi_receive_encoded(/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-file-4.3.0/lib/logstash/outputs/file.rb:119)", "org.jruby.RubyHash.each(org/jruby/RubyHash.java:1415)", "usr.share.logstash.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_output_minus_file_minus_4_dot_3_dot_0.lib.logstash.outputs.file.multi_receive_encoded(/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-file-4.3.0/lib/logstash/outputs/file.rb:118)", "org.jruby.ext.thread.Mutex.synchronize(org/jruby/ext/thread/Mutex.java:164)", "usr.share.logstash.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_output_minus_file_minus_4_dot_3_dot_0.lib.logstash.outputs.file.multi_receive_encoded(/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-file-4.3.0/lib/logstash/outputs/file.rb:117)", "usr.share.logstash.logstash_minus_core.lib.logstash.outputs.base.multi_receive(/usr/share/logstash/logstash-core/lib/logstash/outputs/base.rb:103)", "org.logstash.config.ir.compiler.OutputStrategyExt$AbstractOutputStrategyExt.multi_receive(org/logstash/config/ir/compiler/OutputStrategyExt.java:143)", "org.logstash.config.ir.compiler.AbstractOutputDelegatorExt.multi_receive(org/logstash/config/ir/compiler/AbstractOutputDelegatorExt.java:121)", "usr.share.logstash.logstash_minus_core.lib.logstash.java_pipeline.start_workers(/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:299)"], :thread=>"#<Thread:0x75bae988 sleep>"}
config
containers:
- env:
- name: LS_JAVA_OPTS
value: -Xmx2g -Xms2g
image: docker.elastic.co/logstash/logstash:7.16.3
logstashPipeline:
logstash.conf: |
input {
beats {
port => 5044
}
}
filter {
if [k8s][nameSpace] == "ingress-nginx" {
grok {
ecs_compatibility => "v1"
match => { "message" => '(?<nginxaccess>\{"time".*?\})' }
}
if [nginxaccess] {
json {
source => "nginxaccess"
remove_field => [ "@version", "agent", "host", "ecs", "ephemeral_id", "nginxaccess", "message"]
add_field => { "logstate" => "good" }
}
} else {
mutate { add_field => { "logstate" => "bad" } }
}
} else if [k8s][nameSpace] == "spf-admin-us" or
[k8s][nameSpace] == "spf-seller-us" or
[k8s][nameSpace] == "cdc-us" {
clone {
ecs_compatibility => "v1"
clones => ["copy"]
add_field => { "copy_id" => "copymessage" }
}
if ![copy_id] {
grok {
ecs_compatibility => "v1"
match => { "message" => '%{LOGLEVEL:level}\s+' }
}
}
} else {
clone {
ecs_compatibility => "v1"
clones => ["copy"]
add_field => { "copy_id" => "copymessage" }
}
}
if ![copy_id] {
truncate {
fields => "message"
length_bytes => 5000
}
}
grok {
ecs_compatibility => "v1"
match => { "systemtime" => '(?<logdate>\d{4}-\d{2}-\d{2})' }
}
grok {
ecs_compatibility => "v1"
match => { "systemtime" => '(?<indexSuffix_month>\d{4}-\d{2})' }
}
}
output {
if [copy_id] and [k8s][containerName] {
file {
codec => "line"
path => "/usr/share/logstash/logging/${HOSTNAME}/%{logdate}/%{[k8s][nameSpace]}/%{[k8s][containerName]}/%{[k8s][podName]}.log"
}
}
}