Im using logstash with 6 replication in kubernetes.
Here is my config file
Pipeline Filter
filter {
if [kubernetes][container][name] == "envoy" {
grok {
match => { "message" => '\[%{DATA:envoy_log_type}\]'}
}
if "ACCESS_LOG" in [envoy_log_type] {
grok {
match => { "message" => '\[%{TIMESTAMP_ISO8601:timestamp}\] \"%{DATA:method} (?:%{URIPATH:uri_path}(?:%{URIPARAM:uri_param})?|%{DATA:}) %{DATA:protocol}\" %{NUMBER:status_code} %{DATA:response_flags} %{NUMBER:bytes_sent} %{NUMBER:bytes_received} %{NUMBER:duration} (?:%{NUMBER:upstream_service_time}|%{DATA:tcp_service_time}) \"%{DATA:forwarded_for}\" \"%{DATA:user_agent}\" \"%{DATA:request_id}\" \"%{DATA:authority}\" \"%{DATA:upstream_service}\" \"%{DATA:downstream_addr}\"'}
}
if [uri_path] == "/v1/me" {
ruby {
# Cancel 90% of events
code => "event.cancel if rand <= 0.90"
}
}
geoip {
source => "downstream_addr"
}
mutate {
convert => {"duration" => "integer"}
}
}
}
}
Output
output {
if "ACCESS_LOG" in [envoy_log_type] {
elasticsearch {
hosts => "elasticsearch.default:9200"
index => "envoy-access-log-%{+YYYY.MM.dd}"
}
}
}
Im getting the following error exactly in 3rd replication out of 6. Other instances are running fine so its safe to say no issue with configuration.
If I scale down to 2 instances then no problem. But if i scale up to 3 instance I'm getting this only on 3rd.
[2018-06-26T14:29:48,425][ERROR][logstash.agent ] Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_id:main, :exception=>"IOError", :message=>"org.logstash.LockException: The queue failed to obtain exclusive access, cause: Lock held by this virtual machine on lock path: /usr/share/logstash/data/queue/main/.lock", :backtrace=>["org/logstash/ackedqueue/Queue.java:258:in `open'", "org/logstash/ackedqueue/ext/AbstractJRubyQueue.java:88:in `open'", "/usr/share/logstash/logstash-core/lib/logstash/util/wrapped_acked_queue.rb:39:in `with_queue'", "/usr/share/logstash/logstash-core/lib/logstash/util/wrapped_acked_queue.rb:28:in `create_file_based'", "/usr/share/logstash/logstash-core/lib/logstash/queue_factory.rb:29:in `create'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:172:in `initialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline_action/create.rb:40:in `execute'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:315:in `block in converge_state'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:141:in `with_pipelines'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:312:in `block in converge_state'", "org/jruby/RubyArray.java:1734:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:299:in `converge_state'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:166:in `block in converge_state_and_update'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:141:in `with_pipelines'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:164:in `converge_state_and_update'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:105:in `block in execute'", "org/jruby/RubyProc.java:289:in `call'", "org/jruby/RubyProc.java:273:in `call19'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/interval.rb:18:in `interval'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:94:in `execute'", "usr/share/logstash/logstash_minus_core/lib/logstash//usr/share/logstash/logstash-core/lib/logstash/agent.rb:-1:in `RUBY$method$execute$0$__VARARGS__'", "/usr/share/logstash/logstash-core/lib/logstash/runner.rb:348:in `block in execute'", "org/jruby/RubyProc.java:289:in `call'", "org/jruby/RubyProc.java:273:in `call19'", "org/jruby/RubyProc$INVOKER$i$0$0$call19.gen:-1:in `call'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:24:in `block in initialize'", "org/jruby/RubyProc.java:289:in `call'", "org/jruby/RubyProc.java:246:in `call'", "java/lang/Thread.java:748:in `run'"]}
[2018-06-26T14:29:51,543][ERROR][logstash.pipeline ] Logstash failed to create queue {:pipeline_id=>"main", "exception"=>"org.logstash.LockException: The queue failed to obtain exclusive access, cause: Lock held by this virtual machine on lock path: /usr/share/logstash/data/queue/main/.lock", "backtrace"=>["org/logstash/ackedqueue/Queue.java:258:in `open'", "org/logstash/ackedqueue/ext/AbstractJRubyQueue.java:88:in `open'", "/usr/share/logstash/logstash-core/lib/logstash/util/wrapped_acked_queue.rb:39:in `with_queue'", "/usr/share/logstash/logstash-core/lib/logstash/util/wrapped_acked_queue.rb:28:in `create_file_based'", "/usr/share/logstash/logstash-core/lib/logstash/queue_factory.rb:29:in `create'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:172:in `initialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline_action/create.rb:40:in `execute'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:315:in `block in converge_state'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:141:in `with_pipelines'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:312:in `block in converge_state'", "org/jruby/RubyArray.java:1734:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:299:in `converge_state'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:166:in `block in converge_state_and_update'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:141:in `with_pipelines'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:164:in `converge_state_and_update'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:105:in `block in execute'", "org/jruby/RubyProc.java:289:in `call'", "org/jruby/RubyProc.java:273:in `call19'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/interval.rb:18:in `interval'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:94:in `execute'", "usr/share/logstash/logstash_minus_core/lib/logstash//usr/share/logstash/logstash-core/lib/logstash/agent.rb:-1:in `RUBY$method$execute$0$__VARARGS__'", "/usr/share/logstash/logstash-core/lib/logstash/runner.rb:348:in `block in execute'", "org/jruby/RubyProc.java:289:in `call'", "org/jruby/RubyProc.java:273:in `call19'", "org/jruby/RubyProc$INVOKER$i$0$0$call19.gen:-1:in `call'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:24:in `block in initialize'", "org/jruby/RubyProc.java:289:in `call'", "org/jruby/RubyProc.java:246:in `call'", "java/lang/Thread.java:748:in `run'"]}