org.logstash.LockException


(Siva Kumar) #1

Im using logstash with 6 replication in kubernetes.

Here is my config file

Pipeline Filter

filter {
  if [kubernetes][container][name] == "envoy" {
      grok {
          match => { "message" => '\[%{DATA:envoy_log_type}\]'}
      }

      if "ACCESS_LOG" in [envoy_log_type] {

          grok {
              match => { "message" => '\[%{TIMESTAMP_ISO8601:timestamp}\] \"%{DATA:method} (?:%{URIPATH:uri_path}(?:%{URIPARAM:uri_param})?|%{DATA:}) %{DATA:protocol}\" %{NUMBER:status_code} %{DATA:response_flags} %{NUMBER:bytes_sent} %{NUMBER:bytes_received} %{NUMBER:duration} (?:%{NUMBER:upstream_service_time}|%{DATA:tcp_service_time}) \"%{DATA:forwarded_for}\" \"%{DATA:user_agent}\" \"%{DATA:request_id}\" \"%{DATA:authority}\" \"%{DATA:upstream_service}\" \"%{DATA:downstream_addr}\"'}
          }

          if [uri_path] == "/v1/me" {
             ruby {
                # Cancel 90% of events
                code => "event.cancel if rand <= 0.90"
              }
          }

          geoip {
              source => "downstream_addr"
          }

          mutate {
              convert => {"duration" => "integer"}
          }
      }
    }
}

Output

output {
    if "ACCESS_LOG" in [envoy_log_type] {
        elasticsearch {
            hosts => "elasticsearch.default:9200"
            index => "envoy-access-log-%{+YYYY.MM.dd}"
        }
    } 
}

Im getting the following error exactly in 3rd replication out of 6. Other instances are running fine so its safe to say no issue with configuration.

If I scale down to 2 instances then no problem. But if i scale up to 3 instance I'm getting this only on 3rd.

[2018-06-26T14:29:48,425][ERROR][logstash.agent           ] Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_id:main, :exception=>"IOError", :message=>"org.logstash.LockException: The queue failed to obtain exclusive access, cause: Lock held by this virtual machine on lock path: /usr/share/logstash/data/queue/main/.lock", :backtrace=>["org/logstash/ackedqueue/Queue.java:258:in `open'", "org/logstash/ackedqueue/ext/AbstractJRubyQueue.java:88:in `open'", "/usr/share/logstash/logstash-core/lib/logstash/util/wrapped_acked_queue.rb:39:in `with_queue'", "/usr/share/logstash/logstash-core/lib/logstash/util/wrapped_acked_queue.rb:28:in `create_file_based'", "/usr/share/logstash/logstash-core/lib/logstash/queue_factory.rb:29:in `create'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:172:in `initialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline_action/create.rb:40:in `execute'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:315:in `block in converge_state'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:141:in `with_pipelines'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:312:in `block in converge_state'", "org/jruby/RubyArray.java:1734:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:299:in `converge_state'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:166:in `block in converge_state_and_update'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:141:in `with_pipelines'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:164:in `converge_state_and_update'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:105:in `block in execute'", "org/jruby/RubyProc.java:289:in `call'", "org/jruby/RubyProc.java:273:in `call19'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/interval.rb:18:in `interval'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:94:in `execute'", "usr/share/logstash/logstash_minus_core/lib/logstash//usr/share/logstash/logstash-core/lib/logstash/agent.rb:-1:in `RUBY$method$execute$0$__VARARGS__'", "/usr/share/logstash/logstash-core/lib/logstash/runner.rb:348:in `block in execute'", "org/jruby/RubyProc.java:289:in `call'", "org/jruby/RubyProc.java:273:in `call19'", "org/jruby/RubyProc$INVOKER$i$0$0$call19.gen:-1:in `call'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:24:in `block in initialize'", "org/jruby/RubyProc.java:289:in `call'", "org/jruby/RubyProc.java:246:in `call'", "java/lang/Thread.java:748:in `run'"]}
[2018-06-26T14:29:51,543][ERROR][logstash.pipeline        ] Logstash failed to create queue {:pipeline_id=>"main", "exception"=>"org.logstash.LockException: The queue failed to obtain exclusive access, cause: Lock held by this virtual machine on lock path: /usr/share/logstash/data/queue/main/.lock", "backtrace"=>["org/logstash/ackedqueue/Queue.java:258:in `open'", "org/logstash/ackedqueue/ext/AbstractJRubyQueue.java:88:in `open'", "/usr/share/logstash/logstash-core/lib/logstash/util/wrapped_acked_queue.rb:39:in `with_queue'", "/usr/share/logstash/logstash-core/lib/logstash/util/wrapped_acked_queue.rb:28:in `create_file_based'", "/usr/share/logstash/logstash-core/lib/logstash/queue_factory.rb:29:in `create'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:172:in `initialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline_action/create.rb:40:in `execute'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:315:in `block in converge_state'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:141:in `with_pipelines'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:312:in `block in converge_state'", "org/jruby/RubyArray.java:1734:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:299:in `converge_state'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:166:in `block in converge_state_and_update'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:141:in `with_pipelines'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:164:in `converge_state_and_update'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:105:in `block in execute'", "org/jruby/RubyProc.java:289:in `call'", "org/jruby/RubyProc.java:273:in `call19'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/interval.rb:18:in `interval'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:94:in `execute'", "usr/share/logstash/logstash_minus_core/lib/logstash//usr/share/logstash/logstash-core/lib/logstash/agent.rb:-1:in `RUBY$method$execute$0$__VARARGS__'", "/usr/share/logstash/logstash-core/lib/logstash/runner.rb:348:in `block in execute'", "org/jruby/RubyProc.java:289:in `call'", "org/jruby/RubyProc.java:273:in `call19'", "org/jruby/RubyProc$INVOKER$i$0$0$call19.gen:-1:in `call'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:24:in `block in initialize'", "org/jruby/RubyProc.java:289:in `call'", "org/jruby/RubyProc.java:246:in `call'", "java/lang/Thread.java:748:in `run'"]}

(system) #2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.