i am getting error while integrating couchdb databse with logstash i.e. forwarded to elasticsearch database and kibana.
my config file for logstash is
couchdb_changes {
db => "demo"
#type => "test"
#host => "localhost"
keep_revision => true
host => "listening-dog-hlf-couchdb"
port => 5984
initial_sequence => 0
sequence_path => ".demo_couchdb_seq"
codec => "rubydebug"
}
output {
elasticsearch {
codec => rubydebug { metadata => true }
hosts => ["{ELASTICSEARCH_HOST}:{ELASTICSEARCH_PORT}"]
manage_template => true
index => "ranjan_index"
}
stdout { codec => rubydebug }
}
logs are :-1
Sending Logstash logs to /usr/share/logstash/logs which is now configured via log4j2.properties
[2019-07-12T04:38:28,712][INFO ][logstash.setting.writabledirectory] Creating directory {:setting=>"path.queue", :path=>"/usr/share/logstash/data/queue"}
[2019-07-12T04:38:28,759][INFO ][logstash.setting.writabledirectory] Creating directory {:setting=>"path.dead_letter_queue", :path=>"/usr/share/logstash/data/dead_letter_queue"}
[2019-07-12T04:38:29,526][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2019-07-12T04:38:29,541][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"7.1.1"}
[2019-07-12T04:38:29,595][INFO ][logstash.agent ] No persistent UUID file found. Generating new UUID {:uuid=>"7e41fba6-02a1-450a-a225-bb970b49e9c0", :path=>"/usr/share/logstash/data/uuid"}
[2019-07-12T04:38:40,118][WARN ][logstash.runner ] SIGTERM received. Shutting down.
[2019-07-12T04:38:45,930][INFO ][org.logstash.ackedqueue.QueueUpgrade] No PQ version file found, upgrading to PQ v2.
[2019-07-12T04:38:47,179][INFO ][logstash.outputs.elasticsearch] Elasticsearch pool URLs updated {:changes=>{:removed=>, :added=>[http://elasticsearch-master:9200/]}}
[2019-07-12T04:38:47,506][WARN ][logstash.outputs.elasticsearch] Restored connection to ES instance {:url=>"http://elasticsearch-master:9200/"}
[2019-07-12T04:38:47,627][INFO ][logstash.outputs.elasticsearch] ES Output version determined {:es_version=>7}
[2019-07-12T04:38:47,637][WARN ][logstash.outputs.elasticsearch] Detected a 6.x and above cluster: the type event field won't be used to determine the document _type {:es_version=>7}
[2019-07-12T04:38:47,704][INFO ][logstash.outputs.elasticsearch] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["//elasticsearch-master:9200"]}
[2019-07-12T04:38:47,747][INFO ][logstash.outputs.elasticsearch] Using default mapping template
[2019-07-12T04:38:47,771][INFO ][logstash.javapipeline ] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>250, :thread=>"#<Thread:0x7a91fe8d run>"}
[2019-07-12T04:38:47,827][INFO ][logstash.inputs.couchdbchanges] initial_sequence is set, writing to filesystem ... {:initial_sequence=>0, :sequence_path=>".demo_couchdb_seq"}
[2019-07-12T04:38:47,870][INFO ][logstash.javapipeline ] Pipeline started {"pipeline.id"=>"main"}
[2019-07-12T04:38:48,050][INFO ][logstash.inputs.couchdbchanges] Connecting to CouchDB _changes stream at: {:host=>"listening-dog-hlf-couchdb", :port=>"5984", :db=>"demo"}
[2019-07-12T04:38:48,181][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>}
[2019-07-12T04:38:48,225][INFO ][logstash.outputs.elasticsearch] Attempting to install template {:manage_template=>{"index_patterns"=>"logstash-", "version"=>60001, "settings"=>{"index.refresh_interval"=>"5s", "number_of_shards"=>1}, "mappings"=>{"dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"keyword"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}
[2019-07-12T04:38:48,503][INFO ][logstash.inputs.couchdbchanges] Using service uri : {:uri=>#<URI::HTTP http://listening-dog-hlf-couchdb:5984/demo/_changes?feed=continuous&include_docs=true&since=0&heartbeat=1000>}
[2019-07-12T04:38:49,297][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
[2019-07-12T04:38:55,228][WARN ][org.logstash.execution.ShutdownWatcherExt] {"inflight_count"=>0, "stalling_threads_info"=>{"other"=>[{"thread_id"=>27, "name"=>"[main]<couchdb_changes", "current_call"=>"uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/net/protocol.rb:181:in wait_readable'"}, {"thread_id"=>25, "name"=>"[main]>worker0", "current_call"=>"[...]/logstash-core/lib/logstash/java_pipeline.rb:235:inblock in start_workers'"}, {"thread_id"=>26, "name"=>"[main]>worker1", "current_call"=>"[...]/logstash-core/lib/logstash/java_pipeline.rb:235:in block in start_workers'"}]}} [2019-07-12T04:38:55,231][ERROR][org.logstash.execution.ShutdownWatcherExt] The shutdown process appears to be stalled due to busy or blocked plugins. Check the logs for more information. [2019-07-12T04:39:00,281][WARN ][org.logstash.execution.ShutdownWatcherExt] {"inflight_count"=>0, "stalling_threads_info"=>{"other"=>[{"thread_id"=>27, "name"=>"[main]<couchdb_changes", "current_call"=>"uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/net/protocol.rb:181:inwait_readable'"}, {"thread_id"=>25, "name"=>"[main]>worker0", "current_call"=>"[...]/logstash-core/lib/logstash/java_pipeline.rb:235:in block in start_workers'"}, {"thread_id"=>26, "name"=>"[main]>worker1", "current_call"=>"[...]/logstash-core/lib/logstash/java_pipeline.rb:235:inblock in start_workers'"}]}}
[2019-07-12T04:39:05,326][WARN ][org.logstash.execution.ShutdownWatcherExt] {"inflight_count"=>0, "stalling_threads_info"=>{"other"=>[{"thread_id"=>27, "name"=>"[main]<couchdb_changes", "current_call"=>"uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/net/protocol.rb:181:in wait_readable'"}, {"thread_id"=>25, "name"=>"[main]>worker0", "current_call"=>"[...]/logstash-core/lib/logstash/java_pipeline.rb:235:inblock in start_workers'"}, {"thread_id"=>26, "name"=>"[main]>worker1", "current_call"=>"[...]/logstash-core/lib/logstash/java_pipeline.rb:235:in `block in start_workers'"}]}}