Filtering data in logstash config file for couchdb

Elastic Stack Logstash
1

I have views in my couchdb databases. WIth logstash config file, can I filter to get only valid documents in my indices in ElasticSearch and not the the views? I have multiple databases in couchdb and I have created indices for all of them. How would my filter look like in my config file? Do I need to create filter for each of the each index meaning for input/output section?

Thanks.

2

Since the documents that are views don't have a field "type" in them. I think I do something like this:

filter {
if [type] == "" {
drop { }
}
}

But its not working. Can some one shed some light on this?

3

What does your input look like?

4

Here is my my config file:

input {
# 1 attributes
couchdb_changes {
db => "attributes"
host => "127.0.0.1"
port => 5984
sequence_path => "seq_files\attributes_couchdb_seq"
tags => ["attributes"]
}
# 2 audits
couchdb_changes {
db => "audits"
host => "127.0.0.1"
port => 5984
sequence_path => "seq_files\audits_couchdb_seq"
tags => ["audits"]
}
couchdb_changes {
db => "clips"
host => "127.0.0.1"
port => 5984
sequence_path => "seq_files\clips_couchdb_seq"
tags => ["clips"]
}
couchdb_changes {
db => "events"
host => "127.0.0.1"
port => 5984
sequence_path => "seq_files\events_couchdb_seq"
tags => ["events"]
}
couchdb_changes {
db => "folder"
host => "127.0.0.1"
port => 5984
sequence_path => "seq_files\folder_couchdb_seq"
tags => ["folder"]
}
couchdb_changes {
db => "icons"
host => "127.0.0.1"
port => 5984
sequence_path => "seq_files\icons_couchdb_seq"
tags => ["icons"]
}
couchdb_changes {
db => "menus"
host => "127.0.0.1"
port => 5984
sequence_path => "seq_files\menus_couchdb_seq"
tags => ["menus"]
}
couchdb_changes {
db => "users"
host => "127.0.0.1"
port => 5984
sequence_path => "seq_files\users_couchdb_seq"
tags => ["users"]
}
couchdb_changes {
db => "roles"
host => "127.0.0.1"
port => 5984
sequence_path => "seq_files\roles_couchdb_seq"
tags => ["roles"]
}
couchdb_changes {
db => "cameras"
host => "127.0.0.1"
port => 5984
sequence_path => "seq_files\cameras_couchdb_seq"
tags => ["cameras"]
}
couchdb_changes {
db => "devices"
host => "127.0.0.1"
port => 5984
sequence_path => "seq_files\devices_couchdb_seq"
tags => ["devices"]
}
}
output {
if "attributes" in [tags] {
elasticsearch {
document_id => "%{[@metadata][_id]}"
host => "127.0.0.1"
index => "attributes_index"
protocol => "http"
port => 9200
}
}
if "audits" in [tags] {
elasticsearch {
document_id => "%{[@metadata][_id]}"
host => "127.0.0.1"
index => "audits_index"
protocol => "http"
port => 9200
}
}
if "clips" in [tags] {
elasticsearch {
document_id => "%{[@metadata][_id]}"
host => "127.0.0.1"
index => "clips_index"
protocol => "http"
port => 9200
}
}
if "events" in [tags] {
elasticsearch {
document_id => "%{[@metadata][_id]}"
host => "127.0.0.1"
index => "events_index"
protocol => "http"
port => 9200
}
}
if "folder" in [tags] {
elasticsearch {
document_id => "%{[@metadata][_id]}"
host => "127.0.0.1"
index => "folder_index"
protocol => "http"
port => 9200
}
}
if "icons" in [tags] {
elasticsearch {
document_id => "%{[@metadata][_id]}"
host => "127.0.0.1"
index => "icons_index"
protocol => "http"
port => 9200
}
}
if "menus" in [tags] {
elasticsearch {
document_id => "%{[@metadata][_id]}"
host => "127.0.0.1"
index => "menus_index"
protocol => "http"
port => 9200
}
}
if "users" in [tags] {
elasticsearch {
document_id => "%{[@metadata][_id]}"
host => "127.0.0.1"
index => "users_index"
protocol => "http"
host => '127.0.0.1'
port => 9200
}
}
if "roles" in [tags] {
elasticsearch {
document_id => "%{[@metadata][_id]}"
host => "127.0.0.1"
index => "roles_index"
protocol => "http"
host => '127.0.0.1'
port => 9200
}
}
if "cameras" in [tags] {
elasticsearch {
document_id => "%{[@metadata][_id]}"
host => "127.0.0.1"
index => "cameras_index"
protocol => "http"
host => '127.0.0.1'
port => 9200
}
}
if "devices" in [tags] {
elasticsearch {
document_id => "%{[@metadata][_id]}"
host => "127.0.0.1"
index => "devices_index"
protocol => "http"
host => '127.0.0.1'
port => 9200
}
}
}

5

still looking for a solution for this, will appreciate any help.