Hi,
I am trying to extract data from elasticsearch to mongodb. I am using the inbuilt elasticsearch input plugin and 3.1.5 mongodb output plugin. I have multiple indexes in elasticsearch whose data needs to be extracted into different collections based on the index.
What I have done is write a input config consisting of multiple inputs and a output consisting of multiple outputs, while using if and else conditions.
input {
elasticsearch {
hosts => "localhost:9243"
user => "elastic"
password => "abc"
index => "index1"
query => '{ "query": { "match_all": {} }}'
size => 100
docinfo => true
ssl => true
scroll => "1d"
}
elasticsearch {
hosts => "localhost:9243"
user => "elastic"
password => "abc"
index => "index2"
query => '{ "query": { "match_all": {} }}'
size => 100
docinfo => true
ssl => true
scroll => "1d"
}
elasticsearch {
hosts => "localhost:9243"
user => "elastic"
password => "abc"
index => "index3"
query => '{ "query": { "match_all": {} }}'
size => 100
docinfo => true
ssl => true
scroll => "1d"
}
}
output {
if [index] == "index1" {
mongodb {
id => "id1"
bulk => true
bulk_size => 100
collection => "events"
database => "database1"
uri => "mongodb://user:password@localhost:27017/database1"
codec => "json"
}
}
else if [index] == "index2" {
mongodb {
id => "id2"
bulk => true
bulk_size => 100
collection => "sessions"
database => "database2"
uri => "mongodb://user:password@localhost:27017/database2"
codec => "json"
}
}
else if [index] == "index3" {
mongodb {
id => "id3"
bulk => true
bulk_size => 100
collection => "profile"
database => "database3"
uri => "mongodb://user:password@localhost:27017/database3"
codec => "json"
}
}
}
However, this does not work and throws an error. But if I don't have if-else, only one output it works, but all three indexes data is extracted to a single collection. stdout on individual if-else, doesn't give any output and starts without error. It would be really great if someone could help.