Using ES 2.4.4 and Logstash 2.4.1 with the following config logstash appears to "loop" though the query results, ending up with the same documents being written over and over. Can someone help me figure out what I'm doing wrong?
input {
elasticsearch {
hosts => "es-analytics-int.dev.company.com"
index => "logstash-event-*"
size => 100
query => '{"query":{"range":{"timestamp":{"gte":"2017-02-10T00:00:00.000Z","lte":"2017-03- 12T23:59:59.999Z"}}}}'
}}
filter {
mutate {
remove_field => [ "@timestamp" ]
remove_field => [ "@version" ]
}
date {
locale => en
match => [ "sentTimestamp" , "dd/MMM/yyyy:HH:mm:ss Z" , "yyyy-MM-dd HH:mm:ss,SSS" , "yyy-MM-dd HH:mm:ss,SSSZ" , "ISO8601" ]
}
}
output {
elasticsearch {
hosts => "127.0.0.1"
index => "logstash-%{type}-%{+YYYY.MM}"
}
}