Documents are getting duplicated in mongodb

Hi,
I am trying to fetch data from mysql using logstash and storing them into Elasticsearch and mongodb.Document is getting stored in Elasticsearch properly and no duplicate documents are inserted,but in mongodb same document gets inserted again and again.How can I avoid data duplication in mongodb.

This is my logstash configuration pipeline:-
#This configuration pipeline will ingest data periodically from mysql database and it will store the output in Elasticsearch and mongodb

input {

    jdbc {

        jdbc_connection_string => "jdbc:mysql://localhost:3306/testdb"

        jdbc_user => "root"

        jdbc_password => "password"

        record_last_run => "true"

        use_column_value => "true"

        tracking_column => "Date"

        last_run_metadata_path => "/root/mysql"

        clean_run => "false"

        jdbc_driver_library => "/usr/share/java/mysql-connector-java-8.0.27.jar"

        jdbc_driver_class => "com.mysql.cj.jdbc.Driver"

        statement => "SELECT * FROM testtable where Date > :sql_last_value order by Date"

        schedule => "* * * * *"

    }

}

output {

    mongodb {

        uri => "mongodb://username:password@10.160.0.6:27017/data?authSource=data"

        database => "data"

        collection => "info"

    }

    elasticsearch {

        hosts => ["http://10.160.0.2:9200"]

        user => "elastic"

        password => "password"

        index => "mysql-data"

        document_id => "%{personid}"

    }

}

This are my Mongodb documents:-

{ "_id" : ObjectId("6199156a81c3b0bd84000001"), "date" : ISODate("2016-05-23T10:42:03.568Z"), "city" : "Cape Town", "@timestamp" : "\"2021-11-20T15:34:01.611Z\"", "firstname" : "Jaques", "personid" : 4005, "lastname" : "Kallis", "@version" : "1" }
{ "_id" : ObjectId("619915a481c3b0cafd000002"), "date" : ISODate("2016-05-23T10:42:03.568Z"), "city" : "Cape Town", "@timestamp" : "\"2021-11-20T15:35:00.440Z\"", "firstname" : "Jaques", "personid" : 4005, "lastname" : "Kallis", "@version" : "1" }
{ "_id" : ObjectId("619915e081c3b0bd84000003"), "date" : ISODate("2016-05-23T10:42:03.568Z"), "city" : "Cape Town", "@timestamp" : "\"2021-11-20T15:36:00.209Z\"", "firstname" : "Jaques", "personid" : 4005, "lastname" : "Kallis", "@version" : "1" }

How can I avoid duplicates?
Needed Help ASAP

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.