logstash version:6.4.1
i am trying to pull records from mysql to elastic.
the onetime operation is successful.However, i want that afterwards, only changed or new records should be inserted instead of inserting all again.
// i am using following conf file. with this file all records are inserted again. Also sql_last_value in .logstash.jdbc.last.run is --- 0
input {
jdbc {
jdbc_driver_library => "/home/nitu/learnmysql/mysql-connector-java-5.1.47.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_connection_string => "jdbc:mysql://127.0.0.1:3307/trgdb"
jdbc_user => "root"
jdbc_password => "password"
or jdbc_password_filepath => "/path/to/my/password_file"
schedule => "* * * * *"
statement => "SELECT * from studentlist where datamodified >:sql_last_value"
# statement => "SELECT * from studentlist "
jdbc_paging_enabled => "true"
jdbc_page_size => "50000"
use_column_value => true
tracking_column => datamodified
# tracking_column_type => "timestamp"
schedule => "* * * * *"
}
}
output {
stdout {
codec => rubydebug
}
elasticsearch {
hosts => "localhost:9200"
index => "trgelasticindex"
document_id => "%{studentid}"
}
}