Why is the below query not getting the updated value of some columns even after there is an updated value for couple of columns ? Here is my logstash pipeline.
input {
jdbc {
jdbc_driver_library => "mysql-connector-j-8.0.33.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_connection_string => "abc?useTimezone=true&useLegacyDatetimeCode=false&serverTimezone=UTC"
jdbc_user => "cricket"
jdbc_paging_enabled => true
tracking_column => "process_update_dttm"
use_column_value => true
tracking_column_type => "timestamp"
schedule => "*/5 * * * *"
last_run_metadata_path => "/etc/logstash/java-libs/.logstash_jdbc_last_run_7"
statement_filepath => "/etc/logstash/java-libs/test.sql"
}
}
output {
elasticsearch {
hosts => ["xyx"]
ssl => true
action => "create"
cacert => "CA.pem"
data_stream => true
data_stream_type => "logs"
data_stream_dataset => "something"
data_stream_namespace => "ds"
document_id => "%{process_id}_%{process_update_dttm}"
scripted_upsert => true # This ensures that if the document exists, it is updated with the script
script => '%{[@metadata][script]}' # Executes the script defined in the filter section
script_params => {
"Received" => "%{Received}",
"Sent" => "%{Sent}"
}
}
stdout{ codec => rubydebug }
}