Syncing SQL data with Elastic

Elastic Stack Logstash
1

Hello Everyone,
I am new to elastic and facing issue to keep my sql database in sync with elastic. Here is my logstash config file.

input {
  jdbc {
    jdbc_driver_library => "C:\Users\satyang.sw\Desktop\ELK_Stack\logstash-7.16.2\mysql-connector-java.jar"
    jdbc_driver_class => "com.mysql.jdbc.Driver"
    jdbc_connection_string => "jdbc:mysql://xxx.xxx.xxx.xxx:3306/cpv1_mamcdb_cloud1_15062021?zeroDateTimeBehavior=convertToNull"
    jdbc_user => "xxx"
    jdbc_password => "xxx"
	jdbc_paging_enabled => true
	tracking_column => "last_update_value"
	use_column_value => true
	tracking_column_type => "timestamp"
	schedule => "*/5 * * * * *"
    statement => "SELECT *, UNIX_TIMESTAMP(LastUpdated) AS last_update_value FROM channels WHERE (UNIX_TIMESTAMP(LastUpdated) > :sql_last_value AND LastUpdated < NOW()) ORDER BY LastUpdated ASC"
  }
}

filter {
  mutate {
    copy => { "UID" => "[@metadata][_id]"}
    remove_field => ["UID", "last_update_value"]
  }
}

output {
  stdout { codec =>  "rubydebug"}
  elasticsearch {
    hosts => "http://localhost:9200"
    index => "auto-sync-sql-channels"
	document_id => "%{[@metadata][_id]}"
  }
}

It executes without any error but fails to create docs with desired "_id" and creates like this instead -
image

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.