configuration below could not finish if I dont specify a limit clause in mysql statement.
input {
jdbc {
jdbc_driver_library => "mysql-connector-java-5.1.38/mysql-connector-java-5.1.38-bin.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_connection_string => "jdbc:mysql://localhost:3306/data"
jdbc_validate_connection => true
jdbc_user => "root"
jdbc_password => "123456"
jdbc_paging_enabled => "true"
jdbc_page_size => "5000"
schedule => "* * * * *"
statement => "select * from recommend_refresh_data4 where time > 20151215"
}
}
filter {
mutate {
replace => {"@timestamp" => "%{time}"}
remove_field => [time]
convert => { "newsScore" => "float" }
}
}
output {
stdout { codec => json_lines }
elasticsearch {
index => "recommend_data"
document_type => "bi"
document_id => "%{id}"
hosts => ["10.2.4.126:9200"]
}
}
after start logstash, it is stucked,
Settings: Default filter workers: 2
Logstash startup completed
no record is fetched from mysql. After I press Ctrl + c, it gives warnings:
{:level=>:warn, "INFLIGHT_EVENT_COUNT"=>{"total"=>0}, "STALLING_THREADS"=>{["LogStash::Inputs::Jdbc", {"jdbc_driver_library"=>"mysql-connector-java-5.1.38/mysql-connector-java-5.1.38-bin.jar", "jdbc_driver_class"=>"com.mysql.jdbc.Driver", "jdbc_connection_string"=>"jdbc:mysql://localhost:3306/data", "jdbc_validate_connection"=>"true", "jdbc_user"=>"root", "jdbc_password"=>"123456", "jdbc_paging_enabled"=>"true", "jdbc_page_size"=>"5000", "statement"=>"select * from recommend_refresh_data4 where time > 20151215"}]=>[{"thread_id"=>17, "name"=>"<jdbc", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/sequel-4.29.0/lib/sequel/adapters/jdbc.rb:251:in `execute'"}]}}
seems like the pagination is not working. Tried to set jdbc_fetch_size , no luck.