Hello i have a mysql database and i want to ingest it's data into elasticsearch using logstash
after configuring the input and output i am getting the following error:
org/jruby/RubyArray.java:1865:in `each'
/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.3.0/lib/logstash/plugin_mixins/jdbc/common.rb:54:in `load_driver_jars'
/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.3.0/lib/logstash/plugin_mixins/jdbc/common.rb:34:in `load_driver'
/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.3.0/lib/logstash/inputs/jdbc.rb:320:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:410:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:401:in `block in start_input'
[ERROR] 2022-09-05 15:34:09.771 [[main]<jdbc] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
Pipeline_id:main
Plugin: <LogStash::Inputs::Jdbc jdbc_user=>"houssem", tracking_column=>"unix_ts_in_secs", use_column_value=>true, schedule=>"*/5 * * * * *", jdbc_paging_enabled=>true, jdbc_password=><password>, statement=>"SELECT * FROM log_cdr_2022_07_09", tracking_column_type=>"numeric", jdbc_driver_library=>"mysql-connector-java-8.0.30.jar", jdbc_connection_string=>"jdbc:mysql://localhost:3306/logsql", id=>"64b64830b6afe551dca20bfc82cacb431d3c065672d38f53bac902ea2aac02a5", jdbc_driver_class=>"com.mysql.jdbc.Driver", enable_metric=>true, codec=><LogStash::Codecs::Plain id=>"plain_5d2c9893-d646-4d46-b898-ec28b48a336b", enable_metric=>true, charset=>"UTF-8">, jdbc_paging_mode=>"auto", jdbc_page_size=>100000, jdbc_validate_connection=>false, jdbc_validation_timeout=>3600, jdbc_pool_timeout=>5, sql_log_level=>"info", connection_retry_attempts=>1, connection_retry_attempts_wait_time=>0.5, plugin_timezone=>"utc", clean_run=>false, record_last_run=>true, lowercase_column_names=>true, use_prepared_statements=>false>
Error: unable to load mysql-connector-java-8.0.30.jar from :jdbc_driver_library, no such file to load -- mysql-connector-java-8.0.30.jar
Exception: LogStash::PluginLoadingError
Stack: /usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.3.0/lib/logstash/plugin_mixins/jdbc/common.rb:65:in `block in load_driver_jars'
org/jruby/RubyArray.java:1865:in `each'
/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.3.0/lib/logstash/plugin_mixins/jdbc/common.rb:54:in `load_driver_jars'
/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.3.0/lib/logstash/plugin_mixins/jdbc/common.rb:34:in `load_driver'
/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.3.0/lib/logstash/inputs/jdbc.rb:320:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:410:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:401:in `block in start_input'
[ERROR] 2022-09-05 15:34:10.783 [[main]<jdbc] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
Pipeline_id:main
Plugin: <LogStash::Inputs::Jdbc jdbc_user=>"houssem", tracking_column=>"unix_ts_in_secs", use_column_value=>true, schedule=>"*/5 * * * * *", jdbc_paging_enabled=>true, jdbc_password=><password>, statement=>"SELECT * FROM log_cdr_2022_07_09", tracking_column_type=>"numeric", jdbc_driver_library=>"mysql-connector-java-8.0.30.jar", jdbc_connection_string=>"jdbc:mysql://localhost:3306/logsql", id=>"64b64830b6afe551dca20bfc82cacb431d3c065672d38f53bac902ea2aac02a5", jdbc_driver_class=>"com.mysql.jdbc.Driver", enable_metric=>true, codec=><LogStash::Codecs::Plain id=>"plain_5d2c9893-d646-4d46-b898-ec28b48a336b", enable_metric=>true, charset=>"UTF-8">, jdbc_paging_mode=>"auto", jdbc_page_size=>100000, jdbc_validate_connection=>false, jdbc_validation_timeout=>3600, jdbc_pool_timeout=>5, sql_log_level=>"info", connection_retry_attempts=>1, connection_retry_attempts_wait_time=>0.5, plugin_timezone=>"utc", clean_run=>false, record_last_run=>true, lowercase_column_names=>true, use_prepared_statements=>false>
Error: unable to load mysql-connector-java-8.0.30.jar from :jdbc_driver_library, no such file to load -- mysql-connector-java-8.0.30.jar
Exception: LogStash::PluginLoadingError
Stack: /usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.3.0/lib/logstash/plugin_mixins/jdbc/common.rb:65:in `block in load_driver_jars'
org/jruby/RubyArray.java:1865:in `each'
/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.3.0/lib/logstash/plugin_mixins/jdbc/common.rb:54:in `load_driver_jars'
/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.3.0/lib/logstash/plugin_mixins/jdbc/common.rb:34:in `load_driver'
/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.3.0/lib/logstash/inputs/jdbc.rb:320:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:410:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:401:in `block in start_input'
[ERROR] 2022-09-05 15:34:11.798 [[main]<jdbc] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
Pipeline_id:main
Plugin: <LogStash::Inputs::Jdbc jdbc_user=>"houssem", tracking_column=>"unix_ts_in_secs", use_column_value=>true, schedule=>"*/5 * * * * *", jdbc_paging_enabled=>true, jdbc_password=><password>, statement=>"SELECT * FROM log_cdr_2022_07_09", tracking_column_type=>"numeric", jdbc_driver_library=>"mysql-connector-java-8.0.30.jar", jdbc_connection_string=>"jdbc:mysql://localhost:3306/logsql", id=>"64b64830b6afe551dca20bfc82cacb431d3c065672d38f53bac902ea2aac02a5", jdbc_driver_class=>"com.mysql.jdbc.Driver", enable_metric=>true, codec=><LogStash::Codecs::Plain id=>"plain_5d2c9893-d646-4d46-b898-ec28b48a336b", enable_metric=>true, charset=>"UTF-8">, jdbc_paging_mode=>"auto", jdbc_page_size=>100000, jdbc_validate_connection=>false, jdbc_validation_timeout=>3600, jdbc_pool_timeout=>5, sql_log_level=>"info", connection_retry_attempts=>1, connection_retry_attempts_wait_time=>0.5, plugin_timezone=>"utc", clean_run=>false, record_last_run=>true, lowercase_column_names=>true, use_prepared_statements=>false>
Error: unable to load mysql-connector-java-8.0.30.jar from :jdbc_driver_library, no such file to load -- mysql-connector-java-8.0.30.jar
Exception: LogStash::PluginLoadingError
Stack: /usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.3.0/lib/logstash/plugin_mixins/jdbc/common.rb:65:in `block in load_driver_jars'
org/jruby/RubyArray.java:1865:in `each'
/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.3.0/lib/logstash/plugin_mixins/jdbc/common.rb:54:in `load_driver_jars'
/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.3.0/lib/logstash/plugin_mixins/jdbc/common.rb:34:in `load_driver'
/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-integration-jdbc-5.3.0/lib/logstash/inputs/jdbc.rb:320:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:410:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:401:in `block in start_input'
Following is my config file called jdbc.conf:
input {
jdbc {
jdbc_driver_library => "/etc/logstash/usr/share/java/mysql-connector-java-8.0.30.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_connection_string => "jdbc:mysql://localhost:3306/logsql"
jdbc_user => "houssem"
jdbc_password => "Oypoyp@202"
statement => "SELECT * from CONFIG_CUSTOMER"
}
}
output{
file {
path => "/etc/logstash/new_data.txt"
}
}
~
~
and i run this command to start the pipeline:
sudo /usr/share/logstash/bin/logstash -f jdbc.conf